mirror/libbpf
1
0
Fork 0
mirror of https://github.com/netdata/libbpf.git synced 2026-03-24 10:19:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'libbpf:master' into master

Browse Source
This commit is contained in:
thiagoftsm
2022-05-17 02:16:02 +00:00
committed by GitHub
parent 74c16e9a0c 86eb09863c
commit e4f2e6e865
42 changed files with 84605 additions and 82066 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161138
.github/actions/build-selftests/vmlinux.h vendored
View File

File diff suppressed because it is too large Load Diff

5
.readthedocs.yaml
View File

@@ -10,6 +10,11 @@ sphinx:
builder: html
configuration: docs/conf.py
formats:
- htmlzip
- pdf
- epub
# Optionally set the version of Python and requirements required to build your docs
python:
version: 3.7

2
BPF-CHECKPOINT-COMMIT
View File

@@ -1 +1 @@
fe68195daf34d5dddacd3f93dd3eafc4beca3a0e
f3f19f939c11925dadd3f4776f99f8c278a7017b

2
CHECKPOINT-COMMIT
View File

@@ -1 +1 @@
dc37dc617fabfb1c3a16d49f5d8cc20e9e3608ca
ac6a65868a5a45db49d5ee8524df3b701110d844

2
LICENSE.BSD-2-Clause
View File

@@ -7,7 +7,7 @@ Usage-Guide:
SPDX-License-Identifier: BSD-2-Clause
License-Text:
Copyright (c) <year> <owner> . All rights reserved.
Copyright (c) 2015 The Libbpf Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

58
README.md
View File

@@ -73,34 +73,6 @@ $ cd src
$ PKG_CONFIG_PATH=/build/root/lib64/pkgconfig DESTDIR=/build/root make install
```
Distributions
=============
Distributions packaging libbpf from this mirror:
- [Fedora](https://src.fedoraproject.org/rpms/libbpf)
- [Gentoo](https://packages.gentoo.org/packages/dev-libs/libbpf)
- [Debian](https://packages.debian.org/source/sid/libbpf)
- [Arch](https://www.archlinux.org/packages/extra/x86_64/libbpf/)
- [Ubuntu](https://packages.ubuntu.com/source/impish/libbpf)
- [Alpine](https://pkgs.alpinelinux.org/packages?name=libbpf)
Benefits of packaging from the mirror over packaging from kernel sources:
- Consistent versioning across distributions.
- No ties to any specific kernel, transparent handling of older kernels.
Libbpf is designed to be kernel-agnostic and work across multitude of
kernel versions. It has built-in mechanisms to gracefully handle older
kernels, that are missing some of the features, by working around or
gracefully degrading functionality. Thus libbpf is not tied to a specific
kernel version and can/should be packaged and versioned independently.
- Continuous integration testing via
[TravisCI](https://travis-ci.org/libbpf/libbpf).
- Static code analysis via [LGTM](https://lgtm.com/projects/g/libbpf/libbpf)
and [Coverity](https://scan.coverity.com/projects/libbpf).
Package dependencies of libbpf, package names may vary across distros:
- zlib
- libelf
BPF CO-RE (Compile Once – Run Everywhere)
=========================================
@@ -154,6 +126,36 @@ use it:
converting some more to both contribute to the BPF community and gain some
more experience with it.
Distributions
=============
Distributions packaging libbpf from this mirror:
- [Fedora](https://src.fedoraproject.org/rpms/libbpf)
- [Gentoo](https://packages.gentoo.org/packages/dev-libs/libbpf)
- [Debian](https://packages.debian.org/source/sid/libbpf)
- [Arch](https://www.archlinux.org/packages/extra/x86_64/libbpf/)
- [Ubuntu](https://packages.ubuntu.com/source/impish/libbpf)
- [Alpine](https://pkgs.alpinelinux.org/packages?name=libbpf)
Benefits of packaging from the mirror over packaging from kernel sources:
- Consistent versioning across distributions.
- No ties to any specific kernel, transparent handling of older kernels.
Libbpf is designed to be kernel-agnostic and work across multitude of
kernel versions. It has built-in mechanisms to gracefully handle older
kernels, that are missing some of the features, by working around or
gracefully degrading functionality. Thus libbpf is not tied to a specific
kernel version and can/should be packaged and versioned independently.
- Continuous integration testing via
[TravisCI](https://travis-ci.org/libbpf/libbpf).
- Static code analysis via [LGTM](https://lgtm.com/projects/g/libbpf/libbpf)
and [Coverity](https://scan.coverity.com/projects/libbpf).
Package dependencies of libbpf, package names may vary across distros:
- zlib
- libelf
[![libbpf distro packaging status](https://repology.org/badge/vertical-allrepos/libbpf.svg)](https://repology.org/project/libbpf/versions)
License
=======

3
docs/index.rst
View File

@@ -6,14 +6,13 @@ libbpf
.. toctree::
:maxdepth: 1
API Documentation <https://libbpf.readthedocs.io/en/latest/api.html>
libbpf_naming_convention
libbpf_build
This is documentation for libbpf, a userspace library for loading and
interacting with bpf programs.
For API documentation see the `versioned API documentation site <https://libbpf.readthedocs.io/en/latest/api.html>`_.
All general BPF questions, including kernel functionality, libbpf APIs and
their application, should be sent to bpf@vger.kernel.org mailing list.
You can `subscribe <http://vger.kernel.org/vger-lists.html#bpf>`_ to the

121
include/uapi/linux/bpf.h
View File

@@ -997,6 +997,7 @@ enum bpf_attach_type {
BPF_SK_REUSEPORT_SELECT,
BPF_SK_REUSEPORT_SELECT_OR_MIGRATE,
BPF_PERF_EVENT,
BPF_TRACE_KPROBE_MULTI,
__MAX_BPF_ATTACH_TYPE
};
@@ -1011,6 +1012,8 @@ enum bpf_link_type {
BPF_LINK_TYPE_NETNS = 5,
BPF_LINK_TYPE_XDP = 6,
BPF_LINK_TYPE_PERF_EVENT = 7,
BPF_LINK_TYPE_KPROBE_MULTI = 8,
BPF_LINK_TYPE_STRUCT_OPS = 9,
MAX_BPF_LINK_TYPE,
};
@@ -1118,6 +1121,11 @@ enum bpf_link_type {
*/
#define BPF_F_XDP_HAS_FRAGS (1U << 5)
/* link_create.kprobe_multi.flags used in LINK_CREATE command for
* BPF_TRACE_KPROBE_MULTI attach type to create return probe.
*/
#define BPF_F_KPROBE_MULTI_RETURN (1U << 0)
/* When BPF ldimm64's insn[0].src_reg != 0 then this can have
* the following extensions:
*
@@ -1232,6 +1240,8 @@ enum {
/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
/* If set, XDP frames will be transmitted after processing */
#define BPF_F_TEST_XDP_LIVE_FRAMES (1U << 1)
/* type for BPF_ENABLE_STATS */
enum bpf_stats_type {
@@ -1393,6 +1403,7 @@ union bpf_attr {
__aligned_u64 ctx_out;
__u32 flags;
__u32 cpu;
__u32 batch_size;
} test;
struct { /* anonymous struct used by BPF_*_GET_*_ID */
@@ -1472,6 +1483,22 @@ union bpf_attr {
*/
__u64 bpf_cookie;
} perf_event;
struct {
__u32 flags;
__u32 cnt;
__aligned_u64 syms;
__aligned_u64 addrs;
__aligned_u64 cookies;
} kprobe_multi;
struct {
/* this is overlaid with the target_btf_id above. */
__u32 target_btf_id;
/* black box user-provided value passed through
* to BPF program at the execution time and
* accessible through bpf_get_attach_cookie() BPF helper
*/
__u64 cookie;
} tracing;
};
} link_create;
@@ -2299,8 +2326,8 @@ union bpf_attr {
* Return
* The return value depends on the result of the test, and can be:
*
* * 0, if current task belongs to the cgroup2.
* * 1, if current task does not belong to the cgroup2.
* * 1, if current task belongs to the cgroup2.
* * 0, if current task does not belong to the cgroup2.
* * A negative error code, if an error occurred.
*
* long bpf_skb_change_tail(struct sk_buff *skb, u32 len, u64 flags)
@@ -2992,8 +3019,8 @@ union bpf_attr {
*
* # sysctl kernel.perf_event_max_stack=<new value>
* Return
* A non-negative value equal to or less than *size* on success,
* or a negative error in case of failure.
* The non-negative copied *buf* length equal to or less than
* *size* on success, or a negative error in case of failure.
*
* long bpf_skb_load_bytes_relative(const void *skb, u32 offset, void *to, u32 len, u32 start_header)
* Description
@@ -4299,8 +4326,8 @@ union bpf_attr {
*
* # sysctl kernel.perf_event_max_stack=<new value>
* Return
* A non-negative value equal to or less than *size* on success,
* or a negative error in case of failure.
* The non-negative copied *buf* length equal to or less than
* *size* on success, or a negative error in case of failure.
*
* long bpf_load_hdr_opt(struct bpf_sock_ops *skops, void *searchby_res, u32 len, u64 flags)
* Description
@@ -5086,6 +5113,65 @@ union bpf_attr {
* Return
* 0 on success, or a negative error in case of failure. On error
* *dst* buffer is zeroed out.
*
* long bpf_skb_set_tstamp(struct sk_buff *skb, u64 tstamp, u32 tstamp_type)
* Description
* Change the __sk_buff->tstamp_type to *tstamp_type*
* and set *tstamp* to the __sk_buff->tstamp together.
*
* If there is no need to change the __sk_buff->tstamp_type,
* the tstamp value can be directly written to __sk_buff->tstamp
* instead.
*
* BPF_SKB_TSTAMP_DELIVERY_MONO is the only tstamp that
* will be kept during bpf_redirect_*(). A non zero
* *tstamp* must be used with the BPF_SKB_TSTAMP_DELIVERY_MONO
* *tstamp_type*.
*
* A BPF_SKB_TSTAMP_UNSPEC *tstamp_type* can only be used
* with a zero *tstamp*.
*
* Only IPv4 and IPv6 skb->protocol are supported.
*
* This function is most useful when it needs to set a
* mono delivery time to __sk_buff->tstamp and then
* bpf_redirect_*() to the egress of an iface. For example,
* changing the (rcv) timestamp in __sk_buff->tstamp at
* ingress to a mono delivery time and then bpf_redirect_*()
* to sch_fq@phy-dev.
* Return
* 0 on success.
* **-EINVAL** for invalid input
* **-EOPNOTSUPP** for unsupported protocol
*
* long bpf_ima_file_hash(struct file *file, void *dst, u32 size)
* Description
* Returns a calculated IMA hash of the *file*.
* If the hash is larger than *size*, then only *size*
* bytes will be copied to *dst*
* Return
* The **hash_algo** is returned on success,
* **-EOPNOTSUP** if the hash calculation failed or **-EINVAL** if
* invalid arguments are passed.
*
* void *bpf_kptr_xchg(void *map_value, void *ptr)
* Description
* Exchange kptr at pointer *map_value* with *ptr*, and return the
* old value. *ptr* can be NULL, otherwise it must be a referenced
* pointer which will be released when this helper is called.
* Return
* The old value of kptr (which can be NULL). The returned pointer
* if not NULL, is a reference which must be released using its
* corresponding release function, or moved into a BPF map before
* program exit.
*
* void *bpf_map_lookup_percpu_elem(struct bpf_map *map, const void *key, u32 cpu)
* Description
* Perform a lookup in *percpu map* for an entry associated to
* *key* on *cpu*.
* Return
* Map value associated to *key* on *cpu*, or **NULL** if no entry
* was found or *cpu* is invalid.
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@@ -5280,6 +5366,10 @@ union bpf_attr {
FN(xdp_load_bytes), \
FN(xdp_store_bytes), \
FN(copy_from_user_task), \
FN(skb_set_tstamp), \
FN(ima_file_hash), \
FN(kptr_xchg), \
FN(map_lookup_percpu_elem), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5469,6 +5559,15 @@ union { \
__u64 :64; \
} __attribute__((aligned(8)))
enum {
BPF_SKB_TSTAMP_UNSPEC,
BPF_SKB_TSTAMP_DELIVERY_MONO, /* tstamp has mono delivery time */
/* For any BPF_SKB_TSTAMP_* that the bpf prog cannot handle,
* the bpf prog should handle it like BPF_SKB_TSTAMP_UNSPEC
* and try to deduce it by ingress, egress or skb->sk->sk_clockid.
*/
};
/* user accessible mirror of in-kernel sk_buff.
* new fields can only be added to the end of this structure
*/
@@ -5509,7 +5608,8 @@ struct __sk_buff {
__u32 gso_segs;
__bpf_md_ptr(struct bpf_sock *, sk);
__u32 gso_size;
__u32 :32; /* Padding, future use. */
__u8 tstamp_type;
__u32 :24; /* Padding, future use. */
__u64 hwtstamp;
};
@@ -5523,6 +5623,10 @@ struct bpf_tunnel_key {
__u8 tunnel_ttl;
__u16 tunnel_ext; /* Padding, future use. */
__u32 tunnel_label;
union {
__u32 local_ipv4;
__u32 local_ipv6[4];
};
};
/* user accessible mirror of in-kernel xfrm_state.
@@ -6453,7 +6557,8 @@ struct bpf_sk_lookup {
__u32 protocol; /* IP protocol (IPPROTO_TCP, IPPROTO_UDP) */
__u32 remote_ip4; /* Network byte order */
__u32 remote_ip6[4]; /* Network byte order */
__u32 remote_port; /* Network byte order */
__be16 remote_port; /* Network byte order */
__u16 :16; /* Zero padding */
__u32 local_ip4; /* Network byte order */
__u32 local_ip6[4]; /* Network byte order */
__u32 local_port; /* Host byte order */

4
include/uapi/linux/btf.h
View File

@@ -33,8 +33,8 @@ struct btf_type {
/* "info" bits arrangement
* bits 0-15: vlen (e.g. # of struct's members)
* bits 16-23: unused
* bits 24-27: kind (e.g. int, ptr, array...etc)
* bits 28-30: unused
* bits 24-28: kind (e.g. int, ptr, array...etc)
* bits 29-30: unused
* bit 31: kind_flag, currently used by
* struct, union and fwd
*/

1
include/uapi/linux/if_link.h
View File

@@ -860,6 +860,7 @@ enum {
IFLA_BOND_PEER_NOTIF_DELAY,
IFLA_BOND_AD_LACP_ACTIVE,
IFLA_BOND_MISSED_MAX,
IFLA_BOND_NS_IP6_TARGET,
__IFLA_BOND_MAX,
};

4
include/uapi/linux/perf_event.h
View File

@@ -251,6 +251,8 @@ enum {
PERF_BR_SYSRET = 8, /* syscall return */
PERF_BR_COND_CALL = 9, /* conditional function call */
PERF_BR_COND_RET = 10, /* conditional function return */
PERF_BR_ERET = 11, /* exception return */
PERF_BR_IRQ = 12, /* irq */
PERF_BR_MAX,
};
@@ -465,6 +467,8 @@ struct perf_event_attr {
/*
* User provided data if sigtrap=1, passed back to user via
* siginfo_t::si_perf_data, e.g. to permit user to identify the event.
* Note, siginfo_t::si_perf_data is long-sized, and sig_data will be
* truncated accordingly on 32 bit architectures.
*/
__u64 sig_data;
};

25
scripts/build-fuzzers.sh
View File

@@ -17,6 +17,24 @@ mkdir -p "$OUT"
export LIB_FUZZING_ENGINE=${LIB_FUZZING_ENGINE:--fsanitize=fuzzer}
# libelf is compiled with _FORTIFY_SOURCE by default and it
# isn't compatible with MSan. It was borrowed
# from https://github.com/google/oss-fuzz/pull/7422
if [[ "$SANITIZER" == memory ]]; then
CFLAGS+=" -U_FORTIFY_SOURCE"
CXXFLAGS+=" -U_FORTIFY_SOURCE"
fi
# The alignment check is turned off by default on OSS-Fuzz/CFLite so it should be
# turned on explicitly there. It was borrowed from
# https://github.com/google/oss-fuzz/pull/7092
if [[ "$SANITIZER" == undefined ]]; then
additional_ubsan_checks=alignment
UBSAN_FLAGS="-fsanitize=$additional_ubsan_checks -fno-sanitize-recover=$additional_ubsan_checks"
CFLAGS+=" $UBSAN_FLAGS"
CXXFLAGS+=" $UBSAN_FLAGS"
fi
# Ideally libbelf should be built using release tarballs available
# at https://sourceware.org/elfutils/ftp/. Unfortunately sometimes they
# fail to compile (for example, elfutils-0.185 fails to compile with LDFLAGS enabled
@@ -26,7 +44,7 @@ rm -rf elfutils
git clone git://sourceware.org/git/elfutils.git
(
cd elfutils
git checkout 983e86fd89e8bf02f2d27ba5dce5bf078af4ceda
git checkout 83251d4091241acddbdcf16f814e3bc6ef3df49a
git log --oneline -1
# ASan isn't compatible with -Wl,--no-undefined: https://github.com/google/sanitizers/issues/380
@@ -36,6 +54,11 @@ find -name Makefile.am | xargs sed -i 's/,--no-undefined//'
# https://clang.llvm.org/docs/AddressSanitizer.html#usage
sed -i 's/^\(ZDEFS_LDFLAGS=\).*/\1/' configure.ac
if [[ "$SANITIZER" == undefined ]]; then
# That's basicaly what --enable-sanitize-undefined does to turn off unaligned access
# elfutils heavily relies on on i386/x86_64 but without changing compiler flags along the way
sed -i 's/\(check_undefined_val\)=[0-9]/\1=1/' configure.ac
fi
autoreconf -i -f
if ! ./configure --enable-maintainer-mode --disable-debuginfod --disable-libdebuginfod \

24
src/Makefile
View File

@@ -8,10 +8,15 @@ else
msg = @printf ' %-8s %s%s\n' "$(1)" "$(2)" "$(if $(3), $(3))";
endif
LIBBPF_VERSION := $(shell \
grep -oE '^LIBBPF_([0-9.]+)' libbpf.map | \
sort -rV | head -n1 | cut -d'_' -f2)
LIBBPF_MAJOR_VERSION := $(firstword $(subst ., ,$(LIBBPF_VERSION)))
LIBBPF_MAJOR_VERSION := 0
LIBBPF_MINOR_VERSION := 8
LIBBPF_PATCH_VERSION := 0
LIBBPF_VERSION := $(LIBBPF_MAJOR_VERSION).$(LIBBPF_MINOR_VERSION).$(LIBBPF_PATCH_VERSION)
LIBBPF_MAJMIN_VERSION := $(LIBBPF_MAJOR_VERSION).$(LIBBPF_MINOR_VERSION).0
LIBBPF_MAP_VERSION := $(shell grep -oE '^LIBBPF_([0-9.]+)' libbpf.map | sort -rV | head -n1 | cut -d'_' -f2)
ifneq ($(LIBBPF_MAJMIN_VERSION), $(LIBBPF_MAP_VERSION))
$(error Libbpf release ($(LIBBPF_VERSION)) and map ($(LIBBPF_MAP_VERSION)) versions are out of sync!)
endif
TOPDIR = ..
@@ -37,7 +42,7 @@ STATIC_OBJDIR := $(OBJDIR)/staticobjs
OBJS := bpf.o btf.o libbpf.o libbpf_errno.o netlink.o \
nlattr.o str_error.o libbpf_probes.o bpf_prog_linfo.o xsk.o \
btf_dump.o hashmap.o ringbuf.o strset.o linker.o gen_loader.o \
relo_core.o
relo_core.o usdt.o
SHARED_OBJS := $(addprefix $(SHARED_OBJDIR)/,$(OBJS))
STATIC_OBJS := $(addprefix $(STATIC_OBJDIR)/,$(OBJS))
@@ -51,7 +56,8 @@ endif
HEADERS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \
bpf_helpers.h bpf_helper_defs.h bpf_tracing.h \
bpf_endian.h bpf_core_read.h skel_internal.h libbpf_version.h
bpf_endian.h bpf_core_read.h skel_internal.h libbpf_version.h \
usdt.bpf.h
UAPI_HEADERS := $(addprefix $(TOPDIR)/include/uapi/linux/,\
bpf.h bpf_common.h btf.h)
@@ -99,7 +105,7 @@ $(OBJDIR)/libbpf.so.$(LIBBPF_VERSION): $(SHARED_OBJS)
-Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \
$^ $(ALL_LDFLAGS) -o $@
$(OBJDIR)/libbpf.pc:
$(OBJDIR)/libbpf.pc: force
$(Q)sed -e "s|@PREFIX@|$(PREFIX)|" \
-e "s|@LIBDIR@|$(LIBDIR_PC)|" \
-e "s|@VERSION@|$(LIBBPF_VERSION)|" \
@@ -152,7 +158,7 @@ clean:
$(call msg,CLEAN)
$(Q)rm -rf *.o *.a *.so *.so.* *.pc $(SHARED_OBJDIR) $(STATIC_OBJDIR)
.PHONY: cscope tags
.PHONY: cscope tags force
cscope:
$(call msg,CSCOPE)
$(Q)ls *.c *.h > cscope.files
@@ -162,3 +168,5 @@ tags:
$(call msg,CTAGS)
$(Q)rm -f TAGS tags
$(Q)ls *.c *.h | xargs $(TAGS_PROG) -a
force:

69
src/bpf.c
View File

@@ -29,6 +29,7 @@
#include <errno.h>
#include <linux/bpf.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <limits.h>
#include <sys/resource.h>
#include "bpf.h"
@@ -111,7 +112,7 @@ int probe_memcg_account(void)
BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns),
BPF_EXIT_INSN(),
};
size_t insn_cnt = sizeof(insns) / sizeof(insns[0]);
size_t insn_cnt = ARRAY_SIZE(insns);
union bpf_attr attr;
int prog_fd;
@@ -638,6 +639,20 @@ int bpf_map_delete_elem(int fd, const void *key)
return libbpf_err_errno(ret);
}
int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags)
{
union bpf_attr attr;
int ret;
memset(&attr, 0, sizeof(attr));
attr.map_fd = fd;
attr.key = ptr_to_u64(key);
attr.flags = flags;
ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
return libbpf_err_errno(ret);
}
int bpf_map_get_next_key(int fd, const void *key, void *next_key)
{
union bpf_attr attr;
@@ -816,7 +831,7 @@ int bpf_link_create(int prog_fd, int target_fd,
{
__u32 target_btf_id, iter_info_len;
union bpf_attr attr;
int fd;
int fd, err;
if (!OPTS_VALID(opts, bpf_link_create_opts))
return libbpf_err(-EINVAL);
@@ -853,6 +868,23 @@ int bpf_link_create(int prog_fd, int target_fd,
if (!OPTS_ZEROED(opts, perf_event))
return libbpf_err(-EINVAL);
break;
case BPF_TRACE_KPROBE_MULTI:
attr.link_create.kprobe_multi.flags = OPTS_GET(opts, kprobe_multi.flags, 0);
attr.link_create.kprobe_multi.cnt = OPTS_GET(opts, kprobe_multi.cnt, 0);
attr.link_create.kprobe_multi.syms = ptr_to_u64(OPTS_GET(opts, kprobe_multi.syms, 0));
attr.link_create.kprobe_multi.addrs = ptr_to_u64(OPTS_GET(opts, kprobe_multi.addrs, 0));
attr.link_create.kprobe_multi.cookies = ptr_to_u64(OPTS_GET(opts, kprobe_multi.cookies, 0));
if (!OPTS_ZEROED(opts, kprobe_multi))
return libbpf_err(-EINVAL);
break;
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
case BPF_LSM_MAC:
attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0);
if (!OPTS_ZEROED(opts, tracing))
return libbpf_err(-EINVAL);
break;
default:
if (!OPTS_ZEROED(opts, flags))
return libbpf_err(-EINVAL);
@@ -860,7 +892,37 @@ int bpf_link_create(int prog_fd, int target_fd,
}
proceed:
fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, sizeof(attr));
return libbpf_err_errno(fd);
if (fd >= 0)
return fd;
/* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry
* and other similar programs
*/
err = -errno;
if (err != -EINVAL)
return libbpf_err(err);
/* if user used features not supported by
* BPF_RAW_TRACEPOINT_OPEN command, then just give up immediately
*/
if (attr.link_create.target_fd || attr.link_create.target_btf_id)
return libbpf_err(err);
if (!OPTS_ZEROED(opts, sz))
return libbpf_err(err);
/* otherwise, for few select kinds of programs that can be
* attached using BPF_RAW_TRACEPOINT_OPEN command, try that as
* a fallback for older kernels
*/
switch (attach_type) {
case BPF_TRACE_RAW_TP:
case BPF_LSM_MAC:
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
case BPF_MODIFY_RETURN:
return bpf_raw_tracepoint_open(NULL, prog_fd);
default:
return libbpf_err(err);
}
}
int bpf_link_detach(int link_fd)
@@ -994,6 +1056,7 @@ int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
memset(&attr, 0, sizeof(attr));
attr.test.prog_fd = prog_fd;
attr.test.batch_size = OPTS_GET(opts, batch_size, 0);
attr.test.cpu = OPTS_GET(opts, cpu, 0);
attr.test.flags = OPTS_GET(opts, flags, 0);
attr.test.repeat = OPTS_GET(opts, repeat, 0);

16
src/bpf.h
View File

@@ -244,6 +244,7 @@ LIBBPF_API int bpf_map_lookup_and_delete_elem(int fd, const void *key,
LIBBPF_API int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key,
void *value, __u64 flags);
LIBBPF_API int bpf_map_delete_elem(int fd, const void *key);
LIBBPF_API int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags);
LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key);
LIBBPF_API int bpf_map_freeze(int fd);
@@ -413,10 +414,20 @@ struct bpf_link_create_opts {
struct {
__u64 bpf_cookie;
} perf_event;
struct {
__u32 flags;
__u32 cnt;
const char **syms;
const unsigned long *addrs;
const __u64 *cookies;
} kprobe_multi;
struct {
__u64 cookie;
} tracing;
};
size_t :0;
};
#define bpf_link_create_opts__last_field perf_event
#define bpf_link_create_opts__last_field kprobe_multi.cookies
LIBBPF_API int bpf_link_create(int prog_fd, int target_fd,
enum bpf_attach_type attach_type,
@@ -512,8 +523,9 @@ struct bpf_test_run_opts {
__u32 duration; /* out: average per repetition in ns */
__u32 flags;
__u32 cpu;
__u32 batch_size;
};
#define bpf_test_run_opts__last_field cpu
#define bpf_test_run_opts__last_field batch_size
LIBBPF_API int bpf_prog_test_run_opts(int prog_fd,
struct bpf_test_run_opts *opts);

37
src/bpf_core_read.h
View File

@@ -110,21 +110,50 @@ enum bpf_enum_value_kind {
val; \
})
#define ___bpf_field_ref1(field) (field)
#define ___bpf_field_ref2(type, field) (((typeof(type) *)0)->field)
#define ___bpf_field_ref(args...) \
___bpf_apply(___bpf_field_ref, ___bpf_narg(args))(args)
/*
* Convenience macro to check that field actually exists in target kernel's.
* Returns:
* 1, if matching field is present in target kernel;
* 0, if no matching field found.
*
* Supports two forms:
* - field reference through variable access:
* bpf_core_field_exists(p->my_field);
* - field reference through type and field names:
* bpf_core_field_exists(struct my_type, my_field).
*/
#define bpf_core_field_exists(field) \
__builtin_preserve_field_info(field, BPF_FIELD_EXISTS)
#define bpf_core_field_exists(field...) \
__builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_EXISTS)
/*
* Convenience macro to get the byte size of a field. Works for integers,
* struct/unions, pointers, arrays, and enums.
*
* Supports two forms:
* - field reference through variable access:
* bpf_core_field_size(p->my_field);
* - field reference through type and field names:
* bpf_core_field_size(struct my_type, my_field).
*/
#define bpf_core_field_size(field) \
__builtin_preserve_field_info(field, BPF_FIELD_BYTE_SIZE)
#define bpf_core_field_size(field...) \
__builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_SIZE)
/*
* Convenience macro to get field's byte offset.
*
* Supports two forms:
* - field reference through variable access:
* bpf_core_field_offset(p->my_field);
* - field reference through type and field names:
* bpf_core_field_offset(struct my_type, my_field).
*/
#define bpf_core_field_offset(field...) \
__builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_OFFSET)
/*
* Convenience macro to get BTF type ID of a specified type, using a local BTF

87
src/bpf_helper_defs.h
View File

@@ -961,8 +961,8 @@ static long (*bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (vo
* Returns
* The return value depends on the result of the test, and can be:
*
* * 0, if current task belongs to the cgroup2.
* * 1, if current task does not belong to the cgroup2.
* * 1, if current task belongs to the cgroup2.
* * 0, if current task does not belong to the cgroup2.
* * A negative error code, if an error occurred.
*/
static long (*bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37;
@@ -1752,8 +1752,8 @@ static long (*bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct
* # sysctl kernel.perf_event_max_stack=<new value>
*
* Returns
* A non-negative value equal to or less than *size* on success,
* or a negative error in case of failure.
* The non-negative copied *buf* length equal to or less than
* *size* on success, or a negative error in case of failure.
*/
static long (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67;
@@ -3305,8 +3305,8 @@ static struct udp6_sock *(*bpf_skc_to_udp6_sock)(void *sk) = (void *) 140;
* # sysctl kernel.perf_event_max_stack=<new value>
*
* Returns
* A non-negative value equal to or less than *size* on success,
* or a negative error in case of failure.
* The non-negative copied *buf* length equal to or less than
* *size* on success, or a negative error in case of failure.
*/
static long (*bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141;
@@ -4295,4 +4295,79 @@ static long (*bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, void *bu
*/
static long (*bpf_copy_from_user_task)(void *dst, __u32 size, const void *user_ptr, struct task_struct *tsk, __u64 flags) = (void *) 191;
/*
* bpf_skb_set_tstamp
*
* Change the __sk_buff->tstamp_type to *tstamp_type*
* and set *tstamp* to the __sk_buff->tstamp together.
*
* If there is no need to change the __sk_buff->tstamp_type,
* the tstamp value can be directly written to __sk_buff->tstamp
* instead.
*
* BPF_SKB_TSTAMP_DELIVERY_MONO is the only tstamp that
* will be kept during bpf_redirect_*(). A non zero
* *tstamp* must be used with the BPF_SKB_TSTAMP_DELIVERY_MONO
* *tstamp_type*.
*
* A BPF_SKB_TSTAMP_UNSPEC *tstamp_type* can only be used
* with a zero *tstamp*.
*
* Only IPv4 and IPv6 skb->protocol are supported.
*
* This function is most useful when it needs to set a
* mono delivery time to __sk_buff->tstamp and then
* bpf_redirect_*() to the egress of an iface. For example,
* changing the (rcv) timestamp in __sk_buff->tstamp at
* ingress to a mono delivery time and then bpf_redirect_*()
* to sch_fq@phy-dev.
*
* Returns
* 0 on success.
* **-EINVAL** for invalid input
* **-EOPNOTSUPP** for unsupported protocol
*/
static long (*bpf_skb_set_tstamp)(struct __sk_buff *skb, __u64 tstamp, __u32 tstamp_type) = (void *) 192;
/*
* bpf_ima_file_hash
*
* Returns a calculated IMA hash of the *file*.
* If the hash is larger than *size*, then only *size*
* bytes will be copied to *dst*
*
* Returns
* The **hash_algo** is returned on success,
* **-EOPNOTSUP** if the hash calculation failed or **-EINVAL** if
* invalid arguments are passed.
*/
static long (*bpf_ima_file_hash)(struct file *file, void *dst, __u32 size) = (void *) 193;
/*
* bpf_kptr_xchg
*
* Exchange kptr at pointer *map_value* with *ptr*, and return the
* old value. *ptr* can be NULL, otherwise it must be a referenced
* pointer which will be released when this helper is called.
*
* Returns
* The old value of kptr (which can be NULL). The returned pointer
* if not NULL, is a reference which must be released using its
* corresponding release function, or moved into a BPF map before
* program exit.
*/
static void *(*bpf_kptr_xchg)(void *map_value, void *ptr) = (void *) 194;
/*
* bpf_map_lookup_percpu_elem
*
* Perform a lookup in *percpu map* for an entry associated to
* *key* on *cpu*.
*
* Returns
* Map value associated to *key* on *cpu*, or **NULL** if no entry
* was found or *cpu* is invalid.
*/
static void *(*bpf_map_lookup_percpu_elem)(void *map, const void *key, __u32 cpu) = (void *) 195;

26
src/bpf_helpers.h
View File

@@ -75,6 +75,30 @@
})
#endif
/*
* Compiler (optimization) barrier.
*/
#ifndef barrier
#define barrier() asm volatile("" ::: "memory")
#endif
/* Variable-specific compiler (optimization) barrier. It's a no-op which makes
* compiler believe that there is some black box modification of a given
* variable and thus prevents compiler from making extra assumption about its
* value and potential simplifications and optimizations on this variable.
*
* E.g., compiler might often delay or even omit 32-bit to 64-bit casting of
* a variable, making some code patterns unverifiable. Putting barrier_var()
* in place will ensure that cast is performed before the barrier_var()
* invocation, because compiler has to pessimistically assume that embedded
* asm section might perform some extra operations on that variable.
*
* This is a variable-specific variant of more global barrier().
*/
#ifndef barrier_var
#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var))
#endif
/*
* Helper macro to throw a compilation error if __bpf_unreachable() gets
* built into the resulting code. This works given BPF back end does not
@@ -149,6 +173,8 @@ enum libbpf_tristate {
#define __kconfig __attribute__((section(".kconfig")))
#define __ksym __attribute__((section(".ksyms")))
#define __kptr __attribute__((btf_type_tag("kptr")))
#define __kptr_ref __attribute__((btf_type_tag("kptr_ref")))
#ifndef ___bpf_concat
#define ___bpf_concat(a, b) a ## b

23
src/bpf_tracing.h
View File

@@ -27,6 +27,9 @@
#elif defined(__TARGET_ARCH_riscv)
#define bpf_target_riscv
#define bpf_target_defined
#elif defined(__TARGET_ARCH_arc)
#define bpf_target_arc
#define bpf_target_defined
#else
/* Fall back to what the compiler says */
@@ -54,6 +57,9 @@
#elif defined(__riscv) && __riscv_xlen == 64
#define bpf_target_riscv
#define bpf_target_defined
#elif defined(__arc__)
#define bpf_target_arc
#define bpf_target_defined
#endif /* no compiler target */
#endif
@@ -233,6 +239,23 @@ struct pt_regs___arm64 {
/* riscv does not select ARCH_HAS_SYSCALL_WRAPPER. */
#define PT_REGS_SYSCALL_REGS(ctx) ctx
#elif defined(bpf_target_arc)
/* arc provides struct user_pt_regs instead of struct pt_regs to userspace */
#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x))
#define __PT_PARM1_REG scratch.r0
#define __PT_PARM2_REG scratch.r1
#define __PT_PARM3_REG scratch.r2
#define __PT_PARM4_REG scratch.r3
#define __PT_PARM5_REG scratch.r4
#define __PT_RET_REG scratch.blink
#define __PT_FP_REG __unsupported__
#define __PT_RC_REG scratch.r0
#define __PT_SP_REG scratch.sp
#define __PT_IP_REG scratch.ret
/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */
#define PT_REGS_SYSCALL_REGS(ctx) ctx
#endif
#if defined(bpf_target_defined)

15
src/btf.c
View File

@@ -2626,6 +2626,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext,
const struct btf_ext_info_sec *sinfo;
struct btf_ext_info *ext_info;
__u32 info_left, record_size;
size_t sec_cnt = 0;
/* The start of the info sec (including the __u32 record_size). */
void *info;
@@ -2689,8 +2690,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext,
return -EINVAL;
}
total_record_size = sec_hdrlen +
(__u64)num_records * record_size;
total_record_size = sec_hdrlen + (__u64)num_records * record_size;
if (info_left < total_record_size) {
pr_debug("%s section has incorrect num_records in .BTF.ext\n",
ext_sec->desc);
@@ -2699,12 +2699,14 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext,
info_left -= total_record_size;
sinfo = (void *)sinfo + total_record_size;
sec_cnt++;
}
ext_info = ext_sec->ext_info;
ext_info->len = ext_sec->len - sizeof(__u32);
ext_info->rec_size = record_size;
ext_info->info = info + sizeof(__u32);
ext_info->sec_cnt = sec_cnt;
return 0;
}
@@ -2788,6 +2790,9 @@ void btf_ext__free(struct btf_ext *btf_ext)
{
if (IS_ERR_OR_NULL(btf_ext))
return;
free(btf_ext->func_info.sec_idxs);
free(btf_ext->line_info.sec_idxs);
free(btf_ext->core_relo_info.sec_idxs);
free(btf_ext->data);
free(btf_ext);
}
@@ -2826,10 +2831,8 @@ struct btf_ext *btf_ext__new(const __u8 *data, __u32 size)
if (err)
goto done;
if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) {
err = -EINVAL;
goto done;
}
if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
goto done; /* skip core relos parsing */
err = btf_ext_setup_core_relos(btf_ext);
if (err)

5
src/btf_dump.c
View File

@@ -1505,6 +1505,11 @@ static const char *btf_dump_resolve_name(struct btf_dump *d, __u32 id,
if (s->name_resolved)
return *cached_name ? *cached_name : orig_name;
if (btf_is_fwd(t) || (btf_is_enum(t) && btf_vlen(t) == 0)) {
s->name_resolved = 1;
return orig_name;
}
dup_cnt = btf_dump_name_dups(d, name_map, orig_name);
if (dup_cnt > 1) {
const size_t max_len = 256;

15
src/gen_loader.c
View File

@@ -1043,18 +1043,27 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue,
value = add_data(gen, pvalue, value_size);
key = add_data(gen, &zero, sizeof(zero));
/* if (map_desc[map_idx].initial_value)
* copy_from_user(value, initial_value, value_size);
/* if (map_desc[map_idx].initial_value) {
* if (ctx->flags & BPF_SKEL_KERNEL)
* bpf_probe_read_kernel(value, value_size, initial_value);
* else
* bpf_copy_from_user(value, value_size, initial_value);
* }
*/
emit(gen, BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_6,
sizeof(struct bpf_loader_ctx) +
sizeof(struct bpf_map_desc) * map_idx +
offsetof(struct bpf_map_desc, initial_value)));
emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 4));
emit(gen, BPF_JMP_IMM(BPF_JEQ, BPF_REG_3, 0, 8));
emit2(gen, BPF_LD_IMM64_RAW_FULL(BPF_REG_1, BPF_PSEUDO_MAP_IDX_VALUE,
0, 0, 0, value));
emit(gen, BPF_MOV64_IMM(BPF_REG_2, value_size));
emit(gen, BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_6,
offsetof(struct bpf_loader_ctx, flags)));
emit(gen, BPF_JMP_IMM(BPF_JSET, BPF_REG_0, BPF_SKEL_KERNEL, 2));
emit(gen, BPF_EMIT_CALL(BPF_FUNC_copy_from_user));
emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 1));
emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
map_update_attr = add_data(gen, &attr, attr_size);
move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,

2125
src/libbpf.c
View File

File diff suppressed because it is too large Load Diff

440
src/libbpf.h
View File

@@ -323,6 +323,24 @@ struct bpf_insn;
* different.
*/
LIBBPF_API const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog);
/**
* @brief **bpf_program__set_insns()** can set BPF program's underlying
* BPF instructions.
*
* WARNING: This is a very advanced libbpf API and users need to know
* what they are doing. This should be used from prog_prepare_load_fn
* callback only.
*
* @param prog BPF program for which to return instructions
* @param new_insns a pointer to an array of BPF instructions
* @param new_insn_cnt number of `struct bpf_insn`'s that form
* specified BPF program
* @return 0, on success; negative error code, otherwise
*/
LIBBPF_API int bpf_program__set_insns(struct bpf_program *prog,
struct bpf_insn *new_insns, size_t new_insn_cnt);
/**
* @brief **bpf_program__insn_cnt()** returns number of `struct bpf_insn`'s
* that form specified BPF program.
@@ -378,7 +396,31 @@ struct bpf_link;
LIBBPF_API struct bpf_link *bpf_link__open(const char *path);
LIBBPF_API int bpf_link__fd(const struct bpf_link *link);
LIBBPF_API const char *bpf_link__pin_path(const struct bpf_link *link);
/**
* @brief **bpf_link__pin()** pins the BPF link to a file
* in the BPF FS specified by a path. This increments the links
* reference count, allowing it to stay loaded after the process
* which loaded it has exited.
*
* @param link BPF link to pin, must already be loaded
* @param path file path in a BPF file system
* @return 0, on success; negative error code, otherwise
*/
LIBBPF_API int bpf_link__pin(struct bpf_link *link, const char *path);
/**
* @brief **bpf_link__unpin()** unpins the BPF link from a file
* in the BPFFS specified by a path. This decrements the links
* reference count.
*
* The file pinning the BPF link can also be unlinked by a different
* process in which case this function will return an error.
*
* @param prog BPF program to unpin
* @param path file path to the pin in a BPF file system
* @return 0, on success; negative error code, otherwise
*/
LIBBPF_API int bpf_link__unpin(struct bpf_link *link);
LIBBPF_API int bpf_link__update_program(struct bpf_link *link,
struct bpf_program *prog);
@@ -386,6 +428,22 @@ LIBBPF_API void bpf_link__disconnect(struct bpf_link *link);
LIBBPF_API int bpf_link__detach(struct bpf_link *link);
LIBBPF_API int bpf_link__destroy(struct bpf_link *link);
/**
* @brief **bpf_program__attach()** is a generic function for attaching
* a BPF program based on auto-detection of program type, attach type,
* and extra paremeters, where applicable.
*
* @param prog BPF program to attach
* @return Reference to the newly created BPF link; or NULL is returned on error,
* error code is stored in errno
*
* This is supported for:
* - kprobe/kretprobe (depends on SEC() definition)
* - uprobe/uretprobe (depends on SEC() definition)
* - tracepoint
* - raw tracepoint
* - tracing programs (typed raw TP/fentry/fexit/fmod_ret)
*/
LIBBPF_API struct bpf_link *
bpf_program__attach(const struct bpf_program *prog);
@@ -425,6 +483,29 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
const char *func_name,
const struct bpf_kprobe_opts *opts);
struct bpf_kprobe_multi_opts {
/* size of this struct, for forward/backward compatibility */
size_t sz;
/* array of function symbols to attach */
const char **syms;
/* array of function addresses to attach */
const unsigned long *addrs;
/* array of user-provided values fetchable through bpf_get_attach_cookie */
const __u64 *cookies;
/* number of elements in syms/addrs/cookies arrays */
size_t cnt;
/* create return kprobes */
bool retprobe;
size_t :0;
};
#define bpf_kprobe_multi_opts__last_field retprobe
LIBBPF_API struct bpf_link *
bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
const char *pattern,
const struct bpf_kprobe_multi_opts *opts);
struct bpf_uprobe_opts {
/* size of this struct, for forward/backward compatiblity */
size_t sz;
@@ -436,9 +517,17 @@ struct bpf_uprobe_opts {
__u64 bpf_cookie;
/* uprobe is return probe, invoked at function return time */
bool retprobe;
/* Function name to attach to. Could be an unqualified ("abc") or library-qualified
* "abc@LIBXYZ" name. To specify function entry, func_name should be set while
* func_offset argument to bpf_prog__attach_uprobe_opts() should be 0. To trace an
* offset within a function, specify func_name and use func_offset argument to specify
* offset within the function. Shared library functions must specify the shared library
* binary_path.
*/
const char *func_name;
size_t :0;
};
#define bpf_uprobe_opts__last_field retprobe
#define bpf_uprobe_opts__last_field func_name
/**
* @brief **bpf_program__attach_uprobe()** attaches a BPF program
@@ -480,6 +569,37 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
const char *binary_path, size_t func_offset,
const struct bpf_uprobe_opts *opts);
struct bpf_usdt_opts {
/* size of this struct, for forward/backward compatibility */
size_t sz;
/* custom user-provided value accessible through usdt_cookie() */
__u64 usdt_cookie;
size_t :0;
};
#define bpf_usdt_opts__last_field usdt_cookie
/**
* @brief **bpf_program__attach_usdt()** is just like
* bpf_program__attach_uprobe_opts() except it covers USDT (User-space
* Statically Defined Tracepoint) attachment, instead of attaching to
* user-space function entry or exit.
*
* @param prog BPF program to attach
* @param pid Process ID to attach the uprobe to, 0 for self (own process),
* -1 for all processes
* @param binary_path Path to binary that contains provided USDT probe
* @param usdt_provider USDT provider name
* @param usdt_name USDT probe name
* @param opts Options for altering program attachment
* @return Reference to the newly created BPF link; or NULL is returned on error,
* error code is stored in errno
*/
LIBBPF_API struct bpf_link *
bpf_program__attach_usdt(const struct bpf_program *prog,
pid_t pid, const char *binary_path,
const char *usdt_provider, const char *usdt_name,
const struct bpf_usdt_opts *opts);
struct bpf_tracepoint_opts {
/* size of this struct, for forward/backward compatiblity */
size_t sz;
@@ -501,8 +621,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
LIBBPF_API struct bpf_link *
bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
const char *tp_name);
struct bpf_trace_opts {
/* size of this struct, for forward/backward compatibility */
size_t sz;
/* custom user-provided value fetchable through bpf_get_attach_cookie() */
__u64 cookie;
};
#define bpf_trace_opts__last_field cookie
LIBBPF_API struct bpf_link *
bpf_program__attach_trace(const struct bpf_program *prog);
LIBBPF_API struct bpf_link *
bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts);
LIBBPF_API struct bpf_link *
bpf_program__attach_lsm(const struct bpf_program *prog);
LIBBPF_API struct bpf_link *
@@ -624,12 +756,37 @@ LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead")
LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog);
LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog);
LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
enum bpf_prog_type type);
/**
* @brief **bpf_program__set_type()** sets the program
* type of the passed BPF program.
* @param prog BPF program to set the program type for
* @param type program type to set the BPF map to have
* @return error code; or 0 if no error. An error occurs
* if the object is already loaded.
*
* This must be called before the BPF object is loaded,
* otherwise it has no effect and an error is returned.
*/
LIBBPF_API int bpf_program__set_type(struct bpf_program *prog,
enum bpf_prog_type type);
LIBBPF_API enum bpf_attach_type
bpf_program__expected_attach_type(const struct bpf_program *prog);
LIBBPF_API void
/**
* @brief **bpf_program__set_expected_attach_type()** sets the
* attach type of the passed BPF program. This is used for
* auto-detection of attachment when programs are loaded.
* @param prog BPF program to set the attach type for
* @param type attach type to set the BPF map to have
* @return error code; or 0 if no error. An error occurs
* if the object is already loaded.
*
* This must be called before the BPF object is loaded,
* otherwise it has no effect and an error is returned.
*/
LIBBPF_API int
bpf_program__set_expected_attach_type(struct bpf_program *prog,
enum bpf_attach_type type);
@@ -645,6 +802,17 @@ LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_le
LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size);
LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size);
/**
* @brief **bpf_program__set_attach_target()** sets BTF-based attach target
* for supported BPF program types:
* - BTF-aware raw tracepoints (tp_btf);
* - fentry/fexit/fmod_ret;
* - lsm;
* - freplace.
* @param prog BPF program to set the attach type for
* @param type attach type to set the BPF map to have
* @return error code; or 0 if no error occurred.
*/
LIBBPF_API int
bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd,
const char *attach_func_name);
@@ -728,6 +896,28 @@ struct bpf_map *bpf_map__prev(const struct bpf_map *map, const struct bpf_object
LIBBPF_API struct bpf_map *
bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map);
/**
* @brief **bpf_map__set_autocreate()** sets whether libbpf has to auto-create
* BPF map during BPF object load phase.
* @param map the BPF map instance
* @param autocreate whether to create BPF map during BPF object load
* @return 0 on success; -EBUSY if BPF object was already loaded
*
* **bpf_map__set_autocreate()** allows to opt-out from libbpf auto-creating
* BPF map. By default, libbpf will attempt to create every single BPF map
* defined in BPF object file using BPF_MAP_CREATE command of bpf() syscall
* and fill in map FD in BPF instructions.
*
* This API allows to opt-out of this process for specific map instance. This
* can be useful if host kernel doesn't support such BPF map type or used
* combination of flags and user application wants to avoid creating such
* a map in the first place. User is still responsible to make sure that their
* BPF-side code that expects to use such missing BPF map is recognized by BPF
* verifier as dead code, otherwise BPF verifier will reject such BPF program.
*/
LIBBPF_API int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate);
LIBBPF_API bool bpf_map__autocreate(const struct bpf_map *map);
/**
* @brief **bpf_map__fd()** gets the file descriptor of the passed
* BPF map
@@ -800,6 +990,110 @@ LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path);
LIBBPF_API int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd);
LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map);
/**
* @brief **bpf_map__lookup_elem()** allows to lookup BPF map value
* corresponding to provided key.
* @param map BPF map to lookup element in
* @param key pointer to memory containing bytes of the key used for lookup
* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size**
* @param value pointer to memory in which looked up value will be stored
* @param value_sz size in byte of value data memory; it has to match BPF map
* definition's **value_size**. For per-CPU BPF maps value size has to be
* a product of BPF map value size and number of possible CPUs in the system
* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for
* per-CPU values value size has to be aligned up to closest 8 bytes for
* alignment reasons, so expected size is: `round_up(value_size, 8)
* * libbpf_num_possible_cpus()`.
* @flags extra flags passed to kernel for this operation
* @return 0, on success; negative error, otherwise
*
* **bpf_map__lookup_elem()** is high-level equivalent of
* **bpf_map_lookup_elem()** API with added check for key and value size.
*/
LIBBPF_API int bpf_map__lookup_elem(const struct bpf_map *map,
const void *key, size_t key_sz,
void *value, size_t value_sz, __u64 flags);
/**
* @brief **bpf_map__update_elem()** allows to insert or update value in BPF
* map that corresponds to provided key.
* @param map BPF map to insert to or update element in
* @param key pointer to memory containing bytes of the key
* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size**
* @param value pointer to memory containing bytes of the value
* @param value_sz size in byte of value data memory; it has to match BPF map
* definition's **value_size**. For per-CPU BPF maps value size has to be
* a product of BPF map value size and number of possible CPUs in the system
* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for
* per-CPU values value size has to be aligned up to closest 8 bytes for
* alignment reasons, so expected size is: `round_up(value_size, 8)
* * libbpf_num_possible_cpus()`.
* @flags extra flags passed to kernel for this operation
* @return 0, on success; negative error, otherwise
*
* **bpf_map__update_elem()** is high-level equivalent of
* **bpf_map_update_elem()** API with added check for key and value size.
*/
LIBBPF_API int bpf_map__update_elem(const struct bpf_map *map,
const void *key, size_t key_sz,
const void *value, size_t value_sz, __u64 flags);
/**
* @brief **bpf_map__delete_elem()** allows to delete element in BPF map that
* corresponds to provided key.
* @param map BPF map to delete element from
* @param key pointer to memory containing bytes of the key
* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size**
* @flags extra flags passed to kernel for this operation
* @return 0, on success; negative error, otherwise
*
* **bpf_map__delete_elem()** is high-level equivalent of
* **bpf_map_delete_elem()** API with added check for key size.
*/
LIBBPF_API int bpf_map__delete_elem(const struct bpf_map *map,
const void *key, size_t key_sz, __u64 flags);
/**
* @brief **bpf_map__lookup_and_delete_elem()** allows to lookup BPF map value
* corresponding to provided key and atomically delete it afterwards.
* @param map BPF map to lookup element in
* @param key pointer to memory containing bytes of the key used for lookup
* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size**
* @param value pointer to memory in which looked up value will be stored
* @param value_sz size in byte of value data memory; it has to match BPF map
* definition's **value_size**. For per-CPU BPF maps value size has to be
* a product of BPF map value size and number of possible CPUs in the system
* (could be fetched with **libbpf_num_possible_cpus()**). Note also that for
* per-CPU values value size has to be aligned up to closest 8 bytes for
* alignment reasons, so expected size is: `round_up(value_size, 8)
* * libbpf_num_possible_cpus()`.
* @flags extra flags passed to kernel for this operation
* @return 0, on success; negative error, otherwise
*
* **bpf_map__lookup_and_delete_elem()** is high-level equivalent of
* **bpf_map_lookup_and_delete_elem()** API with added check for key and value size.
*/
LIBBPF_API int bpf_map__lookup_and_delete_elem(const struct bpf_map *map,
const void *key, size_t key_sz,
void *value, size_t value_sz, __u64 flags);
/**
* @brief **bpf_map__get_next_key()** allows to iterate BPF map keys by
* fetching next key that follows current key.
* @param map BPF map to fetch next key from
* @param cur_key pointer to memory containing bytes of current key or NULL to
* fetch the first key
* @param next_key pointer to memory to write next key into
* @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size**
* @return 0, on success; -ENOENT if **cur_key** is the last key in BPF map;
* negative error, otherwise
*
* **bpf_map__get_next_key()** is high-level equivalent of
* **bpf_map_get_next_key()** API with added check for key size.
*/
LIBBPF_API int bpf_map__get_next_key(const struct bpf_map *map,
const void *cur_key, void *next_key, size_t key_sz);
/**
* @brief **libbpf_get_error()** extracts the error code from the passed
* pointer
@@ -1289,6 +1583,35 @@ LIBBPF_API int bpf_object__attach_skeleton(struct bpf_object_skeleton *s);
LIBBPF_API void bpf_object__detach_skeleton(struct bpf_object_skeleton *s);
LIBBPF_API void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s);
struct bpf_var_skeleton {
const char *name;
struct bpf_map **map;
void **addr;
};
struct bpf_object_subskeleton {
size_t sz; /* size of this struct, for forward/backward compatibility */
const struct bpf_object *obj;
int map_cnt;
int map_skel_sz; /* sizeof(struct bpf_map_skeleton) */
struct bpf_map_skeleton *maps;
int prog_cnt;
int prog_skel_sz; /* sizeof(struct bpf_prog_skeleton) */
struct bpf_prog_skeleton *progs;
int var_cnt;
int var_skel_sz; /* sizeof(struct bpf_var_skeleton) */
struct bpf_var_skeleton *vars;
};
LIBBPF_API int
bpf_object__open_subskeleton(struct bpf_object_subskeleton *s);
LIBBPF_API void
bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s);
struct gen_loader_opts {
size_t sz; /* size of this struct, for forward/backward compatiblity */
const char *data;
@@ -1328,6 +1651,115 @@ LIBBPF_API int bpf_linker__add_file(struct bpf_linker *linker,
LIBBPF_API int bpf_linker__finalize(struct bpf_linker *linker);
LIBBPF_API void bpf_linker__free(struct bpf_linker *linker);
/*
* Custom handling of BPF program's SEC() definitions
*/
struct bpf_prog_load_opts; /* defined in bpf.h */
/* Called during bpf_object__open() for each recognized BPF program. Callback
* can use various bpf_program__set_*() setters to adjust whatever properties
* are necessary.
*/
typedef int (*libbpf_prog_setup_fn_t)(struct bpf_program *prog, long cookie);
/* Called right before libbpf performs bpf_prog_load() to load BPF program
* into the kernel. Callback can adjust opts as necessary.
*/
typedef int (*libbpf_prog_prepare_load_fn_t)(struct bpf_program *prog,
struct bpf_prog_load_opts *opts, long cookie);
/* Called during skeleton attach or through bpf_program__attach(). If
* auto-attach is not supported, callback should return 0 and set link to
* NULL (it's not considered an error during skeleton attach, but it will be
* an error for bpf_program__attach() calls). On error, error should be
* returned directly and link set to NULL. On success, return 0 and set link
* to a valid struct bpf_link.
*/
typedef int (*libbpf_prog_attach_fn_t)(const struct bpf_program *prog, long cookie,
struct bpf_link **link);
struct libbpf_prog_handler_opts {
/* size of this struct, for forward/backward compatiblity */
size_t sz;
/* User-provided value that is passed to prog_setup_fn,
* prog_prepare_load_fn, and prog_attach_fn callbacks. Allows user to
* register one set of callbacks for multiple SEC() definitions and
* still be able to distinguish them, if necessary. For example,
* libbpf itself is using this to pass necessary flags (e.g.,
* sleepable flag) to a common internal SEC() handler.
*/
long cookie;
/* BPF program initialization callback (see libbpf_prog_setup_fn_t).
* Callback is optional, pass NULL if it's not necessary.
*/
libbpf_prog_setup_fn_t prog_setup_fn;
/* BPF program loading callback (see libbpf_prog_prepare_load_fn_t).
* Callback is optional, pass NULL if it's not necessary.
*/
libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
/* BPF program attach callback (see libbpf_prog_attach_fn_t).
* Callback is optional, pass NULL if it's not necessary.
*/
libbpf_prog_attach_fn_t prog_attach_fn;
};
#define libbpf_prog_handler_opts__last_field prog_attach_fn
/**
* @brief **libbpf_register_prog_handler()** registers a custom BPF program
* SEC() handler.
* @param sec section prefix for which custom handler is registered
* @param prog_type BPF program type associated with specified section
* @param exp_attach_type Expected BPF attach type associated with specified section
* @param opts optional cookie, callbacks, and other extra options
* @return Non-negative handler ID is returned on success. This handler ID has
* to be passed to *libbpf_unregister_prog_handler()* to unregister such
* custom handler. Negative error code is returned on error.
*
* *sec* defines which SEC() definitions are handled by this custom handler
* registration. *sec* can have few different forms:
* - if *sec* is just a plain string (e.g., "abc"), it will match only
* SEC("abc"). If BPF program specifies SEC("abc/whatever") it will result
* in an error;
* - if *sec* is of the form "abc/", proper SEC() form is
* SEC("abc/something"), where acceptable "something" should be checked by
* *prog_init_fn* callback, if there are additional restrictions;
* - if *sec* is of the form "abc+", it will successfully match both
* SEC("abc") and SEC("abc/whatever") forms;
* - if *sec* is NULL, custom handler is registered for any BPF program that
* doesn't match any of the registered (custom or libbpf's own) SEC()
* handlers. There could be only one such generic custom handler registered
* at any given time.
*
* All custom handlers (except the one with *sec* == NULL) are processed
* before libbpf's own SEC() handlers. It is allowed to "override" libbpf's
* SEC() handlers by registering custom ones for the same section prefix
* (i.e., it's possible to have custom SEC("perf_event/LLC-load-misses")
* handler).
*
* Note, like much of global libbpf APIs (e.g., libbpf_set_print(),
* libbpf_set_strict_mode(), etc)) these APIs are not thread-safe. User needs
* to ensure synchronization if there is a risk of running this API from
* multiple threads simultaneously.
*/
LIBBPF_API int libbpf_register_prog_handler(const char *sec,
enum bpf_prog_type prog_type,
enum bpf_attach_type exp_attach_type,
const struct libbpf_prog_handler_opts *opts);
/**
* @brief *libbpf_unregister_prog_handler()* unregisters previously registered
* custom BPF program SEC() handler.
* @param handler_id handler ID returned by *libbpf_register_prog_handler()*
* after successful registration
* @return 0 on success, negative error code if handler isn't found
*
* Note, like much of global libbpf APIs (e.g., libbpf_set_print(),
* libbpf_set_strict_mode(), etc)) these APIs are not thread-safe. User needs
* to ensure synchronization if there is a risk of running this API from
* multiple threads simultaneously.
*/
LIBBPF_API int libbpf_unregister_prog_handler(int handler_id);
#ifdef __cplusplus
} /* extern "C" */
#endif

22
src/libbpf.map
View File

@@ -438,4 +438,24 @@ LIBBPF_0.7.0 {
libbpf_probe_bpf_map_type;
libbpf_probe_bpf_prog_type;
libbpf_set_memlock_rlim_max;
};
} LIBBPF_0.6.0;
LIBBPF_0.8.0 {
global:
bpf_map__autocreate;
bpf_map__get_next_key;
bpf_map__delete_elem;
bpf_map__lookup_and_delete_elem;
bpf_map__lookup_elem;
bpf_map__set_autocreate;
bpf_map__update_elem;
bpf_map_delete_elem_flags;
bpf_object__destroy_subskeleton;
bpf_object__open_subskeleton;
bpf_program__attach_kprobe_multi_opts;
bpf_program__attach_trace_opts;
bpf_program__attach_usdt;
bpf_program__set_insns;
libbpf_register_prog_handler;
libbpf_unregister_prog_handler;
} LIBBPF_0.7.0;

51
src/libbpf_internal.h
View File

@@ -103,6 +103,17 @@
#define str_has_pfx(str, pfx) \
(strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0)
/* suffix check */
static inline bool str_has_sfx(const char *str, const char *sfx)
{
size_t str_len = strlen(str);
size_t sfx_len = strlen(sfx);
if (sfx_len <= str_len)
return strcmp(str + str_len - sfx_len, sfx);
return false;
}
/* Symbol versioning is different between static and shared library.
* Properly versioned symbols are needed for shared library, but
* only the symbol of the new version is needed for static library.
@@ -148,6 +159,15 @@ do { \
#ifndef __has_builtin
#define __has_builtin(x) 0
#endif
struct bpf_link {
int (*detach)(struct bpf_link *link);
void (*dealloc)(struct bpf_link *link);
char *pin_path; /* NULL, if not pinned */
int fd; /* hook FD, -1 if not applicable */
bool disconnected;
};
/*
* Re-implement glibc's reallocarray() for libbpf internal-only use.
* reallocarray(), unfortunately, is not available in all versions of glibc,
@@ -329,6 +349,8 @@ enum kern_feature_id {
FEAT_BTF_TYPE_TAG,
/* memcg-based accounting for BPF maps and progs */
FEAT_MEMCG_ACCOUNT,
/* BPF cookie (bpf_get_attach_cookie() BPF helper) support */
FEAT_BPF_COOKIE,
__FEAT_CNT,
};
@@ -354,6 +376,13 @@ struct btf_ext_info {
void *info;
__u32 rec_size;
__u32 len;
/* optional (maintained internally by libbpf) mapping between .BTF.ext
* section and corresponding ELF section. This is used to join
* information like CO-RE relocation records with corresponding BPF
* programs defined in ELF sections
*/
__u32 *sec_idxs;
int sec_cnt;
};
#define for_each_btf_ext_sec(seg, sec) \
@@ -449,6 +478,11 @@ __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name,
extern enum libbpf_strict_mode libbpf_mode;
typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type,
const char *sym_name, void *ctx);
int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *arg);
/* handle direct returned errors */
static inline int libbpf_err(int ret)
{
@@ -529,4 +563,21 @@ static inline int ensure_good_fd(int fd)
return fd;
}
/* The following two functions are exposed to bpftool */
int bpf_core_add_cands(struct bpf_core_cand *local_cand,
size_t local_essent_len,
const struct btf *targ_btf,
const char *targ_btf_name,
int targ_start_id,
struct bpf_core_cand_list *cands);
void bpf_core_free_cands(struct bpf_core_cand_list *cands);
struct usdt_manager *usdt_manager_new(struct bpf_object *obj);
void usdt_manager_free(struct usdt_manager *man);
struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man,
const struct bpf_program *prog,
pid_t pid, const char *path,
const char *usdt_provider, const char *usdt_name,
__u64 usdt_cookie);
#endif /* __LIBBPF_LIBBPF_INTERNAL_H */

4
src/libbpf_legacy.h
View File

@@ -54,6 +54,10 @@ enum libbpf_strict_mode {
*
* Note, in this mode the program pin path will be based on the
* function name instead of section name.
*
* Additionally, routines in the .text section are always considered
* sub-programs. Legacy behavior allows for a single routine in .text
* to be a program.
*/
LIBBPF_STRICT_SEC_NAME = 0x04,
/*

2
src/libbpf_version.h
View File

@@ -4,6 +4,6 @@
#define __LIBBPF_VERSION_H
#define LIBBPF_MAJOR_VERSION 0
#define LIBBPF_MINOR_VERSION 7
#define LIBBPF_MINOR_VERSION 8
#endif /* __LIBBPF_VERSION_H */

63
src/netlink.c
View File

@@ -87,29 +87,75 @@ enum {
NL_DONE,
};
static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags)
{
int len;
do {
len = recvmsg(sock, mhdr, flags);
} while (len < 0 && (errno == EINTR || errno == EAGAIN));
if (len < 0)
return -errno;
return len;
}
static int alloc_iov(struct iovec *iov, int len)
{
void *nbuf;
nbuf = realloc(iov->iov_base, len);
if (!nbuf)
return -ENOMEM;
iov->iov_base = nbuf;
iov->iov_len = len;
return 0;
}
static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq,
__dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn,
void *cookie)
{
struct iovec iov = {};
struct msghdr mhdr = {
.msg_iov = &iov,
.msg_iovlen = 1,
};
bool multipart = true;
struct nlmsgerr *err;
struct nlmsghdr *nh;
char buf[4096];
int len, ret;
ret = alloc_iov(&iov, 4096);
if (ret)
goto done;
while (multipart) {
start:
multipart = false;
len = recv(sock, buf, sizeof(buf), 0);
len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC);
if (len < 0) {
ret = -errno;
ret = len;
goto done;
}
if (len > iov.iov_len) {
ret = alloc_iov(&iov, len);
if (ret)
goto done;
}
len = netlink_recvmsg(sock, &mhdr, 0);
if (len < 0) {
ret = len;
goto done;
}
if (len == 0)
break;
for (nh = (struct nlmsghdr *)buf; NLMSG_OK(nh, len);
for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len);
nh = NLMSG_NEXT(nh, len)) {
if (nh->nlmsg_pid != nl_pid) {
ret = -LIBBPF_ERRNO__WRNGPID;
@@ -130,7 +176,8 @@ start:
libbpf_nla_dump_errormsg(nh);
goto done;
case NLMSG_DONE:
return 0;
ret = 0;
goto done;
default:
break;
}
@@ -142,15 +189,17 @@ start:
case NL_NEXT:
goto start;
case NL_DONE:
return 0;
ret = 0;
goto done;
default:
return ret;
goto done;
}
}
}
}
ret = 0;
done:
free(iov.iov_base);
return ret;
}

183
src/relo_core.c
View File

@@ -178,29 +178,28 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind)
* Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access
* string to specify enumerator's value index that need to be relocated.
*/
static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
__u32 type_id,
const char *spec_str,
enum bpf_core_relo_kind relo_kind,
struct bpf_core_spec *spec)
int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
const struct bpf_core_relo *relo,
struct bpf_core_spec *spec)
{
int access_idx, parsed_len, i;
struct bpf_core_accessor *acc;
const struct btf_type *t;
const char *name;
const char *name, *spec_str;
__u32 id;
__s64 sz;
spec_str = btf__name_by_offset(btf, relo->access_str_off);
if (str_is_empty(spec_str) || *spec_str == ':')
return -EINVAL;
memset(spec, 0, sizeof(*spec));
spec->btf = btf;
spec->root_type_id = type_id;
spec->relo_kind = relo_kind;
spec->root_type_id = relo->type_id;
spec->relo_kind = relo->kind;
/* type-based relocations don't have a field access string */
if (core_relo_is_type_based(relo_kind)) {
if (core_relo_is_type_based(relo->kind)) {
if (strcmp(spec_str, "0"))
return -EINVAL;
return 0;
@@ -221,7 +220,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
if (spec->raw_len == 0)
return -EINVAL;
t = skip_mods_and_typedefs(btf, type_id, &id);
t = skip_mods_and_typedefs(btf, relo->type_id, &id);
if (!t)
return -EINVAL;
@@ -231,7 +230,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
acc->idx = access_idx;
spec->len++;
if (core_relo_is_enumval_based(relo_kind)) {
if (core_relo_is_enumval_based(relo->kind)) {
if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
return -EINVAL;
@@ -240,7 +239,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
return 0;
}
if (!core_relo_is_field_based(relo_kind))
if (!core_relo_is_field_based(relo->kind))
return -EINVAL;
sz = btf__resolve_size(btf, id);
@@ -301,7 +300,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
spec->bit_offset += access_idx * sz * 8;
} else {
pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n",
prog_name, type_id, spec_str, i, id, btf_kind_str(t));
prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t));
return -EINVAL;
}
}
@@ -775,31 +774,6 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
return 0;
}
struct bpf_core_relo_res
{
/* expected value in the instruction, unless validate == false */
__u32 orig_val;
/* new value that needs to be patched up to */
__u32 new_val;
/* relocation unsuccessful, poison instruction, but don't fail load */
bool poison;
/* some relocations can't be validated against orig_val */
bool validate;
/* for field byte offset relocations or the forms:
* *(T *)(rX + <off>) = rY
* rX = *(T *)(rY + <off>),
* we remember original and resolved field size to adjust direct
* memory loads of pointers and integers; this is necessary for 32-bit
* host kernel architectures, but also allows to automatically
* relocate fields that were resized from, e.g., u32 to u64, etc.
*/
bool fail_memsz_adjust;
__u32 orig_sz;
__u32 orig_type_id;
__u32 new_sz;
__u32 new_type_id;
};
/* Calculate original and target relocation values, given local and target
* specs and relocation kind. These values are calculated for each candidate.
* If there are multiple candidates, resulting values should all be consistent
@@ -951,9 +925,9 @@ static int insn_bytes_to_bpf_size(__u32 sz)
* 5. *(T *)(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
* 6. *(T *)(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
*/
static int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
int insn_idx, const struct bpf_core_relo *relo,
int relo_idx, const struct bpf_core_relo_res *res)
int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
int insn_idx, const struct bpf_core_relo *relo,
int relo_idx, const struct bpf_core_relo_res *res)
{
__u32 orig_val, new_val;
__u8 class;
@@ -1080,55 +1054,70 @@ poison:
* [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>,
* where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b
*/
static void bpf_core_dump_spec(const char *prog_name, int level, const struct bpf_core_spec *spec)
int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)
{
const struct btf_type *t;
const struct btf_enum *e;
const char *s;
__u32 type_id;
int i;
int i, len = 0;
#define append_buf(fmt, args...) \
({ \
int r; \
r = snprintf(buf, buf_sz, fmt, ##args); \
len += r; \
if (r >= buf_sz) \
r = buf_sz; \
buf += r; \
buf_sz -= r; \
})
type_id = spec->root_type_id;
t = btf_type_by_id(spec->btf, type_id);
s = btf__name_by_offset(spec->btf, t->name_off);
libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
append_buf("<%s> [%u] %s %s",
core_relo_kind_str(spec->relo_kind),
type_id, btf_kind_str(t), str_is_empty(s) ? "<anon>" : s);
if (core_relo_is_type_based(spec->relo_kind))
return;
return len;
if (core_relo_is_enumval_based(spec->relo_kind)) {
t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
e = btf_enum(t) + spec->raw_spec[0];
s = btf__name_by_offset(spec->btf, e->name_off);
libbpf_print(level, "::%s = %u", s, e->val);
return;
append_buf("::%s = %u", s, e->val);
return len;
}
if (core_relo_is_field_based(spec->relo_kind)) {
for (i = 0; i < spec->len; i++) {
if (spec->spec[i].name)
libbpf_print(level, ".%s", spec->spec[i].name);
append_buf(".%s", spec->spec[i].name);
else if (i > 0 || spec->spec[i].idx > 0)
libbpf_print(level, "[%u]", spec->spec[i].idx);
append_buf("[%u]", spec->spec[i].idx);
}
libbpf_print(level, " (");
append_buf(" (");
for (i = 0; i < spec->raw_len; i++)
libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]);
if (spec->bit_offset % 8)
libbpf_print(level, " @ offset %u.%u)",
spec->bit_offset / 8, spec->bit_offset % 8);
append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8);
else
libbpf_print(level, " @ offset %u)", spec->bit_offset / 8);
return;
append_buf(" @ offset %u)", spec->bit_offset / 8);
return len;
}
return len;
#undef append_buf
}
/*
* CO-RE relocate single instruction.
* Calculate CO-RE relocation target result.
*
* The outline and important points of the algorithm:
* 1. For given local type, find corresponding candidate target types.
@@ -1177,22 +1166,22 @@ static void bpf_core_dump_spec(const char *prog_name, int level, const struct bp
* between multiple relocations for the same type ID and is updated as some
* of the candidates are pruned due to structural incompatibility.
*/
int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
int insn_idx,
const struct bpf_core_relo *relo,
int relo_idx,
const struct btf *local_btf,
struct bpf_core_cand_list *cands,
struct bpf_core_spec *specs_scratch)
int bpf_core_calc_relo_insn(const char *prog_name,
const struct bpf_core_relo *relo,
int relo_idx,
const struct btf *local_btf,
struct bpf_core_cand_list *cands,
struct bpf_core_spec *specs_scratch,
struct bpf_core_relo_res *targ_res)
{
struct bpf_core_spec *local_spec = &specs_scratch[0];
struct bpf_core_spec *cand_spec = &specs_scratch[1];
struct bpf_core_spec *targ_spec = &specs_scratch[2];
struct bpf_core_relo_res cand_res, targ_res;
struct bpf_core_relo_res cand_res;
const struct btf_type *local_type;
const char *local_name;
__u32 local_id;
const char *spec_str;
char spec_buf[256];
int i, j, err;
local_id = relo->type_id;
@@ -1201,38 +1190,34 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
if (!local_name)
return -EINVAL;
spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
if (str_is_empty(spec_str))
return -EINVAL;
err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str,
relo->kind, local_spec);
err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec);
if (err) {
const char *spec_str;
spec_str = btf__name_by_offset(local_btf, relo->access_str_off);
pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n",
prog_name, relo_idx, local_id, btf_kind_str(local_type),
str_is_empty(local_name) ? "<anon>" : local_name,
spec_str, err);
spec_str ?: "<?>", err);
return -EINVAL;
}
pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name,
relo_idx, core_relo_kind_str(relo->kind), relo->kind);
bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, local_spec);
libbpf_print(LIBBPF_DEBUG, "\n");
bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec);
pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf);
/* TYPE_ID_LOCAL relo is special and doesn't need candidate search */
if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) {
/* bpf_insn's imm value could get out of sync during linking */
memset(&targ_res, 0, sizeof(targ_res));
targ_res.validate = false;
targ_res.poison = false;
targ_res.orig_val = local_spec->root_type_id;
targ_res.new_val = local_spec->root_type_id;
goto patch_insn;
memset(targ_res, 0, sizeof(*targ_res));
targ_res->validate = false;
targ_res->poison = false;
targ_res->orig_val = local_spec->root_type_id;
targ_res->new_val = local_spec->root_type_id;
return 0;
}
/* libbpf doesn't support candidate search for anonymous types */
if (str_is_empty(spec_str)) {
if (str_is_empty(local_name)) {
pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n",
prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind);
return -EOPNOTSUPP;
@@ -1242,17 +1227,15 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
err = bpf_core_spec_match(local_spec, cands->cands[i].btf,
cands->cands[i].id, cand_spec);
if (err < 0) {
pr_warn("prog '%s': relo #%d: error matching candidate #%d ",
prog_name, relo_idx, i);
bpf_core_dump_spec(prog_name, LIBBPF_WARN, cand_spec);
libbpf_print(LIBBPF_WARN, ": %d\n", err);
bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ",
prog_name, relo_idx, i, spec_buf, err);
return err;
}
pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name,
relo_idx, err == 0 ? "non-matching" : "matching", i);
bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, cand_spec);
libbpf_print(LIBBPF_DEBUG, "\n");
bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name,
relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf);
if (err == 0)
continue;
@@ -1262,7 +1245,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
return err;
if (j == 0) {
targ_res = cand_res;
*targ_res = cand_res;
*targ_spec = *cand_spec;
} else if (cand_spec->bit_offset != targ_spec->bit_offset) {
/* if there are many field relo candidates, they
@@ -1272,7 +1255,8 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
prog_name, relo_idx, cand_spec->bit_offset,
targ_spec->bit_offset);
return -EINVAL;
} else if (cand_res.poison != targ_res.poison || cand_res.new_val != targ_res.new_val) {
} else if (cand_res.poison != targ_res->poison ||
cand_res.new_val != targ_res->new_val) {
/* all candidates should result in the same relocation
* decision and value, otherwise it's dangerous to
* proceed due to ambiguity
@@ -1280,7 +1264,7 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
prog_name, relo_idx,
cand_res.poison ? "failure" : "success", cand_res.new_val,
targ_res.poison ? "failure" : "success", targ_res.new_val);
targ_res->poison ? "failure" : "success", targ_res->new_val);
return -EINVAL;
}
@@ -1314,19 +1298,10 @@ int bpf_core_apply_relo_insn(const char *prog_name, struct bpf_insn *insn,
prog_name, relo_idx);
/* calculate single target relo result explicitly */
err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, &targ_res);
err = bpf_core_calc_relo(prog_name, relo, relo_idx, local_spec, NULL, targ_res);
if (err)
return err;
}
patch_insn:
/* bpf_core_patch_insn() should know how to handle missing targ_spec */
err = bpf_core_patch_insn(prog_name, insn, insn_idx, relo, relo_idx, &targ_res);
if (err) {
pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
prog_name, relo_idx, relo->insn_off / 8, err);
return -EINVAL;
}
return 0;
}

48
src/relo_core.h
View File

@@ -44,14 +44,50 @@ struct bpf_core_spec {
__u32 bit_offset;
};
int bpf_core_apply_relo_insn(const char *prog_name,
struct bpf_insn *insn, int insn_idx,
const struct bpf_core_relo *relo, int relo_idx,
const struct btf *local_btf,
struct bpf_core_cand_list *cands,
struct bpf_core_spec *specs_scratch);
struct bpf_core_relo_res {
/* expected value in the instruction, unless validate == false */
__u32 orig_val;
/* new value that needs to be patched up to */
__u32 new_val;
/* relocation unsuccessful, poison instruction, but don't fail load */
bool poison;
/* some relocations can't be validated against orig_val */
bool validate;
/* for field byte offset relocations or the forms:
* *(T *)(rX + <off>) = rY
* rX = *(T *)(rY + <off>),
* we remember original and resolved field size to adjust direct
* memory loads of pointers and integers; this is necessary for 32-bit
* host kernel architectures, but also allows to automatically
* relocate fields that were resized from, e.g., u32 to u64, etc.
*/
bool fail_memsz_adjust;
__u32 orig_sz;
__u32 orig_type_id;
__u32 new_sz;
__u32 new_type_id;
};
int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id,
const struct btf *targ_btf, __u32 targ_id);
size_t bpf_core_essential_name_len(const char *name);
int bpf_core_calc_relo_insn(const char *prog_name,
const struct bpf_core_relo *relo, int relo_idx,
const struct btf *local_btf,
struct bpf_core_cand_list *cands,
struct bpf_core_spec *specs_scratch,
struct bpf_core_relo_res *targ_res);
int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn,
int insn_idx, const struct bpf_core_relo *relo,
int relo_idx, const struct bpf_core_relo_res *res);
int bpf_core_parse_spec(const char *prog_name, const struct btf *btf,
const struct bpf_core_relo *relo,
struct bpf_core_spec *spec);
int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec);
#endif

185
src/skel_internal.h
View File

@@ -3,9 +3,19 @@
#ifndef __SKEL_INTERNAL_H
#define __SKEL_INTERNAL_H
#ifdef __KERNEL__
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <linux/slab.h>
#include <linux/bpf.h>
#else
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/mman.h>
#include <stdlib.h>
#include "bpf.h"
#endif
#ifndef __NR_bpf
# if defined(__mips__) && defined(_ABIO32)
@@ -25,24 +35,23 @@
* requested during loader program generation.
*/
struct bpf_map_desc {
union {
/* input for the loader prog */
struct {
__aligned_u64 initial_value;
__u32 max_entries;
};
/* output of the loader prog */
struct {
int map_fd;
};
};
/* output of the loader prog */
int map_fd;
/* input for the loader prog */
__u32 max_entries;
__aligned_u64 initial_value;
};
struct bpf_prog_desc {
int prog_fd;
};
enum {
BPF_SKEL_KERNEL = (1ULL << 0),
};
struct bpf_loader_ctx {
size_t sz;
__u32 sz;
__u32 flags;
__u32 log_level;
__u32 log_size;
__u64 log_buf;
@@ -57,12 +66,144 @@ struct bpf_load_and_run_opts {
const char *errstr;
};
long bpf_sys_bpf(__u32 cmd, void *attr, __u32 attr_size);
static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr,
unsigned int size)
{
#ifdef __KERNEL__
return bpf_sys_bpf(cmd, attr, size);
#else
return syscall(__NR_bpf, cmd, attr, size);
#endif
}
#ifdef __KERNEL__
static inline int close(int fd)
{
return close_fd(fd);
}
static inline void *skel_alloc(size_t size)
{
struct bpf_loader_ctx *ctx = kzalloc(size, GFP_KERNEL);
if (!ctx)
return NULL;
ctx->flags |= BPF_SKEL_KERNEL;
return ctx;
}
static inline void skel_free(const void *p)
{
kfree(p);
}
/* skel->bss/rodata maps are populated the following way:
*
* For kernel use:
* skel_prep_map_data() allocates kernel memory that kernel module can directly access.
* Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
* The loader program will perform probe_read_kernel() from maps.rodata.initial_value.
* skel_finalize_map_data() sets skel->rodata to point to actual value in a bpf map and
* does maps.rodata.initial_value = ~0ULL to signal skel_free_map_data() that kvfree
* is not nessary.
*
* For user space:
* skel_prep_map_data() mmaps anon memory into skel->rodata that can be accessed directly.
* Generated lskel stores the pointer in skel->rodata and in skel->maps.rodata.initial_value.
* The loader program will perform copy_from_user() from maps.rodata.initial_value.
* skel_finalize_map_data() remaps bpf array map value from the kernel memory into
* skel->rodata address.
*
* The "bpftool gen skeleton -L" command generates lskel.h that is suitable for
* both kernel and user space. The generated loader program does
* either bpf_probe_read_kernel() or bpf_copy_from_user() from initial_value
* depending on bpf_loader_ctx->flags.
*/
static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
{
if (addr != ~0ULL)
kvfree(p);
/* When addr == ~0ULL the 'p' points to
* ((struct bpf_array *)map)->value. See skel_finalize_map_data.
*/
}
static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
{
void *addr;
addr = kvmalloc(val_sz, GFP_KERNEL);
if (!addr)
return NULL;
memcpy(addr, val, val_sz);
return addr;
}
static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
{
struct bpf_map *map;
void *addr = NULL;
kvfree((void *) (long) *init_val);
*init_val = ~0ULL;
/* At this point bpf_load_and_run() finished without error and
* 'fd' is a valid bpf map FD. All sanity checks below should succeed.
*/
map = bpf_map_get(fd);
if (IS_ERR(map))
return NULL;
if (map->map_type != BPF_MAP_TYPE_ARRAY)
goto out;
addr = ((struct bpf_array *)map)->value;
/* the addr stays valid, since FD is not closed */
out:
bpf_map_put(map);
return addr;
}
#else
static inline void *skel_alloc(size_t size)
{
return calloc(1, size);
}
static inline void skel_free(void *p)
{
free(p);
}
static inline void skel_free_map_data(void *p, __u64 addr, size_t sz)
{
munmap(p, sz);
}
static inline void *skel_prep_map_data(const void *val, size_t mmap_sz, size_t val_sz)
{
void *addr;
addr = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (addr == (void *) -1)
return NULL;
memcpy(addr, val, val_sz);
return addr;
}
static inline void *skel_finalize_map_data(__u64 *init_val, size_t mmap_sz, int flags, int fd)
{
void *addr;
addr = mmap((void *) (long) *init_val, mmap_sz, flags, MAP_SHARED | MAP_FIXED, fd, 0);
if (addr == (void *) -1)
return NULL;
return addr;
}
#endif
static inline int skel_closenz(int fd)
{
if (fd > 0)
@@ -136,22 +277,28 @@ static inline int skel_link_create(int prog_fd, int target_fd,
return skel_sys_bpf(BPF_LINK_CREATE, &attr, attr_sz);
}
#ifdef __KERNEL__
#define set_err
#else
#define set_err err = -errno
#endif
static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
{
int map_fd = -1, prog_fd = -1, key = 0, err;
union bpf_attr attr;
map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1);
err = map_fd = skel_map_create(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, opts->data_sz, 1);
if (map_fd < 0) {
opts->errstr = "failed to create loader map";
err = -errno;
set_err;
goto out;
}
err = skel_map_update_elem(map_fd, &key, opts->data, 0);
if (err < 0) {
opts->errstr = "failed to update loader map";
err = -errno;
set_err;
goto out;
}
@@ -166,10 +313,10 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
attr.log_size = opts->ctx->log_size;
attr.log_buf = opts->ctx->log_buf;
attr.prog_flags = BPF_F_SLEEPABLE;
prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
if (prog_fd < 0) {
opts->errstr = "failed to load loader prog";
err = -errno;
set_err;
goto out;
}
@@ -181,10 +328,12 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
if (err < 0 || (int)attr.test.retval < 0) {
opts->errstr = "failed to execute loader prog";
if (err < 0) {
err = -errno;
set_err;
} else {
err = (int)attr.test.retval;
#ifndef __KERNEL__
errno = -err;
#endif
}
goto out;
}

259
src/usdt.bpf.h Normal file
View File

@@ -0,0 +1,259 @@
/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
#ifndef __USDT_BPF_H__
#define __USDT_BPF_H__
#include <linux/errno.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
/* Below types and maps are internal implementation details of libbpf's USDT
* support and are subjects to change. Also, bpf_usdt_xxx() API helpers should
* be considered an unstable API as well and might be adjusted based on user
* feedback from using libbpf's USDT support in production.
*/
/* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal
* map that keeps track of USDT argument specifications. This might be
* necessary if there are a lot of USDT attachments.
*/
#ifndef BPF_USDT_MAX_SPEC_CNT
#define BPF_USDT_MAX_SPEC_CNT 256
#endif
/* User can override BPF_USDT_MAX_IP_CNT to change default size of internal
* map that keeps track of IP (memory address) mapping to USDT argument
* specification.
* Note, if kernel supports BPF cookies, this map is not used and could be
* resized all the way to 1 to save a bit of memory.
*/
#ifndef BPF_USDT_MAX_IP_CNT
#define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT)
#endif
/* We use BPF CO-RE to detect support for BPF cookie from BPF side. This is
* the only dependency on CO-RE, so if it's undesirable, user can override
* BPF_USDT_HAS_BPF_COOKIE to specify whether to BPF cookie is supported or not.
*/
#ifndef BPF_USDT_HAS_BPF_COOKIE
#define BPF_USDT_HAS_BPF_COOKIE \
bpf_core_enum_value_exists(enum bpf_func_id___usdt, BPF_FUNC_get_attach_cookie___usdt)
#endif
enum __bpf_usdt_arg_type {
BPF_USDT_ARG_CONST,
BPF_USDT_ARG_REG,
BPF_USDT_ARG_REG_DEREF,
};
struct __bpf_usdt_arg_spec {
/* u64 scalar interpreted depending on arg_type, see below */
__u64 val_off;
/* arg location case, see bpf_udst_arg() for details */
enum __bpf_usdt_arg_type arg_type;
/* offset of referenced register within struct pt_regs */
short reg_off;
/* whether arg should be interpreted as signed value */
bool arg_signed;
/* number of bits that need to be cleared and, optionally,
* sign-extended to cast arguments that are 1, 2, or 4 bytes
* long into final 8-byte u64/s64 value returned to user
*/
char arg_bitshift;
};
/* should match USDT_MAX_ARG_CNT in usdt.c exactly */
#define BPF_USDT_MAX_ARG_CNT 12
struct __bpf_usdt_spec {
struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT];
__u64 usdt_cookie;
short arg_cnt;
};
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, BPF_USDT_MAX_SPEC_CNT);
__type(key, int);
__type(value, struct __bpf_usdt_spec);
} __bpf_usdt_specs SEC(".maps") __weak;
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, BPF_USDT_MAX_IP_CNT);
__type(key, long);
__type(value, __u32);
} __bpf_usdt_ip_to_spec_id SEC(".maps") __weak;
/* don't rely on user's BPF code to have latest definition of bpf_func_id */
enum bpf_func_id___usdt {
BPF_FUNC_get_attach_cookie___usdt = 0xBAD, /* value doesn't matter */
};
static __always_inline
int __bpf_usdt_spec_id(struct pt_regs *ctx)
{
if (!BPF_USDT_HAS_BPF_COOKIE) {
long ip = PT_REGS_IP(ctx);
int *spec_id_ptr;
spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip);
return spec_id_ptr ? *spec_id_ptr : -ESRCH;
}
return bpf_get_attach_cookie(ctx);
}
/* Return number of USDT arguments defined for currently traced USDT. */
__weak __hidden
int bpf_usdt_arg_cnt(struct pt_regs *ctx)
{
struct __bpf_usdt_spec *spec;
int spec_id;
spec_id = __bpf_usdt_spec_id(ctx);
if (spec_id < 0)
return -ESRCH;
spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
if (!spec)
return -ESRCH;
return spec->arg_cnt;
}
/* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res.
* Returns 0 on success; negative error, otherwise.
* On error *res is guaranteed to be set to zero.
*/
__weak __hidden
int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res)
{
struct __bpf_usdt_spec *spec;
struct __bpf_usdt_arg_spec *arg_spec;
unsigned long val;
int err, spec_id;
*res = 0;
spec_id = __bpf_usdt_spec_id(ctx);
if (spec_id < 0)
return -ESRCH;
spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
if (!spec)
return -ESRCH;
if (arg_num >= BPF_USDT_MAX_ARG_CNT || arg_num >= spec->arg_cnt)
return -ENOENT;
arg_spec = &spec->args[arg_num];
switch (arg_spec->arg_type) {
case BPF_USDT_ARG_CONST:
/* Arg is just a constant ("-4@$-9" in USDT arg spec).
* value is recorded in arg_spec->val_off directly.
*/
val = arg_spec->val_off;
break;
case BPF_USDT_ARG_REG:
/* Arg is in a register (e.g, "8@%rax" in USDT arg spec),
* so we read the contents of that register directly from
* struct pt_regs. To keep things simple user-space parts
* record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off.
*/
err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
if (err)
return err;
break;
case BPF_USDT_ARG_REG_DEREF:
/* Arg is in memory addressed by register, plus some offset
* (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is
* identified like with BPF_USDT_ARG_REG case, and the offset
* is in arg_spec->val_off. We first fetch register contents
* from pt_regs, then do another user-space probe read to
* fetch argument value itself.
*/
err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off);
if (err)
return err;
err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off);
if (err)
return err;
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
val >>= arg_spec->arg_bitshift;
#endif
break;
default:
return -EINVAL;
}
/* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing
* necessary upper arg_bitshift bits, with sign extension if argument
* is signed
*/
val <<= arg_spec->arg_bitshift;
if (arg_spec->arg_signed)
val = ((long)val) >> arg_spec->arg_bitshift;
else
val = val >> arg_spec->arg_bitshift;
*res = val;
return 0;
}
/* Retrieve user-specified cookie value provided during attach as
* bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie
* returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself
* utilizing BPF cookies internally, so user can't use BPF cookie directly
* for USDT programs and has to use bpf_usdt_cookie() API instead.
*/
__weak __hidden
long bpf_usdt_cookie(struct pt_regs *ctx)
{
struct __bpf_usdt_spec *spec;
int spec_id;
spec_id = __bpf_usdt_spec_id(ctx);
if (spec_id < 0)
return 0;
spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
if (!spec)
return 0;
return spec->usdt_cookie;
}
/* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */
#define ___bpf_usdt_args0() ctx
#define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); (void *)_x; })
#define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); (void *)_x; })
#define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); (void *)_x; })
#define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); (void *)_x; })
#define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); (void *)_x; })
#define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); (void *)_x; })
#define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); (void *)_x; })
#define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); (void *)_x; })
#define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); (void *)_x; })
#define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); (void *)_x; })
#define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); (void *)_x; })
#define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); (void *)_x; })
#define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args)
/*
* BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for
* tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes.
* Original struct pt_regs * context is preserved as 'ctx' argument.
*/
#define BPF_USDT(name, args...) \
name(struct pt_regs *ctx); \
static __attribute__((always_inline)) typeof(name(0)) \
____##name(struct pt_regs *ctx, ##args); \
typeof(name(0)) name(struct pt_regs *ctx) \
{ \
_Pragma("GCC diagnostic push") \
_Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
return ____##name(___bpf_usdt_args(args)); \
_Pragma("GCC diagnostic pop") \
} \
static __attribute__((always_inline)) typeof(name(0)) \
____##name(struct pt_regs *ctx, ##args)
#endif /* __USDT_BPF_H__ */

1518
src/usdt.c Normal file
View File

File diff suppressed because it is too large Load Diff

15
src/xsk.c
View File

@@ -481,8 +481,8 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk)
BPF_EMIT_CALL(BPF_FUNC_redirect_map),
BPF_EXIT_INSN(),
};
size_t insns_cnt[] = {sizeof(prog) / sizeof(struct bpf_insn),
sizeof(prog_redirect_flags) / sizeof(struct bpf_insn),
size_t insns_cnt[] = {ARRAY_SIZE(prog),
ARRAY_SIZE(prog_redirect_flags),
};
struct bpf_insn *progs[] = {prog, prog_redirect_flags};
enum xsk_prog option = get_xsk_prog();
@@ -1193,12 +1193,23 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname,
int xsk_umem__delete(struct xsk_umem *umem)
{
struct xdp_mmap_offsets off;
int err;
if (!umem)
return 0;
if (umem->refcount)
return -EBUSY;
err = xsk_get_mmap_offsets(umem->fd, &off);
if (!err && umem->fill_save && umem->comp_save) {
munmap(umem->fill_save->ring - off.fr.desc,
off.fr.desc + umem->config.fill_size * sizeof(__u64));
munmap(umem->comp_save->ring - off.cr.desc,
off.cr.desc + umem->config.comp_size * sizeof(__u64));
}
close(umem->fd);
free(umem);

1
travis-ci/rootfs/mkrootfs_arch.sh
View File

@@ -80,6 +80,7 @@ packages=(
# selftests test_progs dependencies.
binutils
elfutils
ethtool
glibc
iproute2
# selftests test_verifier dependencies.

9
travis-ci/rootfs/mkrootfs_debian.sh
View File

@@ -2,6 +2,11 @@
# This script builds a Debian root filesystem image for testing libbpf in a
# virtual machine. Requires debootstrap >= 1.0.95 and zstd.
# Use e.g. ./mkrootfs_debian.sh --arch=s390x to generate a rootfs for a
# foreign architecture. Requires configured binfmt_misc, e.g. using
# Debian/Ubuntu's qemu-user-binfmt package or
# https://github.com/multiarch/qemu-user-static.
set -e -u -x -o pipefail
# Check whether we are root now in order to avoid confusing errors later.
@@ -15,8 +20,8 @@ root=$(mktemp -d -p "$PWD")
trap 'rm -r "$root"' EXIT
# Install packages.
packages=binutils,busybox,elfutils,iproute2,libcap2,libelf1,strace,zlib1g
debootstrap --include="$packages" --variant=minbase bullseye "$root"
packages=binutils,busybox,elfutils,ethtool,iproute2,libcap2,libelf1,strace,zlib1g
debootstrap --include="$packages" --variant=minbase "$@" bookworm "$root"
# Remove the init scripts (tests use their own). Also remove various
# unnecessary files in order to save space.

37
travis-ci/rootfs/s390x-self-hosted-builder/README.md
View File

@@ -1,16 +1,25 @@
# IBM Z self-hosted builder
libbpf CI uses an IBM-provided z15 self-hosted builder. There are no IBM Z
builds of GitHub Actions runner, and stable qemu-user has problems with .NET
builds of GitHub (GH) Actions runner, and stable qemu-user has problems with .NET
apps, so the builder runs the x86_64 runner version with qemu-user built from
the master branch.
We are currently supporting runners for the following repositories:
* libbpf/libbpf
* kernel-patches/bpf
* kernel-patches/vmtest
Below instructions are directly applicable to libbpf, and require minor
modifications for kernel-patches repos. Currently, qemu-user-static Docker
image is shared between all GitHub runners, but separate actions-runner-\*
service / Docker image is created for each runner type.
## Configuring the builder.
### Install prerequisites.
```
$ sudo dnf install docker # RHEL
$ sudo apt install -y docker.io # Ubuntu
```
@@ -35,6 +44,10 @@ for details.
### Autostart the x86_64 emulation support.
This step is important, you would not be able to build docker container
without having this service running. If container build fails, make sure
service is running properly.
```
$ sudo systemctl enable --now qemu-user-static
```
@@ -72,3 +85,23 @@ $ sudo systemctl stop actions-runner-libbpf
$ sudo docker rm -f actions-runner-libbpf
$ sudo docker volume rm actions-runner-libbpf
```
## Troubleshooting
In order to check if service is running, use the following command:
```
$ sudo systemctl status <service name>
```
In order to get logs for service:
```
$ journalctl -u <service name>
```
In order to check which containers are currently active:
```
$ sudo docker ps
```

8
travis-ci/vmtest/configs/blacklist/BLACKLIST-latest.s390x
View File

@@ -25,11 +25,13 @@ ksyms_module_libbpf # JIT does not support calling kernel f
ksyms_module_lskel # test_ksyms_module_lskel__open_and_load unexpected error: -9 (?)
modify_return # modify_return attach failed: -524 (trampoline)
module_attach # skel_attach skeleton attach failed: -524 (trampoline)
kprobe_multi_test # relies on fentry
netcnt # failed to load BPF skeleton 'netcnt_prog': -7 (?)
probe_user # check_kprobe_res wrong kprobe res from probe read (?)
recursion # skel_attach unexpected error: -524 (trampoline)
ringbuf # skel_load skeleton load failed (?)
sk_assign # Can't read on server: Invalid argument (?)
sk_lookup # endianness problem
sk_storage_tracing # test_sk_storage_tracing__attach unexpected error: -524 (trampoline)
skc_to_unix_sock # could not attach BPF object unexpected error: -524 (trampoline)
socket_cookie # prog_attach unexpected error: -524 (trampoline)
@@ -44,6 +46,7 @@ test_lsm # failed to find kernel BTF type ID of
test_overhead # attach_fentry unexpected error: -524 (trampoline)
test_profiler # unknown func bpf_probe_read_str#45 (overlapping)
timer # failed to auto-attach program 'test1': -524 (trampoline)
timer_crash # trampoline
timer_mim # failed to auto-attach program 'test1': -524 (trampoline)
trace_ext # failed to auto-attach program 'test_pkt_md_access_new': -524 (trampoline)
trace_printk # trace_printk__load unexpected error: -2 (errno 2) (?)
@@ -54,3 +57,8 @@ vmlinux # failed to auto-attach program 'handle
xdp_adjust_tail # case-128 err 0 errno 28 retval 1 size 128 expect-size 3520 (?)
xdp_bonding # failed to auto-attach program 'trace_on_entry': -524 (trampoline)
xdp_bpf2bpf # failed to auto-attach program 'trace_on_entry': -524 (trampoline)
map_kptr # failed to open_and_load program: -524 (trampoline)
bpf_cookie # failed to open_and_load program: -524 (trampoline)
xdp_do_redirect # prog_run_max_size unexpected error: -22 (errno 22)
send_signal # intermittently fails to receive signal
select_reuseport # intermittently fails on new s390x setup

7
travis-ci/vmtest/configs/config-latest.s390x
View File

@@ -784,6 +784,7 @@ CONFIG_NETFILTER_NETLINK=y
# CONFIG_NETFILTER_NETLINK_OSF is not set
CONFIG_NF_CONNTRACK=y
# CONFIG_NF_LOG_SYSLOG is not set
CONFIG_NETFILTER_SYNPROXY=y
CONFIG_NF_TABLES=y
# CONFIG_NF_TABLES_INET is not set
# CONFIG_NF_TABLES_NETDEV is not set
@@ -814,6 +815,7 @@ CONFIG_NETFILTER_XT_MARK=y
#
# CONFIG_NETFILTER_XT_TARGET_AUDIT is not set
# CONFIG_NETFILTER_XT_TARGET_CLASSIFY is not set
CONFIG_NETFILTER_XT_TARGET_CT=y
# CONFIG_NETFILTER_XT_TARGET_HMARK is not set
# CONFIG_NETFILTER_XT_TARGET_IDLETIMER is not set
# CONFIG_NETFILTER_XT_TARGET_LOG is not set
@@ -858,6 +860,7 @@ CONFIG_NETFILTER_XT_MATCH_BPF=y
# CONFIG_NETFILTER_XT_MATCH_RECENT is not set
# CONFIG_NETFILTER_XT_MATCH_SCTP is not set
# CONFIG_NETFILTER_XT_MATCH_SOCKET is not set
CONFIG_NETFILTER_XT_MATCH_STATE=y
# CONFIG_NETFILTER_XT_MATCH_STATISTIC is not set
# CONFIG_NETFILTER_XT_MATCH_STRING is not set
# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
@@ -885,9 +888,10 @@ CONFIG_IP_NF_IPTABLES=y
# CONFIG_IP_NF_MATCH_ECN is not set
# CONFIG_IP_NF_MATCH_TTL is not set
CONFIG_IP_NF_FILTER=y
CONFIG_IP_NF_TARGET_SYNPROXY=y
# CONFIG_IP_NF_TARGET_REJECT is not set
# CONFIG_IP_NF_MANGLE is not set
# CONFIG_IP_NF_RAW is not set
CONFIG_IP_NF_RAW=y
# CONFIG_IP_NF_SECURITY is not set
# CONFIG_IP_NF_ARPTABLES is not set
# end of IP: Netfilter Configuration
@@ -2586,6 +2590,7 @@ CONFIG_FUNCTION_TRACER=y
CONFIG_FUNCTION_GRAPH_TRACER=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_DYNAMIC_FTRACE_WITH_REGS=y
CONFIG_FPROBE=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_STACK_TRACER=y
# CONFIG_IRQSOFF_TRACER is not set

10
travis-ci/vmtest/configs/config-latest.x86_64
View File

@@ -979,6 +979,7 @@ CONFIG_NETFILTER_NETLINK_LOG=y
# CONFIG_NETFILTER_NETLINK_OSF is not set
CONFIG_NF_CONNTRACK=y
# CONFIG_NF_LOG_NETDEV is not set
CONFIG_NETFILTER_SYNPROXY=y
# CONFIG_NF_TABLES is not set
CONFIG_NETFILTER_XTABLES=y
@@ -992,6 +993,7 @@ CONFIG_NETFILTER_XTABLES=y
#
# CONFIG_NETFILTER_XT_TARGET_AUDIT is not set
# CONFIG_NETFILTER_XT_TARGET_CLASSIFY is not set
CONFIG_NETFILTER_XT_TARGET_CT=y
# CONFIG_NETFILTER_XT_TARGET_HMARK is not set
# CONFIG_NETFILTER_XT_TARGET_IDLETIMER is not set
# CONFIG_NETFILTER_XT_TARGET_LOG is not set
@@ -1037,6 +1039,7 @@ CONFIG_NETFILTER_XT_MATCH_BPF=y
# CONFIG_NETFILTER_XT_MATCH_RECENT is not set
# CONFIG_NETFILTER_XT_MATCH_SCTP is not set
# CONFIG_NETFILTER_XT_MATCH_SOCKET is not set
CONFIG_NETFILTER_XT_MATCH_STATE=y
CONFIG_NETFILTER_XT_MATCH_STATISTIC=y
# CONFIG_NETFILTER_XT_MATCH_STRING is not set
# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
@@ -1061,9 +1064,10 @@ CONFIG_IP_NF_IPTABLES=y
# CONFIG_IP_NF_MATCH_AH is not set
# CONFIG_IP_NF_MATCH_ECN is not set
# CONFIG_IP_NF_MATCH_TTL is not set
# CONFIG_IP_NF_FILTER is not set
CONFIG_IP_NF_FILTER=y
CONFIG_IP_NF_TARGET_SYNPROXY=y
# CONFIG_IP_NF_MANGLE is not set
# CONFIG_IP_NF_RAW is not set
CONFIG_IP_NF_RAW=y
# CONFIG_IP_NF_SECURITY is not set
# CONFIG_IP_NF_ARPTABLES is not set
# end of IP: Netfilter Configuration
@@ -2789,6 +2793,7 @@ CONFIG_DEBUG_BUGVERBOSE=y
# Compile-time checks and compiler options
#
CONFIG_DEBUG_INFO=y
CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
# CONFIG_DEBUG_INFO_REDUCED is not set
# CONFIG_DEBUG_INFO_COMPRESSED is not set
# CONFIG_DEBUG_INFO_SPLIT is not set
@@ -2976,6 +2981,7 @@ CONFIG_FUNCTION_GRAPH_TRACER=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_DYNAMIC_FTRACE_WITH_REGS=y
CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS=y
CONFIG_FPROBE=y
# CONFIG_FUNCTION_PROFILER is not set
# CONFIG_STACK_TRACER is not set
# CONFIG_IRQSOFF_TRACER is not set

2
travis-ci/vmtest/configs/whitelist/WHITELIST-5.5.0
View File

@@ -43,13 +43,13 @@ spinlock
stacktrace_map
stacktrace_map_raw_tp
static_linked
subprogs
task_fd_query_rawtp
task_fd_query_tp
tc_bpf
tcp_estats
tcp_rtt
tp_attach_query
usdt/urand_pid_attach
xdp
xdp_info
xdp_noinline