diff options
Diffstat (limited to 'tools/lib/bpf')
| -rw-r--r-- | tools/lib/bpf/Build | 4 | ||||
| -rw-r--r-- | tools/lib/bpf/Makefile | 12 | ||||
| -rw-r--r-- | tools/lib/bpf/README.rst | 3 | ||||
| -rw-r--r-- | tools/lib/bpf/bpf.c | 8 | ||||
| -rw-r--r-- | tools/lib/bpf/bpf.h | 1 | ||||
| -rw-r--r-- | tools/lib/bpf/bpf_prog_linfo.c | 5 | ||||
| -rw-r--r-- | tools/lib/bpf/btf.c | 332 | ||||
| -rw-r--r-- | tools/lib/bpf/btf.h | 20 | ||||
| -rw-r--r-- | tools/lib/bpf/btf_dump.c | 1333 | ||||
| -rw-r--r-- | tools/lib/bpf/hashmap.c | 229 | ||||
| -rw-r--r-- | tools/lib/bpf/hashmap.h | 173 | ||||
| -rw-r--r-- | tools/lib/bpf/libbpf.c | 1855 | ||||
| -rw-r--r-- | tools/lib/bpf/libbpf.h | 155 | ||||
| -rw-r--r-- | tools/lib/bpf/libbpf.map | 20 | ||||
| -rw-r--r-- | tools/lib/bpf/libbpf_internal.h | 9 | ||||
| -rw-r--r-- | tools/lib/bpf/libbpf_probes.c | 1 | ||||
| -rw-r--r-- | tools/lib/bpf/str_error.c | 2 | ||||
| -rw-r--r-- | tools/lib/bpf/xsk.c | 116 | ||||
| -rw-r--r-- | tools/lib/bpf/xsk.h | 2 |
19 files changed, 3709 insertions, 571 deletions
diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index ee9d5362f35b..e3962cfbc9a6 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1 +1,3 @@ -libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o bpf_prog_linfo.o libbpf_probes.o xsk.o +libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \ + netlink.o bpf_prog_linfo.o libbpf_probes.o xsk.o hashmap.o \ + btf_dump.o diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index f91639bf5650..9312066a1ae3 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -3,7 +3,7 @@ BPF_VERSION = 0 BPF_PATCHLEVEL = 0 -BPF_EXTRAVERSION = 3 +BPF_EXTRAVERSION = 4 MAKEFLAGS += --no-print-directory @@ -204,6 +204,16 @@ check_abi: $(OUTPUT)libbpf.so "versioned symbols in $^ ($(VERSIONED_SYM_COUNT))." \ "Please make sure all LIBBPF_API symbols are" \ "versioned in $(VERSION_SCRIPT)." >&2; \ + readelf -s --wide $(OUTPUT)libbpf-in.o | \ + awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$8}'| \ + sort -u > $(OUTPUT)libbpf_global_syms.tmp; \ + readelf -s --wide $(OUTPUT)libbpf.so | \ + grep -Eo '[^ ]+@LIBBPF_' | cut -d@ -f1 | \ + sort -u > $(OUTPUT)libbpf_versioned_syms.tmp; \ + diff -u $(OUTPUT)libbpf_global_syms.tmp \ + $(OUTPUT)libbpf_versioned_syms.tmp; \ + rm $(OUTPUT)libbpf_global_syms.tmp \ + $(OUTPUT)libbpf_versioned_syms.tmp; \ exit 1; \ fi diff --git a/tools/lib/bpf/README.rst b/tools/lib/bpf/README.rst index cef7b77eab69..8928f7787f2d 100644 --- a/tools/lib/bpf/README.rst +++ b/tools/lib/bpf/README.rst @@ -9,7 +9,8 @@ described here. It's recommended to follow these conventions whenever a new function or type is added to keep libbpf API clean and consistent. All types and functions provided by libbpf API should have one of the -following prefixes: ``bpf_``, ``btf_``, ``libbpf_``, ``xsk_``. +following prefixes: ``bpf_``, ``btf_``, ``libbpf_``, ``xsk_``, +``perf_buffer_``. System call wrappers -------------------- diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index c4a48086dc9a..c7d7993c44bb 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -26,10 +26,11 @@ #include <memory.h> #include <unistd.h> #include <asm/unistd.h> +#include <errno.h> #include <linux/bpf.h> #include "bpf.h" #include "libbpf.h" -#include <errno.h> +#include "libbpf_internal.h" /* * When building perf, unistd.h is overridden. __NR_bpf is @@ -53,10 +54,6 @@ # endif #endif -#ifndef min -#define min(x, y) ((x) < (y) ? (x) : (y)) -#endif - static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; @@ -256,6 +253,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, if (load_attr->name) memcpy(attr.prog_name, load_attr->name, min(strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1)); + attr.prog_flags = load_attr->prog_flags; fd = sys_bpf_prog_load(&attr, sizeof(attr)); if (fd >= 0) diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 9593fec75652..ff42ca043dc8 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -87,6 +87,7 @@ struct bpf_load_program_attr { const void *line_info; __u32 line_info_cnt; __u32 log_level; + __u32 prog_flags; }; /* Flags to direct loading requirements */ diff --git a/tools/lib/bpf/bpf_prog_linfo.c b/tools/lib/bpf/bpf_prog_linfo.c index 6978314ea7f6..8c67561c93b0 100644 --- a/tools/lib/bpf/bpf_prog_linfo.c +++ b/tools/lib/bpf/bpf_prog_linfo.c @@ -6,10 +6,7 @@ #include <linux/err.h> #include <linux/bpf.h> #include "libbpf.h" - -#ifndef min -#define min(x, y) ((x) < (y) ? (x) : (y)) -#endif +#include "libbpf_internal.h" struct bpf_prog_linfo { void *raw_linfo; diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 03348c4d6bd4..467224feb43b 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -4,17 +4,17 @@ #include <stdio.h> #include <stdlib.h> #include <string.h> +#include <fcntl.h> #include <unistd.h> #include <errno.h> #include <linux/err.h> #include <linux/btf.h> +#include <gelf.h> #include "btf.h" #include "bpf.h" #include "libbpf.h" #include "libbpf_internal.h" - -#define max(a, b) ((a) > (b) ? (a) : (b)) -#define min(a, b) ((a) < (b) ? (a) : (b)) +#include "hashmap.h" #define BTF_MAX_NR_TYPES 0x7fffffff #define BTF_MAX_STR_OFFSET 0x7fffffff @@ -417,6 +417,132 @@ done: return btf; } +static bool btf_check_endianness(const GElf_Ehdr *ehdr) +{ +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return ehdr->e_ident[EI_DATA] == ELFDATA2LSB; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return ehdr->e_ident[EI_DATA] == ELFDATA2MSB; +#else +# error "Unrecognized __BYTE_ORDER__" +#endif +} + +struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext) +{ + Elf_Data *btf_data = NULL, *btf_ext_data = NULL; + int err = 0, fd = -1, idx = 0; + struct btf *btf = NULL; + Elf_Scn *scn = NULL; + Elf *elf = NULL; + GElf_Ehdr ehdr; + + if (elf_version(EV_CURRENT) == EV_NONE) { + pr_warning("failed to init libelf for %s\n", path); + return ERR_PTR(-LIBBPF_ERRNO__LIBELF); + } + + fd = open(path, O_RDONLY); + if (fd < 0) { + err = -errno; + pr_warning("failed to open %s: %s\n", path, strerror(errno)); + return ERR_PTR(err); + } + + err = -LIBBPF_ERRNO__FORMAT; + + elf = elf_begin(fd, ELF_C_READ, NULL); + if (!elf) { + pr_warning("failed to open %s as ELF file\n", path); + goto done; + } + if (!gelf_getehdr(elf, &ehdr)) { + pr_warning("failed to get EHDR from %s\n", path); + goto done; + } + if (!btf_check_endianness(&ehdr)) { + pr_warning("non-native ELF endianness is not supported\n"); + goto done; + } + if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) { + pr_warning("failed to get e_shstrndx from %s\n", path); + goto done; + } + + while ((scn = elf_nextscn(elf, scn)) != NULL) { + GElf_Shdr sh; + char *name; + + idx++; + if (gelf_getshdr(scn, &sh) != &sh) { + pr_warning("failed to get section(%d) header from %s\n", + idx, path); + goto done; + } + name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name); + if (!name) { + pr_warning("failed to get section(%d) name from %s\n", + idx, path); + goto done; + } + if (strcmp(name, BTF_ELF_SEC) == 0) { + btf_data = elf_getdata(scn, 0); + if (!btf_data) { + pr_warning("failed to get section(%d, %s) data from %s\n", + idx, name, path); + goto done; + } + continue; + } else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) { + btf_ext_data = elf_getdata(scn, 0); + if (!btf_ext_data) { + pr_warning("failed to get section(%d, %s) data from %s\n", + idx, name, path); + goto done; + } + continue; + } + } + + err = 0; + + if (!btf_data) { + err = -ENOENT; + goto done; + } + btf = btf__new(btf_data->d_buf, btf_data->d_size); + if (IS_ERR(btf)) + goto done; + + if (btf_ext && btf_ext_data) { + *btf_ext = btf_ext__new(btf_ext_data->d_buf, + btf_ext_data->d_size); + if (IS_ERR(*btf_ext)) + goto done; + } else if (btf_ext) { + *btf_ext = NULL; + } +done: + if (elf) + elf_end(elf); + close(fd); + + if (err) + return ERR_PTR(err); + /* + * btf is always parsed before btf_ext, so no need to clean up + * btf_ext, if btf loading failed + */ + if (IS_ERR(btf)) + return btf; + if (btf_ext && IS_ERR(*btf_ext)) { + btf__free(btf); + err = PTR_ERR(*btf_ext); + return ERR_PTR(err); + } + return btf; +} + static int compare_vsi_off(const void *_a, const void *_b) { const struct btf_var_secinfo *a = _a; @@ -1165,16 +1291,9 @@ done: return err; } -#define BTF_DEDUP_TABLE_DEFAULT_SIZE (1 << 14) -#define BTF_DEDUP_TABLE_MAX_SIZE_LOG 31 #define BTF_UNPROCESSED_ID ((__u32)-1) #define BTF_IN_PROGRESS_ID ((__u32)-2) -struct btf_dedup_node { - struct btf_dedup_node *next; - __u32 type_id; -}; - struct btf_dedup { /* .BTF section to be deduped in-place */ struct btf *btf; @@ -1190,7 +1309,7 @@ struct btf_dedup { * candidates, which is fine because we rely on subsequent * btf_xxx_equal() checks to authoritatively verify type equality. */ - struct btf_dedup_node **dedup_table; + struct hashmap *dedup_table; /* Canonical types map */ __u32 *map; /* Hypothetical mapping, used during type graph equivalence checks */ @@ -1215,30 +1334,18 @@ struct btf_str_ptrs { __u32 cap; }; -static inline __u32 hash_combine(__u32 h, __u32 value) +static long hash_combine(long h, long value) { -/* 2^31 + 2^29 - 2^25 + 2^22 - 2^19 - 2^16 + 1 */ -#define GOLDEN_RATIO_PRIME 0x9e370001UL - return h * 37 + value * GOLDEN_RATIO_PRIME; -#undef GOLDEN_RATIO_PRIME + return h * 31 + value; } -#define for_each_dedup_cand(d, hash, node) \ - for (node = d->dedup_table[hash & (d->opts.dedup_table_size - 1)]; \ - node; \ - node = node->next) +#define for_each_dedup_cand(d, node, hash) \ + hashmap__for_each_key_entry(d->dedup_table, node, (void *)hash) -static int btf_dedup_table_add(struct btf_dedup *d, __u32 hash, __u32 type_id) +static int btf_dedup_table_add(struct btf_dedup *d, long hash, __u32 type_id) { - struct btf_dedup_node *node = malloc(sizeof(struct btf_dedup_node)); - int bucket = hash & (d->opts.dedup_table_size - 1); - - if (!node) - return -ENOMEM; - node->type_id = type_id; - node->next = d->dedup_table[bucket]; - d->dedup_table[bucket] = node; - return 0; + return hashmap__append(d->dedup_table, + (void *)hash, (void *)(long)type_id); } static int btf_dedup_hypot_map_add(struct btf_dedup *d, @@ -1267,36 +1374,10 @@ static void btf_dedup_clear_hypot_map(struct btf_dedup *d) d->hypot_cnt = 0; } -static void btf_dedup_table_free(struct btf_dedup *d) -{ - struct btf_dedup_node *head, *tmp; - int i; - - if (!d->dedup_table) - return; - - for (i = 0; i < d->opts.dedup_table_size; i++) { - while (d->dedup_table[i]) { - tmp = d->dedup_table[i]; - d->dedup_table[i] = tmp->next; - free(tmp); - } - - head = d->dedup_table[i]; - while (head) { - tmp = head; - head = head->next; - free(tmp); - } - } - - free(d->dedup_table); - d->dedup_table = NULL; -} - static void btf_dedup_free(struct btf_dedup *d) { - btf_dedup_table_free(d); + hashmap__free(d->dedup_table); + d->dedup_table = NULL; free(d->map); d->map = NULL; @@ -1310,40 +1391,43 @@ static void btf_dedup_free(struct btf_dedup *d) free(d); } -/* Find closest power of two >= to size, capped at 2^max_size_log */ -static __u32 roundup_pow2_max(__u32 size, int max_size_log) +static size_t btf_dedup_identity_hash_fn(const void *key, void *ctx) { - int i; + return (size_t)key; +} - for (i = 0; i < max_size_log && (1U << i) < size; i++) - ; - return 1U << i; +static size_t btf_dedup_collision_hash_fn(const void *key, void *ctx) +{ + return 0; } +static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx) +{ + return k1 == k2; +} static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext, const struct btf_dedup_opts *opts) { struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); + hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; int i, err = 0; - __u32 sz; if (!d) return ERR_PTR(-ENOMEM); d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds; - sz = opts && opts->dedup_table_size ? opts->dedup_table_size - : BTF_DEDUP_TABLE_DEFAULT_SIZE; - sz = roundup_pow2_max(sz, BTF_DEDUP_TABLE_MAX_SIZE_LOG); - d->opts.dedup_table_size = sz; + /* dedup_table_size is now used only to force collisions in tests */ + if (opts && opts->dedup_table_size == 1) + hash_fn = btf_dedup_collision_hash_fn; d->btf = btf; d->btf_ext = btf_ext; - d->dedup_table = calloc(d->opts.dedup_table_size, - sizeof(struct btf_dedup_node *)); - if (!d->dedup_table) { - err = -ENOMEM; + d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); + if (IS_ERR(d->dedup_table)) { + err = PTR_ERR(d->dedup_table); + d->dedup_table = NULL; goto done; } @@ -1662,9 +1746,9 @@ done: return err; } -static __u32 btf_hash_common(struct btf_type *t) +static long btf_hash_common(struct btf_type *t) { - __u32 h; + long h; h = hash_combine(0, t->name_off); h = hash_combine(h, t->info); @@ -1680,10 +1764,10 @@ static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) } /* Calculate type signature hash of INT. */ -static __u32 btf_hash_int(struct btf_type *t) +static long btf_hash_int(struct btf_type *t) { __u32 info = *(__u32 *)(t + 1); - __u32 h; + long h; h = btf_hash_common(t); h = hash_combine(h, info); @@ -1703,9 +1787,9 @@ static bool btf_equal_int(struct btf_type *t1, struct btf_type *t2) } /* Calculate type signature hash of ENUM. */ -static __u32 btf_hash_enum(struct btf_type *t) +static long btf_hash_enum(struct btf_type *t) { - __u32 h; + long h; /* don't hash vlen and enum members to support enum fwd resolving */ h = hash_combine(0, t->name_off); @@ -1757,11 +1841,11 @@ static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) * as referenced type IDs equivalence is established separately during type * graph equivalence check algorithm. */ -static __u32 btf_hash_struct(struct btf_type *t) +static long btf_hash_struct(struct btf_type *t) { struct btf_member *member = (struct btf_member *)(t + 1); __u32 vlen = BTF_INFO_VLEN(t->info); - __u32 h = btf_hash_common(t); + long h = btf_hash_common(t); int i; for (i = 0; i < vlen; i++) { @@ -1804,10 +1888,10 @@ static bool btf_shallow_equal_struct(struct btf_type *t1, struct btf_type *t2) * under assumption that they were already resolved to canonical type IDs and * are not going to change. */ -static __u32 btf_hash_array(struct btf_type *t) +static long btf_hash_array(struct btf_type *t) { struct btf_array *info = (struct btf_array *)(t + 1); - __u32 h = btf_hash_common(t); + long h = btf_hash_common(t); h = hash_combine(h, info->type); h = hash_combine(h, info->index_type); @@ -1858,11 +1942,11 @@ static bool btf_compat_array(struct btf_type *t1, struct btf_type *t2) * under assumption that they were already resolved to canonical type IDs and * are not going to change. */ -static inline __u32 btf_hash_fnproto(struct btf_type *t) +static long btf_hash_fnproto(struct btf_type *t) { struct btf_param *member = (struct btf_param *)(t + 1); __u16 vlen = BTF_INFO_VLEN(t->info); - __u32 h = btf_hash_common(t); + long h = btf_hash_common(t); int i; for (i = 0; i < vlen; i++) { @@ -1880,7 +1964,7 @@ static inline __u32 btf_hash_fnproto(struct btf_type *t) * This function is called during reference types deduplication to compare * FUNC_PROTO to potential canonical representative. */ -static inline bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) +static bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) { struct btf_param *m1, *m2; __u16 vlen; @@ -1906,7 +1990,7 @@ static inline bool btf_equal_fnproto(struct btf_type *t1, struct btf_type *t2) * IDs. This check is performed during type graph equivalence check and * referenced types equivalence is checked separately. */ -static inline bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) +static bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) { struct btf_param *m1, *m2; __u16 vlen; @@ -1937,11 +2021,12 @@ static inline bool btf_compat_fnproto(struct btf_type *t1, struct btf_type *t2) static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) { struct btf_type *t = d->btf->types[type_id]; + struct hashmap_entry *hash_entry; struct btf_type *cand; - struct btf_dedup_node *cand_node; /* if we don't find equivalent type, then we are canonical */ __u32 new_id = type_id; - __u32 h; + __u32 cand_id; + long h; switch (BTF_INFO_KIND(t->info)) { case BTF_KIND_CONST: @@ -1960,10 +2045,11 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_INT: h = btf_hash_int(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_int(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } } @@ -1971,10 +2057,11 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_ENUM: h = btf_hash_enum(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_enum(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } if (d->opts.dont_resolve_fwds) @@ -1982,21 +2069,22 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) if (btf_compat_enum(t, cand)) { if (btf_is_enum_fwd(t)) { /* resolve fwd to full enum */ - new_id = cand_node->type_id; + new_id = cand_id; break; } /* resolve canonical enum fwd to full enum */ - d->map[cand_node->type_id] = type_id; + d->map[cand_id] = type_id; } } break; case BTF_KIND_FWD: h = btf_hash_common(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_common(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } } @@ -2397,12 +2485,12 @@ static void btf_dedup_merge_hypot_map(struct btf_dedup *d) */ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) { - struct btf_dedup_node *cand_node; struct btf_type *cand_type, *t; + struct hashmap_entry *hash_entry; /* if we don't find equivalent type, then we are canonical */ __u32 new_id = type_id; __u16 kind; - __u32 h; + long h; /* already deduped or is in process of deduping (loop detected) */ if (d->map[type_id] <= BTF_MAX_NR_TYPES) @@ -2415,7 +2503,8 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) return 0; h = btf_hash_struct(t); - for_each_dedup_cand(d, h, cand_node) { + for_each_dedup_cand(d, hash_entry, h) { + __u32 cand_id = (__u32)(long)hash_entry->value; int eq; /* @@ -2428,17 +2517,17 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) * creating a loop (FWD -> STRUCT and STRUCT -> FWD), because * FWD and compatible STRUCT/UNION are considered equivalent. */ - cand_type = d->btf->types[cand_node->type_id]; + cand_type = d->btf->types[cand_id]; if (!btf_shallow_equal_struct(t, cand_type)) continue; btf_dedup_clear_hypot_map(d); - eq = btf_dedup_is_equiv(d, type_id, cand_node->type_id); + eq = btf_dedup_is_equiv(d, type_id, cand_id); if (eq < 0) return eq; if (!eq) continue; - new_id = cand_node->type_id; + new_id = cand_id; btf_dedup_merge_hypot_map(d); break; } @@ -2488,12 +2577,12 @@ static int btf_dedup_struct_types(struct btf_dedup *d) */ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) { - struct btf_dedup_node *cand_node; + struct hashmap_entry *hash_entry; + __u32 new_id = type_id, cand_id; struct btf_type *t, *cand; /* if we don't find equivalent type, then we are representative type */ - __u32 new_id = type_id; int ref_type_id; - __u32 h; + long h; if (d->map[type_id] == BTF_IN_PROGRESS_ID) return -ELOOP; @@ -2516,10 +2605,11 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) t->type = ref_type_id; h = btf_hash_common(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_common(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } } @@ -2539,10 +2629,11 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) info->index_type = ref_type_id; h = btf_hash_array(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_array(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } } @@ -2570,10 +2661,11 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) } h = btf_hash_fnproto(t); - for_each_dedup_cand(d, h, cand_node) { - cand = d->btf->types[cand_node->type_id]; + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = d->btf->types[cand_id]; if (btf_equal_fnproto(t, cand)) { - new_id = cand_node->type_id; + new_id = cand_id; break; } } @@ -2600,7 +2692,9 @@ static int btf_dedup_ref_types(struct btf_dedup *d) if (err < 0) return err; } - btf_dedup_table_free(d); + /* we won't need d->dedup_table anymore */ + hashmap__free(d->dedup_table); + d->dedup_table = NULL; return 0; } diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index c7b399e81fce..88a52ae56fc6 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -4,6 +4,7 @@ #ifndef __LIBBPF_BTF_H #define __LIBBPF_BTF_H +#include <stdarg.h> #include <linux/types.h> #ifdef __cplusplus @@ -16,6 +17,7 @@ extern "C" { #define BTF_ELF_SEC ".BTF" #define BTF_EXT_ELF_SEC ".BTF.ext" +#define MAPS_ELF_SEC ".maps" struct btf; struct btf_ext; @@ -59,6 +61,8 @@ struct btf_ext_header { LIBBPF_API void btf__free(struct btf *btf); LIBBPF_API struct btf *btf__new(__u8 *data, __u32 size); +LIBBPF_API struct btf *btf__parse_elf(const char *path, + struct btf_ext **btf_ext); LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf); LIBBPF_API int btf__load(struct btf *btf); LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, @@ -100,6 +104,22 @@ struct btf_dedup_opts { LIBBPF_API int btf__dedup(struct btf *btf, struct btf_ext *btf_ext, const struct btf_dedup_opts *opts); +struct btf_dump; + +struct btf_dump_opts { + void *ctx; +}; + +typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args); + +LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf, + const struct btf_ext *btf_ext, + const struct btf_dump_opts *opts, + btf_dump_printf_fn_t printf_fn); +LIBBPF_API void btf_dump__free(struct btf_dump *d); + +LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c new file mode 100644 index 000000000000..7065bb5b2752 --- /dev/null +++ b/tools/lib/bpf/btf_dump.c @@ -0,0 +1,1333 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +/* + * BTF-to-C type converter. + * + * Copyright (c) 2019 Facebook + */ + +#include <stdbool.h> +#include <stddef.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <linux/err.h> +#include <linux/btf.h> +#include "btf.h" +#include "hashmap.h" +#include "libbpf.h" +#include "libbpf_internal.h" + +static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t"; +static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1; + +static const char *pfx(int lvl) +{ + return lvl >= PREFIX_CNT ? PREFIXES : &PREFIXES[PREFIX_CNT - lvl]; +} + +enum btf_dump_type_order_state { + NOT_ORDERED, + ORDERING, + ORDERED, +}; + +enum btf_dump_type_emit_state { + NOT_EMITTED, + EMITTING, + EMITTED, +}; + +/* per-type auxiliary state */ +struct btf_dump_type_aux_state { + /* topological sorting state */ + enum btf_dump_type_order_state order_state: 2; + /* emitting state used to determine the need for forward declaration */ + enum btf_dump_type_emit_state emit_state: 2; + /* whether forward declaration was already emitted */ + __u8 fwd_emitted: 1; + /* whether unique non-duplicate name was already assigned */ + __u8 name_resolved: 1; +}; + +struct btf_dump { + const struct btf *btf; + const struct btf_ext *btf_ext; + btf_dump_printf_fn_t printf_fn; + struct btf_dump_opts opts; + + /* per-type auxiliary state */ + struct btf_dump_type_aux_state *type_states; + /* per-type optional cached unique name, must be freed, if present */ + const char **cached_names; + + /* topo-sorted list of dependent type definitions */ + __u32 *emit_queue; + int emit_queue_cap; + int emit_queue_cnt; + + /* + * stack of type declarations (e.g., chain of modifiers, arrays, + * funcs, etc) + */ + __u32 *decl_stack; + int decl_stack_cap; + int decl_stack_cnt; + + /* maps struct/union/enum name to a number of name occurrences */ + struct hashmap *type_names; + /* + * maps typedef identifiers and enum value names to a number of such + * name occurrences + */ + struct hashmap *ident_names; +}; + +static size_t str_hash_fn(const void *key, void *ctx) +{ + const char *s = key; + size_t h = 0; + + while (*s) { + h = h * 31 + *s; + s++; + } + return h; +} + +static bool str_equal_fn(const void *a, const void *b, void *ctx) +{ + return strcmp(a, b) == 0; +} + +static __u16 btf_kind_of(const struct btf_type *t) +{ + return BTF_INFO_KIND(t->info); +} + +static __u16 btf_vlen_of(const struct btf_type *t) +{ + return BTF_INFO_VLEN(t->info); +} + +static bool btf_kflag_of(const struct btf_type *t) +{ + return BTF_INFO_KFLAG(t->info); +} + +static const char *btf_name_of(const struct btf_dump *d, __u32 name_off) +{ + return btf__name_by_offset(d->btf, name_off); +} + +static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + d->printf_fn(d->opts.ctx, fmt, args); + va_end(args); +} + +struct btf_dump *btf_dump__new(const struct btf *btf, + const struct btf_ext *btf_ext, + const struct btf_dump_opts *opts, + btf_dump_printf_fn_t printf_fn) +{ + struct btf_dump *d; + int err; + + d = calloc(1, sizeof(struct btf_dump)); + if (!d) + return ERR_PTR(-ENOMEM); + + d->btf = btf; + d->btf_ext = btf_ext; + d->printf_fn = printf_fn; + d->opts.ctx = opts ? opts->ctx : NULL; + + d->type_names = hashmap__new(str_hash_fn, str_equal_fn, NULL); + if (IS_ERR(d->type_names)) { + err = PTR_ERR(d->type_names); + d->type_names = NULL; + btf_dump__free(d); + return ERR_PTR(err); + } + d->ident_names = hashmap__new(str_hash_fn, str_equal_fn, NULL); + if (IS_ERR(d->ident_names)) { + err = PTR_ERR(d->ident_names); + d->ident_names = NULL; + btf_dump__free(d); + return ERR_PTR(err); + } + + return d; +} + +void btf_dump__free(struct btf_dump *d) +{ + int i, cnt; + + if (!d) + return; + + free(d->type_states); + if (d->cached_names) { + /* any set cached name is owned by us and should be freed */ + for (i = 0, cnt = btf__get_nr_types(d->btf); i <= cnt; i++) { + if (d->cached_names[i]) + free((void *)d->cached_names[i]); + } + } + free(d->cached_names); + free(d->emit_queue); + free(d->decl_stack); + hashmap__free(d->type_names); + hashmap__free(d->ident_names); + + free(d); +} + +static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr); +static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id); + +/* + * Dump BTF type in a compilable C syntax, including all the necessary + * dependent types, necessary for compilation. If some of the dependent types + * were already emitted as part of previous btf_dump__dump_type() invocation + * for another type, they won't be emitted again. This API allows callers to + * filter out BTF types according to user-defined criterias and emitted only + * minimal subset of types, necessary to compile everything. Full struct/union + * definitions will still be emitted, even if the only usage is through + * pointer and could be satisfied with just a forward declaration. + * + * Dumping is done in two high-level passes: + * 1. Topologically sort type definitions to satisfy C rules of compilation. + * 2. Emit type definitions in C syntax. + * + * Returns 0 on success; <0, otherwise. + */ +int btf_dump__dump_type(struct btf_dump *d, __u32 id) +{ + int err, i; + + if (id > btf__get_nr_types(d->btf)) + return -EINVAL; + + /* type states are lazily allocated, as they might not be needed */ + if (!d->type_states) { + d->type_states = calloc(1 + btf__get_nr_types(d->btf), + sizeof(d->type_states[0])); + if (!d->type_states) + return -ENOMEM; + d->cached_names = calloc(1 + btf__get_nr_types(d->btf), + sizeof(d->cached_names[0])); + if (!d->cached_names) + return -ENOMEM; + + /* VOID is special */ + d->type_states[0].order_state = ORDERED; + d->type_states[0].emit_state = EMITTED; + } + + d->emit_queue_cnt = 0; + err = btf_dump_order_type(d, id, false); + if (err < 0) + return err; + + for (i = 0; i < d->emit_queue_cnt; i++) + btf_dump_emit_type(d, d->emit_queue[i], 0 /*top-level*/); + + return 0; +} + +static int btf_dump_add_emit_queue_id(struct btf_dump *d, __u32 id) +{ + __u32 *new_queue; + size_t new_cap; + + if (d->emit_queue_cnt >= d->emit_queue_cap) { + new_cap = max(16, d->emit_queue_cap * 3 / 2); + new_queue = realloc(d->emit_queue, + new_cap * sizeof(new_queue[0])); + if (!new_queue) + return -ENOMEM; + d->emit_queue = new_queue; + d->emit_queue_cap = new_cap; + } + + d->emit_queue[d->emit_queue_cnt++] = id; + return 0; +} + +/* + * Determine order of emitting dependent types and specified type to satisfy + * C compilation rules. This is done through topological sorting with an + * additional complication which comes from C rules. The main idea for C is + * that if some type is "embedded" into a struct/union, it's size needs to be + * known at the time of definition of containing type. E.g., for: + * + * struct A {}; + * struct B { struct A x; } + * + * struct A *HAS* to be defined before struct B, because it's "embedded", + * i.e., it is part of struct B layout. But in the following case: + * + * struct A; + * struct B { struct A *x; } + * struct A {}; + * + * it's enough to just have a forward declaration of struct A at the time of + * struct B definition, as struct B has a pointer to struct A, so the size of + * field x is known without knowing struct A size: it's sizeof(void *). + * + * Unfortunately, there are some trickier cases we need to handle, e.g.: + * + * struct A {}; // if this was forward-declaration: compilation error + * struct B { + * struct { // anonymous struct + * struct A y; + * } *x; + * }; + * + * In this case, struct B's field x is a pointer, so it's size is known + * regardless of the size of (anonymous) struct it points to. But because this + * struct is anonymous and thus defined inline inside struct B, *and* it + * embeds struct A, compiler requires full definition of struct A to be known + * before struct B can be defined. This creates a transitive dependency + * between struct A and struct B. If struct A was forward-declared before + * struct B definition and fully defined after struct B definition, that would + * trigger compilation error. + * + * All this means that while we are doing topological sorting on BTF type + * graph, we need to determine relationships between different types (graph + * nodes): + * - weak link (relationship) between X and Y, if Y *CAN* be + * forward-declared at the point of X definition; + * - strong link, if Y *HAS* to be fully-defined before X can be defined. + * + * The rule is as follows. Given a chain of BTF types from X to Y, if there is + * BTF_KIND_PTR type in the chain and at least one non-anonymous type + * Z (excluding X, including Y), then link is weak. Otherwise, it's strong. + * Weak/strong relationship is determined recursively during DFS traversal and + * is returned as a result from btf_dump_order_type(). + * + * btf_dump_order_type() is trying to avoid unnecessary forward declarations, + * but it is not guaranteeing that no extraneous forward declarations will be + * emitted. + * + * To avoid extra work, algorithm marks some of BTF types as ORDERED, when + * it's done with them, but not for all (e.g., VOLATILE, CONST, RESTRICT, + * ARRAY, FUNC_PROTO), as weak/strong semantics for those depends on the + * entire graph path, so depending where from one came to that BTF type, it + * might cause weak or strong ordering. For types like STRUCT/UNION/INT/ENUM, + * once they are processed, there is no need to do it again, so they are + * marked as ORDERED. We can mark PTR as ORDERED as well, as it semi-forces + * weak link, unless subsequent referenced STRUCT/UNION/ENUM is anonymous. But + * in any case, once those are processed, no need to do it again, as the + * result won't change. + * + * Returns: + * - 1, if type is part of strong link (so there is strong topological + * ordering requirements); + * - 0, if type is part of weak link (so can be satisfied through forward + * declaration); + * - <0, on error (e.g., unsatisfiable type loop detected). + */ +static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) +{ + /* + * Order state is used to detect strong link cycles, but only for BTF + * kinds that are or could be an independent definition (i.e., + * stand-alone fwd decl, enum, typedef, struct, union). Ptrs, arrays, + * func_protos, modifiers are just means to get to these definitions. + * Int/void don't need definitions, they are assumed to be always + * properly defined. We also ignore datasec, var, and funcs for now. + * So for all non-defining kinds, we never even set ordering state, + * for defining kinds we set ORDERING and subsequently ORDERED if it + * forms a strong link. + */ + struct btf_dump_type_aux_state *tstate = &d->type_states[id]; + const struct btf_type *t; + __u16 kind, vlen; + int err, i; + + /* return true, letting typedefs know that it's ok to be emitted */ + if (tstate->order_state == ORDERED) + return 1; + + t = btf__type_by_id(d->btf, id); + kind = btf_kind_of(t); + + if (tstate->order_state == ORDERING) { + /* type loop, but resolvable through fwd declaration */ + if ((kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION) && + through_ptr && t->name_off != 0) + return 0; + pr_warning("unsatisfiable type cycle, id:[%u]\n", id); + return -ELOOP; + } + + switch (kind) { + case BTF_KIND_INT: + tstate->order_state = ORDERED; + return 0; + + case BTF_KIND_PTR: + err = btf_dump_order_type(d, t->type, true); + tstate->order_state = ORDERED; + return err; + + case BTF_KIND_ARRAY: { + const struct btf_array *a = (void *)(t + 1); + + return btf_dump_order_type(d, a->type, through_ptr); + } + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: { + const struct btf_member *m = (void *)(t + 1); + /* + * struct/union is part of strong link, only if it's embedded + * (so no ptr in a path) or it's anonymous (so has to be + * defined inline, even if declared through ptr) + */ + if (through_ptr && t->name_off != 0) + return 0; + + tstate->order_state = ORDERING; + + vlen = btf_vlen_of(t); + for (i = 0; i < vlen; i++, m++) { + err = btf_dump_order_type(d, m->type, false); + if (err < 0) + return err; + } + + if (t->name_off != 0) { + err = btf_dump_add_emit_queue_id(d, id); + if (err < 0) + return err; + } + + tstate->order_state = ORDERED; + return 1; + } + case BTF_KIND_ENUM: + case BTF_KIND_FWD: + if (t->name_off != 0) { + err = btf_dump_add_emit_queue_id(d, id); + if (err) + return err; + } + tstate->order_state = ORDERED; + return 1; + + case BTF_KIND_TYPEDEF: { + int is_strong; + + is_strong = btf_dump_order_type(d, t->type, through_ptr); + if (is_strong < 0) + return is_strong; + + /* typedef is similar to struct/union w.r.t. fwd-decls */ + if (through_ptr && !is_strong) + return 0; + + /* typedef is always a named definition */ + err = btf_dump_add_emit_queue_id(d, id); + if (err) + return err; + + d->type_states[id].order_state = ORDERED; + return 1; + } + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + return btf_dump_order_type(d, t->type, through_ptr); + + case BTF_KIND_FUNC_PROTO: { + const struct btf_param *p = (void *)(t + 1); + bool is_strong; + + err = btf_dump_order_type(d, t->type, through_ptr); + if (err < 0) + return err; + is_strong = err > 0; + + vlen = btf_vlen_of(t); + for (i = 0; i < vlen; i++, p++) { + err = btf_dump_order_type(d, p->type, through_ptr); + if (err < 0) + return err; + if (err > 0) + is_strong = true; + } + return is_strong; + } + case BTF_KIND_FUNC: + case BTF_KIND_VAR: + case BTF_KIND_DATASEC: + d->type_states[id].order_state = ORDERED; + return 0; + + default: + return -EINVAL; + } +} + +static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, + const struct btf_type *t); +static void btf_dump_emit_struct_def(struct btf_dump *d, __u32 id, + const struct btf_type *t, int lvl); + +static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, + const struct btf_type *t); +static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, + const struct btf_type *t, int lvl); + +static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id, + const struct btf_type *t); + +static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, + const struct btf_type *t, int lvl); + +/* a local view into a shared stack */ +struct id_stack { + const __u32 *ids; + int cnt; +}; + +static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, + const char *fname, int lvl); +static void btf_dump_emit_type_chain(struct btf_dump *d, + struct id_stack *decl_stack, + const char *fname, int lvl); + +static const char *btf_dump_type_name(struct btf_dump *d, __u32 id); +static const char *btf_dump_ident_name(struct btf_dump *d, __u32 id); +static size_t btf_dump_name_dups(struct btf_dump *d, struct hashmap *name_map, + const char *orig_name); + +static bool btf_dump_is_blacklisted(struct btf_dump *d, __u32 id) +{ + const struct btf_type *t = btf__type_by_id(d->btf, id); + + /* __builtin_va_list is a compiler built-in, which causes compilation + * errors, when compiling w/ different compiler, then used to compile + * original code (e.g., GCC to compile kernel, Clang to use generated + * C header from BTF). As it is built-in, it should be already defined + * properly internally in compiler. + */ + if (t->name_off == 0) + return false; + return strcmp(btf_name_of(d, t->name_off), "__builtin_va_list") == 0; +} + +/* + * Emit C-syntax definitions of types from chains of BTF types. + * + * High-level handling of determining necessary forward declarations are handled + * by btf_dump_emit_type() itself, but all nitty-gritty details of emitting type + * declarations/definitions in C syntax are handled by a combo of + * btf_dump_emit_type_decl()/btf_dump_emit_type_chain() w/ delegation to + * corresponding btf_dump_emit_*_{def,fwd}() functions. + * + * We also keep track of "containing struct/union type ID" to determine when + * we reference it from inside and thus can avoid emitting unnecessary forward + * declaration. + * + * This algorithm is designed in such a way, that even if some error occurs + * (either technical, e.g., out of memory, or logical, i.e., malformed BTF + * that doesn't comply to C rules completely), algorithm will try to proceed + * and produce as much meaningful output as possible. + */ +static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) +{ + struct btf_dump_type_aux_state *tstate = &d->type_states[id]; + bool top_level_def = cont_id == 0; + const struct btf_type *t; + __u16 kind; + + if (tstate->emit_state == EMITTED) + return; + + t = btf__type_by_id(d->btf, id); + kind = btf_kind_of(t); + + if (top_level_def && t->name_off == 0) { + pr_warning("unexpected nameless definition, id:[%u]\n", id); + return; + } + + if (tstate->emit_state == EMITTING) { + if (tstate->fwd_emitted) + return; + + switch (kind) { + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + /* + * if we are referencing a struct/union that we are + * part of - then no need for fwd declaration + */ + if (id == cont_id) + return; + if (t->name_off == 0) { + pr_warning("anonymous struct/union loop, id:[%u]\n", + id); + return; + } + btf_dump_emit_struct_fwd(d, id, t); + btf_dump_printf(d, ";\n\n"); + tstate->fwd_emitted = 1; + break; + case BTF_KIND_TYPEDEF: + /* + * for typedef fwd_emitted means typedef definition + * was emitted, but it can be used only for "weak" + * references through pointer only, not for embedding + */ + if (!btf_dump_is_blacklisted(d, id)) { + btf_dump_emit_typedef_def(d, id, t, 0); + btf_dump_printf(d, ";\n\n"); + }; + tstate->fwd_emitted = 1; + break; + default: + break; + } + + return; + } + + switch (kind) { + case BTF_KIND_INT: + tstate->emit_state = EMITTED; + break; + case BTF_KIND_ENUM: + if (top_level_def) { + btf_dump_emit_enum_def(d, id, t, 0); + btf_dump_printf(d, ";\n\n"); + } + tstate->emit_state = EMITTED; + break; + case BTF_KIND_PTR: + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + btf_dump_emit_type(d, t->type, cont_id); + break; + case BTF_KIND_ARRAY: { + const struct btf_array *a = (void *)(t + 1); + + btf_dump_emit_type(d, a->type, cont_id); + break; + } + case BTF_KIND_FWD: + btf_dump_emit_fwd_def(d, id, t); + btf_dump_printf(d, ";\n\n"); + tstate->emit_state = EMITTED; + break; + case BTF_KIND_TYPEDEF: + tstate->emit_state = EMITTING; + btf_dump_emit_type(d, t->type, id); + /* + * typedef can server as both definition and forward + * declaration; at this stage someone depends on + * typedef as a forward declaration (refers to it + * through pointer), so unless we already did it, + * emit typedef as a forward declaration + */ + if (!tstate->fwd_emitted && !btf_dump_is_blacklisted(d, id)) { + btf_dump_emit_typedef_def(d, id, t, 0); + btf_dump_printf(d, ";\n\n"); + } + tstate->emit_state = EMITTED; + break; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + tstate->emit_state = EMITTING; + /* if it's a top-level struct/union definition or struct/union + * is anonymous, then in C we'll be emitting all fields and + * their types (as opposed to just `struct X`), so we need to + * make sure that all types, referenced from struct/union + * members have necessary forward-declarations, where + * applicable + */ + if (top_level_def || t->name_off == 0) { + const struct btf_member *m = (void *)(t + 1); + __u16 vlen = btf_vlen_of(t); + int i, new_cont_id; + + new_cont_id = t->name_off == 0 ? cont_id : id; + for (i = 0; i < vlen; i++, m++) + btf_dump_emit_type(d, m->type, new_cont_id); + } else if (!tstate->fwd_emitted && id != cont_id) { + btf_dump_emit_struct_fwd(d, id, t); + btf_dump_printf(d, ";\n\n"); + tstate->fwd_emitted = 1; + } + + if (top_level_def) { + btf_dump_emit_struct_def(d, id, t, 0); + btf_dump_printf(d, ";\n\n"); + tstate->emit_state = EMITTED; + } else { + tstate->emit_state = NOT_EMITTED; + } + break; + case BTF_KIND_FUNC_PROTO: { + const struct btf_param *p = (void *)(t + 1); + __u16 vlen = btf_vlen_of(t); + int i; + + btf_dump_emit_type(d, t->type, cont_id); + for (i = 0; i < vlen; i++, p++) + btf_dump_emit_type(d, p->type, cont_id); + + break; + } + default: + break; + } +} + +static int btf_align_of(const struct btf *btf, __u32 id) +{ + const struct btf_type *t = btf__type_by_id(btf, id); + __u16 kind = btf_kind_of(t); + + switch (kind) { + case BTF_KIND_INT: + case BTF_KIND_ENUM: + return min(sizeof(void *), t->size); + case BTF_KIND_PTR: + return sizeof(void *); + case BTF_KIND_TYPEDEF: + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + return btf_align_of(btf, t->type); + case BTF_KIND_ARRAY: { + const struct btf_array *a = (void *)(t + 1); + + return btf_align_of(btf, a->type); + } + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: { + const struct btf_member *m = (void *)(t + 1); + __u16 vlen = btf_vlen_of(t); + int i, align = 1; + + for (i = 0; i < vlen; i++, m++) + align = max(align, btf_align_of(btf, m->type)); + + return align; + } + default: + pr_warning("unsupported BTF_KIND:%u\n", btf_kind_of(t)); + return 1; + } +} + +static bool btf_is_struct_packed(const struct btf *btf, __u32 id, + const struct btf_type *t) +{ + const struct btf_member *m; + int align, i, bit_sz; + __u16 vlen; + bool kflag; + + align = btf_align_of(btf, id); + /* size of a non-packed struct has to be a multiple of its alignment*/ + if (t->size % align) + return true; + + m = (void *)(t + 1); + kflag = btf_kflag_of(t); + vlen = btf_vlen_of(t); + /* all non-bitfield fields have to be naturally aligned */ + for (i = 0; i < vlen; i++, m++) { + align = btf_align_of(btf, m->type); + bit_sz = kflag ? BTF_MEMBER_BITFIELD_SIZE(m->offset) : 0; + if (bit_sz == 0 && m->offset % (8 * align) != 0) + return true; + } + + /* + * if original struct was marked as packed, but its layout is + * naturally aligned, we'll detect that it's not packed + */ + return false; +} + +static int chip_away_bits(int total, int at_most) +{ + return total % at_most ? : at_most; +} + +static void btf_dump_emit_bit_padding(const struct btf_dump *d, + int cur_off, int m_off, int m_bit_sz, + int align, int lvl) +{ + int off_diff = m_off - cur_off; + int ptr_bits = sizeof(void *) * 8; + + if (off_diff <= 0) + /* no gap */ + return; + if (m_bit_sz == 0 && off_diff < align * 8) + /* natural padding will take care of a gap */ + return; + + while (off_diff > 0) { + const char *pad_type; + int pad_bits; + + if (ptr_bits > 32 && off_diff > 32) { + pad_type = "long"; + pad_bits = chip_away_bits(off_diff, ptr_bits); + } else if (off_diff > 16) { + pad_type = "int"; + pad_bits = chip_away_bits(off_diff, 32); + } else if (off_diff > 8) { + pad_type = "short"; + pad_bits = chip_away_bits(off_diff, 16); + } else { + pad_type = "char"; + pad_bits = chip_away_bits(off_diff, 8); + } + btf_dump_printf(d, "\n%s%s: %d;", pfx(lvl), pad_type, pad_bits); + off_diff -= pad_bits; + } +} + +static void btf_dump_emit_struct_fwd(struct btf_dump *d, __u32 id, + const struct btf_type *t) +{ + btf_dump_printf(d, "%s %s", + btf_kind_of(t) == BTF_KIND_STRUCT ? "struct" : "union", + btf_dump_type_name(d, id)); +} + +static void btf_dump_emit_struct_def(struct btf_dump *d, + __u32 id, + const struct btf_type *t, + int lvl) +{ + const struct btf_member *m = (void *)(t + 1); + bool kflag = btf_kflag_of(t), is_struct; + int align, i, packed, off = 0; + __u16 vlen = btf_vlen_of(t); + + is_struct = btf_kind_of(t) == BTF_KIND_STRUCT; + packed = is_struct ? btf_is_struct_packed(d->btf, id, t) : 0; + align = packed ? 1 : btf_align_of(d->btf, id); + + btf_dump_printf(d, "%s%s%s {", + is_struct ? "struct" : "union", + t->name_off ? " " : "", + btf_dump_type_name(d, id)); + + for (i = 0; i < vlen; i++, m++) { + const char *fname; + int m_off, m_sz; + + fname = btf_name_of(d, m->name_off); + m_sz = kflag ? BTF_MEMBER_BITFIELD_SIZE(m->offset) : 0; + m_off = kflag ? BTF_MEMBER_BIT_OFFSET(m->offset) : m->offset; + align = packed ? 1 : btf_align_of(d->btf, m->type); + + btf_dump_emit_bit_padding(d, off, m_off, m_sz, align, lvl + 1); + btf_dump_printf(d, "\n%s", pfx(lvl + 1)); + btf_dump_emit_type_decl(d, m->type, fname, lvl + 1); + + if (m_sz) { + btf_dump_printf(d, ": %d", m_sz); + off = m_off + m_sz; + } else { + m_sz = max(0, btf__resolve_size(d->btf, m->type)); + off = m_off + m_sz * 8; + } + btf_dump_printf(d, ";"); + } + + if (vlen) + btf_dump_printf(d, "\n"); + btf_dump_printf(d, "%s}", pfx(lvl)); + if (packed) + btf_dump_printf(d, " __attribute__((packed))"); +} + +static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, + const struct btf_type *t) +{ + btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id)); +} + +static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, + const struct btf_type *t, + int lvl) +{ + const struct btf_enum *v = (void *)(t+1); + __u16 vlen = btf_vlen_of(t); + const char *name; + size_t dup_cnt; + int i; + + btf_dump_printf(d, "enum%s%s", + t->name_off ? " " : "", + btf_dump_type_name(d, id)); + + if (vlen) { + btf_dump_printf(d, " {"); + for (i = 0; i < vlen; i++, v++) { + name = btf_name_of(d, v->name_off); + /* enumerators share namespace with typedef idents */ + dup_cnt = btf_dump_name_dups(d, d->ident_names, name); + if (dup_cnt > 1) { + btf_dump_printf(d, "\n%s%s___%zu = %d,", + pfx(lvl + 1), name, dup_cnt, + (__s32)v->val); + } else { + btf_dump_printf(d, "\n%s%s = %d,", + pfx(lvl + 1), name, + (__s32)v->val); + } + } + btf_dump_printf(d, "\n%s}", pfx(lvl)); + } +} + +static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id, + const struct btf_type *t) +{ + const char *name = btf_dump_type_name(d, id); + + if (btf_kflag_of(t)) + btf_dump_printf(d, "union %s", name); + else + btf_dump_printf(d, "struct %s", name); +} + +static void btf_dump_emit_typedef_def(struct btf_dump *d, __u32 id, + const struct btf_type *t, int lvl) +{ + const char *name = btf_dump_ident_name(d, id); + + btf_dump_printf(d, "typedef "); + btf_dump_emit_type_decl(d, t->type, name, lvl); +} + +static int btf_dump_push_decl_stack_id(struct btf_dump *d, __u32 id) +{ + __u32 *new_stack; + size_t new_cap; + + if (d->decl_stack_cnt >= d->decl_stack_cap) { + new_cap = max(16, d->decl_stack_cap * 3 / 2); + new_stack = realloc(d->decl_stack, + new_cap * sizeof(new_stack[0])); + if (!new_stack) + return -ENOMEM; + d->decl_stack = new_stack; + d->decl_stack_cap = new_cap; + } + + d->decl_stack[d->decl_stack_cnt++] = id; + + return 0; +} + +/* + * Emit type declaration (e.g., field type declaration in a struct or argument + * declaration in function prototype) in correct C syntax. + * + * For most types it's trivial, but there are few quirky type declaration + * cases worth mentioning: + * - function prototypes (especially nesting of function prototypes); + * - arrays; + * - const/volatile/restrict for pointers vs other types. + * + * For a good discussion of *PARSING* C syntax (as a human), see + * Peter van der Linden's "Expert C Programming: Deep C Secrets", + * Ch.3 "Unscrambling Declarations in C". + * + * It won't help with BTF to C conversion much, though, as it's an opposite + * problem. So we came up with this algorithm in reverse to van der Linden's + * parsing algorithm. It goes from structured BTF representation of type + * declaration to a valid compilable C syntax. + * + * For instance, consider this C typedef: + * typedef const int * const * arr[10] arr_t; + * It will be represented in BTF with this chain of BTF types: + * [typedef] -> [array] -> [ptr] -> [const] -> [ptr] -> [const] -> [int] + * + * Notice how [const] modifier always goes before type it modifies in BTF type + * graph, but in C syntax, const/volatile/restrict modifiers are written to + * the right of pointers, but to the left of other types. There are also other + * quirks, like function pointers, arrays of them, functions returning other + * functions, etc. + * + * We handle that by pushing all the types to a stack, until we hit "terminal" + * type (int/enum/struct/union/fwd). Then depending on the kind of a type on + * top of a stack, modifiers are handled differently. Array/function pointers + * have also wildly different syntax and how nesting of them are done. See + * code for authoritative definition. + * + * To avoid allocating new stack for each independent chain of BTF types, we + * share one bigger stack, with each chain working only on its own local view + * of a stack frame. Some care is required to "pop" stack frames after + * processing type declaration chain. + */ +static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, + const char *fname, int lvl) +{ + struct id_stack decl_stack; + const struct btf_type *t; + int err, stack_start; + __u16 kind; + + stack_start = d->decl_stack_cnt; + for (;;) { + err = btf_dump_push_decl_stack_id(d, id); + if (err < 0) { + /* + * if we don't have enough memory for entire type decl + * chain, restore stack, emit warning, and try to + * proceed nevertheless + */ + pr_warning("not enough memory for decl stack:%d", err); + d->decl_stack_cnt = stack_start; + return; + } + + /* VOID */ + if (id == 0) + break; + + t = btf__type_by_id(d->btf, id); + kind = btf_kind_of(t); + switch (kind) { + case BTF_KIND_PTR: + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + case BTF_KIND_FUNC_PROTO: + id = t->type; + break; + case BTF_KIND_ARRAY: { + const struct btf_array *a = (void *)(t + 1); + + id = a->type; + break; + } + case BTF_KIND_INT: + case BTF_KIND_ENUM: + case BTF_KIND_FWD: + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + case BTF_KIND_TYPEDEF: + goto done; + default: + pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n", + kind, id); + goto done; + } + } +done: + /* + * We might be inside a chain of declarations (e.g., array of function + * pointers returning anonymous (so inlined) structs, having another + * array field). Each of those needs its own "stack frame" to handle + * emitting of declarations. Those stack frames are non-overlapping + * portions of shared btf_dump->decl_stack. To make it a bit nicer to + * handle this set of nested stacks, we create a view corresponding to + * our own "stack frame" and work with it as an independent stack. + * We'll need to clean up after emit_type_chain() returns, though. + */ + decl_stack.ids = d->decl_stack + stack_start; + decl_stack.cnt = d->decl_stack_cnt - stack_start; + btf_dump_emit_type_chain(d, &decl_stack, fname, lvl); + /* + * emit_type_chain() guarantees that it will pop its entire decl_stack + * frame before returning. But it works with a read-only view into + * decl_stack, so it doesn't actually pop anything from the + * perspective of shared btf_dump->decl_stack, per se. We need to + * reset decl_stack state to how it was before us to avoid it growing + * all the time. + */ + d->decl_stack_cnt = stack_start; +} + +static void btf_dump_emit_mods(struct btf_dump *d, struct id_stack *decl_stack) +{ + const struct btf_type *t; + __u32 id; + + while (decl_stack->cnt) { + id = decl_stack->ids[decl_stack->cnt - 1]; + t = btf__type_by_id(d->btf, id); + + switch (btf_kind_of(t)) { + case BTF_KIND_VOLATILE: + btf_dump_printf(d, "volatile "); + break; + case BTF_KIND_CONST: + btf_dump_printf(d, "const "); + break; + case BTF_KIND_RESTRICT: + btf_dump_printf(d, "restrict "); + break; + default: + return; + } + decl_stack->cnt--; + } +} + +static bool btf_is_mod_kind(const struct btf *btf, __u32 id) +{ + const struct btf_type *t = btf__type_by_id(btf, id); + + switch (btf_kind_of(t)) { + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + return true; + default: + return false; + } +} + +static void btf_dump_emit_name(const struct btf_dump *d, + const char *name, bool last_was_ptr) +{ + bool separate = name[0] && !last_was_ptr; + + btf_dump_printf(d, "%s%s", separate ? " " : "", name); +} + +static void btf_dump_emit_type_chain(struct btf_dump *d, + struct id_stack *decls, + const char *fname, int lvl) +{ + /* + * last_was_ptr is used to determine if we need to separate pointer + * asterisk (*) from previous part of type signature with space, so + * that we get `int ***`, instead of `int * * *`. We default to true + * for cases where we have single pointer in a chain. E.g., in ptr -> + * func_proto case. func_proto will start a new emit_type_chain call + * with just ptr, which should be emitted as (*) or (*<fname>), so we + * don't want to prepend space for that last pointer. + */ + bool last_was_ptr = true; + const struct btf_type *t; + const char *name; + __u16 kind; + __u32 id; + + while (decls->cnt) { + id = decls->ids[--decls->cnt]; + if (id == 0) { + /* VOID is a special snowflake */ + btf_dump_emit_mods(d, decls); + btf_dump_printf(d, "void"); + last_was_ptr = false; + continue; + } + + t = btf__type_by_id(d->btf, id); + kind = btf_kind_of(t); + + switch (kind) { + case BTF_KIND_INT: + btf_dump_emit_mods(d, decls); + name = btf_name_of(d, t->name_off); + btf_dump_printf(d, "%s", name); + break; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + btf_dump_emit_mods(d, decls); + /* inline anonymous struct/union */ + if (t->name_off == 0) + btf_dump_emit_struct_def(d, id, t, lvl); + else + btf_dump_emit_struct_fwd(d, id, t); + break; + case BTF_KIND_ENUM: + btf_dump_emit_mods(d, decls); + /* inline anonymous enum */ + if (t->name_off == 0) + btf_dump_emit_enum_def(d, id, t, lvl); + else + btf_dump_emit_enum_fwd(d, id, t); + break; + case BTF_KIND_FWD: + btf_dump_emit_mods(d, decls); + btf_dump_emit_fwd_def(d, id, t); + break; + case BTF_KIND_TYPEDEF: + btf_dump_emit_mods(d, decls); + btf_dump_printf(d, "%s", btf_dump_ident_name(d, id)); + break; + case BTF_KIND_PTR: + btf_dump_printf(d, "%s", last_was_ptr ? "*" : " *"); + break; + case BTF_KIND_VOLATILE: + btf_dump_printf(d, " volatile"); + break; + case BTF_KIND_CONST: + btf_dump_printf(d, " const"); + break; + case BTF_KIND_RESTRICT: + btf_dump_printf(d, " restrict"); + break; + case BTF_KIND_ARRAY: { + const struct btf_array *a = (void *)(t + 1); + const struct btf_type *next_t; + __u32 next_id; + bool multidim; + /* + * GCC has a bug + * (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=8354) + * which causes it to emit extra const/volatile + * modifiers for an array, if array's element type has + * const/volatile modifiers. Clang doesn't do that. + * In general, it doesn't seem very meaningful to have + * a const/volatile modifier for array, so we are + * going to silently skip them here. + */ + while (decls->cnt) { + next_id = decls->ids[decls->cnt - 1]; + if (btf_is_mod_kind(d->btf, next_id)) + decls->cnt--; + else + break; + } + + if (decls->cnt == 0) { + btf_dump_emit_name(d, fname, last_was_ptr); + btf_dump_printf(d, "[%u]", a->nelems); + return; + } + + next_t = btf__type_by_id(d->btf, next_id); + multidim = btf_kind_of(next_t) == BTF_KIND_ARRAY; + /* we need space if we have named non-pointer */ + if (fname[0] && !last_was_ptr) + btf_dump_printf(d, " "); + /* no parentheses for multi-dimensional array */ + if (!multidim) + btf_dump_printf(d, "("); + btf_dump_emit_type_chain(d, decls, fname, lvl); + if (!multidim) + btf_dump_printf(d, ")"); + btf_dump_printf(d, "[%u]", a->nelems); + return; + } + case BTF_KIND_FUNC_PROTO: { + const struct btf_param *p = (void *)(t + 1); + __u16 vlen = btf_vlen_of(t); + int i; + + btf_dump_emit_mods(d, decls); + if (decls->cnt) { + btf_dump_printf(d, " ("); + btf_dump_emit_type_chain(d, decls, fname, lvl); + btf_dump_printf(d, ")"); + } else { + btf_dump_emit_name(d, fname, last_was_ptr); + } + btf_dump_printf(d, "("); + /* + * Clang for BPF target generates func_proto with no + * args as a func_proto with a single void arg (e.g., + * `int (*f)(void)` vs just `int (*f)()`). We are + * going to pretend there are no args for such case. + */ + if (vlen == 1 && p->type == 0) { + btf_dump_printf(d, ")"); + return; + } + + for (i = 0; i < vlen; i++, p++) { + if (i > 0) + btf_dump_printf(d, ", "); + + /* last arg of type void is vararg */ + if (i == vlen - 1 && p->type == 0) { + btf_dump_printf(d, "..."); + break; + } + + name = btf_name_of(d, p->name_off); + btf_dump_emit_type_decl(d, p->type, name, lvl); + } + + btf_dump_printf(d, ")"); + return; + } + default: + pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n", + kind, id); + return; + } + + last_was_ptr = kind == BTF_KIND_PTR; + } + + btf_dump_emit_name(d, fname, last_was_ptr); +} + +/* return number of duplicates (occurrences) of a given name */ +static size_t btf_dump_name_dups(struct btf_dump *d, struct hashmap *name_map, + const char *orig_name) +{ + size_t dup_cnt = 0; + + hashmap__find(name_map, orig_name, (void **)&dup_cnt); + dup_cnt++; + hashmap__set(name_map, orig_name, (void *)dup_cnt, NULL, NULL); + + return dup_cnt; +} + +static const char *btf_dump_resolve_name(struct btf_dump *d, __u32 id, + struct hashmap *name_map) +{ + struct btf_dump_type_aux_state *s = &d->type_states[id]; + const struct btf_type *t = btf__type_by_id(d->btf, id); + const char *orig_name = btf_name_of(d, t->name_off); + const char **cached_name = &d->cached_names[id]; + size_t dup_cnt; + + if (t->name_off == 0) + return ""; + + if (s->name_resolved) + return *cached_name ? *cached_name : orig_name; + + dup_cnt = btf_dump_name_dups(d, name_map, orig_name); + if (dup_cnt > 1) { + const size_t max_len = 256; + char new_name[max_len]; + + snprintf(new_name, max_len, "%s___%zu", orig_name, dup_cnt); + *cached_name = strdup(new_name); + } + + s->name_resolved = 1; + return *cached_name ? *cached_name : orig_name; +} + +static const char *btf_dump_type_name(struct btf_dump *d, __u32 id) +{ + return btf_dump_resolve_name(d, id, d->type_names); +} + +static const char *btf_dump_ident_name(struct btf_dump *d, __u32 id) +{ + return btf_dump_resolve_name(d, id, d->ident_names); +} diff --git a/tools/lib/bpf/hashmap.c b/tools/lib/bpf/hashmap.c new file mode 100644 index 000000000000..6122272943e6 --- /dev/null +++ b/tools/lib/bpf/hashmap.c @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#include <stdint.h> +#include <stdlib.h> +#include <stdio.h> +#include <errno.h> +#include <linux/err.h> +#include "hashmap.h" + +/* start with 4 buckets */ +#define HASHMAP_MIN_CAP_BITS 2 + +static void hashmap_add_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + entry->next = *pprev; + *pprev = entry; +} + +static void hashmap_del_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + *pprev = entry->next; + entry->next = NULL; +} + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx) +{ + map->hash_fn = hash_fn; + map->equal_fn = equal_fn; + map->ctx = ctx; + + map->buckets = NULL; + map->cap = 0; + map->cap_bits = 0; + map->sz = 0; +} + +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx) +{ + struct hashmap *map = malloc(sizeof(struct hashmap)); + + if (!map) + return ERR_PTR(-ENOMEM); + hashmap__init(map, hash_fn, equal_fn, ctx); + return map; +} + +void hashmap__clear(struct hashmap *map) +{ + free(map->buckets); + map->cap = map->cap_bits = map->sz = 0; +} + +void hashmap__free(struct hashmap *map) +{ + if (!map) + return; + + hashmap__clear(map); + free(map); +} + +size_t hashmap__size(const struct hashmap *map) +{ + return map->sz; +} + +size_t hashmap__capacity(const struct hashmap *map) +{ + return map->cap; +} + +static bool hashmap_needs_to_grow(struct hashmap *map) +{ + /* grow if empty or more than 75% filled */ + return (map->cap == 0) || ((map->sz + 1) * 4 / 3 > map->cap); +} + +static int hashmap_grow(struct hashmap *map) +{ + struct hashmap_entry **new_buckets; + struct hashmap_entry *cur, *tmp; + size_t new_cap_bits, new_cap; + size_t h; + int bkt; + + new_cap_bits = map->cap_bits + 1; + if (new_cap_bits < HASHMAP_MIN_CAP_BITS) + new_cap_bits = HASHMAP_MIN_CAP_BITS; + + new_cap = 1UL << new_cap_bits; + new_buckets = calloc(new_cap, sizeof(new_buckets[0])); + if (!new_buckets) + return -ENOMEM; + + hashmap__for_each_entry_safe(map, cur, tmp, bkt) { + h = hash_bits(map->hash_fn(cur->key, map->ctx), new_cap_bits); + hashmap_add_entry(&new_buckets[h], cur); + } + + map->cap = new_cap; + map->cap_bits = new_cap_bits; + free(map->buckets); + map->buckets = new_buckets; + + return 0; +} + +static bool hashmap_find_entry(const struct hashmap *map, + const void *key, size_t hash, + struct hashmap_entry ***pprev, + struct hashmap_entry **entry) +{ + struct hashmap_entry *cur, **prev_ptr; + + if (!map->buckets) + return false; + + for (prev_ptr = &map->buckets[hash], cur = *prev_ptr; + cur; + prev_ptr = &cur->next, cur = cur->next) { + if (map->equal_fn(cur->key, key, map->ctx)) { + if (pprev) + *pprev = prev_ptr; + *entry = cur; + return true; + } + } + + return false; +} + +int hashmap__insert(struct hashmap *map, const void *key, void *value, + enum hashmap_insert_strategy strategy, + const void **old_key, void **old_value) +{ + struct hashmap_entry *entry; + size_t h; + int err; + + if (old_key) + *old_key = NULL; + if (old_value) + *old_value = NULL; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (strategy != HASHMAP_APPEND && + hashmap_find_entry(map, key, h, NULL, &entry)) { + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + if (strategy == HASHMAP_SET || strategy == HASHMAP_UPDATE) { + entry->key = key; + entry->value = value; + return 0; + } else if (strategy == HASHMAP_ADD) { + return -EEXIST; + } + } + + if (strategy == HASHMAP_UPDATE) + return -ENOENT; + + if (hashmap_needs_to_grow(map)) { + err = hashmap_grow(map); + if (err) + return err; + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + } + + entry = malloc(sizeof(struct hashmap_entry)); + if (!entry) + return -ENOMEM; + + entry->key = key; + entry->value = value; + hashmap_add_entry(&map->buckets[h], entry); + map->sz++; + + return 0; +} + +bool hashmap__find(const struct hashmap *map, const void *key, void **value) +{ + struct hashmap_entry *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, NULL, &entry)) + return false; + + if (value) + *value = entry->value; + return true; +} + +bool hashmap__delete(struct hashmap *map, const void *key, + const void **old_key, void **old_value) +{ + struct hashmap_entry **pprev, *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, &pprev, &entry)) + return false; + + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + hashmap_del_entry(pprev, entry); + free(entry); + map->sz--; + + return true; +} + diff --git a/tools/lib/bpf/hashmap.h b/tools/lib/bpf/hashmap.h new file mode 100644 index 000000000000..03748a742146 --- /dev/null +++ b/tools/lib/bpf/hashmap.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#ifndef __LIBBPF_HASHMAP_H +#define __LIBBPF_HASHMAP_H + +#include <stdbool.h> +#include <stddef.h> +#include "libbpf_internal.h" + +static inline size_t hash_bits(size_t h, int bits) +{ + /* shuffle bits and return requested number of upper bits */ + return (h * 11400714819323198485llu) >> (__WORDSIZE - bits); +} + +typedef size_t (*hashmap_hash_fn)(const void *key, void *ctx); +typedef bool (*hashmap_equal_fn)(const void *key1, const void *key2, void *ctx); + +struct hashmap_entry { + const void *key; + void *value; + struct hashmap_entry *next; +}; + +struct hashmap { + hashmap_hash_fn hash_fn; + hashmap_equal_fn equal_fn; + void *ctx; + + struct hashmap_entry **buckets; + size_t cap; + size_t cap_bits; + size_t sz; +}; + +#define HASHMAP_INIT(hash_fn, equal_fn, ctx) { \ + .hash_fn = (hash_fn), \ + .equal_fn = (equal_fn), \ + .ctx = (ctx), \ + .buckets = NULL, \ + .cap = 0, \ + .cap_bits = 0, \ + .sz = 0, \ +} + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx); +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx); +void hashmap__clear(struct hashmap *map); +void hashmap__free(struct hashmap *map); + +size_t hashmap__size(const struct hashmap *map); +size_t hashmap__capacity(const struct hashmap *map); + +/* + * Hashmap insertion strategy: + * - HASHMAP_ADD - only add key/value if key doesn't exist yet; + * - HASHMAP_SET - add key/value pair if key doesn't exist yet; otherwise, + * update value; + * - HASHMAP_UPDATE - update value, if key already exists; otherwise, do + * nothing and return -ENOENT; + * - HASHMAP_APPEND - always add key/value pair, even if key already exists. + * This turns hashmap into a multimap by allowing multiple values to be + * associated with the same key. Most useful read API for such hashmap is + * hashmap__for_each_key_entry() iteration. If hashmap__find() is still + * used, it will return last inserted key/value entry (first in a bucket + * chain). + */ +enum hashmap_insert_strategy { + HASHMAP_ADD, + HASHMAP_SET, + HASHMAP_UPDATE, + HASHMAP_APPEND, +}; + +/* + * hashmap__insert() adds key/value entry w/ various semantics, depending on + * provided strategy value. If a given key/value pair replaced already + * existing key/value pair, both old key and old value will be returned + * through old_key and old_value to allow calling code do proper memory + * management. + */ +int hashmap__insert(struct hashmap *map, const void *key, void *value, + enum hashmap_insert_strategy strategy, + const void **old_key, void **old_value); + +static inline int hashmap__add(struct hashmap *map, + const void *key, void *value) +{ + return hashmap__insert(map, key, value, HASHMAP_ADD, NULL, NULL); +} + +static inline int hashmap__set(struct hashmap *map, + const void *key, void *value, + const void **old_key, void **old_value) +{ + return hashmap__insert(map, key, value, HASHMAP_SET, + old_key, old_value); +} + +static inline int hashmap__update(struct hashmap *map, + const void *key, void *value, + const void **old_key, void **old_value) +{ + return hashmap__insert(map, key, value, HASHMAP_UPDATE, + old_key, old_value); +} + +static inline int hashmap__append(struct hashmap *map, + const void *key, void *value) +{ + return hashmap__insert(map, key, value, HASHMAP_APPEND, NULL, NULL); +} + +bool hashmap__delete(struct hashmap *map, const void *key, + const void **old_key, void **old_value); + +bool hashmap__find(const struct hashmap *map, const void *key, void **value); + +/* + * hashmap__for_each_entry - iterate over all entries in hashmap + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry(map, cur, bkt) \ + for (bkt = 0; bkt < map->cap; bkt++) \ + for (cur = map->buckets[bkt]; cur; cur = cur->next) + +/* + * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe + * against removals + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @tmp: struct hashmap_entry * used as a temporary next cursor storage + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \ + for (bkt = 0; bkt < map->cap; bkt++) \ + for (cur = map->buckets[bkt]; \ + cur && ({tmp = cur->next; true; }); \ + cur = tmp) + +/* + * hashmap__for_each_key_entry - iterate over entries associated with given key + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @key: key to iterate entries for + */ +#define hashmap__for_each_key_entry(map, cur, _key) \ + for (cur = ({ size_t bkt = hash_bits(map->hash_fn((_key), map->ctx),\ + map->cap_bits); \ + map->buckets ? map->buckets[bkt] : NULL; }); \ + cur; \ + cur = cur->next) \ + if (map->equal_fn(cur->key, (_key), map->ctx)) + +#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \ + for (cur = ({ size_t bkt = hash_bits(map->hash_fn((_key), map->ctx),\ + map->cap_bits); \ + cur = map->buckets ? map->buckets[bkt] : NULL; }); \ + cur && ({ tmp = cur->next; true; }); \ + cur = tmp) \ + if (map->equal_fn(cur->key, (_key), map->ctx)) + +#endif /* __LIBBPF_HASHMAP_H */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 151f7ac1882e..ed07789b3e62 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -32,6 +32,9 @@ #include <linux/limits.h> #include <linux/perf_event.h> #include <linux/ring_buffer.h> +#include <sys/epoll.h> +#include <sys/ioctl.h> +#include <sys/mman.h> #include <sys/stat.h> #include <sys/types.h> #include <sys/vfs.h> @@ -188,6 +191,7 @@ struct bpf_program { void *line_info; __u32 line_info_rec_size; __u32 line_info_cnt; + __u32 prog_flags; }; enum libbpf_map_type { @@ -206,7 +210,8 @@ static const char * const libbpf_type_to_btf_name[] = { struct bpf_map { int fd; char *name; - size_t offset; + int sec_idx; + size_t sec_offset; int map_ifindex; int inner_map_fd; struct bpf_map_def def; @@ -233,6 +238,7 @@ struct bpf_object { size_t nr_programs; struct bpf_map *maps; size_t nr_maps; + size_t maps_cap; struct bpf_secdata sections; bool loaded; @@ -259,6 +265,7 @@ struct bpf_object { } *reloc; int nr_reloc; int maps_shndx; + int btf_maps_shndx; int text_shndx; int data_shndx; int rodata_shndx; @@ -348,8 +355,11 @@ static int bpf_program__init(void *data, size_t size, char *section_name, int idx, struct bpf_program *prog) { - if (size < sizeof(struct bpf_insn)) { - pr_warning("corrupted section '%s'\n", section_name); + const size_t bpf_insn_sz = sizeof(struct bpf_insn); + + if (size == 0 || size % bpf_insn_sz) { + pr_warning("corrupted section '%s', size: %zu\n", + section_name, size); return -EINVAL; } @@ -375,9 +385,8 @@ bpf_program__init(void *data, size_t size, char *section_name, int idx, section_name); goto errout; } - prog->insns_cnt = size / sizeof(struct bpf_insn); - memcpy(prog->insns, data, - prog->insns_cnt * sizeof(struct bpf_insn)); + prog->insns_cnt = size / bpf_insn_sz; + memcpy(prog->insns, data, size); prog->idx = idx; prog->instances.fds = NULL; prog->instances.nr = -1; @@ -494,15 +503,14 @@ static struct bpf_object *bpf_object__new(const char *path, strcpy(obj->path, path); /* Using basename() GNU version which doesn't modify arg. */ - strncpy(obj->name, basename((void *)path), - sizeof(obj->name) - 1); + strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); end = strchr(obj->name, '.'); if (end) *end = 0; obj->efile.fd = -1; /* - * Caller of this function should also calls + * Caller of this function should also call * bpf_object__elf_finish() after data collection to return * obj_buf to user. If not, we should duplicate the buffer to * avoid user freeing them before elf finish. @@ -510,6 +518,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.maps_shndx = -1; + obj->efile.btf_maps_shndx = -1; obj->efile.data_shndx = -1; obj->efile.rodata_shndx = -1; obj->efile.bss_shndx = -1; @@ -562,38 +571,35 @@ static int bpf_object__elf_init(struct bpf_object *obj) } else { obj->efile.fd = open(obj->path, O_RDONLY); if (obj->efile.fd < 0) { - char errmsg[STRERR_BUFSIZE]; - char *cp = libbpf_strerror_r(errno, errmsg, - sizeof(errmsg)); + char errmsg[STRERR_BUFSIZE], *cp; + err = -errno; + cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); pr_warning("failed to open %s: %s\n", obj->path, cp); - return -errno; + return err; } obj->efile.elf = elf_begin(obj->efile.fd, - LIBBPF_ELF_C_READ_MMAP, - NULL); + LIBBPF_ELF_C_READ_MMAP, NULL); } if (!obj->efile.elf) { - pr_warning("failed to open %s as ELF file\n", - obj->path); + pr_warning("failed to open %s as ELF file\n", obj->path); err = -LIBBPF_ERRNO__LIBELF; goto errout; } if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { - pr_warning("failed to get EHDR from %s\n", - obj->path); + pr_warning("failed to get EHDR from %s\n", obj->path); err = -LIBBPF_ERRNO__FORMAT; goto errout; } ep = &obj->efile.ehdr; /* Old LLVM set e_machine to EM_NONE */ - if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { - pr_warning("%s is not an eBPF object file\n", - obj->path); + if (ep->e_type != ET_REL || + (ep->e_machine && ep->e_machine != EM_BPF)) { + pr_warning("%s is not an eBPF object file\n", obj->path); err = -LIBBPF_ERRNO__FORMAT; goto errout; } @@ -604,47 +610,31 @@ errout: return err; } -static int -bpf_object__check_endianness(struct bpf_object *obj) +static int bpf_object__check_endianness(struct bpf_object *obj) { - static unsigned int const endian = 1; - - switch (obj->efile.ehdr.e_ident[EI_DATA]) { - case ELFDATA2LSB: - /* We are big endian, BPF obj is little endian. */ - if (*(unsigned char const *)&endian != 1) - goto mismatch; - break; - - case ELFDATA2MSB: - /* We are little endian, BPF obj is big endian. */ - if (*(unsigned char const *)&endian != 0) - goto mismatch; - break; - default: - return -LIBBPF_ERRNO__ENDIAN; - } - - return 0; - -mismatch: - pr_warning("Error: endianness mismatch.\n"); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) + return 0; +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) + return 0; +#else +# error "Unrecognized __BYTE_ORDER__" +#endif + pr_warning("endianness mismatch.\n"); return -LIBBPF_ERRNO__ENDIAN; } static int -bpf_object__init_license(struct bpf_object *obj, - void *data, size_t size) +bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) { - memcpy(obj->license, data, - min(size, sizeof(obj->license) - 1)); + memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); pr_debug("license of %s is %s\n", obj->path, obj->license); return 0; } static int -bpf_object__init_kversion(struct bpf_object *obj, - void *data, size_t size) +bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) { __u32 kver; @@ -654,8 +644,7 @@ bpf_object__init_kversion(struct bpf_object *obj, } memcpy(&kver, data, sizeof(kver)); obj->kern_version = kver; - pr_debug("kernel version of %s is %x\n", obj->path, - obj->kern_version); + pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); return 0; } @@ -664,7 +653,9 @@ static int compare_bpf_map(const void *_a, const void *_b) const struct bpf_map *a = _a; const struct bpf_map *b = _b; - return a->offset - b->offset; + if (a->sec_idx != b->sec_idx) + return a->sec_idx - b->sec_idx; + return a->sec_offset - b->sec_offset; } static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) @@ -781,24 +772,55 @@ int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, return -ENOENT; } -static bool bpf_object__has_maps(const struct bpf_object *obj) +static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) { - return obj->efile.maps_shndx >= 0 || - obj->efile.data_shndx >= 0 || - obj->efile.rodata_shndx >= 0 || - obj->efile.bss_shndx >= 0; + struct bpf_map *new_maps; + size_t new_cap; + int i; + + if (obj->nr_maps < obj->maps_cap) + return &obj->maps[obj->nr_maps++]; + + new_cap = max((size_t)4, obj->maps_cap * 3 / 2); + new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); + if (!new_maps) { + pr_warning("alloc maps for object failed\n"); + return ERR_PTR(-ENOMEM); + } + + obj->maps_cap = new_cap; + obj->maps = new_maps; + + /* zero out new maps */ + memset(obj->maps + obj->nr_maps, 0, + (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); + /* + * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) + * when failure (zclose won't close negative fd)). + */ + for (i = obj->nr_maps; i < obj->maps_cap; i++) { + obj->maps[i].fd = -1; + obj->maps[i].inner_map_fd = -1; + } + + return &obj->maps[obj->nr_maps++]; } static int -bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, - enum libbpf_map_type type, Elf_Data *data, - void **data_buff) +bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, + int sec_idx, Elf_Data *data, void **data_buff) { - struct bpf_map_def *def = &map->def; char map_name[BPF_OBJ_NAME_LEN]; + struct bpf_map_def *def; + struct bpf_map *map; + + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); map->libbpf_type = type; - map->offset = ~(typeof(map->offset))0; + map->sec_idx = sec_idx; + map->sec_offset = 0; snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, libbpf_type_to_btf_name[type]); map->name = strdup(map_name); @@ -806,13 +828,15 @@ bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, pr_warning("failed to alloc map name\n"); return -ENOMEM; } + pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", + map_name, map->sec_idx, map->sec_offset); + def = &map->def; def->type = BPF_MAP_TYPE_ARRAY; def->key_size = sizeof(int); def->value_size = data->d_size; def->max_entries = 1; - def->map_flags = type == LIBBPF_MAP_RODATA ? - BPF_F_RDONLY_PROG : 0; + def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; if (data_buff) { *data_buff = malloc(data->d_size); if (!*data_buff) { @@ -827,30 +851,61 @@ bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map, return 0; } -static int -bpf_object__init_maps(struct bpf_object *obj, int flags) +static int bpf_object__init_global_data_maps(struct bpf_object *obj) +{ + int err; + + if (!obj->caps.global_data) + return 0; + /* + * Populate obj->maps with libbpf internal maps. + */ + if (obj->efile.data_shndx >= 0) { + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, + obj->efile.data_shndx, + obj->efile.data, + &obj->sections.data); + if (err) + return err; + } + if (obj->efile.rodata_shndx >= 0) { + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, + obj->efile.rodata_shndx, + obj->efile.rodata, + &obj->sections.rodata); + if (err) + return err; + } + if (obj->efile.bss_shndx >= 0) { + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, + obj->efile.bss_shndx, + obj->efile.bss, NULL); + if (err) + return err; + } + return 0; +} + +static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) { - int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0; - bool strict = !(flags & MAPS_RELAX_COMPAT); Elf_Data *symbols = obj->efile.symbols; + int i, map_def_sz = 0, nr_maps = 0, nr_syms; Elf_Data *data = NULL; - int ret = 0; + Elf_Scn *scn; + + if (obj->efile.maps_shndx < 0) + return 0; if (!symbols) return -EINVAL; - nr_syms = symbols->d_size / sizeof(GElf_Sym); - if (obj->efile.maps_shndx >= 0) { - Elf_Scn *scn = elf_getscn(obj->efile.elf, - obj->efile.maps_shndx); - - if (scn) - data = elf_getdata(scn, NULL); - if (!scn || !data) { - pr_warning("failed to get Elf_Data from map section %d\n", - obj->efile.maps_shndx); - return -EINVAL; - } + scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); + if (scn) + data = elf_getdata(scn, NULL); + if (!scn || !data) { + pr_warning("failed to get Elf_Data from map section %d\n", + obj->efile.maps_shndx); + return -EINVAL; } /* @@ -860,16 +915,8 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) * * TODO: Detect array of map and report error. */ - if (obj->caps.global_data) { - if (obj->efile.data_shndx >= 0) - nr_maps_glob++; - if (obj->efile.rodata_shndx >= 0) - nr_maps_glob++; - if (obj->efile.bss_shndx >= 0) - nr_maps_glob++; - } - - for (i = 0; data && i < nr_syms; i++) { + nr_syms = symbols->d_size / sizeof(GElf_Sym); + for (i = 0; i < nr_syms; i++) { GElf_Sym sym; if (!gelf_getsym(symbols, i, &sym)) @@ -878,74 +925,59 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) continue; nr_maps++; } - - if (!nr_maps && !nr_maps_glob) - return 0; - /* Assume equally sized map definitions */ - if (data) { - pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, - nr_maps, data->d_size); - - map_def_sz = data->d_size / nr_maps; - if (!data->d_size || (data->d_size % nr_maps) != 0) { - pr_warning("unable to determine map definition size " - "section %s, %d maps in %zd bytes\n", - obj->path, nr_maps, data->d_size); - return -EINVAL; - } - } - - nr_maps += nr_maps_glob; - obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); - if (!obj->maps) { - pr_warning("alloc maps for object failed\n"); - return -ENOMEM; - } - obj->nr_maps = nr_maps; - - for (i = 0; i < nr_maps; i++) { - /* - * fill all fd with -1 so won't close incorrect - * fd (fd=0 is stdin) when failure (zclose won't close - * negative fd)). - */ - obj->maps[i].fd = -1; - obj->maps[i].inner_map_fd = -1; + pr_debug("maps in %s: %d maps in %zd bytes\n", + obj->path, nr_maps, data->d_size); + + map_def_sz = data->d_size / nr_maps; + if (!data->d_size || (data->d_size % nr_maps) != 0) { + pr_warning("unable to determine map definition size " + "section %s, %d maps in %zd bytes\n", + obj->path, nr_maps, data->d_size); + return -EINVAL; } - /* - * Fill obj->maps using data in "maps" section. - */ - for (i = 0, map_idx = 0; data && i < nr_syms; i++) { + /* Fill obj->maps using data in "maps" section. */ + for (i = 0; i < nr_syms; i++) { GElf_Sym sym; const char *map_name; struct bpf_map_def *def; + struct bpf_map *map; if (!gelf_getsym(symbols, i, &sym)) continue; if (sym.st_shndx != obj->efile.maps_shndx) continue; - map_name = elf_strptr(obj->efile.elf, - obj->efile.strtabidx, + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); + + map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, sym.st_name); + if (!map_name) { + pr_warning("failed to get map #%d name sym string for obj %s\n", + i, obj->path); + return -LIBBPF_ERRNO__FORMAT; + } - obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC; - obj->maps[map_idx].offset = sym.st_value; + map->libbpf_type = LIBBPF_MAP_UNSPEC; + map->sec_idx = sym.st_shndx; + map->sec_offset = sym.st_value; + pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", + map_name, map->sec_idx, map->sec_offset); if (sym.st_value + map_def_sz > data->d_size) { pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", obj->path, map_name); return -EINVAL; } - obj->maps[map_idx].name = strdup(map_name); - if (!obj->maps[map_idx].name) { + map->name = strdup(map_name); + if (!map->name) { pr_warning("failed to alloc map name\n"); return -ENOMEM; } - pr_debug("map %d is \"%s\"\n", map_idx, - obj->maps[map_idx].name); + pr_debug("map %d is \"%s\"\n", i, map->name); def = (struct bpf_map_def *)(data->d_buf + sym.st_value); /* * If the definition of the map in the object file fits in @@ -954,7 +986,7 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) * calloc above. */ if (map_def_sz <= sizeof(struct bpf_map_def)) { - memcpy(&obj->maps[map_idx].def, def, map_def_sz); + memcpy(&map->def, def, map_def_sz); } else { /* * Here the map structure being read is bigger than what @@ -974,37 +1006,338 @@ bpf_object__init_maps(struct bpf_object *obj, int flags) return -EINVAL; } } - memcpy(&obj->maps[map_idx].def, def, - sizeof(struct bpf_map_def)); + memcpy(&map->def, def, sizeof(struct bpf_map_def)); } - map_idx++; } + return 0; +} - if (!obj->caps.global_data) - goto finalize; +static const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, + __u32 id) +{ + const struct btf_type *t = btf__type_by_id(btf, id); - /* - * Populate rest of obj->maps with libbpf internal maps. - */ - if (obj->efile.data_shndx >= 0) - ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], - LIBBPF_MAP_DATA, - obj->efile.data, - &obj->sections.data); - if (!ret && obj->efile.rodata_shndx >= 0) - ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], - LIBBPF_MAP_RODATA, - obj->efile.rodata, - &obj->sections.rodata); - if (!ret && obj->efile.bss_shndx >= 0) - ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++], - LIBBPF_MAP_BSS, - obj->efile.bss, NULL); -finalize: - if (!ret) + while (true) { + switch (BTF_INFO_KIND(t->info)) { + case BTF_KIND_VOLATILE: + case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: + case BTF_KIND_TYPEDEF: + t = btf__type_by_id(btf, t->type); + break; + default: + return t; + } + } +} + +/* + * Fetch integer attribute of BTF map definition. Such attributes are + * represented using a pointer to an array, in which dimensionality of array + * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; + * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF + * type definition, while using only sizeof(void *) space in ELF data section. + */ +static bool get_map_field_int(const char *map_name, const struct btf *btf, + const struct btf_type *def, + const struct btf_member *m, __u32 *res) { + const struct btf_type *t = skip_mods_and_typedefs(btf, m->type); + const char *name = btf__name_by_offset(btf, m->name_off); + const struct btf_array *arr_info; + const struct btf_type *arr_t; + + if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { + pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", + map_name, name, BTF_INFO_KIND(t->info)); + return false; + } + + arr_t = btf__type_by_id(btf, t->type); + if (!arr_t) { + pr_warning("map '%s': attr '%s': type [%u] not found.\n", + map_name, name, t->type); + return false; + } + if (BTF_INFO_KIND(arr_t->info) != BTF_KIND_ARRAY) { + pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", + map_name, name, BTF_INFO_KIND(arr_t->info)); + return false; + } + arr_info = (const void *)(arr_t + 1); + *res = arr_info->nelems; + return true; +} + +static int bpf_object__init_user_btf_map(struct bpf_object *obj, + const struct btf_type *sec, + int var_idx, int sec_idx, + const Elf_Data *data, bool strict) +{ + const struct btf_type *var, *def, *t; + const struct btf_var_secinfo *vi; + const struct btf_var *var_extra; + const struct btf_member *m; + const char *map_name; + struct bpf_map *map; + int vlen, i; + + vi = (const struct btf_var_secinfo *)(const void *)(sec + 1) + var_idx; + var = btf__type_by_id(obj->btf, vi->type); + var_extra = (const void *)(var + 1); + map_name = btf__name_by_offset(obj->btf, var->name_off); + vlen = BTF_INFO_VLEN(var->info); + + if (map_name == NULL || map_name[0] == '\0') { + pr_warning("map #%d: empty name.\n", var_idx); + return -EINVAL; + } + if ((__u64)vi->offset + vi->size > data->d_size) { + pr_warning("map '%s' BTF data is corrupted.\n", map_name); + return -EINVAL; + } + if (BTF_INFO_KIND(var->info) != BTF_KIND_VAR) { + pr_warning("map '%s': unexpected var kind %u.\n", + map_name, BTF_INFO_KIND(var->info)); + return -EINVAL; + } + if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && + var_extra->linkage != BTF_VAR_STATIC) { + pr_warning("map '%s': unsupported var linkage %u.\n", + map_name, var_extra->linkage); + return -EOPNOTSUPP; + } + + def = skip_mods_and_typedefs(obj->btf, var->type); + if (BTF_INFO_KIND(def->info) != BTF_KIND_STRUCT) { + pr_warning("map '%s': unexpected def kind %u.\n", + map_name, BTF_INFO_KIND(var->info)); + return -EINVAL; + } + if (def->size > vi->size) { + pr_warning("map '%s': invalid def size.\n", map_name); + return -EINVAL; + } + + map = bpf_object__add_map(obj); + if (IS_ERR(map)) + return PTR_ERR(map); + map->name = strdup(map_name); + if (!map->name) { + pr_warning("map '%s': failed to alloc map name.\n", map_name); + return -ENOMEM; + } + map->libbpf_type = LIBBPF_MAP_UNSPEC; + map->def.type = BPF_MAP_TYPE_UNSPEC; + map->sec_idx = sec_idx; + map->sec_offset = vi->offset; + pr_debug("map '%s': at sec_idx %d, offset %zu.\n", + map_name, map->sec_idx, map->sec_offset); + + vlen = BTF_INFO_VLEN(def->info); + m = (const void *)(def + 1); + for (i = 0; i < vlen; i++, m++) { + const char *name = btf__name_by_offset(obj->btf, m->name_off); + + if (!name) { + pr_warning("map '%s': invalid field #%d.\n", + map_name, i); + return -EINVAL; + } + if (strcmp(name, "type") == 0) { + if (!get_map_field_int(map_name, obj->btf, def, m, + &map->def.type)) + return -EINVAL; + pr_debug("map '%s': found type = %u.\n", + map_name, map->def.type); + } else if (strcmp(name, "max_entries") == 0) { + if (!get_map_field_int(map_name, obj->btf, def, m, + &map->def.max_entries)) + return -EINVAL; + pr_debug("map '%s': found max_entries = %u.\n", + map_name, map->def.max_entries); + } else if (strcmp(name, "map_flags") == 0) { + if (!get_map_field_int(map_name, obj->btf, def, m, + &map->def.map_flags)) + return -EINVAL; + pr_debug("map '%s': found map_flags = %u.\n", + map_name, map->def.map_flags); + } else if (strcmp(name, "key_size") == 0) { + __u32 sz; + + if (!get_map_field_int(map_name, obj->btf, def, m, + &sz)) + return -EINVAL; + pr_debug("map '%s': found key_size = %u.\n", + map_name, sz); + if (map->def.key_size && map->def.key_size != sz) { + pr_warning("map '%s': conflicting key size %u != %u.\n", + map_name, map->def.key_size, sz); + return -EINVAL; + } + map->def.key_size = sz; + } else if (strcmp(name, "key") == 0) { + __s64 sz; + + t = btf__type_by_id(obj->btf, m->type); + if (!t) { + pr_warning("map '%s': key type [%d] not found.\n", + map_name, m->type); + return -EINVAL; + } + if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { + pr_warning("map '%s': key spec is not PTR: %u.\n", + map_name, BTF_INFO_KIND(t->info)); + return -EINVAL; + } + sz = btf__resolve_size(obj->btf, t->type); + if (sz < 0) { + pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", + map_name, t->type, sz); + return sz; + } + pr_debug("map '%s': found key [%u], sz = %lld.\n", + map_name, t->type, sz); + if (map->def.key_size && map->def.key_size != sz) { + pr_warning("map '%s': conflicting key size %u != %lld.\n", + map_name, map->def.key_size, sz); + return -EINVAL; + } + map->def.key_size = sz; + map->btf_key_type_id = t->type; + } else if (strcmp(name, "value_size") == 0) { + __u32 sz; + + if (!get_map_field_int(map_name, obj->btf, def, m, + &sz)) + return -EINVAL; + pr_debug("map '%s': found value_size = %u.\n", + map_name, sz); + if (map->def.value_size && map->def.value_size != sz) { + pr_warning("map '%s': conflicting value size %u != %u.\n", + map_name, map->def.value_size, sz); + return -EINVAL; + } + map->def.value_size = sz; + } else if (strcmp(name, "value") == 0) { + __s64 sz; + + t = btf__type_by_id(obj->btf, m->type); + if (!t) { + pr_warning("map '%s': value type [%d] not found.\n", + map_name, m->type); + return -EINVAL; + } + if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) { + pr_warning("map '%s': value spec is not PTR: %u.\n", + map_name, BTF_INFO_KIND(t->info)); + return -EINVAL; + } + sz = btf__resolve_size(obj->btf, t->type); + if (sz < 0) { + pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", + map_name, t->type, sz); + return sz; + } + pr_debug("map '%s': found value [%u], sz = %lld.\n", + map_name, t->type, sz); + if (map->def.value_size && map->def.value_size != sz) { + pr_warning("map '%s': conflicting value size %u != %lld.\n", + map_name, map->def.value_size, sz); + return -EINVAL; + } + map->def.value_size = sz; + map->btf_value_type_id = t->type; + } else { + if (strict) { + pr_warning("map '%s': unknown field '%s'.\n", + map_name, name); + return -ENOTSUP; + } + pr_debug("map '%s': ignoring unknown field '%s'.\n", + map_name, name); + } + } + + if (map->def.type == BPF_MAP_TYPE_UNSPEC) { + pr_warning("map '%s': map type isn't specified.\n", map_name); + return -EINVAL; + } + + return 0; +} + +static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) +{ + const struct btf_type *sec = NULL; + int nr_types, i, vlen, err; + const struct btf_type *t; + const char *name; + Elf_Data *data; + Elf_Scn *scn; + + if (obj->efile.btf_maps_shndx < 0) + return 0; + + scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); + if (scn) + data = elf_getdata(scn, NULL); + if (!scn || !data) { + pr_warning("failed to get Elf_Data from map section %d (%s)\n", + obj->efile.maps_shndx, MAPS_ELF_SEC); + return -EINVAL; + } + + nr_types = btf__get_nr_types(obj->btf); + for (i = 1; i <= nr_types; i++) { + t = btf__type_by_id(obj->btf, i); + if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC) + continue; + name = btf__name_by_offset(obj->btf, t->name_off); + if (strcmp(name, MAPS_ELF_SEC) == 0) { + sec = t; + break; + } + } + + if (!sec) { + pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); + return -ENOENT; + } + + vlen = BTF_INFO_VLEN(sec->info); + for (i = 0; i < vlen; i++) { + err = bpf_object__init_user_btf_map(obj, sec, i, + obj->efile.btf_maps_shndx, + data, strict); + if (err) + return err; + } + + return 0; +} + +static int bpf_object__init_maps(struct bpf_object *obj, int flags) +{ + bool strict = !(flags & MAPS_RELAX_COMPAT); + int err; + + err = bpf_object__init_user_maps(obj, strict); + if (err) + return err; + + err = bpf_object__init_user_btf_maps(obj, strict); + if (err) + return err; + + err = bpf_object__init_global_data_maps(obj); + if (err) + return err; + + if (obj->nr_maps) { qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); - return ret; + } + return 0; } static bool section_have_execinstr(struct bpf_object *obj, int idx) @@ -1093,6 +1426,86 @@ static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) } } +static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) +{ + return obj->efile.btf_maps_shndx >= 0; +} + +static int bpf_object__init_btf(struct bpf_object *obj, + Elf_Data *btf_data, + Elf_Data *btf_ext_data) +{ + bool btf_required = bpf_object__is_btf_mandatory(obj); + int err = 0; + + if (btf_data) { + obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); + if (IS_ERR(obj->btf)) { + pr_warning("Error loading ELF section %s: %d.\n", + BTF_ELF_SEC, err); + goto out; + } + err = btf__finalize_data(obj, obj->btf); + if (err) { + pr_warning("Error finalizing %s: %d.\n", + BTF_ELF_SEC, err); + goto out; + } + } + if (btf_ext_data) { + if (!obj->btf) { + pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", + BTF_EXT_ELF_SEC, BTF_ELF_SEC); + goto out; + } + obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, + btf_ext_data->d_size); + if (IS_ERR(obj->btf_ext)) { + pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", + BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); + obj->btf_ext = NULL; + goto out; + } + } +out: + if (err || IS_ERR(obj->btf)) { + if (btf_required) + err = err ? : PTR_ERR(obj->btf); + else + err = 0; + if (!IS_ERR_OR_NULL(obj->btf)) + btf__free(obj->btf); + obj->btf = NULL; + } + if (btf_required && !obj->btf) { + pr_warning("BTF is required, but is missing or corrupted.\n"); + return err == 0 ? -ENOENT : err; + } + return 0; +} + +static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) +{ + int err = 0; + + if (!obj->btf) + return 0; + + bpf_object__sanitize_btf(obj); + bpf_object__sanitize_btf_ext(obj); + + err = btf__load(obj->btf); + if (err) { + pr_warning("Error loading %s into kernel: %d.\n", + BTF_ELF_SEC, err); + btf__free(obj->btf); + obj->btf = NULL; + if (bpf_object__is_btf_mandatory(obj)) + return err; + } + return 0; +} + static int bpf_object__elf_collect(struct bpf_object *obj, int flags) { Elf *elf = obj->efile.elf; @@ -1104,8 +1517,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) /* Elf is corrupted/truncated, avoid calling elf_strptr. */ if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { - pr_warning("failed to get e_shstrndx from %s\n", - obj->path); + pr_warning("failed to get e_shstrndx from %s\n", obj->path); return -LIBBPF_ERRNO__FORMAT; } @@ -1118,24 +1530,21 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) if (gelf_getshdr(scn, &sh) != &sh) { pr_warning("failed to get section(%d) header from %s\n", idx, obj->path); - err = -LIBBPF_ERRNO__FORMAT; - goto out; + return -LIBBPF_ERRNO__FORMAT; } name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); if (!name) { pr_warning("failed to get section(%d) name from %s\n", idx, obj->path); - err = -LIBBPF_ERRNO__FORMAT; - goto out; + return -LIBBPF_ERRNO__FORMAT; } data = elf_getdata(scn, 0); if (!data) { pr_warning("failed to get section(%d) data from %s(%s)\n", idx, name, obj->path); - err = -LIBBPF_ERRNO__FORMAT; - goto out; + return -LIBBPF_ERRNO__FORMAT; } pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", idx, name, (unsigned long)data->d_size, @@ -1146,12 +1555,18 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) err = bpf_object__init_license(obj, data->d_buf, data->d_size); + if (err) + return err; } else if (strcmp(name, "version") == 0) { err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); + if (err) + return err; } else if (strcmp(name, "maps") == 0) { obj->efile.maps_shndx = idx; + } else if (strcmp(name, MAPS_ELF_SEC) == 0) { + obj->efile.btf_maps_shndx = idx; } else if (strcmp(name, BTF_ELF_SEC) == 0) { btf_data = data; } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { @@ -1160,11 +1575,10 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) if (obj->efile.symbols) { pr_warning("bpf: multiple SYMTAB in %s\n", obj->path); - err = -LIBBPF_ERRNO__FORMAT; - } else { - obj->efile.symbols = data; - obj->efile.strtabidx = sh.sh_link; + return -LIBBPF_ERRNO__FORMAT; } + obj->efile.symbols = data; + obj->efile.strtabidx = sh.sh_link; } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { if (sh.sh_flags & SHF_EXECINSTR) { if (strcmp(name, ".text") == 0) @@ -1178,6 +1592,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) pr_warning("failed to alloc program %s (%s): %s", name, obj->path, cp); + return err; } } else if (strcmp(name, ".data") == 0) { obj->efile.data = data; @@ -1189,8 +1604,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) pr_debug("skip section(%d) %s\n", idx, name); } } else if (sh.sh_type == SHT_REL) { + int nr_reloc = obj->efile.nr_reloc; void *reloc = obj->efile.reloc; - int nr_reloc = obj->efile.nr_reloc + 1; int sec = sh.sh_info; /* points to other section */ /* Only do relo for section with exec instructions */ @@ -1200,79 +1615,37 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) continue; } - reloc = reallocarray(reloc, nr_reloc, + reloc = reallocarray(reloc, nr_reloc + 1, sizeof(*obj->efile.reloc)); if (!reloc) { pr_warning("realloc failed\n"); - err = -ENOMEM; - } else { - int n = nr_reloc - 1; + return -ENOMEM; + } - obj->efile.reloc = reloc; - obj->efile.nr_reloc = nr_reloc; + obj->efile.reloc = reloc; + obj->efile.nr_reloc++; - obj->efile.reloc[n].shdr = sh; - obj->efile.reloc[n].data = data; - } + obj->efile.reloc[nr_reloc].shdr = sh; + obj->efile.reloc[nr_reloc].data = data; } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { obj->efile.bss = data; obj->efile.bss_shndx = idx; } else { pr_debug("skip section(%d) %s\n", idx, name); } - if (err) - goto out; } if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { pr_warning("Corrupted ELF file: index of strtab invalid\n"); - return LIBBPF_ERRNO__FORMAT; - } - if (btf_data) { - obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); - if (IS_ERR(obj->btf)) { - pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", - BTF_ELF_SEC, PTR_ERR(obj->btf)); - obj->btf = NULL; - } else { - err = btf__finalize_data(obj, obj->btf); - if (!err) { - bpf_object__sanitize_btf(obj); - err = btf__load(obj->btf); - } - if (err) { - pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n", - BTF_ELF_SEC, err); - btf__free(obj->btf); - obj->btf = NULL; - err = 0; - } - } - } - if (btf_ext_data) { - if (!obj->btf) { - pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", - BTF_EXT_ELF_SEC, BTF_ELF_SEC); - } else { - obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, - btf_ext_data->d_size); - if (IS_ERR(obj->btf_ext)) { - pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", - BTF_EXT_ELF_SEC, - PTR_ERR(obj->btf_ext)); - obj->btf_ext = NULL; - } else { - bpf_object__sanitize_btf_ext(obj); - } - } + return -LIBBPF_ERRNO__FORMAT; } - if (bpf_object__has_maps(obj)) { + err = bpf_object__init_btf(obj, btf_data, btf_ext_data); + if (!err) err = bpf_object__init_maps(obj, flags); - if (err) - goto out; - } - err = bpf_object__init_prog_names(obj); -out: + if (!err) + err = bpf_object__sanitize_and_load_btf(obj); + if (!err) + err = bpf_object__init_prog_names(obj); return err; } @@ -1291,7 +1664,8 @@ bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) } struct bpf_program * -bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) +bpf_object__find_program_by_title(const struct bpf_object *obj, + const char *title) { struct bpf_program *pos; @@ -1313,7 +1687,8 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, int shndx) { - return shndx == obj->efile.maps_shndx; + return shndx == obj->efile.maps_shndx || + shndx == obj->efile.btf_maps_shndx; } static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, @@ -1346,8 +1721,7 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, size_t nr_maps = obj->nr_maps; int i, nrels; - pr_debug("collecting relocating info for: '%s'\n", - prog->section_name); + pr_debug("collecting relocating info for: '%s'\n", prog->section_name); nrels = shdr->sh_size / shdr->sh_entsize; prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); @@ -1358,23 +1732,21 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, prog->nr_reloc = nrels; for (i = 0; i < nrels; i++) { - GElf_Sym sym; - GElf_Rel rel; - unsigned int insn_idx; - unsigned int shdr_idx; struct bpf_insn *insns = prog->insns; enum libbpf_map_type type; + unsigned int insn_idx; + unsigned int shdr_idx; const char *name; size_t map_idx; + GElf_Sym sym; + GElf_Rel rel; if (!gelf_getrel(data, i, &rel)) { pr_warning("relocation: failed to get %d reloc\n", i); return -LIBBPF_ERRNO__FORMAT; } - if (!gelf_getsym(symbols, - GELF_R_SYM(rel.r_info), - &sym)) { + if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { pr_warning("relocation: symbol %"PRIx64" not found\n", GELF_R_SYM(rel.r_info)); return -LIBBPF_ERRNO__FORMAT; @@ -1435,16 +1807,19 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, if (maps[map_idx].libbpf_type != type) continue; if (type != LIBBPF_MAP_UNSPEC || - (type == LIBBPF_MAP_UNSPEC && - maps[map_idx].offset == sym.st_value)) { - pr_debug("relocation: find map %zd (%s) for insn %u\n", - map_idx, maps[map_idx].name, insn_idx); + (maps[map_idx].sec_idx == sym.st_shndx && + maps[map_idx].sec_offset == sym.st_value)) { + pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", + map_idx, maps[map_idx].name, + maps[map_idx].sec_idx, + maps[map_idx].sec_offset, + insn_idx); break; } } if (map_idx >= nr_maps) { - pr_warning("bpf relocation: map_idx %d large than %d\n", + pr_warning("bpf relocation: map_idx %d larger than %d\n", (int)map_idx, (int)nr_maps - 1); return -LIBBPF_ERRNO__RELOC; } @@ -1458,14 +1833,18 @@ bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, return 0; } -static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) +static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) { struct bpf_map_def *def = &map->def; __u32 key_type_id = 0, value_type_id = 0; int ret; + /* if it's BTF-defined map, we don't need to search for type IDs */ + if (map->sec_idx == obj->efile.btf_maps_shndx) + return 0; + if (!bpf_map__is_internal(map)) { - ret = btf__get_map_kv_tids(btf, map->name, def->key_size, + ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, def->value_size, &key_type_id, &value_type_id); } else { @@ -1473,7 +1852,7 @@ static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) * LLVM annotates global data differently in BTF, that is, * only as '.data', '.bss' or '.rodata'. */ - ret = btf__find_by_name(btf, + ret = btf__find_by_name(obj->btf, libbpf_type_to_btf_name[map->libbpf_type]); } if (ret < 0) @@ -1737,6 +2116,7 @@ static int bpf_object__create_maps(struct bpf_object *obj) { struct bpf_create_map_attr create_attr = {}; + int nr_cpus = 0; unsigned int i; int err; @@ -1759,7 +2139,22 @@ bpf_object__create_maps(struct bpf_object *obj) create_attr.map_flags = def->map_flags; create_attr.key_size = def->key_size; create_attr.value_size = def->value_size; - create_attr.max_entries = def->max_entries; + if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && + !def->max_entries) { + if (!nr_cpus) + nr_cpus = libbpf_num_possible_cpus(); + if (nr_cpus < 0) { + pr_warning("failed to determine number of system CPUs: %d\n", + nr_cpus); + err = nr_cpus; + goto err_out; + } + pr_debug("map '%s': setting size to %d\n", + map->name, nr_cpus); + create_attr.max_entries = nr_cpus; + } else { + create_attr.max_entries = def->max_entries; + } create_attr.btf_fd = 0; create_attr.btf_key_type_id = 0; create_attr.btf_value_type_id = 0; @@ -1767,17 +2162,19 @@ bpf_object__create_maps(struct bpf_object *obj) map->inner_map_fd >= 0) create_attr.inner_map_fd = map->inner_map_fd; - if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { + if (obj->btf && !bpf_map_find_btf_info(obj, map)) { create_attr.btf_fd = btf__fd(obj->btf); create_attr.btf_key_type_id = map->btf_key_type_id; create_attr.btf_value_type_id = map->btf_value_type_id; } *pfd = bpf_create_map_xattr(&create_attr); - if (*pfd < 0 && create_attr.btf_key_type_id) { - cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); + if (*pfd < 0 && (create_attr.btf_key_type_id || + create_attr.btf_value_type_id)) { + err = -errno; + cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", - map->name, cp, errno); + map->name, cp, err); create_attr.btf_fd = 0; create_attr.btf_key_type_id = 0; create_attr.btf_value_type_id = 0; @@ -1789,11 +2186,11 @@ bpf_object__create_maps(struct bpf_object *obj) if (*pfd < 0) { size_t j; - err = *pfd; + err = -errno; err_out: - cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); - pr_warning("failed to create map (name: '%s'): %s\n", - map->name, cp); + cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); + pr_warning("failed to create map (name: '%s'): %s(%d)\n", + map->name, cp, err); for (j = 0; j < i; j++) zclose(obj->maps[j].fd); return err; @@ -1807,7 +2204,7 @@ err_out: } } - pr_debug("create map %s: fd=%d\n", map->name, *pfd); + pr_debug("created map %s: fd=%d\n", map->name, *pfd); } return 0; @@ -1828,18 +2225,14 @@ check_btf_ext_reloc_err(struct bpf_program *prog, int err, if (btf_prog_info) { /* * Some info has already been found but has problem - * in the last btf_ext reloc. Must have to error - * out. + * in the last btf_ext reloc. Must have to error out. */ pr_warning("Error in relocating %s for sec %s.\n", info_name, prog->section_name); return err; } - /* - * Have problem loading the very first info. Ignore - * the rest. - */ + /* Have problem loading the very first info. Ignore the rest. */ pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", info_name, prog->section_name, info_name); return 0; @@ -2043,9 +2436,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) return -LIBBPF_ERRNO__RELOC; } - err = bpf_program__collect_reloc(prog, - shdr, data, - obj); + err = bpf_program__collect_reloc(prog, shdr, data, obj); if (err) return err; } @@ -2062,6 +2453,9 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, char *log_buf; int ret; + if (!insns || !insns_cnt) + return -EINVAL; + memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); load_attr.prog_type = prog->type; load_attr.expected_attach_type = prog->expected_attach_type; @@ -2080,8 +2474,7 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_attr.line_info_rec_size = prog->line_info_rec_size; load_attr.line_info_cnt = prog->line_info_cnt; load_attr.log_level = prog->log_level; - if (!load_attr.insns || !load_attr.insns_cnt) - return -EINVAL; + load_attr.prog_flags = prog->prog_flags; retry_load: log_buf = malloc(log_buf_size); @@ -2219,14 +2612,14 @@ out: return err; } -static bool bpf_program__is_function_storage(struct bpf_program *prog, - struct bpf_object *obj) +static bool bpf_program__is_function_storage(const struct bpf_program *prog, + const struct bpf_object *obj) { return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; } static int -bpf_object__load_progs(struct bpf_object *obj) +bpf_object__load_progs(struct bpf_object *obj, int log_level) { size_t i; int err; @@ -2234,6 +2627,7 @@ bpf_object__load_progs(struct bpf_object *obj) for (i = 0; i < obj->nr_programs; i++) { if (bpf_program__is_function_storage(&obj->programs[i], obj)) continue; + obj->programs[i].log_level |= log_level; err = bpf_program__load(&obj->programs[i], obj->license, obj->kern_version); @@ -2270,6 +2664,7 @@ static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: case BPF_PROG_TYPE_PERF_EVENT: case BPF_PROG_TYPE_CGROUP_SYSCTL: + case BPF_PROG_TYPE_CGROUP_SOCKOPT: return false; case BPF_PROG_TYPE_KPROBE: default: @@ -2360,11 +2755,9 @@ struct bpf_object *bpf_object__open_buffer(void *obj_buf, snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", (unsigned long)obj_buf, (unsigned long)obj_buf_sz); - tmp_name[sizeof(tmp_name) - 1] = '\0'; name = tmp_name; } - pr_debug("loading object '%s' from buffer\n", - name); + pr_debug("loading object '%s' from buffer\n", name); return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); } @@ -2385,10 +2778,14 @@ int bpf_object__unload(struct bpf_object *obj) return 0; } -int bpf_object__load(struct bpf_object *obj) +int bpf_object__load_xattr(struct bpf_object_load_attr *attr) { + struct bpf_object *obj; int err; + if (!attr) + return -EINVAL; + obj = attr->obj; if (!obj) return -EINVAL; @@ -2401,7 +2798,7 @@ int bpf_object__load(struct bpf_object *obj) CHECK_ERR(bpf_object__create_maps(obj), err, out); CHECK_ERR(bpf_object__relocate(obj), err, out); - CHECK_ERR(bpf_object__load_progs(obj), err, out); + CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); return 0; out: @@ -2410,6 +2807,15 @@ out: return err; } +int bpf_object__load(struct bpf_object *obj) +{ + struct bpf_object_load_attr attr = { + .obj = obj, + }; + + return bpf_object__load_xattr(&attr); +} + static int check_path(const char *path) { char *cp, errmsg[STRERR_BUFSIZE]; @@ -2914,17 +3320,17 @@ bpf_object__next(struct bpf_object *prev) return next; } -const char *bpf_object__name(struct bpf_object *obj) +const char *bpf_object__name(const struct bpf_object *obj) { return obj ? obj->path : ERR_PTR(-EINVAL); } -unsigned int bpf_object__kversion(struct bpf_object *obj) +unsigned int bpf_object__kversion(const struct bpf_object *obj) { return obj ? obj->kern_version : 0; } -struct btf *bpf_object__btf(struct bpf_object *obj) +struct btf *bpf_object__btf(const struct bpf_object *obj) { return obj ? obj->btf : NULL; } @@ -2945,13 +3351,14 @@ int bpf_object__set_priv(struct bpf_object *obj, void *priv, return 0; } -void *bpf_object__priv(struct bpf_object *obj) +void *bpf_object__priv(const struct bpf_object *obj) { return obj ? obj->priv : ERR_PTR(-EINVAL); } static struct bpf_program * -__bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) +__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, + bool forward) { size_t nr_programs = obj->nr_programs; ssize_t idx; @@ -2976,7 +3383,7 @@ __bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward) } struct bpf_program * -bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) +bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) { struct bpf_program *prog = prev; @@ -2988,7 +3395,7 @@ bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) } struct bpf_program * -bpf_program__prev(struct bpf_program *next, struct bpf_object *obj) +bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) { struct bpf_program *prog = next; @@ -3010,7 +3417,7 @@ int bpf_program__set_priv(struct bpf_program *prog, void *priv, return 0; } -void *bpf_program__priv(struct bpf_program *prog) +void *bpf_program__priv(const struct bpf_program *prog) { return prog ? prog->priv : ERR_PTR(-EINVAL); } @@ -3020,7 +3427,7 @@ void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) prog->prog_ifindex = ifindex; } -const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) +const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) { const char *title; @@ -3036,7 +3443,7 @@ const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) return title; } -int bpf_program__fd(struct bpf_program *prog) +int bpf_program__fd(const struct bpf_program *prog) { return bpf_program__nth_fd(prog, 0); } @@ -3069,7 +3476,7 @@ int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, return 0; } -int bpf_program__nth_fd(struct bpf_program *prog, int n) +int bpf_program__nth_fd(const struct bpf_program *prog, int n) { int fd; @@ -3097,25 +3504,25 @@ void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) prog->type = type; } -static bool bpf_program__is_type(struct bpf_program *prog, +static bool bpf_program__is_type(const struct bpf_program *prog, enum bpf_prog_type type) { return prog ? (prog->type == type) : false; } -#define BPF_PROG_TYPE_FNS(NAME, TYPE) \ -int bpf_program__set_##NAME(struct bpf_program *prog) \ -{ \ - if (!prog) \ - return -EINVAL; \ - bpf_program__set_type(prog, TYPE); \ - return 0; \ -} \ - \ -bool bpf_program__is_##NAME(struct bpf_program *prog) \ -{ \ - return bpf_program__is_type(prog, TYPE); \ -} \ +#define BPF_PROG_TYPE_FNS(NAME, TYPE) \ +int bpf_program__set_##NAME(struct bpf_program *prog) \ +{ \ + if (!prog) \ + return -EINVAL; \ + bpf_program__set_type(prog, TYPE); \ + return 0; \ +} \ + \ +bool bpf_program__is_##NAME(const struct bpf_program *prog) \ +{ \ + return bpf_program__is_type(prog, TYPE); \ +} \ BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); @@ -3216,6 +3623,10 @@ static const struct { BPF_CGROUP_UDP6_RECVMSG), BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, BPF_CGROUP_SYSCTL), + BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, + BPF_CGROUP_GETSOCKOPT), + BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, + BPF_CGROUP_SETSOCKOPT), }; #undef BPF_PROG_SEC_IMPL @@ -3314,17 +3725,17 @@ bpf_program__identify_section(struct bpf_program *prog, expected_attach_type); } -int bpf_map__fd(struct bpf_map *map) +int bpf_map__fd(const struct bpf_map *map) { return map ? map->fd : -EINVAL; } -const struct bpf_map_def *bpf_map__def(struct bpf_map *map) +const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) { return map ? &map->def : ERR_PTR(-EINVAL); } -const char *bpf_map__name(struct bpf_map *map) +const char *bpf_map__name(const struct bpf_map *map) { return map ? map->name : NULL; } @@ -3355,17 +3766,17 @@ int bpf_map__set_priv(struct bpf_map *map, void *priv, return 0; } -void *bpf_map__priv(struct bpf_map *map) +void *bpf_map__priv(const struct bpf_map *map) { return map ? map->priv : ERR_PTR(-EINVAL); } -bool bpf_map__is_offload_neutral(struct bpf_map *map) +bool bpf_map__is_offload_neutral(const struct bpf_map *map) { return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; } -bool bpf_map__is_internal(struct bpf_map *map) +bool bpf_map__is_internal(const struct bpf_map *map) { return map->libbpf_type != LIBBPF_MAP_UNSPEC; } @@ -3390,7 +3801,7 @@ int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) } static struct bpf_map * -__bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) +__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) { ssize_t idx; struct bpf_map *s, *e; @@ -3414,7 +3825,7 @@ __bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i) } struct bpf_map * -bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) +bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) { if (prev == NULL) return obj->maps; @@ -3423,7 +3834,7 @@ bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) } struct bpf_map * -bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) +bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) { if (next == NULL) { if (!obj->nr_maps) @@ -3435,7 +3846,7 @@ bpf_map__prev(struct bpf_map *next, struct bpf_object *obj) } struct bpf_map * -bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) +bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) { struct bpf_map *pos; @@ -3447,7 +3858,7 @@ bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) } int -bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) +bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) { return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); } @@ -3455,20 +3866,12 @@ bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name) struct bpf_map * bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) { - int i; - - for (i = 0; i < obj->nr_maps; i++) { - if (obj->maps[i].offset == offset) - return &obj->maps[i]; - } - return ERR_PTR(-ENOENT); + return ERR_PTR(-ENOTSUP); } long libbpf_get_error(const void *ptr) { - if (IS_ERR(ptr)) - return PTR_ERR(ptr); - return 0; + return PTR_ERR_OR_ZERO(ptr); } int bpf_prog_load(const char *file, enum bpf_prog_type type, @@ -3487,10 +3890,7 @@ int bpf_prog_load(const char *file, enum bpf_prog_type type, int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, struct bpf_object **pobj, int *prog_fd) { - struct bpf_object_open_attr open_attr = { - .file = attr->file, - .prog_type = attr->prog_type, - }; + struct bpf_object_open_attr open_attr = {}; struct bpf_program *prog, *first_prog = NULL; enum bpf_attach_type expected_attach_type; enum bpf_prog_type prog_type; @@ -3503,6 +3903,9 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, if (!attr->file) return -EINVAL; + open_attr.file = attr->file; + open_attr.prog_type = attr->prog_type; + obj = bpf_object__open_xattr(&open_attr); if (IS_ERR_OR_NULL(obj)) return -ENOENT; @@ -3529,6 +3932,7 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, expected_attach_type); prog->log_level = attr->log_level; + prog->prog_flags = attr->prog_flags; if (!first_prog) first_prog = prog; } @@ -3555,6 +3959,372 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, return 0; } +struct bpf_link { + int (*destroy)(struct bpf_link *link); +}; + +int bpf_link__destroy(struct bpf_link *link) +{ + int err; + + if (!link) + return 0; + + err = link->destroy(link); + free(link); + + return err; +} + +struct bpf_link_fd { + struct bpf_link link; /* has to be at the top of struct */ + int fd; /* hook FD */ +}; + +static int bpf_link__destroy_perf_event(struct bpf_link *link) +{ + struct bpf_link_fd *l = (void *)link; + int err; + + err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); + if (err) + err = -errno; + + close(l->fd); + return err; +} + +struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, + int pfd) +{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link_fd *link; + int prog_fd, err; + + if (pfd < 0) { + pr_warning("program '%s': invalid perf event FD %d\n", + bpf_program__title(prog, false), pfd); + return ERR_PTR(-EINVAL); + } + prog_fd = bpf_program__fd(prog); + if (prog_fd < 0) { + pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", + bpf_program__title(prog, false)); + return ERR_PTR(-EINVAL); + } + + link = malloc(sizeof(*link)); + if (!link) + return ERR_PTR(-ENOMEM); + link->link.destroy = &bpf_link__destroy_perf_event; + link->fd = pfd; + + if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { + err = -errno; + free(link); + pr_warning("program '%s': failed to attach to pfd %d: %s\n", + bpf_program__title(prog, false), pfd, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return ERR_PTR(err); + } + if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { + err = -errno; + free(link); + pr_warning("program '%s': failed to enable pfd %d: %s\n", + bpf_program__title(prog, false), pfd, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return ERR_PTR(err); + } + return (struct bpf_link *)link; +} + +/* + * this function is expected to parse integer in the range of [0, 2^31-1] from + * given file using scanf format string fmt. If actual parsed value is + * negative, the result might be indistinguishable from error + */ +static int parse_uint_from_file(const char *file, const char *fmt) +{ + char buf[STRERR_BUFSIZE]; + int err, ret; + FILE *f; + + f = fopen(file, "r"); + if (!f) { + err = -errno; + pr_debug("failed to open '%s': %s\n", file, + libbpf_strerror_r(err, buf, sizeof(buf))); + return err; + } + err = fscanf(f, fmt, &ret); + if (err != 1) { + err = err == EOF ? -EIO : -errno; + pr_debug("failed to parse '%s': %s\n", file, + libbpf_strerror_r(err, buf, sizeof(buf))); + fclose(f); + return err; + } + fclose(f); + return ret; +} + +static int determine_kprobe_perf_type(void) +{ + const char *file = "/sys/bus/event_source/devices/kprobe/type"; + + return parse_uint_from_file(file, "%d\n"); +} + +static int determine_uprobe_perf_type(void) +{ + const char *file = "/sys/bus/event_source/devices/uprobe/type"; + + return parse_uint_from_file(file, "%d\n"); +} + +static int determine_kprobe_retprobe_bit(void) +{ + const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; + + return parse_uint_from_file(file, "config:%d\n"); +} + +static int determine_uprobe_retprobe_bit(void) +{ + const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; + + return parse_uint_from_file(file, "config:%d\n"); +} + +static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, + uint64_t offset, int pid) +{ + struct perf_event_attr attr = {}; + char errmsg[STRERR_BUFSIZE]; + int type, pfd, err; + + type = uprobe ? determine_uprobe_perf_type() + : determine_kprobe_perf_type(); + if (type < 0) { + pr_warning("failed to determine %s perf type: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(type, errmsg, sizeof(errmsg))); + return type; + } + if (retprobe) { + int bit = uprobe ? determine_uprobe_retprobe_bit() + : determine_kprobe_retprobe_bit(); + + if (bit < 0) { + pr_warning("failed to determine %s retprobe bit: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(bit, errmsg, + sizeof(errmsg))); + return bit; + } + attr.config |= 1 << bit; + } + attr.size = sizeof(attr); + attr.type = type; + attr.config1 = (uint64_t)(void *)name; /* kprobe_func or uprobe_path */ + attr.config2 = offset; /* kprobe_addr or probe_offset */ + + /* pid filter is meaningful only for uprobes */ + pfd = syscall(__NR_perf_event_open, &attr, + pid < 0 ? -1 : pid /* pid */, + pid == -1 ? 0 : -1 /* cpu */, + -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); + if (pfd < 0) { + err = -errno; + pr_warning("%s perf_event_open() failed: %s\n", + uprobe ? "uprobe" : "kprobe", + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return err; + } + return pfd; +} + +struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, + bool retprobe, + const char *func_name) +{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link *link; + int pfd, err; + + pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, + 0 /* offset */, -1 /* pid */); + if (pfd < 0) { + pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", + bpf_program__title(prog, false), + retprobe ? "kretprobe" : "kprobe", func_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + return ERR_PTR(pfd); + } + link = bpf_program__attach_perf_event(prog, pfd); + if (IS_ERR(link)) { + close(pfd); + err = PTR_ERR(link); + pr_warning("program '%s': failed to attach to %s '%s': %s\n", + bpf_program__title(prog, false), + retprobe ? "kretprobe" : "kprobe", func_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return link; + } + return link; +} + +struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, + bool retprobe, pid_t pid, + const char *binary_path, + size_t func_offset) +{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link *link; + int pfd, err; + + pfd = perf_event_open_probe(true /* uprobe */, retprobe, + binary_path, func_offset, pid); + if (pfd < 0) { + pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", + bpf_program__title(prog, false), + retprobe ? "uretprobe" : "uprobe", + binary_path, func_offset, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + return ERR_PTR(pfd); + } + link = bpf_program__attach_perf_event(prog, pfd); + if (IS_ERR(link)) { + close(pfd); + err = PTR_ERR(link); + pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", + bpf_program__title(prog, false), + retprobe ? "uretprobe" : "uprobe", + binary_path, func_offset, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return link; + } + return link; +} + +static int determine_tracepoint_id(const char *tp_category, + const char *tp_name) +{ + char file[PATH_MAX]; + int ret; + + ret = snprintf(file, sizeof(file), + "/sys/kernel/debug/tracing/events/%s/%s/id", + tp_category, tp_name); + if (ret < 0) + return -errno; + if (ret >= sizeof(file)) { + pr_debug("tracepoint %s/%s path is too long\n", + tp_category, tp_name); + return -E2BIG; + } + return parse_uint_from_file(file, "%d\n"); +} + +static int perf_event_open_tracepoint(const char *tp_category, + const char *tp_name) +{ + struct perf_event_attr attr = {}; + char errmsg[STRERR_BUFSIZE]; + int tp_id, pfd, err; + + tp_id = determine_tracepoint_id(tp_category, tp_name); + if (tp_id < 0) { + pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", + tp_category, tp_name, + libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); + return tp_id; + } + + attr.type = PERF_TYPE_TRACEPOINT; + attr.size = sizeof(attr); + attr.config = tp_id; + + pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, + -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); + if (pfd < 0) { + err = -errno; + pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", + tp_category, tp_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return err; + } + return pfd; +} + +struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, + const char *tp_category, + const char *tp_name) +{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link *link; + int pfd, err; + + pfd = perf_event_open_tracepoint(tp_category, tp_name); + if (pfd < 0) { + pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", + bpf_program__title(prog, false), + tp_category, tp_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + return ERR_PTR(pfd); + } + link = bpf_program__attach_perf_event(prog, pfd); + if (IS_ERR(link)) { + close(pfd); + err = PTR_ERR(link); + pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", + bpf_program__title(prog, false), + tp_category, tp_name, + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + return link; + } + return link; +} + +static int bpf_link__destroy_fd(struct bpf_link *link) +{ + struct bpf_link_fd *l = (void *)link; + + return close(l->fd); +} + +struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, + const char *tp_name) +{ + char errmsg[STRERR_BUFSIZE]; + struct bpf_link_fd *link; + int prog_fd, pfd; + + prog_fd = bpf_program__fd(prog); + if (prog_fd < 0) { + pr_warning("program '%s': can't attach before loaded\n", + bpf_program__title(prog, false)); + return ERR_PTR(-EINVAL); + } + + link = malloc(sizeof(*link)); + if (!link) + return ERR_PTR(-ENOMEM); + link->link.destroy = &bpf_link__destroy_fd; + + pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); + if (pfd < 0) { + pfd = -errno; + free(link); + pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", + bpf_program__title(prog, false), tp_name, + libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); + return ERR_PTR(pfd); + } + link->fd = pfd; + return (struct bpf_link *)link; +} + enum bpf_perf_event_ret bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, void **copy_mem, size_t *copy_size, @@ -3603,6 +4373,370 @@ bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, return ret; } +struct perf_buffer; + +struct perf_buffer_params { + struct perf_event_attr *attr; + /* if event_cb is specified, it takes precendence */ + perf_buffer_event_fn event_cb; + /* sample_cb and lost_cb are higher-level common-case callbacks */ + perf_buffer_sample_fn sample_cb; + perf_buffer_lost_fn lost_cb; + void *ctx; + int cpu_cnt; + int *cpus; + int *map_keys; +}; + +struct perf_cpu_buf { + struct perf_buffer *pb; + void *base; /* mmap()'ed memory */ + void *buf; /* for reconstructing segmented data */ + size_t buf_size; + int fd; + int cpu; + int map_key; +}; + +struct perf_buffer { + perf_buffer_event_fn event_cb; + perf_buffer_sample_fn sample_cb; + perf_buffer_lost_fn lost_cb; + void *ctx; /* passed into callbacks */ + + size_t page_size; + size_t mmap_size; + struct perf_cpu_buf **cpu_bufs; + struct epoll_event *events; + int cpu_cnt; + int epoll_fd; /* perf event FD */ + int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ +}; + +static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, + struct perf_cpu_buf *cpu_buf) +{ + if (!cpu_buf) + return; + if (cpu_buf->base && + munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) + pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); + if (cpu_buf->fd >= 0) { + ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); + close(cpu_buf->fd); + } + free(cpu_buf->buf); + free(cpu_buf); +} + +void perf_buffer__free(struct perf_buffer *pb) +{ + int i; + + if (!pb) + return; + if (pb->cpu_bufs) { + for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { + struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; + + bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); + perf_buffer__free_cpu_buf(pb, cpu_buf); + } + free(pb->cpu_bufs); + } + if (pb->epoll_fd >= 0) + close(pb->epoll_fd); + free(pb->events); + free(pb); +} + +static struct perf_cpu_buf * +perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, + int cpu, int map_key) +{ + struct perf_cpu_buf *cpu_buf; + char msg[STRERR_BUFSIZE]; + int err; + + cpu_buf = calloc(1, sizeof(*cpu_buf)); + if (!cpu_buf) + return ERR_PTR(-ENOMEM); + + cpu_buf->pb = pb; + cpu_buf->cpu = cpu; + cpu_buf->map_key = map_key; + + cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, + -1, PERF_FLAG_FD_CLOEXEC); + if (cpu_buf->fd < 0) { + err = -errno; + pr_warning("failed to open perf buffer event on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + + cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, + PROT_READ | PROT_WRITE, MAP_SHARED, + cpu_buf->fd, 0); + if (cpu_buf->base == MAP_FAILED) { + cpu_buf->base = NULL; + err = -errno; + pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + + if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { + err = -errno; + pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", + cpu, libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + + return cpu_buf; + +error: + perf_buffer__free_cpu_buf(pb, cpu_buf); + return (struct perf_cpu_buf *)ERR_PTR(err); +} + +static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, + struct perf_buffer_params *p); + +struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, + const struct perf_buffer_opts *opts) +{ + struct perf_buffer_params p = {}; + struct perf_event_attr attr = { + .config = PERF_COUNT_SW_BPF_OUTPUT, + .type = PERF_TYPE_SOFTWARE, + .sample_type = PERF_SAMPLE_RAW, + .sample_period = 1, + .wakeup_events = 1, + }; + + p.attr = &attr; + p.sample_cb = opts ? opts->sample_cb : NULL; + p.lost_cb = opts ? opts->lost_cb : NULL; + p.ctx = opts ? opts->ctx : NULL; + + return __perf_buffer__new(map_fd, page_cnt, &p); +} + +struct perf_buffer * +perf_buffer__new_raw(int map_fd, size_t page_cnt, + const struct perf_buffer_raw_opts *opts) +{ + struct perf_buffer_params p = {}; + + p.attr = opts->attr; + p.event_cb = opts->event_cb; + p.ctx = opts->ctx; + p.cpu_cnt = opts->cpu_cnt; + p.cpus = opts->cpus; + p.map_keys = opts->map_keys; + + return __perf_buffer__new(map_fd, page_cnt, &p); +} + +static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, + struct perf_buffer_params *p) +{ + struct bpf_map_info map = {}; + char msg[STRERR_BUFSIZE]; + struct perf_buffer *pb; + __u32 map_info_len; + int err, i; + + if (page_cnt & (page_cnt - 1)) { + pr_warning("page count should be power of two, but is %zu\n", + page_cnt); + return ERR_PTR(-EINVAL); + } + + map_info_len = sizeof(map); + err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); + if (err) { + err = -errno; + pr_warning("failed to get map info for map FD %d: %s\n", + map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); + return ERR_PTR(err); + } + + if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { + pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", + map.name); + return ERR_PTR(-EINVAL); + } + + pb = calloc(1, sizeof(*pb)); + if (!pb) + return ERR_PTR(-ENOMEM); + + pb->event_cb = p->event_cb; + pb->sample_cb = p->sample_cb; + pb->lost_cb = p->lost_cb; + pb->ctx = p->ctx; + + pb->page_size = getpagesize(); + pb->mmap_size = pb->page_size * page_cnt; + pb->map_fd = map_fd; + + pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); + if (pb->epoll_fd < 0) { + err = -errno; + pr_warning("failed to create epoll instance: %s\n", + libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + + if (p->cpu_cnt > 0) { + pb->cpu_cnt = p->cpu_cnt; + } else { + pb->cpu_cnt = libbpf_num_possible_cpus(); + if (pb->cpu_cnt < 0) { + err = pb->cpu_cnt; + goto error; + } + if (map.max_entries < pb->cpu_cnt) + pb->cpu_cnt = map.max_entries; + } + + pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); + if (!pb->events) { + err = -ENOMEM; + pr_warning("failed to allocate events: out of memory\n"); + goto error; + } + pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); + if (!pb->cpu_bufs) { + err = -ENOMEM; + pr_warning("failed to allocate buffers: out of memory\n"); + goto error; + } + + for (i = 0; i < pb->cpu_cnt; i++) { + struct perf_cpu_buf *cpu_buf; + int cpu, map_key; + + cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; + map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; + + cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); + if (IS_ERR(cpu_buf)) { + err = PTR_ERR(cpu_buf); + goto error; + } + + pb->cpu_bufs[i] = cpu_buf; + + err = bpf_map_update_elem(pb->map_fd, &map_key, + &cpu_buf->fd, 0); + if (err) { + err = -errno; + pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", + cpu, map_key, cpu_buf->fd, + libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + + pb->events[i].events = EPOLLIN; + pb->events[i].data.ptr = cpu_buf; + if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, + &pb->events[i]) < 0) { + err = -errno; + pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", + cpu, cpu_buf->fd, + libbpf_strerror_r(err, msg, sizeof(msg))); + goto error; + } + } + + return pb; + +error: + if (pb) + perf_buffer__free(pb); + return ERR_PTR(err); +} + +struct perf_sample_raw { + struct perf_event_header header; + uint32_t size; + char data[0]; +}; + +struct perf_sample_lost { + struct perf_event_header header; + uint64_t id; + uint64_t lost; + uint64_t sample_id; +}; + +static enum bpf_perf_event_ret +perf_buffer__process_record(struct perf_event_header *e, void *ctx) +{ + struct perf_cpu_buf *cpu_buf = ctx; + struct perf_buffer *pb = cpu_buf->pb; + void *data = e; + + /* user wants full control over parsing perf event */ + if (pb->event_cb) + return pb->event_cb(pb->ctx, cpu_buf->cpu, e); + + switch (e->type) { + case PERF_RECORD_SAMPLE: { + struct perf_sample_raw *s = data; + + if (pb->sample_cb) + pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); + break; + } + case PERF_RECORD_LOST: { + struct perf_sample_lost *s = data; + + if (pb->lost_cb) + pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); + break; + } + default: + pr_warning("unknown perf sample type %d\n", e->type); + return LIBBPF_PERF_EVENT_ERROR; + } + return LIBBPF_PERF_EVENT_CONT; +} + +static int perf_buffer__process_records(struct perf_buffer *pb, + struct perf_cpu_buf *cpu_buf) +{ + enum bpf_perf_event_ret ret; + + ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, + pb->page_size, &cpu_buf->buf, + &cpu_buf->buf_size, + perf_buffer__process_record, cpu_buf); + if (ret != LIBBPF_PERF_EVENT_CONT) + return ret; + return 0; +} + +int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) +{ + int i, cnt, err; + + cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); + for (i = 0; i < cnt; i++) { + struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; + + err = perf_buffer__process_records(pb, cpu_buf); + if (err) { + pr_warning("error while processing records: %d\n", err); + return err; + } + } + return cnt < 0 ? -errno : cnt; +} + struct bpf_prog_info_array_desc { int array_offset; /* e.g. offset of jited_prog_insns */ int count_offset; /* e.g. offset of jited_prog_len */ @@ -3848,3 +4982,60 @@ void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) desc->array_offset, addr); } } + +int libbpf_num_possible_cpus(void) +{ + static const char *fcpu = "/sys/devices/system/cpu/possible"; + int len = 0, n = 0, il = 0, ir = 0; + unsigned int start = 0, end = 0; + static int cpus; + char buf[128]; + int error = 0; + int fd = -1; + + if (cpus > 0) + return cpus; + + fd = open(fcpu, O_RDONLY); + if (fd < 0) { + error = errno; + pr_warning("Failed to open file %s: %s\n", + fcpu, strerror(error)); + return -error; + } + len = read(fd, buf, sizeof(buf)); + close(fd); + if (len <= 0) { + error = len ? errno : EINVAL; + pr_warning("Failed to read # of possible cpus from %s: %s\n", + fcpu, strerror(error)); + return -error; + } + if (len == sizeof(buf)) { + pr_warning("File %s size overflow\n", fcpu); + return -EOVERFLOW; + } + buf[len] = '\0'; + + for (ir = 0, cpus = 0; ir <= len; ir++) { + /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ + if (buf[ir] == ',' || buf[ir] == '\0') { + buf[ir] = '\0'; + n = sscanf(&buf[il], "%u-%u", &start, &end); + if (n <= 0) { + pr_warning("Failed to get # CPUs from %s\n", + &buf[il]); + return -EINVAL; + } else if (n == 1) { + end = start; + } + cpus += end - start + 1; + il = ir + 1; + } + } + if (cpus <= 0) { + pr_warning("Invalid #CPUs %d from %s\n", cpus, fcpu); + return -EINVAL; + } + return cpus; +} diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index c5ff00515ce7..5cbf459ece0b 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -89,18 +89,25 @@ LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj, LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path); LIBBPF_API void bpf_object__close(struct bpf_object *object); +struct bpf_object_load_attr { + struct bpf_object *obj; + int log_level; +}; + /* Load/unload object into/from kernel */ LIBBPF_API int bpf_object__load(struct bpf_object *obj); +LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr); LIBBPF_API int bpf_object__unload(struct bpf_object *obj); -LIBBPF_API const char *bpf_object__name(struct bpf_object *obj); -LIBBPF_API unsigned int bpf_object__kversion(struct bpf_object *obj); +LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj); +LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj); struct btf; -LIBBPF_API struct btf *bpf_object__btf(struct bpf_object *obj); +LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj); LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj); LIBBPF_API struct bpf_program * -bpf_object__find_program_by_title(struct bpf_object *obj, const char *title); +bpf_object__find_program_by_title(const struct bpf_object *obj, + const char *title); LIBBPF_API struct bpf_object *bpf_object__next(struct bpf_object *prev); #define bpf_object__for_each_safe(pos, tmp) \ @@ -112,7 +119,7 @@ LIBBPF_API struct bpf_object *bpf_object__next(struct bpf_object *prev); typedef void (*bpf_object_clear_priv_t)(struct bpf_object *, void *); LIBBPF_API int bpf_object__set_priv(struct bpf_object *obj, void *priv, bpf_object_clear_priv_t clear_priv); -LIBBPF_API void *bpf_object__priv(struct bpf_object *prog); +LIBBPF_API void *bpf_object__priv(const struct bpf_object *prog); LIBBPF_API int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, @@ -123,7 +130,7 @@ LIBBPF_API int libbpf_attach_type_by_name(const char *name, /* Accessors of bpf_program */ struct bpf_program; LIBBPF_API struct bpf_program *bpf_program__next(struct bpf_program *prog, - struct bpf_object *obj); + const struct bpf_object *obj); #define bpf_object__for_each_program(pos, obj) \ for ((pos) = bpf_program__next(NULL, (obj)); \ @@ -131,24 +138,23 @@ LIBBPF_API struct bpf_program *bpf_program__next(struct bpf_program *prog, (pos) = bpf_program__next((pos), (obj))) LIBBPF_API struct bpf_program *bpf_program__prev(struct bpf_program *prog, - struct bpf_object *obj); + const struct bpf_object *obj); -typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, - void *); +typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *); LIBBPF_API int bpf_program__set_priv(struct bpf_program *prog, void *priv, bpf_program_clear_priv_t clear_priv); -LIBBPF_API void *bpf_program__priv(struct bpf_program *prog); +LIBBPF_API void *bpf_program__priv(const struct bpf_program *prog); LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex); -LIBBPF_API const char *bpf_program__title(struct bpf_program *prog, +LIBBPF_API const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy); LIBBPF_API int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_version); -LIBBPF_API int bpf_program__fd(struct bpf_program *prog); +LIBBPF_API int bpf_program__fd(const struct bpf_program *prog); LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog, const char *path, int instance); @@ -159,6 +165,27 @@ LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); LIBBPF_API void bpf_program__unload(struct bpf_program *prog); +struct bpf_link; + +LIBBPF_API int bpf_link__destroy(struct bpf_link *link); + +LIBBPF_API struct bpf_link * +bpf_program__attach_perf_event(struct bpf_program *prog, int pfd); +LIBBPF_API struct bpf_link * +bpf_program__attach_kprobe(struct bpf_program *prog, bool retprobe, + const char *func_name); +LIBBPF_API struct bpf_link * +bpf_program__attach_uprobe(struct bpf_program *prog, bool retprobe, + pid_t pid, const char *binary_path, + size_t func_offset); +LIBBPF_API struct bpf_link * +bpf_program__attach_tracepoint(struct bpf_program *prog, + const char *tp_category, + const char *tp_name); +LIBBPF_API struct bpf_link * +bpf_program__attach_raw_tracepoint(struct bpf_program *prog, + const char *tp_name); + struct bpf_insn; /* @@ -221,7 +248,7 @@ typedef int (*bpf_program_prep_t)(struct bpf_program *prog, int n, LIBBPF_API int bpf_program__set_prep(struct bpf_program *prog, int nr_instance, bpf_program_prep_t prep); -LIBBPF_API int bpf_program__nth_fd(struct bpf_program *prog, int n); +LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n); /* * Adjust type of BPF program. Default is kprobe. @@ -240,14 +267,14 @@ LIBBPF_API void bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type); -LIBBPF_API bool bpf_program__is_socket_filter(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_tracepoint(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_raw_tracepoint(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_kprobe(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_sched_cls(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_sched_act(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_xdp(struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_perf_event(struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog); +LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog); /* * No need for __attribute__((packed)), all members of 'bpf_map_def' @@ -269,10 +296,10 @@ struct bpf_map_def { */ struct bpf_map; LIBBPF_API struct bpf_map * -bpf_object__find_map_by_name(struct bpf_object *obj, const char *name); +bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name); LIBBPF_API int -bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name); +bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name); /* * Get bpf_map through the offset of corresponding struct bpf_map_def @@ -282,7 +309,7 @@ LIBBPF_API struct bpf_map * bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset); LIBBPF_API struct bpf_map * -bpf_map__next(struct bpf_map *map, struct bpf_object *obj); +bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj); #define bpf_object__for_each_map(pos, obj) \ for ((pos) = bpf_map__next(NULL, (obj)); \ (pos) != NULL; \ @@ -290,22 +317,22 @@ bpf_map__next(struct bpf_map *map, struct bpf_object *obj); #define bpf_map__for_each bpf_object__for_each_map LIBBPF_API struct bpf_map * -bpf_map__prev(struct bpf_map *map, struct bpf_object *obj); +bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj); -LIBBPF_API int bpf_map__fd(struct bpf_map *map); -LIBBPF_API const struct bpf_map_def *bpf_map__def(struct bpf_map *map); -LIBBPF_API const char *bpf_map__name(struct bpf_map *map); +LIBBPF_API int bpf_map__fd(const struct bpf_map *map); +LIBBPF_API const struct bpf_map_def *bpf_map__def(const struct bpf_map *map); +LIBBPF_API const char *bpf_map__name(const struct bpf_map *map); LIBBPF_API __u32 bpf_map__btf_key_type_id(const struct bpf_map *map); LIBBPF_API __u32 bpf_map__btf_value_type_id(const struct bpf_map *map); typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *); LIBBPF_API int bpf_map__set_priv(struct bpf_map *map, void *priv, bpf_map_clear_priv_t clear_priv); -LIBBPF_API void *bpf_map__priv(struct bpf_map *map); +LIBBPF_API void *bpf_map__priv(const struct bpf_map *map); LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries); -LIBBPF_API bool bpf_map__is_offload_neutral(struct bpf_map *map); -LIBBPF_API bool bpf_map__is_internal(struct bpf_map *map); +LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map); +LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map); LIBBPF_API void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex); LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path); LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path); @@ -320,6 +347,7 @@ struct bpf_prog_load_attr { enum bpf_attach_type expected_attach_type; int ifindex; int log_level; + int prog_flags; }; LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, @@ -330,6 +358,26 @@ LIBBPF_API int bpf_prog_load(const char *file, enum bpf_prog_type type, LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags); LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags); +struct perf_buffer; + +typedef void (*perf_buffer_sample_fn)(void *ctx, int cpu, + void *data, __u32 size); +typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt); + +/* common use perf buffer options */ +struct perf_buffer_opts { + /* if specified, sample_cb is called for each sample */ + perf_buffer_sample_fn sample_cb; + /* if specified, lost_cb is called for each batch of lost samples */ + perf_buffer_lost_fn lost_cb; + /* ctx is provided to sample_cb and lost_cb */ + void *ctx; +}; + +LIBBPF_API struct perf_buffer * +perf_buffer__new(int map_fd, size_t page_cnt, + const struct perf_buffer_opts *opts); + enum bpf_perf_event_ret { LIBBPF_PERF_EVENT_DONE = 0, LIBBPF_PERF_EVENT_ERROR = -1, @@ -337,6 +385,35 @@ enum bpf_perf_event_ret { }; struct perf_event_header; + +typedef enum bpf_perf_event_ret +(*perf_buffer_event_fn)(void *ctx, int cpu, struct perf_event_header *event); + +/* raw perf buffer options, giving most power and control */ +struct perf_buffer_raw_opts { + /* perf event attrs passed directly into perf_event_open() */ + struct perf_event_attr *attr; + /* raw event callback */ + perf_buffer_event_fn event_cb; + /* ctx is provided to event_cb */ + void *ctx; + /* if cpu_cnt == 0, open all on all possible CPUs (up to the number of + * max_entries of given PERF_EVENT_ARRAY map) + */ + int cpu_cnt; + /* if cpu_cnt > 0, cpus is an array of CPUs to open ring buffers on */ + int *cpus; + /* if cpu_cnt > 0, map_keys specify map keys to set per-CPU FDs for */ + int *map_keys; +}; + +LIBBPF_API struct perf_buffer * +perf_buffer__new_raw(int map_fd, size_t page_cnt, + const struct perf_buffer_raw_opts *opts); + +LIBBPF_API void perf_buffer__free(struct perf_buffer *pb); +LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms); + typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, void *private_data); @@ -447,6 +524,22 @@ bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear); LIBBPF_API void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear); +/* + * A helper function to get the number of possible CPUs before looking up + * per-CPU maps. Negative errno is returned on failure. + * + * Example usage: + * + * int ncpus = libbpf_num_possible_cpus(); + * if (ncpus < 0) { + * // error handling + * } + * long values[ncpus]; + * bpf_map_lookup_elem(per_cpu_map_fd, key, values); + * + */ +LIBBPF_API int libbpf_num_possible_cpus(void); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 673001787cba..f9d316e873d8 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -164,3 +164,23 @@ LIBBPF_0.0.3 { bpf_map_freeze; btf__finalize_data; } LIBBPF_0.0.2; + +LIBBPF_0.0.4 { + global: + bpf_link__destroy; + bpf_object__load_xattr; + bpf_program__attach_kprobe; + bpf_program__attach_perf_event; + bpf_program__attach_raw_tracepoint; + bpf_program__attach_tracepoint; + bpf_program__attach_uprobe; + btf_dump__dump_type; + btf_dump__free; + btf_dump__new; + btf__parse_elf; + libbpf_num_possible_cpus; + perf_buffer__free; + perf_buffer__new; + perf_buffer__new_raw; + perf_buffer__poll; +} LIBBPF_0.0.3; diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index dfab8012185c..2ac29bd36226 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -9,6 +9,8 @@ #ifndef __LIBBPF_LIBBPF_INTERNAL_H #define __LIBBPF_LIBBPF_INTERNAL_H +#include "libbpf.h" + #define BTF_INFO_ENC(kind, kind_flag, vlen) \ ((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN)) #define BTF_TYPE_ENC(name, info, size_or_type) (name), (info), (size_or_type) @@ -21,6 +23,13 @@ #define BTF_PARAM_ENC(name, type) (name), (type) #define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size) +#ifndef min +# define min(x, y) ((x) < (y) ? (x) : (y)) +#endif +#ifndef max +# define max(x, y) ((x) < (y) ? (y) : (x)) +#endif + extern void libbpf_print(enum libbpf_print_level level, const char *format, ...) __attribute__((format(printf, 2, 3))); diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 6635a31a7a16..ace1a0708d99 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -101,6 +101,7 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, case BPF_PROG_TYPE_SK_REUSEPORT: case BPF_PROG_TYPE_FLOW_DISSECTOR: case BPF_PROG_TYPE_CGROUP_SYSCTL: + case BPF_PROG_TYPE_CGROUP_SOCKOPT: default: break; } diff --git a/tools/lib/bpf/str_error.c b/tools/lib/bpf/str_error.c index 00e48ac5b806..b8064eedc177 100644 --- a/tools/lib/bpf/str_error.c +++ b/tools/lib/bpf/str_error.c @@ -11,7 +11,7 @@ */ char *libbpf_strerror_r(int err, char *dst, int len) { - int ret = strerror_r(err, dst, len); + int ret = strerror_r(err < 0 ? -err : err, dst, len); if (ret) snprintf(dst, len, "ERROR: strerror_r(%d)=%d", err, ret); return dst; diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c index 38667b62f1fe..b33740221b7e 100644 --- a/tools/lib/bpf/xsk.c +++ b/tools/lib/bpf/xsk.c @@ -60,13 +60,12 @@ struct xsk_socket { struct xsk_umem *umem; struct xsk_socket_config config; int fd; - int xsks_map; int ifindex; int prog_fd; - int qidconf_map_fd; int xsks_map_fd; __u32 queue_id; char ifname[IFNAMSIZ]; + bool zc; }; struct xsk_nl_info { @@ -265,15 +264,11 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) /* This is the C-program: * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx) * { - * int *qidconf, index = ctx->rx_queue_index; + * int index = ctx->rx_queue_index; * * // A set entry here means that the correspnding queue_id * // has an active AF_XDP socket bound to it. - * qidconf = bpf_map_lookup_elem(&qidconf_map, &index); - * if (!qidconf) - * return XDP_ABORTED; - * - * if (*qidconf) + * if (bpf_map_lookup_elem(&xsks_map, &index)) * return bpf_redirect_map(&xsks_map, index, 0); * * return XDP_PASS; @@ -286,15 +281,10 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_1, -4), BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), - BPF_LD_MAP_FD(BPF_REG_1, xsk->qidconf_map_fd), + BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd), BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem), BPF_MOV64_REG(BPF_REG_1, BPF_REG_0), - BPF_MOV32_IMM(BPF_REG_0, 0), - /* if r1 == 0 goto +8 */ - BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 8), BPF_MOV32_IMM(BPF_REG_0, 2), - /* r1 = *(u32 *)(r1 + 0) */ - BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 0), /* if r1 == 0 goto +5 */ BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5), /* r2 = *(u32 *)(r10 - 4) */ @@ -337,7 +327,8 @@ static int xsk_get_max_queues(struct xsk_socket *xsk) channels.cmd = ETHTOOL_GCHANNELS; ifr.ifr_data = (void *)&channels; - strncpy(ifr.ifr_name, xsk->ifname, IFNAMSIZ); + strncpy(ifr.ifr_name, xsk->ifname, IFNAMSIZ - 1); + ifr.ifr_name[IFNAMSIZ - 1] = '\0'; err = ioctl(fd, SIOCETHTOOL, &ifr); if (err && errno != EOPNOTSUPP) { ret = -errno; @@ -366,18 +357,11 @@ static int xsk_create_bpf_maps(struct xsk_socket *xsk) if (max_queues < 0) return max_queues; - fd = bpf_create_map_name(BPF_MAP_TYPE_ARRAY, "qidconf_map", + fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map", sizeof(int), sizeof(int), max_queues, 0); if (fd < 0) return fd; - xsk->qidconf_map_fd = fd; - fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map", - sizeof(int), sizeof(int), max_queues, 0); - if (fd < 0) { - close(xsk->qidconf_map_fd); - return fd; - } xsk->xsks_map_fd = fd; return 0; @@ -385,10 +369,8 @@ static int xsk_create_bpf_maps(struct xsk_socket *xsk) static void xsk_delete_bpf_maps(struct xsk_socket *xsk) { - close(xsk->qidconf_map_fd); + bpf_map_delete_elem(xsk->xsks_map_fd, &xsk->queue_id); close(xsk->xsks_map_fd); - xsk->qidconf_map_fd = -1; - xsk->xsks_map_fd = -1; } static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) @@ -417,10 +399,9 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) if (err) goto out_map_ids; - for (i = 0; i < prog_info.nr_map_ids; i++) { - if (xsk->qidconf_map_fd != -1 && xsk->xsks_map_fd != -1) - break; + xsk->xsks_map_fd = -1; + for (i = 0; i < prog_info.nr_map_ids; i++) { fd = bpf_map_get_fd_by_id(map_ids[i]); if (fd < 0) continue; @@ -431,11 +412,6 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) continue; } - if (!strcmp(map_info.name, "qidconf_map")) { - xsk->qidconf_map_fd = fd; - continue; - } - if (!strcmp(map_info.name, "xsks_map")) { xsk->xsks_map_fd = fd; continue; @@ -445,40 +421,18 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) } err = 0; - if (xsk->qidconf_map_fd < 0 || xsk->xsks_map_fd < 0) { + if (xsk->xsks_map_fd == -1) err = -ENOENT; - xsk_delete_bpf_maps(xsk); - } out_map_ids: free(map_ids); return err; } -static void xsk_clear_bpf_maps(struct xsk_socket *xsk) -{ - int qid = false; - - bpf_map_update_elem(xsk->qidconf_map_fd, &xsk->queue_id, &qid, 0); - bpf_map_delete_elem(xsk->xsks_map_fd, &xsk->queue_id); -} - static int xsk_set_bpf_maps(struct xsk_socket *xsk) { - int qid = true, fd = xsk->fd, err; - - err = bpf_map_update_elem(xsk->qidconf_map_fd, &xsk->queue_id, &qid, 0); - if (err) - goto out; - - err = bpf_map_update_elem(xsk->xsks_map_fd, &xsk->queue_id, &fd, 0); - if (err) - goto out; - - return 0; -out: - xsk_clear_bpf_maps(xsk); - return err; + return bpf_map_update_elem(xsk->xsks_map_fd, &xsk->queue_id, + &xsk->fd, 0); } static int xsk_setup_xdp_prog(struct xsk_socket *xsk) @@ -497,26 +451,27 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk) return err; err = xsk_load_xdp_prog(xsk); - if (err) - goto out_maps; + if (err) { + xsk_delete_bpf_maps(xsk); + return err; + } } else { xsk->prog_fd = bpf_prog_get_fd_by_id(prog_id); err = xsk_lookup_bpf_maps(xsk); - if (err) - goto out_load; + if (err) { + close(xsk->prog_fd); + return err; + } } err = xsk_set_bpf_maps(xsk); - if (err) - goto out_load; + if (err) { + xsk_delete_bpf_maps(xsk); + close(xsk->prog_fd); + return err; + } return 0; - -out_load: - close(xsk->prog_fd); -out_maps: - xsk_delete_bpf_maps(xsk); - return err; } int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname, @@ -527,6 +482,7 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname, void *rx_map = NULL, *tx_map = NULL; struct sockaddr_xdp sxdp = {}; struct xdp_mmap_offsets off; + struct xdp_options opts; struct xsk_socket *xsk; socklen_t optlen; int err; @@ -643,8 +599,16 @@ int xsk_socket__create(struct xsk_socket **xsk_ptr, const char *ifname, goto out_mmap_tx; } - xsk->qidconf_map_fd = -1; - xsk->xsks_map_fd = -1; + xsk->prog_fd = -1; + + optlen = sizeof(opts); + err = getsockopt(xsk->fd, SOL_XDP, XDP_OPTIONS, &opts, &optlen); + if (err) { + err = -errno; + goto out_mmap_tx; + } + + xsk->zc = opts.flags & XDP_OPTIONS_ZEROCOPY; if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) { err = xsk_setup_xdp_prog(xsk); @@ -708,8 +672,10 @@ void xsk_socket__delete(struct xsk_socket *xsk) if (!xsk) return; - xsk_clear_bpf_maps(xsk); - xsk_delete_bpf_maps(xsk); + if (xsk->prog_fd != -1) { + xsk_delete_bpf_maps(xsk); + close(xsk->prog_fd); + } optlen = sizeof(off); err = getsockopt(xsk->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen); diff --git a/tools/lib/bpf/xsk.h b/tools/lib/bpf/xsk.h index 82ea71a0f3ec..833a6e60d065 100644 --- a/tools/lib/bpf/xsk.h +++ b/tools/lib/bpf/xsk.h @@ -167,7 +167,7 @@ LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk); #define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048 #define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048 -#define XSK_UMEM__DEFAULT_FRAME_SHIFT 11 /* 2048 bytes */ +#define XSK_UMEM__DEFAULT_FRAME_SHIFT 12 /* 4096 bytes */ #define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT) #define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0 |