summaryrefslogtreecommitdiff
path: root/tools/bpf
diff options
context:
space:
mode:
Diffstat (limited to 'tools/bpf')
-rw-r--r--tools/bpf/bpftool/Makefile2
-rw-r--r--tools/bpf/bpftool/btf.c13
-rw-r--r--tools/bpf/bpftool/btf_dumper.c14
-rw-r--r--tools/bpf/bpftool/common.c4
-rw-r--r--tools/bpf/bpftool/gen.c55
-rw-r--r--tools/bpf/bpftool/link.c133
-rw-r--r--tools/bpf/bpftool/map.c16
-rw-r--r--tools/bpf/bpftool/sign.c24
-rw-r--r--tools/bpf/bpftool/struct_ops.c4
-rw-r--r--tools/bpf/bpftool/xlated_dumper.c19
-rw-r--r--tools/bpf/resolve_btfids/Makefile5
-rw-r--r--tools/bpf/resolve_btfids/main.c476
12 files changed, 611 insertions, 154 deletions
diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile
index 271a7dc77273..b0f7168e7943 100644
--- a/tools/bpf/bpftool/Makefile
+++ b/tools/bpf/bpftool/Makefile
@@ -99,7 +99,7 @@ endif
HOST_LDFLAGS := $(LDFLAGS)
# Remove warnings for libbpf bootstrap build
-LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat -Wformat-signedness,$(HOST_CFLAGS))
+LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS))
INSTALL ?= install
RM ?= rm -f
diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c
index 6ef908adf3a4..c9589026da8d 100644
--- a/tools/bpf/bpftool/btf.c
+++ b/tools/bpf/bpftool/btf.c
@@ -179,7 +179,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
case BTF_KIND_STRUCT:
case BTF_KIND_UNION: {
const struct btf_member *m = (const void *)(t + 1);
- __u32 i, vlen = BTF_INFO_VLEN(t->info);
+ __u32 i, vlen = btf_vlen(t);
if (json_output) {
jsonw_uint_field(w, "size", t->size);
@@ -193,7 +193,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
const char *name = btf_str(btf, m->name_off);
__u32 bit_off, bit_sz;
- if (BTF_INFO_KFLAG(t->info)) {
+ if (btf_kflag(t)) {
bit_off = BTF_MEMBER_BIT_OFFSET(m->offset);
bit_sz = BTF_MEMBER_BITFIELD_SIZE(m->offset);
} else {
@@ -224,7 +224,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
}
case BTF_KIND_ENUM: {
const struct btf_enum *v = (const void *)(t + 1);
- __u32 i, vlen = BTF_INFO_VLEN(t->info);
+ __u32 i, vlen = btf_vlen(t);
const char *encoding;
encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED";
@@ -300,8 +300,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
break;
}
case BTF_KIND_FWD: {
- const char *fwd_kind = BTF_INFO_KFLAG(t->info) ? "union"
- : "struct";
+ const char *fwd_kind = btf_kflag(t) ? "union" : "struct";
if (json_output)
jsonw_string_field(w, "fwd_kind", fwd_kind);
@@ -322,7 +321,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
}
case BTF_KIND_FUNC_PROTO: {
const struct btf_param *p = (const void *)(t + 1);
- __u32 i, vlen = BTF_INFO_VLEN(t->info);
+ __u32 i, vlen = btf_vlen(t);
if (json_output) {
jsonw_uint_field(w, "ret_type_id", t->type);
@@ -365,7 +364,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id,
case BTF_KIND_DATASEC: {
const struct btf_var_secinfo *v = (const void *)(t + 1);
const struct btf_type *vt;
- __u32 i, vlen = BTF_INFO_VLEN(t->info);
+ __u32 i, vlen = btf_vlen(t);
if (json_output) {
jsonw_uint_field(w, "size", t->size);
diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c
index 9dc8425b1789..e4075824343f 100644
--- a/tools/bpf/bpftool/btf_dumper.c
+++ b/tools/bpf/bpftool/btf_dumper.c
@@ -476,8 +476,8 @@ static int btf_dumper_struct(const struct btf_dumper *d, __u32 type_id,
if (!t)
return -EINVAL;
- kind_flag = BTF_INFO_KFLAG(t->info);
- vlen = BTF_INFO_VLEN(t->info);
+ kind_flag = btf_kflag(t);
+ vlen = btf_vlen(t);
jsonw_start_object(d->jw);
m = (struct btf_member *)(t + 1);
@@ -535,7 +535,7 @@ static int btf_dumper_datasec(const struct btf_dumper *d, __u32 type_id,
if (!t)
return -EINVAL;
- vlen = BTF_INFO_VLEN(t->info);
+ vlen = btf_vlen(t);
vsi = (struct btf_var_secinfo *)(t + 1);
jsonw_start_object(d->jw);
@@ -557,7 +557,7 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id,
{
const struct btf_type *t = btf__type_by_id(d->btf, type_id);
- switch (BTF_INFO_KIND(t->info)) {
+ switch (btf_kind(t)) {
case BTF_KIND_INT:
return btf_dumper_int(t, bit_offset, data, d->jw,
d->is_plain_text);
@@ -631,7 +631,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id,
t = btf__type_by_id(btf, type_id);
- switch (BTF_INFO_KIND(t->info)) {
+ switch (btf_kind(t)) {
case BTF_KIND_INT:
case BTF_KIND_TYPEDEF:
case BTF_KIND_FLOAT:
@@ -661,7 +661,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id,
break;
case BTF_KIND_FWD:
BTF_PRINT_ARG("%s %s ",
- BTF_INFO_KFLAG(t->info) ? "union" : "struct",
+ btf_kflag(t) ? "union" : "struct",
btf__name_by_offset(btf, t->name_off));
break;
case BTF_KIND_VOLATILE:
@@ -718,7 +718,7 @@ static int btf_dump_func(const struct btf *btf, char *func_sig,
BTF_PRINT_ARG("%s(", btf__name_by_offset(btf, func->name_off));
else
BTF_PRINT_ARG("(");
- vlen = BTF_INFO_VLEN(func_proto->info);
+ vlen = btf_vlen(func_proto);
for (i = 0; i < vlen; i++) {
struct btf_param *arg = &((struct btf_param *)(func_proto + 1))[i];
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 8bfcff9e2f63..ef366ccc9650 100644
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -832,6 +832,8 @@ static int prog_fd_by_nametag(void *nametag, int **fds, bool tag)
fd = bpf_prog_get_fd_by_id(id);
if (fd < 0) {
+ if (errno == ENOENT)
+ continue;
p_err("can't get prog by id (%u): %s",
id, strerror(errno));
goto err_close_fds;
@@ -996,6 +998,8 @@ static int map_fd_by_name(char *name, int **fds,
opts_ro.open_flags = BPF_F_RDONLY;
fd = bpf_map_get_fd_by_id_opts(id, &opts_ro);
if (fd < 0) {
+ if (errno == ENOENT)
+ continue;
p_err("can't get map by id (%u): %s",
id, strerror(errno));
goto err_close_fds;
diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
index 6ae7262ebe0c..a50540ef6521 100644
--- a/tools/bpf/bpftool/gen.c
+++ b/tools/bpf/bpftool/gen.c
@@ -101,6 +101,12 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz)
return true;
}
+ if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ snprintf(buf, buf_sz, "%s", name + 1);
+ sanitize_identifier(buf);
+ return true;
+ }
+
for (i = 0, n = ARRAY_SIZE(sfxs); i < n; i++) {
const char *sfx = sfxs[i], *p;
@@ -117,7 +123,7 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz)
static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz)
{
- static const char *pfxs[] = { ".data", ".rodata", ".bss", ".kconfig" };
+ static const char *pfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" };
int i, n;
/* recognize hard coded LLVM section name */
@@ -254,7 +260,7 @@ static const struct btf_type *find_type_for_map(struct btf *btf, const char *map
return NULL;
}
-static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz)
+static bool is_skel_data(const struct bpf_map *map, char *buf, size_t sz)
{
size_t tmp_sz;
@@ -263,13 +269,24 @@ static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz)
return true;
}
- if (!bpf_map__is_internal(map) || !(bpf_map__map_flags(map) & BPF_F_MMAPABLE))
+ if (!bpf_map__is_internal(map))
return false;
if (!get_map_ident(map, buf, sz))
return false;
- return true;
+ if (bpf_map__map_flags(map) & BPF_F_MMAPABLE)
+ return true;
+
+ if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY)
+ return bpf_map__btf_value_type_id(map) != 0;
+
+ return false;
+}
+
+static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz)
+{
+ return is_skel_data(map, buf, sz) && bpf_map__type(map) != BPF_MAP_TYPE_PERCPU_ARRAY;
}
static int codegen_datasecs(struct bpf_object *obj, const char *obj_name)
@@ -287,7 +304,7 @@ static int codegen_datasecs(struct bpf_object *obj, const char *obj_name)
bpf_object__for_each_map(map, obj) {
/* only generate definitions for memory-mapped internal maps */
- if (!is_mmapable_map(map, map_ident, sizeof(map_ident)))
+ if (!is_skel_data(map, map_ident, sizeof(map_ident)))
continue;
sec = find_type_for_map(btf, map_ident);
@@ -517,7 +534,7 @@ static void codegen_asserts(struct bpf_object *obj, const char *obj_name)
", obj_name);
bpf_object__for_each_map(map, obj) {
- if (!is_mmapable_map(map, map_ident, sizeof(map_ident)))
+ if (!is_skel_data(map, map_ident, sizeof(map_ident)))
continue;
sec = find_type_for_map(btf, map_ident);
@@ -668,8 +685,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name)
bpf_object__for_each_map(map, obj) {
if (!get_map_ident(map, ident, sizeof(ident)))
continue;
- if (bpf_map__is_internal(map) &&
- (bpf_map__map_flags(map) & BPF_F_MMAPABLE))
+ if (is_skel_data(map, ident, sizeof(ident)))
printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n",
ident, bpf_map_mmap_sz(map));
codegen("\
@@ -741,7 +757,7 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h
const void *mmap_data = NULL;
size_t mmap_size = 0;
- if (!is_mmapable_map(map, ident, sizeof(ident)))
+ if (!is_skel_data(map, ident, sizeof(ident)))
continue;
codegen("\
@@ -793,6 +809,8 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h
if (sign_progs) {
sopts.insns = opts.insns;
sopts.insns_sz = opts.insns_sz;
+ sopts.data = opts.data;
+ sopts.data_sz = opts.data_sz;
sopts.excl_prog_hash = prog_sha;
sopts.excl_prog_hash_sz = sizeof(prog_sha);
sopts.signature = sig_buf;
@@ -847,9 +865,23 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h
bpf_object__for_each_map(map, obj) {
const char *mmap_flags;
- if (!is_mmapable_map(map, ident, sizeof(ident)))
+ if (!is_skel_data(map, ident, sizeof(ident)))
continue;
+ if (bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY) {
+ codegen("\
+ \n\
+ err = skel_protect_map_data(skel->%1$s, &skel->maps.%1$s.initial_value, %2$zd);\n\
+ if (err) \n\
+ return err; \n\
+ #ifdef __KERNEL__ \n\
+ skel->%1$s = NULL; \n\
+ #endif \n\
+ ",
+ ident, bpf_map_mmap_sz(map));
+ continue;
+ }
+
if (bpf_map__map_flags(map) & BPF_F_RDONLY_PROG)
mmap_flags = "PROT_READ";
else
@@ -953,8 +985,7 @@ codegen_maps_skeleton(struct bpf_object *obj, size_t map_cnt, bool mmaped, bool
map->map = &obj->maps.%s; \n\
",
i, bpf_map__name(map), ident);
- /* memory-mapped internal maps */
- if (mmaped && is_mmapable_map(map, ident, sizeof(ident))) {
+ if (mmaped && is_skel_data(map, ident, sizeof(ident))) {
printf("\tmap->mmaped = (void **)&obj->%s;\n", ident);
}
diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index bdcd717b0348..088d1d206065 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -377,6 +377,25 @@ static __u64 *u64_to_arr(__u64 val)
return (__u64 *) u64_to_ptr(val);
}
+static __u32 *u64_to_u32_arr(__u64 val)
+{
+ return (__u32 *)u64_to_ptr(val);
+}
+
+static struct kernel_sym *find_kernel_sym_by_addr(__u64 addr, bool is_ibt_enabled)
+{
+ struct kernel_sym *sym;
+
+ if (!addr)
+ return NULL;
+
+ sym = kernel_syms_search(&dd, addr);
+ if (!sym && is_ibt_enabled && addr >= 4)
+ sym = kernel_syms_search(&dd, addr - 4);
+
+ return sym;
+}
+
static void
show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
{
@@ -404,6 +423,52 @@ show_uprobe_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
}
static void
+show_tracing_multi_json(struct bpf_link_info *info, json_writer_t *wtr)
+{
+ bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+ __u64 *addrs, *cookies;
+ __u32 i, *ids;
+
+ if (!dd.sym_count)
+ kernel_syms_load(&dd);
+ show_symbol = !!dd.sym_count;
+
+ show_link_attach_type_json(info->tracing_multi.attach_type, wtr);
+ jsonw_uint_field(wtr, "func_cnt", info->tracing_multi.count);
+ jsonw_uint_field(wtr, "btf_obj_id", info->tracing_multi.btf_obj_id);
+ jsonw_name(wtr, "funcs");
+
+ jsonw_start_array(wtr);
+
+ ids = u64_to_u32_arr(info->tracing_multi.ids);
+ addrs = u64_to_arr(info->tracing_multi.addrs);
+ cookies = u64_to_arr(info->tracing_multi.cookies);
+
+ for (i = 0; i < info->tracing_multi.count; i++) {
+ struct kernel_sym *sym;
+ __u64 addr = addrs[i];
+
+ sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+ jsonw_start_object(wtr);
+ jsonw_uint_field(wtr, "id", ids[i]);
+ jsonw_uint_field(wtr, "addr", addr);
+ if (sym) {
+ jsonw_string_field(wtr, "func", sym->name);
+ if (sym->module[0] == '\0') {
+ jsonw_name(wtr, "module");
+ jsonw_null(wtr);
+ } else {
+ jsonw_string_field(wtr, "module", sym->module);
+ }
+ }
+ jsonw_uint_field(wtr, "cookie", cookies[i]);
+ jsonw_end_object(wtr);
+ }
+ jsonw_end_array(wtr);
+}
+
+static void
show_perf_event_kprobe_json(struct bpf_link_info *info, json_writer_t *wtr)
{
jsonw_bool_field(wtr, "retprobe", info->perf_event.type == BPF_PERF_EVENT_KRETPROBE);
@@ -589,6 +654,9 @@ static int show_link_close_json(int fd, struct bpf_link_info *info)
case BPF_LINK_TYPE_UPROBE_MULTI:
show_uprobe_multi_json(info, json_wtr);
break;
+ case BPF_LINK_TYPE_TRACING_MULTI:
+ show_tracing_multi_json(info, json_wtr);
+ break;
case BPF_LINK_TYPE_PERF_EVENT:
switch (info->perf_event.type) {
case BPF_PERF_EVENT_EVENT:
@@ -833,6 +901,46 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info)
}
}
+static void show_tracing_multi_plain(struct bpf_link_info *info)
+{
+ bool is_ibt_enabled = is_x86_ibt_enabled(), show_symbol;
+ __u64 *addrs, *cookies;
+ __u32 i, *ids;
+
+ if (!info->tracing_multi.count)
+ return;
+
+ if (!dd.sym_count)
+ kernel_syms_load(&dd);
+ show_symbol = !!dd.sym_count;
+
+ printf("\n\t");
+ show_link_attach_type_plain(info->tracing_multi.attach_type);
+ printf("btf_obj_id %u ", info->tracing_multi.btf_obj_id);
+ printf("count %u ", info->tracing_multi.count);
+
+ printf("\n\t%-16s %-16s %-16s %s",
+ "btf_id", "addr", "cookie", "func [module]");
+
+ ids = u64_to_u32_arr(info->tracing_multi.ids);
+ addrs = u64_to_arr(info->tracing_multi.addrs);
+ cookies = u64_to_arr(info->tracing_multi.cookies);
+
+ for (i = 0; i < info->tracing_multi.count; i++) {
+ __u64 addr = addrs[i];
+ struct kernel_sym *sym;
+
+ sym = show_symbol ? find_kernel_sym_by_addr(addr, is_ibt_enabled) : NULL;
+
+ printf("\n\t%-16u %016llx %-16llu", ids[i], addr, cookies[i]);
+ if (sym) {
+ printf(" %s", sym->name);
+ if (sym->module[0] != '\0')
+ printf(" [%s]", sym->module);
+ }
+ }
+}
+
static void show_perf_event_kprobe_plain(struct bpf_link_info *info)
{
const char *buf;
@@ -989,6 +1097,9 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
case BPF_LINK_TYPE_UPROBE_MULTI:
show_uprobe_multi_plain(info);
break;
+ case BPF_LINK_TYPE_TRACING_MULTI:
+ show_tracing_multi_plain(info);
+ break;
case BPF_LINK_TYPE_PERF_EVENT:
switch (info->perf_event.type) {
case BPF_PERF_EVENT_EVENT:
@@ -1029,6 +1140,7 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info)
static int do_show_link(int fd)
{
__u64 *ref_ctr_offsets = NULL, *offsets = NULL, *cookies = NULL;
+ __u32 *ids = NULL;
struct bpf_link_info info;
__u32 len = sizeof(info);
char path_buf[PATH_MAX];
@@ -1114,6 +1226,26 @@ again:
goto again;
}
}
+ if (info.type == BPF_LINK_TYPE_TRACING_MULTI && !info.tracing_multi.ids) {
+ count = info.tracing_multi.count;
+ if (count) {
+ ids = calloc(count, sizeof(__u32));
+ addrs = calloc(count, sizeof(__u64));
+ cookies = calloc(count, sizeof(__u64));
+ if (!ids || !addrs || !cookies) {
+ p_err("mem alloc failed");
+ close(fd);
+ free(cookies);
+ free(addrs);
+ free(ids);
+ return -ENOMEM;
+ }
+ info.tracing_multi.ids = ptr_to_u64(ids);
+ info.tracing_multi.addrs = ptr_to_u64(addrs);
+ info.tracing_multi.cookies = ptr_to_u64(cookies);
+ goto again;
+ }
+ }
if (info.type == BPF_LINK_TYPE_PERF_EVENT) {
switch (info.perf_event.type) {
case BPF_PERF_EVENT_TRACEPOINT:
@@ -1153,6 +1285,7 @@ again:
free(cookies);
free(offsets);
free(addrs);
+ free(ids);
close(fd);
return 0;
}
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index 71a45d96617e..684a8fb72414 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv)
show_map_close_json(fds[i], &info);
else
show_map_close_plain(fds[i], &info);
-
- close(fds[i]);
}
if (json_output && nb_fds > 1)
jsonw_end_array(json_wtr); /* root array */
@@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds)
static struct btf *btf_vmlinux;
+static void free_btf_vmlinux(void)
+{
+ btf__free(btf_vmlinux);
+ btf_vmlinux = NULL;
+}
+
static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf)
{
int err = 0;
@@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr,
exit_free:
free(key);
free(value);
- close(fd);
free_map_kv_btf(btf);
return err;
@@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv)
for (i = 0; i < nb_fds; i++) {
if (bpf_map_get_info_by_fd(fds[i], &info, &len)) {
p_err("can't get map info: %s", strerror(errno));
+ err = -1;
break;
}
err = map_dump(fds[i], &info, wtr, nb_fds > 1);
@@ -958,7 +962,7 @@ exit_close:
close(fds[i]);
exit_free:
free(fds);
- btf__free(btf_vmlinux);
+ free_btf_vmlinux();
return err;
}
@@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key,
btf_wtr = get_btf_writer();
if (!btf_wtr) {
p_info("failed to create json writer for btf. falling back to plain output");
- btf__free(btf);
+ free_map_kv_btf(btf);
btf = NULL;
print_entry_plain(info, key, value);
} else {
@@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key,
} else {
print_entry_plain(info, key, value);
}
- btf__free(btf);
+ free_map_kv_btf(btf);
}
static int do_lookup(int argc, char **argv)
diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c
index f9b742f4bb10..88726a6db6d0 100644
--- a/tools/bpf/bpftool/sign.c
+++ b/tools/bpf/bpftool/sign.c
@@ -135,9 +135,21 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
CMS_ContentInfo *cms = NULL;
long actual_sig_len = 0;
X509 *x509 = NULL;
+ void *data = NULL;
+ size_t data_sz;
int err = 0;
- bd_in = BIO_new_mem_buf(opts->insns, opts->insns_sz);
+ data_sz = (size_t)opts->insns_sz + opts->data_sz;
+ data = malloc(data_sz);
+ if (!data) {
+ err = -ENOMEM;
+ goto cleanup;
+ }
+ memcpy(data, opts->insns, opts->insns_sz);
+ if (opts->data_sz)
+ memcpy((char *)data + opts->insns_sz, opts->data, opts->data_sz);
+
+ bd_in = BIO_new_mem_buf(data, data_sz);
if (!bd_in) {
err = -ENOMEM;
goto cleanup;
@@ -175,10 +187,13 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts)
goto cleanup;
}
- EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash,
- &opts->excl_prog_hash_sz, EVP_sha256(), NULL);
+ if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash,
+ &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) {
+ err = -EIO;
+ goto cleanup;
+ }
- bd_out = BIO_new(BIO_s_mem());
+ bd_out = BIO_new(BIO_s_mem());
if (!bd_out) {
err = -ENOMEM;
goto cleanup;
@@ -212,6 +227,7 @@ cleanup:
X509_free(x509);
EVP_PKEY_free(private_key);
BIO_free(bd_in);
+ free(data);
DISPLAY_OSSL_ERR(err < 0);
return err;
}
diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c
index aa43dead249c..835e5e561f7f 100644
--- a/tools/bpf/bpftool/struct_ops.c
+++ b/tools/bpf/bpftool/struct_ops.c
@@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv)
err = cmd_select(cmds, argc, argv, do_help);
btf__free(btf_vmlinux);
+ btf_vmlinux = NULL;
+ map_info_type = NULL;
+ map_info_alloc_len = 0;
+ map_info_type_id = 0;
return err;
}
diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c
index 5e7cb8b36fef..5579173a61e3 100644
--- a/tools/bpf/bpftool/xlated_dumper.c
+++ b/tools/bpf/bpftool/xlated_dumper.c
@@ -107,14 +107,7 @@ print_insn_for_graph(void *private_data, const char *fmt, ...)
p = buf;
while (*p != '\0') {
- if (*p == '\n') {
- memmove(p + 3, p, strlen(buf) + 1 - (p - buf));
- /* Align each instruction dump row left. */
- *p++ = '\\';
- *p++ = 'l';
- /* Output multiline concatenation. */
- *p++ = '\\';
- } else if (*p == '<' || *p == '>' || *p == '|' || *p == '&') {
+ if (*p == '<' || *p == '>' || *p == '|' || *p == '&') {
memmove(p + 1, p, strlen(buf) + 1 - (p - buf));
/* Escape special character. */
*p++ = '\\';
@@ -129,16 +122,10 @@ print_insn_for_graph(void *private_data, const char *fmt, ...)
static void __printf(2, 3)
print_insn_json(void *private_data, const char *fmt, ...)
{
- unsigned int l = strlen(fmt);
- char chomped_fmt[l];
va_list args;
va_start(args, fmt);
- if (l > 0) {
- strncpy(chomped_fmt, fmt, l - 1);
- chomped_fmt[l - 1] = '\0';
- }
- jsonw_vprintf_enquote(json_wtr, chomped_fmt, args);
+ jsonw_vprintf_enquote(json_wtr, fmt, args);
va_end(args);
}
@@ -351,6 +338,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len,
printf("%4u: ", i);
print_bpf_insn(&cbs, insn + i, true);
+ printf("\n");
if (opcodes) {
printf(" ");
@@ -417,6 +405,7 @@ void dump_xlated_for_graph(struct dump_data *dd, void *buf_start, void *buf_end,
printf("%u: ", insn_off);
print_bpf_insn(&cbs, cur, true);
+ printf("\\l\\\n");
if (opcodes) {
printf("\\ \\ \\ \\ ");
diff --git a/tools/bpf/resolve_btfids/Makefile b/tools/bpf/resolve_btfids/Makefile
index 7672208f65e4..bb0463b380af 100644
--- a/tools/bpf/resolve_btfids/Makefile
+++ b/tools/bpf/resolve_btfids/Makefile
@@ -40,7 +40,6 @@ LIBBPF_DESTDIR := $(LIBBPF_OUT)
LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include
SUBCMD_DESTDIR := $(SUBCMD_OUT)
-SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include
BINARY := $(OUTPUT)/resolve_btfids
BINARY_IN := $(BINARY)-in.o
@@ -56,7 +55,7 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
$(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
- $(abspath $@) install_headers
+ $(abspath $@)
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) \
@@ -77,7 +76,7 @@ HOSTCFLAGS_resolve_btfids += -g \
-I$(srctree)/tools/include \
-I$(srctree)/tools/include/uapi \
-I$(LIBBPF_INCLUDE) \
- -I$(SUBCMD_INCLUDE) \
+ -I$(srctree)/tools/lib \
$(LIBELF_FLAGS) \
-Wall -Werror
diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c
index f8a91fa7584f..37d7e7224207 100644
--- a/tools/bpf/resolve_btfids/main.c
+++ b/tools/bpf/resolve_btfids/main.c
@@ -58,6 +58,17 @@
* __BTF_ID__func__vfs_fallocate__5:
* .zero 4
* .word (1 << 3) | (1 << 1) | (1 << 2)
+ *
+ * In addition to resolving BTF IDs, resolve_btfids performs kernel-specific
+ * BTF-to-BTF transformations for kfuncs found in BTF_SET8_KFUNCS sets. For
+ * each such kfunc it:
+ *
+ * - emits a "bpf_kfunc" decl tag, and "bpf_fastcall" when KF_FASTCALL is set;
+ * - wraps the return value and/or arguments that use arena pointers
+ * with the "address_space(1)" type attribute;
+ * - rewrites the prototype of KF_IMPLICIT_ARGS kfuncs.
+ *
+ * These kfunc annotations were historically produced by pahole.
*/
#define _GNU_SOURCE
@@ -119,6 +130,11 @@ struct btf_id {
Elf64_Addr addr[ADDR_CNT];
};
+struct addr_sym {
+ Elf64_Addr addr;
+ const char *name;
+};
+
struct object {
const char *path;
const char *btf_path;
@@ -150,12 +166,27 @@ struct object {
int nr_structs;
int nr_unions;
int nr_typedefs;
+
+ struct addr_sym *addr_syms;
+ u32 addr_syms_cnt;
+ u32 addr_syms_cap;
};
+#define DECL_TAG_FASTCALL "bpf_fastcall"
+#define DECL_TAG_KFUNC "bpf_kfunc"
+
+#define KF_FASTCALL (1 << 12)
+#define KF_ARENA_RET (1 << 13)
+#define KF_ARENA_ARG1 (1 << 14)
+#define KF_ARENA_ARG2 (1 << 15)
#define KF_IMPLICIT_ARGS (1 << 16)
#define KF_IMPL_SUFFIX "_impl"
+#define TYPE_ATTR_ARENA "address_space(1)"
+#define PARAM_SUFFIX_ARENA "__arena"
+#define PARAM_SUFFIX_ARENA_NULLABLE "__arena__nullable"
struct kfunc {
+ struct rb_node rb_node;
const char *name;
u32 btf_id;
u32 flags;
@@ -166,9 +197,7 @@ struct btf2btf_context {
u32 *decl_tags;
u32 nr_decl_tags;
u32 max_decl_tags;
- struct kfunc *kfuncs;
- u32 nr_kfuncs;
- u32 max_kfuncs;
+ struct rb_root kfuncs;
};
static int verbose;
@@ -201,6 +230,35 @@ static int eprintf(int level, int var, const char *fmt, ...)
#define pr_info(fmt, ...) \
eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__)
+/*
+ * Grow *data so it can hold at least cnt elements of elem_sz bytes each.
+ * *cap is the capacity in elements and is updated on growth.
+ */
+static int __ensure_mem(void **data, u32 *cap, u32 cnt, size_t elem_sz)
+{
+ u32 new_cap, old_cap = *cap;
+ void *arr;
+
+ if (cnt <= old_cap)
+ return 0;
+
+ new_cap = max(old_cap + 256, old_cap * 2);
+ if (new_cap < cnt)
+ new_cap = cnt;
+
+ arr = realloc(*data, elem_sz * new_cap);
+ if (!arr)
+ return -ENOMEM;
+
+ *data = arr;
+ *cap = new_cap;
+
+ return 0;
+}
+
+#define ensure_mem(arr_ptr, cap_ptr, cnt) \
+ __ensure_mem((void **)(arr_ptr), (cap_ptr), (cnt), sizeof(**(arr_ptr)))
+
static bool is_btf_id(const char *name)
{
return name && !strncmp(name, BTF_ID_PREFIX, sizeof(BTF_ID_PREFIX) - 1);
@@ -480,6 +538,40 @@ static int elf_collect(struct object *obj)
return 0;
}
+static int push_addr_sym(struct object *obj, Elf64_Addr addr, const char *name)
+{
+ if (ensure_mem(&obj->addr_syms, &obj->addr_syms_cap, obj->addr_syms_cnt + 1))
+ return -ENOMEM;
+
+ obj->addr_syms[obj->addr_syms_cnt++] = (struct addr_sym){
+ .addr = addr,
+ .name = name,
+ };
+
+ return 0;
+}
+
+static int cmp_addr_sym(const void *a, const void *b)
+{
+ Elf64_Addr aa = ((const struct addr_sym *)a)->addr;
+ Elf64_Addr ab = ((const struct addr_sym *)b)->addr;
+
+ return (aa > ab) - (aa < ab);
+}
+
+static const char *find_name_by_addr(struct object *obj, Elf64_Addr addr)
+{
+ struct addr_sym key = { .addr = addr };
+ struct addr_sym *res;
+
+ if (!obj->addr_syms_cnt)
+ return NULL;
+
+ res = bsearch(&key, obj->addr_syms, obj->addr_syms_cnt,
+ sizeof(*obj->addr_syms), cmp_addr_sym);
+ return res ? res->name : NULL;
+}
+
static int symbols_collect(struct object *obj)
{
Elf_Scn *scn = NULL;
@@ -573,8 +665,15 @@ static int symbols_collect(struct object *obj)
return -1;
}
id->addr[id->addr_cnt++] = sym.st_value;
+
+ if (push_addr_sym(obj, sym.st_value, id->name))
+ return -1;
}
+ if (obj->addr_syms_cnt)
+ qsort(obj->addr_syms, obj->addr_syms_cnt,
+ sizeof(*obj->addr_syms), cmp_addr_sym);
+
return 0;
}
@@ -890,17 +989,8 @@ static const struct btf_type *btf_type_skip_qualifiers(const struct btf *btf, s3
static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id)
{
- u32 *arr = ctx->decl_tags;
- u32 cap = ctx->max_decl_tags;
-
- if (ctx->nr_decl_tags + 1 > cap) {
- cap = max(cap + 256, cap * 2);
- arr = realloc(arr, sizeof(u32) * cap);
- if (!arr)
- return -ENOMEM;
- ctx->max_decl_tags = cap;
- ctx->decl_tags = arr;
- }
+ if (ensure_mem(&ctx->decl_tags, &ctx->max_decl_tags, ctx->nr_decl_tags + 1))
+ return -ENOMEM;
ctx->decl_tags[ctx->nr_decl_tags++] = decl_tag_id;
@@ -909,23 +999,57 @@ static int push_decl_tag_id(struct btf2btf_context *ctx, u32 decl_tag_id)
static int push_kfunc(struct btf2btf_context *ctx, struct kfunc *kfunc)
{
- struct kfunc *arr = ctx->kfuncs;
- u32 cap = ctx->max_kfuncs;
+ struct rb_node **p = &ctx->kfuncs.rb_node;
+ struct rb_node *parent = NULL;
+ struct kfunc *k;
- if (ctx->nr_kfuncs + 1 > cap) {
- cap = max(cap + 256, cap * 2);
- arr = realloc(arr, sizeof(struct kfunc) * cap);
- if (!arr)
- return -ENOMEM;
- ctx->max_kfuncs = cap;
- ctx->kfuncs = arr;
+ /*
+ * Dedup by BTF ID: collecting the same kfunc twice is a no-op,
+ * UNLESS the kfunc flags are inconsistent, in which case we
+ * fail hard because it indicates a bug in a kfunc set declaration.
+ */
+ while (*p) {
+ parent = *p;
+ k = rb_entry(parent, struct kfunc, rb_node);
+
+ if (kfunc->btf_id < k->btf_id) {
+ p = &(*p)->rb_left;
+ } else if (kfunc->btf_id > k->btf_id) {
+ p = &(*p)->rb_right;
+ } else if (k->flags == kfunc->flags) {
+ return 0;
+ } else {
+ pr_err("ERROR: resolve_btfids: kfunc %s has inconsistent flags across BTF ID sets: 0x%x != 0x%x\n",
+ kfunc->name, k->flags, kfunc->flags);
+ return -EINVAL;
+ }
}
- ctx->kfuncs[ctx->nr_kfuncs++] = *kfunc;
+ k = zalloc(sizeof(*k));
+ if (!k)
+ return -ENOMEM;
+
+ *k = *kfunc;
+ rb_link_node(&k->rb_node, parent, p);
+ rb_insert_color(&k->rb_node, &ctx->kfuncs);
return 0;
}
+static void free_kfuncs(struct rb_root *root)
+{
+ struct rb_node *next;
+ struct kfunc *kfunc;
+
+ next = rb_first(root);
+ while (next) {
+ kfunc = rb_entry(next, struct kfunc, rb_node);
+ next = rb_next(&kfunc->rb_node);
+ rb_erase(&kfunc->rb_node, root);
+ free(kfunc);
+ }
+}
+
static int collect_decl_tags(struct btf2btf_context *ctx)
{
const u32 type_cnt = btf__type_cnt(ctx->btf);
@@ -945,94 +1069,76 @@ static int collect_decl_tags(struct btf2btf_context *ctx)
return 0;
}
-/*
- * To find the kfunc flags having its struct btf_id (with ELF addresses)
- * we need to find the address that is in range of a set8.
- * If a set8 is found, then the flags are located at addr + 4 bytes.
- * Return 0 (no flags!) if not found.
- */
-static u32 find_kfunc_flags(struct object *obj, struct btf_id *kfunc_id)
+static bool param_name_has_suffix(const char *name, const char *suffix)
{
- const u32 *elf_data_ptr = obj->efile.idlist->d_buf;
- u64 set_lower_addr, set_upper_addr, addr;
- struct btf_id *set_id;
- struct rb_node *next;
- u32 flags;
- u64 idx;
+ size_t name_len = strlen(name);
+ size_t suffix_len = strlen(suffix);
- for (next = rb_first(&obj->sets); next; next = rb_next(next)) {
- set_id = rb_entry(next, struct btf_id, rb_node);
- if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1)
- continue;
-
- set_lower_addr = set_id->addr[0];
- set_upper_addr = set_lower_addr + set_id->cnt * sizeof(u64);
+ return name_len >= suffix_len && !strcmp(name + name_len - suffix_len, suffix);
+}
- for (u32 i = 0; i < kfunc_id->addr_cnt; i++) {
- addr = kfunc_id->addr[i];
- /*
- * Lower bound is exclusive to skip the 8-byte header of the set.
- * Upper bound is inclusive to capture the last entry at offset 8*cnt.
- */
- if (set_lower_addr < addr && addr <= set_upper_addr) {
- pr_debug("found kfunc %s in BTF_ID_FLAGS %s\n",
- kfunc_id->name, set_id->name);
- idx = addr - obj->efile.idlist_addr;
- idx = idx / sizeof(u32) + 1;
- flags = elf_data_ptr[idx];
-
- return flags;
- }
- }
- }
+static bool is_arena_param(const struct btf *btf, const struct btf_param *param)
+{
+ const char *name = btf__name_by_offset(btf, param->name_off);
- return 0;
+ return param_name_has_suffix(name, PARAM_SUFFIX_ARENA) ||
+ param_name_has_suffix(name, PARAM_SUFFIX_ARENA_NULLABLE);
}
static int collect_kfuncs(struct object *obj, struct btf2btf_context *ctx)
{
- const char *tag_name, *func_name;
+ Elf_Data *idlist = obj->efile.idlist;
struct btf *btf = ctx->btf;
- const struct btf_type *t;
- u32 flags, func_id;
- struct kfunc kfunc;
- struct btf_id *id;
- int err;
+ struct rb_node *next;
- if (ctx->nr_decl_tags == 0)
+ if (!idlist || !idlist->d_buf)
return 0;
- for (u32 i = 0; i < ctx->nr_decl_tags; i++) {
- t = btf__type_by_id(btf, ctx->decl_tags[i]);
- if (btf_kflag(t) || btf_decl_tag(t)->component_idx != -1)
- continue;
+ for (next = rb_first(&obj->sets); next; next = rb_next(next)) {
+ struct btf_id_set8 *set8;
+ struct btf_id *set_id;
+ u64 set_addr;
- tag_name = btf__name_by_offset(btf, t->name_off);
- if (strcmp(tag_name, "bpf_kfunc") != 0)
+ set_id = rb_entry(next, struct btf_id, rb_node);
+ if (set_id->kind != BTF_ID_KIND_SET8 || set_id->addr_cnt != 1)
continue;
- func_id = t->type;
- t = btf__type_by_id(btf, func_id);
- if (!btf_is_func(t))
+ set_addr = set_id->addr[0];
+ set8 = idlist->d_buf + (set_addr - obj->efile.idlist_addr);
+ if (!(set8->flags & BTF_SET8_KFUNCS))
continue;
- func_name = btf__name_by_offset(btf, t->name_off);
- if (!func_name)
- continue;
+ for (u32 i = 0; i < set_id->cnt; i++) {
+ size_t off = (char *)&set8->pairs[i] - (char *)set8;
+ const char *name = find_name_by_addr(obj, set_addr + off);
+ struct kfunc kfunc;
+ s32 func_id;
+ int err;
- id = btf_id__find(&obj->funcs, func_name);
- if (!id || id->kind != BTF_ID_KIND_SYM)
- continue;
+ if (!name) {
+ pr_err("WARN: resolve_btfids: no BTF ID symbol for %s entry %u\n",
+ set_id->name, i);
+ warnings++;
+ continue;
+ }
- flags = find_kfunc_flags(obj, id);
+ func_id = btf__find_by_name_kind_own(btf, name, BTF_KIND_FUNC);
+ if (func_id < 0) {
+ pr_err("WARN: resolve_btfids: no BTF func for kfunc %s in %s\n",
+ name, set_id->name);
+ warnings++;
+ continue;
+ }
- kfunc.name = id->name;
- kfunc.btf_id = func_id;
- kfunc.flags = flags;
+ pr_debug("found kfunc %s in %s\n", name, set_id->name);
- err = push_kfunc(ctx, &kfunc);
- if (err)
- return err;
+ kfunc.name = name;
+ kfunc.btf_id = func_id;
+ kfunc.flags = set8->pairs[i].flags;
+ err = push_kfunc(ctx, &kfunc);
+ if (err)
+ return err;
+ }
}
return 0;
@@ -1141,7 +1247,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct
return -E2BIG;
}
- if (btf__find_by_name_kind(btf, tmp_name, BTF_KIND_FUNC) > 0) {
+ if (btf__find_by_name_kind_own(btf, tmp_name, BTF_KIND_FUNC) > 0) {
pr_debug("resolve_btfids: function %s already exists in BTF\n", tmp_name);
goto add_new_proto;
}
@@ -1160,7 +1266,7 @@ static int process_kfunc_with_implicit_args(struct btf2btf_context *ctx, struct
continue;
tag_name = btf__name_by_offset(btf, t->name_off);
- if (strcmp(tag_name, "bpf_kfunc") == 0)
+ if (strcmp(tag_name, DECL_TAG_KFUNC) == 0)
continue;
idx = btf_decl_tag(t)->component_idx;
@@ -1211,22 +1317,187 @@ add_new_proto:
return 0;
}
+static bool is_arena_arg(const struct btf *btf, const struct kfunc *kfunc,
+ const struct btf_param *param, u32 idx)
+{
+ if (is_arena_param(btf, param))
+ return true;
+
+ switch (idx) {
+ case 0:
+ return kfunc->flags & KF_ARENA_ARG1;
+ case 1:
+ return kfunc->flags & KF_ARENA_ARG2;
+ default:
+ return false;
+ }
+}
+
+static s32 arena_tag_ptr(struct btf *btf, u32 ptr_id, struct kfunc *kfunc)
+{
+ const struct btf_type *ptr = btf__type_by_id(btf, ptr_id);
+ s32 tag_id, new_ptr_id;
+
+ if (!btf_is_ptr(ptr)) {
+ pr_err("ERROR: resolve_btfids: kfunc %s: arena type is not a pointer\n",
+ kfunc->name);
+ return -EINVAL;
+ }
+
+ tag_id = btf__add_type_attr(btf, TYPE_ATTR_ARENA, ptr->type);
+ if (tag_id < 0) {
+ pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a type attr to BTF: %d\n",
+ kfunc->name, tag_id);
+ return tag_id;
+ }
+
+ new_ptr_id = btf__add_ptr(btf, tag_id);
+ if (new_ptr_id < 0) {
+ pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a pointer to BTF: %d\n",
+ kfunc->name, new_ptr_id);
+ }
+
+ return new_ptr_id;
+}
+
+/*
+ * Add a FUNC_PROTO for @kfunc with each arena pointer tagged with an
+ * "address_space(1)" attribute. The original proto may be shared with
+ * other FUNCs, so it is never modified in place. Returns the original
+ * proto id when @kfunc has no arena return value or arguments.
+ */
+static s32 add_arena_tagged_proto(struct btf *btf, struct kfunc *kfunc)
+{
+ const struct btf_type *func = btf__type_by_id(btf, kfunc->btf_id);
+ u32 proto_id = func->type;
+ const struct btf_type *proto = btf__type_by_id(btf, proto_id);
+ const struct btf_param *params = btf_params(proto);
+ u32 nr_params = btf_vlen(proto);
+ s32 ret_type_id = proto->type;
+ const struct btf_type *t;
+ struct btf_param *tag_params;
+ s32 new_proto_id, id;
+ const char *name;
+ bool has_arena_arg = false;
+ int err, i;
+
+ for (i = 0; i < nr_params; i++) {
+ if (is_arena_arg(btf, kfunc, &params[i], i)) {
+ has_arena_arg = true;
+ break;
+ }
+ }
+
+ if (!(kfunc->flags & KF_ARENA_RET) && !has_arena_arg)
+ return proto_id;
+
+ if (kfunc->flags & KF_ARENA_RET) {
+ ret_type_id = arena_tag_ptr(btf, ret_type_id, kfunc);
+ if (ret_type_id < 0)
+ return ret_type_id;
+ }
+
+ new_proto_id = btf__add_func_proto(btf, ret_type_id);
+ if (new_proto_id < 0) {
+ pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a func proto to BTF: %d\n",
+ kfunc->name, new_proto_id);
+ return new_proto_id;
+ }
+
+ for (i = 0; i < nr_params; i++) {
+ /* btf__add_func_param() below may move the proto, re-fetch */
+ proto = btf__type_by_id(btf, proto_id);
+ name = btf__name_by_offset(btf, btf_params(proto)[i].name_off);
+
+ err = btf__add_func_param(btf, name ?: "", btf_params(proto)[i].type);
+ if (err < 0) {
+ pr_err("ERROR: resolve_btfids: kfunc %s: failed to add a proto param to BTF: %d\n",
+ kfunc->name, err);
+ return err;
+ }
+ }
+
+ for (i = 0; i < nr_params; i++) {
+ t = btf__type_by_id(btf, new_proto_id);
+ tag_params = btf_params(t);
+ if (!is_arena_arg(btf, kfunc, &tag_params[i], i))
+ continue;
+
+ id = arena_tag_ptr(btf, tag_params[i].type, kfunc);
+ if (id < 0)
+ return id;
+
+ t = btf__type_by_id(btf, new_proto_id);
+ tag_params = btf_params(t);
+ tag_params[i].type = id;
+ }
+
+ pr_debug("added arena-tagged proto for kfunc %s: %d\n", kfunc->name, new_proto_id);
+
+ return new_proto_id;
+}
+
+static int process_kfunc_with_arena_attrs(struct btf2btf_context *ctx,
+ struct kfunc *kfunc)
+{
+ struct btf_type *t;
+ s32 proto_id;
+
+ proto_id = add_arena_tagged_proto(ctx->btf, kfunc);
+ if (proto_id < 0)
+ return proto_id;
+
+ t = (struct btf_type *)btf__type_by_id(ctx->btf, kfunc->btf_id);
+ t->type = proto_id;
+
+ return 0;
+}
+
+static int add_decl_tag(struct btf2btf_context *ctx, const char *tag_name,
+ u32 target_btf_id, int component_idx)
+{
+ s32 new_id;
+
+ new_id = btf__add_decl_tag(ctx->btf, tag_name, target_btf_id, component_idx);
+ if (new_id < 0) {
+ pr_err("ERROR: resolve_btfids: failed to add '%s' decl tag for BTF id %u: %d\n",
+ tag_name, target_btf_id, new_id);
+ return new_id;
+ }
+
+ return push_decl_tag_id(ctx, new_id);
+}
+
static int btf2btf(struct object *obj)
{
struct btf2btf_context ctx = {};
+ struct rb_node *next;
int err;
err = build_btf2btf_context(obj, &ctx);
if (err)
goto out;
- for (u32 i = 0; i < ctx.nr_kfuncs; i++) {
- struct kfunc *kfunc = &ctx.kfuncs[i];
+ for (next = rb_first(&ctx.kfuncs); next; next = rb_next(next)) {
+ struct kfunc *kfunc = rb_entry(next, struct kfunc, rb_node);
- if (!(kfunc->flags & KF_IMPLICIT_ARGS))
- continue;
+ err = add_decl_tag(&ctx, DECL_TAG_KFUNC, kfunc->btf_id, -1);
+ if (err)
+ goto out;
+
+ if (kfunc->flags & KF_FASTCALL) {
+ err = add_decl_tag(&ctx, DECL_TAG_FASTCALL, kfunc->btf_id, -1);
+ if (err)
+ goto out;
+ }
+
+ if (kfunc->flags & KF_IMPLICIT_ARGS) {
+ err = process_kfunc_with_implicit_args(&ctx, kfunc);
+ if (err)
+ goto out;
+ }
- err = process_kfunc_with_implicit_args(&ctx, kfunc);
+ err = process_kfunc_with_arena_attrs(&ctx, kfunc);
if (err)
goto out;
}
@@ -1234,7 +1505,7 @@ static int btf2btf(struct object *obj)
err = 0;
out:
free(ctx.decl_tags);
- free(ctx.kfuncs);
+ free_kfuncs(&ctx.kfuncs);
return err;
}
@@ -1309,6 +1580,12 @@ static int finalize_btf(struct object *obj)
struct btf *base_btf = obj->base_btf, *btf = obj->btf;
int err;
+ err = btf__dedup(obj->btf, NULL);
+ if (err) {
+ pr_err("FAILED to dedup BTF: %s\n", strerror(errno));
+ goto out_err;
+ }
+
if (obj->base_btf && obj->distill_base) {
err = btf__distill_base(obj->btf, &base_btf, &btf);
if (err) {
@@ -1575,6 +1852,7 @@ out:
btf_id__free_all(&obj.typedefs);
btf_id__free_all(&obj.funcs);
btf_id__free_all(&obj.sets);
+ free(obj.addr_syms);
if (obj.efile.elf) {
elf_end(obj.efile.elf);
close(obj.efile.fd);