From aed1bf1a352acd4f5ffd2a81e23d2bb6e8e9b636 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 19 Aug 2026 19:20:21 -1000 Subject: bpf: sched_ext: Mark ops argument container pointer fields as trusted Walking an unannotated pointer field of a trusted struct yields a bare PTR_TO_BTF_ID in non-sleepable programs, which kfuncs and helpers accept, but PTR_UNTRUSTED in sleepable programs, which they reject. This gets in the way of making ops.init_task() sleepable, which schedulers want for allocations. For example, passing args->cgroup into bpf_cgrp_storage_get() then fails verification and the only recourse is round-tripping through the cgroup ID with bpf_cgroup_from_id(). The pointer fields in the sched_ext ops argument containers are all pinned by the callers for the duration of the ops calls and are never NULL. Add them to the verifier's trusted-fields whitelist so that they are PTR_TRUSTED in both sleepable and non-sleepable programs. Fixes: 8195136669661 ("sched_ext: Add cgroup support") Signed-off-by: Tejun Heo Acked-by: Kumar Kartikeya Dwivedi Link: https://lore.kernel.org/bpf/95d7ccc17681aa3a4a2eeb1b073f00f7@kernel.org Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/verifier.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e421ea2b80c3..821b47ac75c5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5877,6 +5877,28 @@ BTF_TYPE_SAFE_TRUSTED(struct file) { struct inode *f_inode; }; +/* + * The pointer fields in the sched_ext ops argument containers are pinned by the + * callers for the duration of the ops calls and are never NULL. + */ +BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) { +#ifdef CONFIG_EXT_GROUP_SCHED + struct cgroup *cgroup; +#endif +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) { + struct task_struct *task; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) { + struct sched_ext_ops *ops; +}; + +BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) { + struct sched_ext_ops *ops; +}; + BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) { struct inode *d_inode; }; @@ -5921,6 +5943,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env, BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm)); BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args)); + BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args)); return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted"); } -- cgit v1.2.3 From 4e3fb5bba680c1778e05a120c3793d22204f8fd4 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:18 +0000 Subject: bpf: Simplify cnum contains() by removing redundant branches Bearing in mind that a non-empty cnum represents an inclusive circular range in the corresponding unsigned integer domain, membership could be tested by checking whether the distance from the range base to the queried value is within the range size. This makes the explicit wrapping and non-wrapping cases in contains() unnecessary, since: v - cnum.base <= cnum.size is equivalent for both ordinary ranges and ranges that cross the unsigned wrap boundary. We've also run some benchmarks to compare the performance of the new implementation, for example, for empty ranges, overflowing ranges non-overflowing ranges. We've run the tests with O2 optimization. Results: | Test | contains | contains_new | | -------------------- | ----------- | ------------ | | empty | 1.511 ns/op | 1.479 ns/op | | non-overflow inside | 1.156 ns/op | 1.270 ns/op | | non-overflow outside | 1.053 ns/op | 1.258 ns/op | | overflow high side | 1.469 ns/op | 1.257 ns/op | | overflow low side | 1.469 ns/op | 1.270 ns/op | | overflow outside | 1.478 ns/op | 1.256 ns/op | | singleton inside | 1.049 ns/op | 1.256 ns/op | | singleton outside | 1.051 ns/op | 1.254 ns/op | | full range | 1.053 ns/op | 1.256 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves the soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-2-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index a90e317e3578..ac1f62a0195d 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -210,12 +210,7 @@ bool FN(is_empty)(struct cnum_t cnum) bool FN(contains)(struct cnum_t cnum, ut v) { - if (FN(is_empty)(cnum)) - return false; - if (FN(urange_overflow)(cnum)) - return v >= cnum.base || v <= (ut)cnum.base + cnum.size; - else - return v >= cnum.base && v <= (ut)cnum.base + cnum.size; + return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size; } bool FN(is_const)(struct cnum_t cnum) -- cgit v1.2.3 From b601a4ecbe672371776785bd7ac142af0ac395ac Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:24 +0000 Subject: bpf: Simplify cnum normalize() by dropping unnecessary comparisons In normalize(), there's no need to compare cnum.base with ST_MAX, because any cnum with size == UT_MAX represents the full unsigned domain independently of base, so normalize() can canonicalize all such values to base 0. Also, it's a no-op assigning 0 to cnum.base if the cnum is already unbounded, so we can remove the base != 0 check too. We've also run some benchmarks to compare the performance of the new implementation. We've run the tests with O2 optimization. Results: | Test | normalize | normalize_new | | ----------------------- | ----------- | ------------- | | normal small range | 1.255 ns/op | 1.217 ns/op | | full range normalized | 1.019 ns/op | 1.217 ns/op | | full range nonzero base | 1.031 ns/op | 1.221 ns/op | | empty | 1.023 ns/op | 1.217 ns/op | | signed max full range | 1.020 ns/op | 1.218 ns/op | | max base singleton | 1.218 ns/op | 1.219 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of the operation are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-3-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel') diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index ac1f62a0195d..09535edbbafc 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -181,7 +181,7 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max) static inline struct cnum_t FN(normalize)(struct cnum_t cnum) { - if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX) + if (cnum.size == UT_MAX) cnum.base = 0; return cnum; } -- cgit v1.2.3 From 705249119d942e7228d9207800ddfa13e4601b19 Mon Sep 17 00:00:00 2001 From: Vinicius Sampaio Date: Thu, 30 Jul 2026 10:21:32 +0000 Subject: bpf: Avoid redundant min()/max() in cnum signed bounds Simplify the smin() and smax() calculation by removing the integer min and max calculation on the non-overflowing branch. This is a valid simplification because when the arc does not cross the signed ST_MAX/ST_MIN boundary, the signed order of the endpoints is preserved, so base is the signed minimum and base + size is the signed maximum. Therefore, the min()/max() calls are redundant. We've also run benchmarks to compare the performance of this change. With O2 optimization we have: smin() results: | Test | smin | smin_new | | ------------------------ | ----------- | ----------- | | empty | 1.866 ns/op | 1.477 ns/op | | positive range | 1.870 ns/op | 1.248 ns/op | | negative range | 1.657 ns/op | 1.246 ns/op | | cross zero | 1.870 ns/op | 1.660 ns/op | | signed overflow boundary | 1.463 ns/op | 1.049 ns/op | | signed overflow range | 1.519 ns/op | 1.050 ns/op | | unsigned overflow range | 1.883 ns/op | 1.669 ns/op | | full range | 1.555 ns/op | 1.049 ns/op | smax() results: | Test | smax | smax_new | | ------------------------ | ----------- | ----------- | | empty | 1.462 ns/op | 1.456 ns/op | | positive range | 1.250 ns/op | 1.249 ns/op | | negative range | 1.259 ns/op | 1.297 ns/op | | cross zero | 1.471 ns/op | 1.475 ns/op | | signed overflow boundary | 1.056 ns/op | 1.260 ns/op | | signed overflow range | 1.049 ns/op | 1.259 ns/op | | unsigned overflow range | 1.470 ns/op | 1.477 ns/op | | full range | 1.072 ns/op | 1.263 ns/op | You can check the full implementation of these tests on this repository: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/benches Lean4 proofs showing that the new implementation preserves soundness of these operations are available at: https://github.com/OpenSourceVerif/open-verified-artifacts/tree/cnum-refactor-verif/cnum/lean Co-developed-by: Yazhou Tang Co-developed-by: Shenghao Yuan Signed-off-by: Yazhou Tang Signed-off-by: Shenghao Yuan Signed-off-by: Vinicius Sampaio Acked-by: Shung-Hsi Yu Link: https://patch.msgid.link/20260730102129.15209-4-vldsampaio@pm.me Signed-off-by: Eduard Zingerman --- kernel/bpf/cnum_defs.h | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h index 09535edbbafc..30685e43de04 100644 --- a/kernel/bpf/cnum_defs.h +++ b/kernel/bpf/cnum_defs.h @@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max) return (struct cnum_t){ .base = base, .size = size }; } -/* True if this cnum represents two unsigned ranges. */ +/* + * True if this cnum represents two unsigned ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(urange_overflow)(struct cnum_t cnum) { /* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */ @@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum) * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum. * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range * [0..UT_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_umin / cnum{T}_umax. */ ut FN(umin)(struct cnum_t cnum) { @@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum) } EXPORT_SYMBOL_GPL(FN(umax)); -/* True if this cnum represents two signed ranges. */ +/* + * True if this cnum represents two signed ranges. + * + * The caller should ensure !is_empty(cnum) holds when calling + * this function. + */ static inline bool FN(srange_overflow)(struct cnum_t cnum) { return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN); @@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum) * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum. * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range * [ST_MIN..ST_MAX] is returned. + * + * The caller should ensure !is_empty(cnum) holds when calling + * cnum{T}_smin / cnum{T}_smax. */ st FN(smin)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MIN - : min((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base; } st FN(smax)(struct cnum_t cnum) { - return FN(srange_overflow)(cnum) - ? ST_MAX - : max((st)cnum.base, (st)(cnum.base + cnum.size)); + return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size); } /* -- cgit v1.2.3 From da13c047cda6ef1bbc9355adb040d5b4eecea710 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:44 -0700 Subject: bpf: Factor check_global_ret_scalar_reg() out of the global return check check_global_subprog_return_code() verifies that a global subprogram returns void, an arena pointer, or register R0 holding a scalar value. Later patches in this series add 16-byte aggregate return support, whose second half is returned in R2 and needs the same validation. Factor the per-register check into check_global_ret_scalar_reg(env, regno) so that it can be reused for R2. No functional change. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055244.3294700-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/verifier.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 821b47ac75c5..5265563c8abf 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -17479,37 +17479,45 @@ enforce_retval: return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + + if (subprog_returns_void(env, subprog)) + return 0; + + return check_global_ret_scalar_reg(env, BPF_REG_0); +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) -- cgit v1.2.3 From cf5b96d9554815780f864d6f0014ac384e9be59c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:49 -0700 Subject: bpf: Add helpers to describe the R0: R2 return register pair LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes: such a value comes back in the R0:R2 register pair, with R2 holding the upper half. See LLVM patches [1] and [2]. Later patches teach the JIT, live register analysis and the verifier itself about that convention. They need to answer the same question: does this subprogram return its value in a register pair? Add bpf_ret_reg_pair() up front so it can be used in subsequent patches. It is answered from a per-subprogram flag that bpf_compute_subprog_ret_regs() derives once from the BTF prototype. jit_requested only says that the JIT is enabled, not that it succeeded: bpf_fixup_call_args() falls back to the interpreter when bpf_jit_subprogs() fails with anything other than -EFAULT. So jit_required is still set once the pair is modelled, which turns that fallback into a load failure rather than a silent divergence from what was verified. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055249.3295283-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 7 +++++ kernel/bpf/verifier.c | 66 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5fad59fdab0d..953d8191563a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -819,6 +819,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; @@ -1055,6 +1057,11 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { u8 num_params; bool is_void; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5265563c8abf..2be5867d4412 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -385,27 +385,66 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} - return btf_type_is_void(type); +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; + + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but btf_check_func_type_match() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + if (!type || btf_type_is_void(type)) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -21185,6 +21224,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; -- cgit v1.2.3 From 34c3503cb29ad0f73d413b83a0586e53d832bb90 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:54 -0700 Subject: bpf: Wire up JIT support for 16-byte kfunc returns LLVM 23 returns an __int128, or a struct/union larger than 8 bytes and no larger than 16 bytes, in the BPF R0:R2 register pair. The previous patch added the shared helpers describing that convention; wire up the JIT side so that the second half of the return value actually lands in R2. Placing the second return half into R2 is possible on any JIT, but it needs architecture-specific JIT work. Rather than requiring every JIT to implement it at once, add a bpf_jit_supports_kfunc_ret_reg_pair() capability, defaulting to false in the generic core; an architecture opts in once its JIT handles the R0:R2 pair, and the remaining ones are left for future work. Only the x86-64, arm64 and riscv64 JITs opt in so far. On arm64 and riscv64 the native second return register is already BPF R2 (x1 in bpf2a64[] and a1 in regmap[] respectively), so the upper half needs no move at all, unlike x86-64's RDX->RSI. The lower half is covered by the move into BPF R0 that those JITs already emit after every call, from x0 into x8 and from a0 into a5. This has been tested on x86-64 and arm64. The riscv64 path is expected to work by the same register-mapping reasoning as arm64 but has not been tested. bpf_add_kfunc_call() also rejects a kfunc that is marked KF_FASTCALL and returns more than 8 bytes. The bpf_fastcall contract implemented by mark_fastcall_pattern_for_call() assumes a call clobbers R0 plus the registers holding its arguments, so a return in the R0:R2 pair would clobber an R2 the caller expects the fastcall pattern to preserve. Such a kfunc is rejected with -EOPNOTSUPP as well. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055254.3295610-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ arch/riscv/net/bpf_jit_comp64.c | 5 +++++ arch/x86/net/bpf_jit_comp.c | 27 ++++++++++++++++++++------- include/linux/filter.h | 1 + kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 12 ++++++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..3aa3ea0bc30b 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2388,6 +2388,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 74efe4b138d2..47c7bf431ba8 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -2121,6 +2121,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1a9fb530adc3..48429fae0641 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1689,17 +1689,12 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; int i; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { u8 flags = fm->arg_flags[i]; u32 reg = BPF_REG_1 + i; @@ -2644,6 +2639,8 @@ populate_extable: /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { LOAD_TAIL_CALL_CNT_PTR(stack_depth); @@ -2652,7 +2649,10 @@ populate_extable: if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; @@ -2666,6 +2666,14 @@ populate_extable: return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } @@ -4156,6 +4164,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a9bc6a848f2..6e746b0a0930 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..5db77d7915df 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2be5867d4412..48e31136a7df 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2916,6 +2916,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; -- cgit v1.2.3 From 5bbb242c0859024d6d0ba990c0476b33c7c0be71 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:59 -0700 Subject: bpf: Handle R2 as a return register in precision backtracking Precision backtracking treats only R0 as a return register at a call/return boundary, so once the verifier starts modeling R2 that way, marking the second half of such a return precise would trip the "unexpected regs" checks in backtrack_insn() and reject a valid program. Marking the upper half precise, for example by branching on it after a call to a static subprogram, walks backtracking into the callee and reaches its BPF_EXIT with R2 still set in the mask. Handle R2 like R0 in boundaries where a call defines the return registers. R2 differs from R0 in that it is an argument register as well, so it is part of the BPF_REGMASK_ARGS check and has to be cleared before that check rather than next to R0. Clear it unconditionally, rather than only where the callee or the kfunc really does return a pair. That gives up the "unexpected regs" assertion for R2, and in exchange keeps backtracking free of any BTF lookup. Nothing is lost: a callee that does not return a pair leaves the caller's R2 uninitialized, so the main verification pass has already rejected any program that reads it, and backtracking is never asked for its precision. At BPF_EXIT the return registers are sampled before the callback path clears R1-R5. That clear does not touch R0, but it does cover R2, and running it first would drop a pair return whenever the instruction following the call happens to be one that invokes a callback. Suggested-by: Eduard Zingerman Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055259.3295829-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/backtrack.c | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694..653db80bcc47 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -423,6 +423,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +436,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -506,6 +508,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,7 +524,25 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise, r2_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + /* Sample the return registers before the callback + * handling below clears R1-R5. + */ + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. @@ -533,30 +555,22 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); + + bt_clear_reg(bt, BPF_REG_0); + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ -- cgit v1.2.3 From 991fb7823106bb2faa8a4871bc1d7c745201bac9 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:04 -0700 Subject: bpf: Account R2 of register-pair returns in live register analysis A BPF_EXIT of a subprogram returning a value larger than 8 bytes (a struct/union or an __int128) reads R2 as well as R0, since the second half of the return value is passed back in R2. compute_insn_live_regs() only marked R0 used at exit, so a callee's R2 could be considered dead and cleaned from checkpointed states, which would allow unsound state pruning. Mark R2 as read at the BPF_EXIT of a subprogram that does return a register pair. bpf_compute_live_registers() now loops over the subprograms and, for each, over the [start, end) instruction range from env->subprog_info[], so the return convention is queried once per subprogram through bpf_ret_reg_pair() rather than once per instruction. Marking R2 at every exit instead would be simpler, but R2 would then stay live backwards across any call that is not followed by a write to R2, which is nearly every program, and would needlessly hurt state pruning. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055304.3296353-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/liveness.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 74fc4b3f80d6..71f998c6eb88 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2060,7 +2060,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2073,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); u32 def = 0; u32 use = U32_MAX; @@ -2191,7 +2193,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; @@ -2228,8 +2230,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct insn_live_regs *state; int insn_cnt = env->prog->len; u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, j, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2258,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); -- cgit v1.2.3 From 0630ad00d96d9cf600c5c340cdecb363caa9d1cc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:09 -0700 Subject: bpf: Add verifier support for 16-byte returns in R0: R2 LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes. See LLVM patches [1] and [2]. Before LLVM 23 the BPF backend could not return these values at all. A by-value struct or union return (of any size) was rejected at compile time with: error: aggregate returns are not supported and an __int128 return failed later in the backend with: fatal error: error in backend: unable to allocate function return #1 Both are resolved in LLVM 23, which lowers such returns into the R0:R2 register pair. Model that pair at calls to global and static BPF subprograms and at kfunc calls: R2 is marked alongside R0 at the call, propagated out of a callee at its exit, and held to the same scalar-only and no-stack-pointer rules that already apply to R0. A struct returned by a kfunc must be composed of scalars, since its bytes reach the program as raw register contents and a pointer field would otherwise be laundered into a scalar. An extension program is the one caller of the convention that cannot take part in it: only R0 is checked at its exit, since that is the program exit code, so it has no way to hand back an upper half. Replacing a function whose return value is larger than 8 bytes is therefore rejected in btf_check_func_type_match(). That function compares btf_type->info, which carries no size for an int, so an 8 byte long and a 16 byte __int128 compared equal; sizes up to 8 bytes stay interchangeable, as they always have been, so this only rejects what the R0:R2 convention newly makes incompatible. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055309.3297684-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- kernel/bpf/btf.c | 6 +++++ kernel/bpf/verifier.c | 68 +++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index da36d4b9d31a..a10a070016be 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7677,6 +7677,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 48e31136a7df..930925fada30 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -415,6 +415,9 @@ static u32 ret_regs_cnt(u32 size) return size > 8 && size <= 16 ? 2 : 1; } +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) { const struct btf *btf = env->prog->aux->btf; @@ -9930,6 +9933,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn; + u32 i, nregs; target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); @@ -9985,9 +9989,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, caller->regs, ret_regs[i]); bpf_diag_mod_end(env); } @@ -10347,11 +10356,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10386,9 +10399,13 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ + /* + * return to the caller whatever the callee had in the + * return register(s) + */ bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; + for (i = 0; i < nregs; i++) + caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; bpf_diag_mod_end(env); } @@ -11323,6 +11340,19 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars. Both halves of a register pair are treated the same way. + */ +static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[i]); +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -13991,10 +14021,25 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so it must contain only scalars: + * otherwise a pointer field would be laundered into a scalar + * and escape provenance and reference tracking. + */ + if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + return -EINVAL; + } + mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -17562,11 +17607,20 @@ static int check_global_subprog_return_code(struct bpf_verifier_env *env) { struct bpf_func_state *cur_frame = cur_func(env); u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; if (subprog_returns_void(env, subprog)) return 0; - return check_global_ret_scalar_reg(env, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; } /* Bitmask with 1s for all caller saved registers */ -- cgit v1.2.3 From a2e98481a6396f919ad91ba9c2ff3ec61c4028b3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:14 -0700 Subject: bpf: Enable aggregate return types up to 16 bytes Relax btf_distill_func_proto() to accept a by-value struct or union that the R0:R2 convention added in earlier patches can carry: - a struct or union larger than 8 and up to 16 bytes, returned in the R0:R2 register pair, matching what LLVM emits for the BPF target; - a struct or union up to 8 bytes, returned in R0 alone. A >8 byte scalar (__int128) was already accepted and is unchanged. Everything else stays rejected: a return type larger than 16 bytes, and any type that __get_type_size() cannot return in registers at all (e.g. an array), which it already reports as ret < 0. btf_validate_return_type() is relaxed as well, so that it accepts a by-value struct or union up to 16 bytes in addition to void and scalars. With btf_distill_func_proto() and btf_validate_return_type() relaxed, the verifier, JIT, precision-backtracking and live-register support from the earlier patches becomes reachable: <=16 byte aggregate return values now work end to end. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055314.3298664-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/btf.c | 23 +++++++++++++++++----- kernel/bpf/verifier.c | 18 ++++++++--------- .../testing/selftests/bpf/progs/exceptions_fail.c | 2 +- 4 files changed, 30 insertions(+), 15 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 953d8191563a..004b06785521 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,6 +1488,8 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a10a070016be..91b8ce77f699 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7584,7 +7584,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7963,7 +7963,7 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { u32 tags = 0; int err; @@ -7986,6 +7986,19 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so it may only return scalars by value. A local + * function is verified inline, so a pointer field stays tracked + * and needs no such restriction. + */ + bool local_func = subprog && !is_global; + + if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + return 0; + } + return -EOPNOTSUPP; } @@ -8073,12 +8086,12 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", + "Global function %s() has unsupported return type. " + "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", tname); } return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 930925fada30..e036ae20bf6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11620,9 +11620,9 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11640,7 +11640,7 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } @@ -12039,7 +12039,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12055,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13111,7 @@ check_ok: break; } - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13653,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14032,7 +14032,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e5066..9708efb93683 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); -- cgit v1.2.3 From bd2466ed8fc2ee54408bfe433e75f9f46bfa04f7 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:24 +0800 Subject: bpf: Roll back stream capacity when allocation fails bpf_stream_push_str() accounts the string length before allocating a stream element. If the allocation fails, the length remains charged even though no element is queued and therefore cannot be released by a reader. Repeated failures can exhaust the stream capacity permanently until the BPF program is freed. Refactor bpf_stream_release_capacity() to take a length so the consume and release sides are symmetric, and use it to roll back the charge when creating the stream element fails. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/c0ad7b18c59be6695fcc9e4e1200b2f4d5cbbea2.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index be9ce98e9469..0b157ec4e38e 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -68,10 +68,8 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len) return 0; } -static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem) +static void bpf_stream_release_capacity(struct bpf_stream *stream, int len) { - int len = elem->total_len; - atomic_sub(len, &stream->capacity); } @@ -79,7 +77,14 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l { int ret = bpf_stream_consume_capacity(stream, len); - return ret ?: __bpf_stream_push_str(&stream->log, str, len); + if (ret) + return ret; + + ret = __bpf_stream_push_str(&stream->log, str, len); + if (ret) + bpf_stream_release_capacity(stream, len); + + return ret; } static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux) @@ -188,7 +193,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) if (cont) continue; bpf_stream_backlog_pop(stream); - bpf_stream_release_capacity(stream, elem); + bpf_stream_release_capacity(stream, elem->total_len); bpf_stream_free_elem(elem); } -- cgit v1.2.3 From effa6370ba89e7839c9bfeb802d8f05d6843ddf1 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:25 +0800 Subject: bpf: Fix stream capacity leak in staging path bpf_stream_stage_printk() increments ss->len before pushing the formatted string to the staging log. If element allocation fails, ss->len remains inflated and bpf_stream_stage_commit() permanently charges the stream capacity for data that was never queued. Only account the string length after a successful push, and use vscnprintf() so the staged length is the truncated payload without the trailing NUL. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/46898fb1ae482b2f8b1da459922b187378dfdc1d.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 0b157ec4e38e..a36fc9338967 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -316,17 +316,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...) { struct bpf_bprintf_buffers *buf; va_list args; - int ret; + int len, ret; if (bpf_try_get_buffers(&buf)) return -EBUSY; va_start(args, fmt); - ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); + len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args); va_end(args); - ss->len += ret; /* Exclude NULL byte during push. */ - ret = __bpf_stream_push_str(&ss->log, buf->buf, ret); + ret = __bpf_stream_push_str(&ss->log, buf->buf, len); + if (!ret) + ss->len += len; bpf_put_buffers(); return ret; } -- cgit v1.2.3 From 16790357a50baae10de1f619b608a85145d54bc2 Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:26 +0800 Subject: bpf: Return partial progress from bpf_stream_read on fault bpf_stream_read() pops and frees stream elements after a successful copy_to_user(). If a later copy_to_user() fails, it currently restores only the current element's consumed_len and returns -EFAULT, hiding bytes already delivered to userspace and making the consumed data unrecoverable on retry. On a short copy, keep the successfully copied prefix of the current element and return the number of bytes copied. Return -EFAULT only when no bytes were copied for the call. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/7668c5838c6beafeea719df8d55319e2fe08cfef.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index a36fc9338967..09471a605268 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -167,6 +167,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) while (rem_len) { int pos = len - rem_len; + int chunk, n; bool cont; node = bpf_stream_backlog_peek(stream); @@ -180,13 +181,14 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) cons_len = elem->consumed_len; cont = bpf_stream_consume_elem(elem, &rem_len) == false; - - ret = copy_to_user(buf + pos, elem->str + cons_len, - elem->consumed_len - cons_len); - /* Restore in case of error. */ - if (ret) { - ret = -EFAULT; - elem->consumed_len = cons_len; + chunk = elem->consumed_len - cons_len; + + n = copy_to_user(buf + pos, elem->str + cons_len, chunk); + if (n) { + /* Keep any successfully copied bytes; -EFAULT only if none. */ + elem->consumed_len -= n; + rem_len += n; + ret = (len == rem_len) ? -EFAULT : 0; break; } -- cgit v1.2.3 From 41c0348f6e65c26df4866d5dc96c57b1995fe36c Mon Sep 17 00:00:00 2001 From: Jianlin Shi Date: Sun, 23 Aug 2026 22:17:27 +0800 Subject: bpf: Reject oversized bpf_stream_vprintk output with -E2BIG bstr_printf() returns the would-be length excluding the trailing NUL. When that value is >= MAX_BPRINTF_BUF the message was truncated, but bpf_stream_push_str() still tried to allocate with the inflated length and failed with -ENOMEM. The boundary case of exactly MAX_BPRINTF_BUF could also copy the trailing NUL into the stream element. Reject such lengths with -E2BIG before charging stream capacity, and tighten bpf_stream_elem_alloc() to accept only payloads strictly shorter than the bprintf buffer. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Jianlin Shi Link: https://lore.kernel.org/r/74387ae5024d7e2996095e97211430285095b525.1787492521.git.shijianlin11@foxmail.com Signed-off-by: Kumar Kartikeya Dwivedi --- kernel/bpf/stream.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 09471a605268..7a5c3ac8676b 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -22,11 +22,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len) size_t alloc_size; /* - * Length denotes the amount of data to be written as part of stream element, - * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can - * accomodate, therefore deny allocations that won't fit into them. + * Length is the payload pushed into the stream, excluding the + * trailing NUL of the bprintf buffer. Reject anything that cannot + * fit without copying that NUL into the stream element. */ - if (len < 0 || len > max_len) + if (len < 0 || len >= max_len) return NULL; alloc_size = offsetof(struct bpf_stream_elem, str[len]); @@ -245,6 +245,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo return ret; ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args); + /* Truncation: reject before capacity charge (not -ENOMEM). */ + if (ret >= MAX_BPRINTF_BUF) { + bpf_bprintf_cleanup(&data); + return -E2BIG; + } /* Exclude NULL byte during push. */ ret = bpf_stream_push_str(stream, data.buf, ret); bpf_bprintf_cleanup(&data); -- cgit v1.2.3 From ea0b60fae724aa1147d4a07eda5131d3a9a9977e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:00 +0200 Subject: bpf: Reject oversized stream read buffers BPF_PROG_STREAM_READ_BY_FD exposes stream_buf_len as a u32, but the internal stream reader accepts an int. Values above INT_MAX therefore become negative before bpf_stream_read(), producing negative consumption lengths. The generic usercopy size check prevents the oversized copy, but emits a warning and reports EFAULT for an unchecked argument. Change bpf_prog_stream_read() to accept u32 and reject values that the signed reader cannot represent. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-2-memxor@gmail.com --- include/linux/bpf.h | 2 +- kernel/bpf/stream.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b3cd28d9e3b5..c2027688be3e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4101,7 +4101,7 @@ void bpf_put_buffers(void); void bpf_prog_stream_init(struct bpf_prog *prog); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 7a5c3ac8676b..2b80a0599865 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -203,13 +203,15 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) return ret ? ret : len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } -- cgit v1.2.3 From f62a9b82700c718cdf619fb36aaab31015ff87eb Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:19 -0700 Subject: bpf: Record each half of a paired return value in verifier diagnostics A subprogram returning more than 8 bytes comes back in the R0:R2 register pair. There are three places, check_func_call(), prepare_func_exit() and check_kfunc_call(), where only R0 is tracked by the diagnostic modification scope and R2 is missed. Fix it by opening a diagnostic modification scope for each return register. This way, both return registers are recorded. Fixes: 0630ad00d96d ("bpf: Add verifier support for 16-byte returns in R0: R2") Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061519.1692270-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 56 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 14 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b..5d8162e13c20 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9995,9 +9995,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, */ if (!returns_void) { nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; - for (i = 0; i < nregs; i++) - mark_reg_unknown(env, caller->regs, ret_regs[i]); + mark_reg_unknown(env, caller->regs, ret_regs[0]); bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL, + BPF_DIAG_MOD_WRITE); + mark_reg_unknown(env, caller->regs, ret_regs[i]); + bpf_diag_mod_end(env); + } } if (env->subprog_info[subprog].might_throw) { @@ -10403,10 +10408,14 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * return to the caller whatever the callee had in the * return register(s) */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - for (i = 0; i < nregs; i++) - caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; - bpf_diag_mod_end(env); + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -11342,15 +11351,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* * Mark the register(s) holding a @size byte kfunc return value as unknown - * scalars. Both halves of a register pair are treated the same way. + * scalars and return how many registers that took. Both halves of a register + * pair are treated the same way. + * + * R0 is written in the diagnostic modification scope the caller opened before + * the caller-saved scrub, since R0 gets no scrub record of its own and that + * scope is what carries its pre-call value into the history. A scope tracks a + * single register and does not nest, so close it before recording the rest of + * the return value. A value that fits in R0 leaves the caller's scope open for + * the caller to close, so a later write to R0 still lands in it. */ -static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, - struct bpf_reg_state *regs, u32 size) +static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) { u32 i, nregs = ret_regs_cnt(size); - for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[0]); + if (nregs == 1) + return nregs; + + bpf_diag_mod_end(env); + for (i = 1; i < nregs; i++) { + bpf_diag_mod_begin(env, ®s[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE); mark_reg_unknown(env, regs, ret_regs[i]); + bpf_diag_mod_end(env); + } + + return nregs; } static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) @@ -13767,7 +13794,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *insn_aux; const char *operation; int err, insn_idx = *insn_idx_p; - u32 i, nargs, ptr_type_id; + u32 i, nargs, ptr_type_id, ret_nregs = 1; struct bpf_kfunc_desc *desc; struct btf *desc_btf; int id; @@ -14021,7 +14048,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); @@ -14039,7 +14066,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, btf_name_by_offset(desc_btf, t->name_off)); return -EINVAL; } - mark_kfunc_ret_regs(env, regs, t->size); + ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -14170,7 +14197,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * Record R0 before process_iter_next_call() snapshots the alternate * iterator path's diagnostic position. */ - bpf_diag_mod_end(env); + if (ret_nregs == 1) + bpf_diag_mod_end(env); if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); -- cgit v1.2.3 From 77983371742e37ff3117acdc5dfb1969cedd415c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:24 -0700 Subject: bpf: Drop the recursion depth argument of btf_type_is_scalar_struct() btf_type_is_scalar_struct() recurses into nested struct members and carries the nesting depth in a @rec argument, so every caller has to spell out the 0 that starts the walk. Move the recursion into btf_struct_member_walk() and leave btf_type_is_scalar_struct() as a thin wrapper over it. This is groundwork for the later patches, which give the walk further state that callers should not have to supply: the kinds of member it accepts, and a record of the member that made it fail. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061524.1692957-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 24 +++++++++++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521..3eb61edc8c5e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1489,7 +1489,7 @@ struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699..47d43eb983a5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7995,7 +7995,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (local_func || btf_type_is_scalar_struct(env, btf, t)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d8162e13c20..ed077a5422af 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11647,9 +11647,8 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11667,7 +11666,7 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) return false; continue; } @@ -11686,6 +11685,13 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return true; } +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_member_walk(env, btf, t, 0); +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -12066,7 +12072,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12082,7 +12088,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13138,7 +13144,7 @@ check_ok: break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13680,7 +13686,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14059,7 +14065,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), -- cgit v1.2.3 From 3fc3e7331af6f60979fdf55e776bfe67105b584a Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:29 -0700 Subject: bpf: Add btf_type_is_arena_ptr() Add btf_type_is_arena_ptr() to test whether a BTF type is a pointer carrying the "arena" type tag, and use it to simplify both btf_scan_type_tags() and btf_validate_return_type(). The helper has external linkage because a later patch calls it from verifier.c to decide whether a struct member returned by value may be an arena pointer. btf_scan_type_tags() drove btf_type_tag_walk() with a one-entry match table, so the "multiple type tags" error it checked for could never fire: btf_type_tag_walk() only fails when the walk sets more than one flag. Dropping that error path, and with it the int return, is therefore no functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061529.1693266-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 1 + kernel/bpf/btf.c | 62 +++++++++++++++++++---------------------------------- 2 files changed, 23 insertions(+), 40 deletions(-) (limited to 'kernel') diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..ddd0f4f32d24 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 47d43eb983a5..280530d25886 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -8106,9 +8090,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) -- cgit v1.2.3 From 8d7ea1b752e5e49b00442f27f2a76228423aa74b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:34 -0700 Subject: bpf: Let the by-value struct walk take the kinds of member it accepts btf_struct_member_walk() hardcodes that every member of a struct returned by value must be a scalar. A later patch needs the same walk to also accept arena pointers, so give it a mask of the member kinds it allows. The mask reaches callers through btf_struct_is_composed_of(), which replaces btf_type_is_scalar_struct() as the entry point exported to btf.c. btf_type_is_scalar_struct() stays as a verifier-local wrapper for the call sites that only ever ask about scalars. There is no functional change: btf_member_kind_allowed() is exactly btf_type_is_scalar() when the mask is BTF_MEMBER_SCALAR alone, and the shared check is the last statement of the loop body, so dropping the array branch's continue neither skips a member nor checks one twice. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061534.1693789-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 11 +++++++++-- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 13 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3eb61edc8c5e..ae9f606539f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 280530d25886..b1f4ef614d4c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7979,7 +7979,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t)) + if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed077a5422af..90139f1b78d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,9 +11646,23 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec) + const struct btf_type *t, u32 member_kinds, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11666,7 +11680,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) return false; continue; } @@ -11675,21 +11689,25 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt if (!array->nelems) return false; member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; } - if (!btf_type_is_scalar(member_type)) + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } return true; } -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t) + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) { - return btf_struct_member_walk(env, btf, t, 0); + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } enum kfunc_ptr_arg_type { -- cgit v1.2.3 From 9e0195cd1d312ed99c3107d82f418e97baa45289 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:39 -0700 Subject: bpf: Let a by-value struct nest arrays and structs freely Go through nested array types until it reaches a non-array type. Later code will check such non-array type based on current logic. A zero-length array is still rejected, at whichever dimension it appears. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061539.1695015-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 90139f1b78d1..a377b1995b51 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11675,6 +11675,17 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt const struct btf_array *array; member_type = btf_type_skip_modifiers(btf, member->type, NULL); + /* + * Every element of an array is laid out in the value being + * returned, so an array counts as its element type however many + * dimensions deep that is. + */ + while (btf_type_is_array(member_type)) { + array = btf_array(member_type); + if (!array->nelems) + return false; + member_type = btf_type_skip_modifiers(btf, array->type, NULL); + } if (btf_type_is_struct(member_type)) { if (rec >= 3) { verbose(env, "max struct nesting depth exceeded\n"); @@ -11684,12 +11695,6 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt return false; continue; } - if (btf_type_is_array(member_type)) { - array = btf_array(member_type); - if (!array->nelems) - return false; - member_type = btf_type_skip_modifiers(btf, array->type, NULL); - } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } -- cgit v1.2.3 From 282d07dc6c74a5b4eea19743dd8c487c4b8814da Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:44 -0700 Subject: bpf: Report which member makes a kfunc return type unsupported A kfunc that returns a struct by value may only return scalars, and the message that rejects one names the type but not the member at fault: kernel function bpf_kfunc_call_test_ret_ptr returns STRUCT prog_test_ret_ptr that is not composed of scalars Record the member that made the walk fail and name it. What is recorded is a path, not a single member, because the walk descends up to 4 levels. For struct outer { struct inner { void *p; } in; __u64 tag; }; naming 'p' alone would send the reader looking for a member struct outer does not have, so the actual message reads "member 'in.p' has type PTR". For struct s { void *p[2]; }; naming 'p' alone would blame the member, when it is the element type that was rejected, so the actual message reads "member 'p[]' has type PTR". A zero-length array has no element type to name and is rejected for its length instead, so it reports "member 'p' is a zero-length array". The detailed diagnostics for this failure: Verification failed: Program Structure: Unsupported kernel function return type Reason: bpf_kfunc_call_test_ret_ptr() returns STRUCT prog_test_ret_ptr by value. Its member 'p' is PTR, not a scalar. Only kfuncs returning scalar values, or structures composed of scalar values are supported. ... Suggestion: Call a kernel function that returns only scalars by value. A type nested deeper than the walk descends has no single member to blame, so that case reports the depth instead: Reason: bpf_kfunc_call_test_ret_deep() returns STRUCT prog_test_ret_deep by value. It nests structs more than 4 levels deep. ... Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061545.1695736-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 103 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 94 insertions(+), 9 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a377b1995b51..745daf58b052 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,6 +11646,17 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + bool in_array[BTF_MEMBER_MAX_DEPTH]; + const struct btf_type *bad_type; + int depth; + bool too_deep; +}; + static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, u32 member_kinds) { @@ -11662,10 +11673,12 @@ static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type * element type. */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, u32 member_kinds, int rec) + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; + bool in_array; u32 i; if (!btf_type_is_struct(t)) @@ -11674,6 +11687,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt for_each_member(i, t, member) { const struct btf_array *array; + in_array = false; member_type = btf_type_skip_modifiers(btf, member->type, NULL); /* * Every element of an array is laid out in the value being @@ -11682,30 +11696,48 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt */ while (btf_type_is_array(member_type)) { array = btf_array(member_type); - if (!array->nelems) - return false; + if (!array->nelems) { + member_type = NULL; + goto bad_member; + } member_type = btf_type_skip_modifiers(btf, array->type, NULL); + in_array = true; } if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } if (!btf_member_kind_allowed(btf, member_type, member_kinds)) - return false; + goto bad_member; } return true; + +bad_member: + if (path) { + path->depth = rec + 1; + path->bad_type = member_type; + } +bad_path: + if (path && path->depth) { + path->member[rec] = member; + path->in_array[rec] = in_array; + } + return false; } bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, const struct btf_type *t, u32 member_kinds) { - return btf_struct_member_walk(env, btf, t, member_kinds, 0); + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); } static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, @@ -11715,6 +11747,25 @@ static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) { + const char *name = btf_name_by_offset(btf, path->member[i]->name_off); + + /* An anonymous struct or union has no name to spell. */ + if (!name || !name[0]) + continue; + len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name); + if (path->in_array[i]) + len += scnprintf(buf + len, buf_sz - len, "[]"); + } +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -14082,17 +14133,51 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + /* * The returned struct comes back as raw register bits modeled * as an unknown scalar, so it must contain only scalars: * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t)) { + if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + char bad_name[BTF_MEMBER_PATH_LEN]; + + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + if (!path.bad_type) { + verbose(env, "member '%s' is a zero-length array\n", + bad_name); + member_note = bpf_diag_fmt( + env, " Its member '%s' is a zero-length array.", + bad_name); + } else { + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(path.bad_type)); + member_note = bpf_diag_fmt( + env, " Its member '%s' is %s, not a scalar.", + bad_name, btf_type_str(path.bad_type)); + } + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values, or " + "structures composed of scalar values are " + "supported.", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off), member_note); return -EINVAL; } ret_nregs = mark_kfunc_ret_regs(env, regs, t->size); -- cgit v1.2.3 From ff08e6b79ee1470e736054e9558c47e4150406b1 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:50 -0700 Subject: bpf: Allow a global function to return arena pointers by value A global function may already return an arena pointer on its own, and check_global_ret_scalar_reg() accepts one in either half of the R0:R2 pair. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Only a subprogram gets this: the main program returns to the kernel, which has no arena to cast the address back into, so btf_validate_return_type() keeps rejecting an arena pointer there, whether bare or inside a struct. The rejection message therefore has to name what the function at hand may return, or it would offer the main program an arena pointer it cannot have. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061550.1696869-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/btf.c | 28 +++++++++++++++++----- .../testing/selftests/bpf/progs/exceptions_fail.c | 2 +- 2 files changed, 23 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index b1f4ef614d4c..9c2cab08bb79 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7973,13 +7973,23 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_struct(t) && t->size <= 16) { /* * A global function's caller models the return as an opaque - * scalar pair, so it may only return scalars by value. A local - * function is verified inline, so a pointer field stays tracked - * and needs no such restriction. + * scalar pair, so a pointer member would be laundered into a + * scalar and escape provenance and reference tracking. Only + * scalars and arena pointers are allowed: an arena pointer has + * no provenance to lose, since a program may already derive one + * from any scalar with addr_space_cast(), which confines the + * result to the arena. The main program is the exception: it + * returns to the kernel, which has no arena to cast the address + * back into. A local function is verified inline, so its caller + * receives the real register state and any member is fine. */ bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; - if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; + + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) return 0; } @@ -8073,10 +8083,16 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { + /* Only a subprogram may return arena pointers. */ + const char *supported = subprog ? + "void, scalar, arena pointer, or a struct/union of " + "scalars and arena pointers" : + "void, scalar, or a scalar-only struct/union"; + bpf_log(log, "Global function %s() has unsupported return type. " - "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", - tname); + "Only %s up to 16 bytes is supported.\n", + tname, supported); } return err; } diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9708efb93683..22503cf62e9f 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") +__failure __msg("Only void, scalar, arena pointer, or a struct/union of scalars and arena pointers up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); -- cgit v1.2.3 From 9046109974dd1f3ffc3f62e22555be13eaf7fdb0 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:55 -0700 Subject: bpf: Allow arena pointers in a by-value kfunc return A kfunc may already return an arena pointer on its own, which the program casts back into the arena address space to use. Let the members of a by-value struct it returns be arena pointers as well, rather than scalars only. Such a member carries the arena type tag but not the address space qualifier, so the program has to cast it itself. Handing it back as a scalar costs nothing: an arena address has no provenance to track, and the cast is confined to the caller's arena. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061555.1697503-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 22 +++++++++++++--------- .../selftests/bpf/progs/aggregate_ret_kfunc.c | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 745daf58b052..a725e3917976 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -14138,13 +14138,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, /* * The returned struct comes back as raw register bits modeled - * as an unknown scalar, so it must contain only scalars: - * otherwise a pointer field would be laundered into a scalar - * and escape provenance and reference tracking. + * as an unknown scalar, so a pointer member would be laundered + * into a scalar and escape provenance and reference tracking. + * Only scalars and arena pointers are allowed; see + * btf_validate_return_type() in btf.c for why an arena pointer + * is safe here. */ - if (!btf_struct_member_walk(env, desc_btf, t, BTF_MEMBER_SCALAR, 0, &path)) { + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { verbose(env, - "kernel function %s returns %s %s that is not composed of scalars\n", + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off)); if (path.too_deep) { @@ -14165,16 +14168,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, verbose(env, "member '%s' has type %s\n", bad_name, btf_type_str(path.bad_type)); member_note = bpf_diag_fmt( - env, " Its member '%s' is %s, not a scalar.", + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", bad_name, btf_type_str(path.bad_type)); } } bpf_diag_program_structure( env, insn_idx, "unsupported kernel function return type", - "Call a kernel function that returns only scalars by value.", + "Call a kernel function that returns only scalars or arena pointers by value.", "%s() returns %s %s by value.%s " - "Only kfuncs returning scalar values, or " - "structures composed of scalar values are " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values or arena pointers are " "supported.", func_name, btf_type_str(t), btf_name_by_offset(desc_btf, t->name_off), member_note); diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index d6b422ae9784..f10e5cf6fd89 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -71,7 +71,7 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 -__failure __msg("is not composed of scalars") +__failure __msg("is not composed of scalars or arena pointers") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( -- cgit v1.2.3 From 264d8fd2794fabe76a89abd5d4f5f8c1ed5fad85 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:17 +0200 Subject: bpf, keys: Add a bpf keyring for program signature validation BPF program signatures can currently be verified against one of the system keyrings (builtin, secondary, platform) or against an arbitrary user/session caller-supplied keyring named through keyring_id. There is nothing in between: the system keyrings need a kernel rebuild or a vouched-for enrollment to rotate a key, while a caller-supplied keyring is fully controlled by the loader and therefore carries no trust on its own (unless explicitly combined with BPF LSM to protect against key tampering). Add a dedicated bpf keyring to fill that gap, modelled after the dm-verity keyring which was added in commit 033724b1c627 ("dm-verity: add dm-verity keyring") and which can eventually be used also via systemd [0] through the same enrollment method as in dm-verity's case. It is selected with the new KEY_SPEC_BPF_KEYRING special key id and gives an operator a place to enroll a BPF-only signing key at boot, specifically scoped to BPF program loading and nothing else in the kernel's trust hierarchy. The id is reserved from the KEY_SPEC space so that the latter is not linked into any process keyring, and lookup_user_key() resolves KEY_SPEC_BPF_KEYRING constant instead of having to look it up via /proc/keys first. By default the keyring is sealed empty at init. Systems that want to provision keys pass bpf.keyring_unsealed=1, which leaves the keyring open for the initrd to add keys to. The keyring is only ever consulted once it is both non-empty and restricted. An unrestricted keyring is ignored. Signed-off-by: Daniel Borkmann Link: https://github.com/systemd/systemd/pull/43549 [0] Link: https://lore.kernel.org/r/20260828175227.1537793-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 +++ include/linux/bpf.h | 7 +++ include/linux/key.h | 2 + include/uapi/linux/keyctl.h | 1 + kernel/bpf/Makefile | 3 ++ kernel/bpf/keys.c | 68 +++++++++++++++++++++++++ kernel/bpf/verifier.c | 9 ++++ security/keys/process_keys.c | 25 +++++++++ 8 files changed, 123 insertions(+) create mode 100644 kernel/bpf/keys.c (limited to 'kernel') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e4643634a9b1..2beb61092bb3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -667,6 +667,14 @@ Kernel parameters See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c2027688be3e..039674b2f875 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1735,6 +1735,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -3819,6 +3820,7 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3839,6 +3841,11 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..bd10fe45819d 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -440,6 +440,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..9a92c348bbda 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..08549b3220c1 --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a725e3917976..0c229a226eab 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -21127,6 +21128,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21164,9 +21167,15 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return -EINVAL; if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); + else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + key = bpf_lookup_keyring(); else key = bpf_lookup_user_key(attr->keyring_id, 0); if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index a63c46bb2d14..44358388e395 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -590,6 +593,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -741,6 +758,14 @@ try_again: key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1) -- cgit v1.2.3 From abaa0835b4f381067cc04bfd58180679d813578b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:18 +0200 Subject: bpf: Refuse caller-supplied keyrings when the bpf one is active Nothing changes for systems that do not use the bpf keyring. Without bpf.keyring_unsealed=1 a caller-supplied keyring behaves exactly as before, which also lets it serve as the staging step for software installed onto a running system whose signing key is not enrolled anywhere yet. Passing bpf.keyring_unsealed=1 states that the bpf keyring is the trust anchor for this boot, so from the first program load onwards a caller- supplied keyring is refused with -EPERM. Deriving this from the boot flag rather than from the keyring's runtime state keeps the decision immutable from userspace. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++++++ include/linux/bpf.h | 6 ++++++ kernel/bpf/keys.c | 5 +++++ kernel/bpf/verifier.c | 10 +++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2beb61092bb3..e70fb15757d4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -673,6 +673,14 @@ Kernel parameters keys. Once the keyring is restricted it becomes active and can be used for BPF program signature verification. + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + See Documentation/bpf/signing.rst bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 039674b2f875..3a7eb2185c35 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3821,6 +3821,7 @@ struct bpf_key { struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3846,6 +3847,11 @@ static inline struct bpf_key *bpf_lookup_keyring(void) return NULL; } +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c index 08549b3220c1..bffc356839ac 100644 --- a/kernel/bpf/keys.c +++ b/kernel/bpf/keys.c @@ -18,6 +18,11 @@ static bool bpf_keyring_unsealed __ro_after_init; module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + struct bpf_key *bpf_lookup_keyring(void) { struct bpf_key *bkey; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c229a226eab..dab618836299 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21165,12 +21165,16 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, if (!attr->signature_size || attr->signature_size > KMALLOC_MAX_CACHE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { key = bpf_lookup_keyring(); - else + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { verbose(env, "the bpf keyring is empty or has not been restricted\n"); -- cgit v1.2.3 From 300b348e6d5e8e96c065c42c9999baca224c1661 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:19 +0200 Subject: bpf: Raise the bound on a program's signature size signature_size is bounded by KMALLOC_MAX_CACHE_SIZE, which is 8 KiB on a 4 KiB page system. Back then we chose it somewhat arbitrarily and was picked when a BPF program signature was RSA or ECDSA. ML-DSA (FIPS-204) verification is wired through the X.509 and PKCS#7 parsers, and BPF reaches them too via verify_pkcs7_signature() without having to know the concrete algorithm. The bound becomes a bit too small, thus add an explicit BPF_PROG_MAX_SIGNATURE_SIZE of 64 KiB and use that instead to cover all options. KMALLOC_MAX_CACHE_SIZE is PAGE_SIZE-derived, so what was accepted so far depended on the page size which is not optimal as it should be the same behavior on every configuration. On 64 KiB page kernels this lowers the ceiling from 128 KiB to 64 KiB. Nothing that could have been verified is affected as the largest signature the kernel implements is ML-DSA-87 at 4627 bytes. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-4-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 15 ++++++++++----- tools/testing/selftests/bpf/prog_tests/signed_loader.c | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'kernel') diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dab618836299..8f585ceb2cd5 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21119,6 +21119,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } +/* + * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably + * above the largest signature the kernel can verify, and far below anything + * that would make rejecting a load expensive. Deliberately a fixed number so + * that what the syscall accepts does not depend on PAGE_SIZE. + */ +#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024) + static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) { switch (keyring_id) { @@ -21158,13 +21166,10 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, u64 data_sz; int err = 0; - /* - * Don't attempt to use kmalloc_large or vmalloc for signatures. - * Practical signature for BPF program should be below this limit. - */ if (!attr->signature_size || - attr->signature_size > KMALLOC_MAX_CACHE_SIZE) + attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE) return -EINVAL; + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { diff --git a/tools/testing/selftests/bpf/prog_tests/signed_loader.c b/tools/testing/selftests/bpf/prog_tests/signed_loader.c index 77381d345435..0c5294738d6c 100644 --- a/tools/testing/selftests/bpf/prog_tests/signed_loader.c +++ b/tools/testing/selftests/bpf/prog_tests/signed_loader.c @@ -571,8 +571,9 @@ static void signature_too_large(void) if (gen_loader_fixture_init(&f) == 0) { /* - * signature_size beyond the kernel's bound (KMALLOC_MAX_CACHE_SIZE) - * is rejected before the buffer is read. + * signature_size beyond the kernel's bound + * (BPF_PROG_MAX_SIGNATURE_SIZE) is rejected before the buffer + * is read. */ fd = load_loader(f.gopts.insns, f.gopts.insns_sz, -1, junk, 64 << 20, KEY_SPEC_SESSION_KEYRING, 0); -- cgit v1.2.3 From 802a30c5db726639df7f93b3a1982ecc64202abe Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Mon, 31 Aug 2026 14:32:25 +0800 Subject: bpf: Fix queue/stack map u32 index overflow The queue/stack map addresses elements[] with the product of a u32 head/tail index and value_size, but the storage itself is allocated in 64-bit arithmetic. When max_entries * value_size reaches or exceeds U32_MAX, the product wraps and push/peek/pop operate on the wrong element, corrupting map data and leaking stale values to user space. max_entries == U32_MAX would also make the u32 capacity counter qs->size (max_entries + 1) wrap to 0 and permanently break the map. The original bound check was removed by commit a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps"), which deleted the bpf_map_charge_init() call and with it the U32_MAX - PAGE_SIZE check that had earlier been moved into bpf_map_charge_init() by c85d69135a91. Oversized queue/stack maps can therefore be created again. Restore the bound in queue_stack_map_alloc_check() with a single comparison that rejects any max_entries/value_size combination whose element storage would reach or exceed U32_MAX bytes, keeping the u32 index multiplication overflow-free and the capacity counter valid. Fixes: a37fb7ef24a4 ("bpf: Eliminate rlimit-based memory accounting for queue_stack_maps maps") Signed-off-by: Yuan Chen Signed-off-by: Andrii Nakryiko Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260831063226.621309-2-chenyuan_fl@163.com --- kernel/bpf/queue_stack_maps.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 6e8b18c32a10..1cee2ba6e332 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr) */ return -E2BIG; + if (attr->max_entries >= U32_MAX / attr->value_size) + return -E2BIG; + return 0; } -- cgit v1.2.3 From d02bfbc49c8ef978eac5f39a2de67281b8e00edb Mon Sep 17 00:00:00 2001 From: "Alexis Lothoré (eBPF Foundation)" Date: Thu, 3 Sep 2026 15:51:30 +0200 Subject: bpf: mark instructions accessing program stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare to emit KASAN checks in JITed programs, JIT compilers need to be aware about whether some load/store instructions are targeting the bpf program stack, as those should not be monitored (we already have guard pages for that, and it is difficult anyway to correctly monitor any kind of data passed on stack). To support this need, make the BPF verifier mark the instructions depending on whether they could access or not memory other than stack. As different states in the verifier could lead to different memory types for the same access, just marking an instruction as accessing stack only is not enough (it could be some other memory type in another verifier state), so the algorithm rather sets by default any load/store instruction as stack only, and if _any_ state leads to any memory access type other than PTR_TO_STACK, it overrides this setting. It also takes care about shifting back the instruction marking in adjust_insn_aux_data if the verifier patches instructions. However, if the verifier generates new BPF_ST/BPF_STX/BPF_LDX while patching some instructions, those new ones are systematically marked as non-stack-accessing: this may over-instrument a few memory accessing instructions, but it allows making sure that we will not miss accidentally any. Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-1-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 16 ++++++++++++++++ kernel/bpf/fixups.c | 33 ++++++++++++++++++++++++++++++--- kernel/bpf/verifier.c | 10 ++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) (limited to 'kernel') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae9f606539f4..1a3c44ab06a1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -706,6 +706,7 @@ struct bpf_insn_aux_data { */ u32 calls_callback:1; u32 indirect_target:1; /* if it is an indirect jump target */ + u32 non_stack_access:1; /* instruction can access non-stack memory */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -1671,6 +1672,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..73fb3ffc18e3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -213,7 +213,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -227,8 +228,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -239,8 +247,25 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is @@ -306,6 +331,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + struct bpf_insn original_insn; if (len > 1) { new_data = vrealloc(env->insn_aux_data, @@ -318,6 +344,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -326,7 +353,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f585ceb2cd5..5b51e7ee1a3f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3273,6 +3273,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + #define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) @@ -6417,6 +6422,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6655,6 +6661,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } -- cgit v1.2.3 From 90769ad233cac2ade07f8d2f91214e584842158d Mon Sep 17 00:00:00 2001 From: "Alexis Lothoré (eBPF Foundation)" Date: Thu, 3 Sep 2026 15:51:31 +0200 Subject: bpf: add BPF_JIT_KASAN for KASAN instrumentation of JITed programs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new Kconfig option CONFIG_BPF_JIT_KASAN that automatically enables generic KASAN (Kernel Address SANitizer) memory access checks for JIT-compiled BPF programs as well, when both KASAN (and more specifically, generic KASAN with KASAN_VMALLOC) and JIT compiler are enabled. This new Kconfig is not a user selectable one: it is automatically enabled if KASAN is enabled on a compatible platform. When enabled, the JIT compiler will emit shadow memory checks before memory loads and stores to detect use-after-free or out-of-bounds accesses at runtime. The option is gated behind HAVE_EBPF_JIT_KASAN, as it needs proper arch-specific implementation. Acked-by: Andrey Konovalov Acked-by: Ihor Solodrai Signed-off-by: Alexis Lothoré (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-2-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/Kconfig | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'kernel') diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig index eb3de35734f0..d7d25477ef48 100644 --- a/kernel/bpf/Kconfig +++ b/kernel/bpf/Kconfig @@ -17,6 +17,10 @@ config HAVE_CBPF_JIT config HAVE_EBPF_JIT bool +# KASAN support for JIT compiler +config HAVE_EBPF_JIT_KASAN + bool + # Used by archs to tell that they want the BPF JIT compiler enabled by # default for kernels that were compiled with BPF JIT support. config ARCH_WANT_DEFAULT_BPF_JIT @@ -101,4 +105,17 @@ config BPF_LSM If you are unsure how to answer this question, answer N. +config BPF_JIT_KASAN + bool + depends on HAVE_EBPF_JIT_KASAN + depends on KASAN_GENERIC + depends on KASAN_VMALLOC + depends on BPF_JIT + default y + help + Makes JIT compiler insert generic outline KASAN checks in BPF + programs when they are inserted in the kernel. This feature is + automatically enabled if the needed set of KASAN and BPF + configuration options is enabled. + endmenu # "BPF subsystem" -- cgit v1.2.3