From cf5b96d9554815780f864d6f0014ac384e9be59c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:49 -0700 Subject: bpf: Add helpers to describe the R0: R2 return register pair LLVM 23 added support for returning a value in two registers for an __int128, or a struct/union whose size is greater than 8 but not more than 16 bytes: such a value comes back in the R0:R2 register pair, with R2 holding the upper half. See LLVM patches [1] and [2]. Later patches teach the JIT, live register analysis and the verifier itself about that convention. They need to answer the same question: does this subprogram return its value in a register pair? Add bpf_ret_reg_pair() up front so it can be used in subsequent patches. It is answered from a per-subprogram flag that bpf_compute_subprog_ret_regs() derives once from the BTF prototype. jit_requested only says that the JIT is enabled, not that it succeeded: bpf_fixup_call_args() falls back to the interpreter when bpf_jit_subprogs() fails with anything other than -EFAULT. So jit_required is still set once the pair is modelled, which turns that fallback into a load failure rather than a silent divergence from what was verified. [1] https://github.com/llvm/llvm-project/pull/190894 [2] https://github.com/llvm/llvm-project/pull/206876 Signed-off-by: Yonghong Song Link: https://patch.msgid.link/20260819055249.3295283-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 5fad59fdab0d..953d8191563a 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -819,6 +819,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; @@ -1055,6 +1057,11 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { u8 num_params; bool is_void; -- cgit v1.2.3 From 34c3503cb29ad0f73d413b83a0586e53d832bb90 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:52:54 -0700 Subject: bpf: Wire up JIT support for 16-byte kfunc returns LLVM 23 returns an __int128, or a struct/union larger than 8 bytes and no larger than 16 bytes, in the BPF R0:R2 register pair. The previous patch added the shared helpers describing that convention; wire up the JIT side so that the second half of the return value actually lands in R2. Placing the second return half into R2 is possible on any JIT, but it needs architecture-specific JIT work. Rather than requiring every JIT to implement it at once, add a bpf_jit_supports_kfunc_ret_reg_pair() capability, defaulting to false in the generic core; an architecture opts in once its JIT handles the R0:R2 pair, and the remaining ones are left for future work. Only the x86-64, arm64 and riscv64 JITs opt in so far. On arm64 and riscv64 the native second return register is already BPF R2 (x1 in bpf2a64[] and a1 in regmap[] respectively), so the upper half needs no move at all, unlike x86-64's RDX->RSI. The lower half is covered by the move into BPF R0 that those JITs already emit after every call, from x0 into x8 and from a0 into a5. This has been tested on x86-64 and arm64. The riscv64 path is expected to work by the same register-mapping reasoning as arm64 but has not been tested. bpf_add_kfunc_call() also rejects a kfunc that is marked KF_FASTCALL and returns more than 8 bytes. The bpf_fastcall contract implemented by mark_fastcall_pattern_for_call() assumes a call clobbers R0 plus the registers holding its arguments, so a return in the R0:R2 pair would clobber an R2 the caller expects the fastcall pattern to preserve. Such a kfunc is rejected with -EOPNOTSUPP as well. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055254.3295610-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- arch/arm64/net/bpf_jit_comp.c | 5 +++++ arch/riscv/net/bpf_jit_comp64.c | 5 +++++ arch/x86/net/bpf_jit_comp.c | 27 ++++++++++++++++++++------- include/linux/filter.h | 1 + kernel/bpf/core.c | 5 +++++ kernel/bpf/verifier.c | 12 ++++++++++++ 6 files changed, 48 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41db..3aa3ea0bc30b 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2388,6 +2388,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 74efe4b138d2..47c7bf431ba8 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -2121,6 +2121,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1a9fb530adc3..48429fae0641 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1689,17 +1689,12 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; int i; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { u8 flags = fm->arg_flags[i]; u32 reg = BPF_REG_1 + i; @@ -2644,6 +2639,8 @@ populate_extable: /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { LOAD_TAIL_CALL_CNT_PTR(stack_depth); @@ -2652,7 +2649,10 @@ populate_extable: if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; @@ -2666,6 +2666,14 @@ populate_extable: return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } @@ -4156,6 +4164,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a9bc6a848f2..6e746b0a0930 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75a..5db77d7915df 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2be5867d4412..48e31136a7df 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -2916,6 +2916,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; -- cgit v1.2.3 From a2e98481a6396f919ad91ba9c2ff3ec61c4028b3 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Tue, 18 Aug 2026 22:53:14 -0700 Subject: bpf: Enable aggregate return types up to 16 bytes Relax btf_distill_func_proto() to accept a by-value struct or union that the R0:R2 convention added in earlier patches can carry: - a struct or union larger than 8 and up to 16 bytes, returned in the R0:R2 register pair, matching what LLVM emits for the BPF target; - a struct or union up to 8 bytes, returned in R0 alone. A >8 byte scalar (__int128) was already accepted and is unchanged. Everything else stays rejected: a return type larger than 16 bytes, and any type that __get_type_size() cannot return in registers at all (e.g. an array), which it already reports as ret < 0. btf_validate_return_type() is relaxed as well, so that it accepts a by-value struct or union up to 16 bytes in addition to void and scalars. With btf_distill_func_proto() and btf_validate_return_type() relaxed, the verifier, JIT, precision-backtracking and live-register support from the earlier patches becomes reachable: <=16 byte aggregate return values now work end to end. Signed-off-by: Yonghong Song Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260819055314.3298664-1-yonghong.song@linux.dev Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/btf.c | 23 +++++++++++++++++----- kernel/bpf/verifier.c | 18 ++++++++--------- .../testing/selftests/bpf/progs/exceptions_fail.c | 2 +- 4 files changed, 30 insertions(+), 15 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 953d8191563a..004b06785521 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,6 +1488,8 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a10a070016be..91b8ce77f699 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7584,7 +7584,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7963,7 +7963,7 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { u32 tags = 0; int err; @@ -7986,6 +7986,19 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so it may only return scalars by value. A local + * function is verified inline, so a pointer field stays tracked + * and needs no such restriction. + */ + bool local_func = subprog && !is_global; + + if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + return 0; + } + return -EOPNOTSUPP; } @@ -8073,12 +8086,12 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", + "Global function %s() has unsupported return type. " + "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", tname); } return err; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 930925fada30..e036ae20bf6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11620,9 +11620,9 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11640,7 +11640,7 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } @@ -12039,7 +12039,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12055,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13111,7 @@ check_ok: break; } - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13653,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14032,7 +14032,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!__btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e5066..9708efb93683 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); -- cgit v1.2.3 From 5e289c5a4a526e870efac0ca658635c47138593a Mon Sep 17 00:00:00 2001 From: Quanye Yang Date: Sat, 22 Aug 2026 14:55:33 +0800 Subject: bpf: Annotate bpf_obj_memcpy with data_race syzbot reported KCSAN write-write races when two tasks concurrently update the same map value. Both accesses reach the ordinary memcpy() paths in bpf_obj_memcpy() through copy_map_value(). Unlocked in-place updates of published map values are intentionally not serialized and may produce torn values. Callers requiring consistency must provide synchronization appropriate for the map type. bpf_long_memcpy() already annotates the same behavior for long-aligned copies. Annotate the ordinary memcpy() sites in bpf_obj_memcpy() with data_race(), matching bpf_long_memcpy(). This documents the existing concurrency semantics and suppresses KCSAN reports for these intentional races without changing synchronization or map update behavior. Reported-by: syzbot+44044637ef892e79ca2b@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=44044637ef892e79ca2b Signed-off-by: Quanye Yang Link: https://lore.kernel.org/bpf/20260822-bpf-kcsan-obj-memcpy-v3-1-4227b2edd8b4@proton.me Signed-off-by: Kumar Kartikeya Dwivedi --- include/linux/bpf.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index ffa5626411ac..b3cd28d9e3b5 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -572,7 +572,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +580,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) -- cgit v1.2.3 From ea0b60fae724aa1147d4a07eda5131d3a9a9977e Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sun, 23 Aug 2026 21:32:00 +0200 Subject: bpf: Reject oversized stream read buffers BPF_PROG_STREAM_READ_BY_FD exposes stream_buf_len as a u32, but the internal stream reader accepts an int. Values above INT_MAX therefore become negative before bpf_stream_read(), producing negative consumption lengths. The generic usercopy size check prevents the oversized copy, but emits a warning and reports EFAULT for an unchecked argument. Change bpf_prog_stream_read() to accept u32 and reject values that the signed reader cannot represent. Fixes: 5ab154f1463a ("bpf: Introduce BPF standard streams") Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20260823193202.2389715-2-memxor@gmail.com --- include/linux/bpf.h | 2 +- kernel/bpf/stream.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index b3cd28d9e3b5..c2027688be3e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -4101,7 +4101,7 @@ void bpf_put_buffers(void); void bpf_prog_stream_init(struct bpf_prog *prog); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c index 7a5c3ac8676b..2b80a0599865 100644 --- a/kernel/bpf/stream.c +++ b/kernel/bpf/stream.c @@ -203,13 +203,15 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len) return ret ? ret : len - rem_len; } -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len) +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len) { struct bpf_stream *stream; stream = bpf_stream_get(stream_id, prog->aux); if (!stream) return -ENOENT; + if (len > INT_MAX) + return -EINVAL; return bpf_stream_read(stream, buf, len); } -- cgit v1.2.3 From 77983371742e37ff3117acdc5dfb1969cedd415c Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:24 -0700 Subject: bpf: Drop the recursion depth argument of btf_type_is_scalar_struct() btf_type_is_scalar_struct() recurses into nested struct members and carries the nesting depth in a @rec argument, so every caller has to spell out the 0 that starts the walk. Move the recursion into btf_struct_member_walk() and leave btf_type_is_scalar_struct() as a thin wrapper over it. This is groundwork for the later patches, which give the walk further state that callers should not have to supply: the kinds of member it accepts, and a record of the member that made it fail. No functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061524.1692957-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 2 +- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 24 +++++++++++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521..3eb61edc8c5e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1489,7 +1489,7 @@ struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + const struct btf_type *t); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699..47d43eb983a5 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7995,7 +7995,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (local_func || btf_type_is_scalar_struct(env, btf, t)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d8162e13c20..ed077a5422af 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11647,9 +11647,8 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11667,7 +11666,7 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) return false; continue; } @@ -11686,6 +11685,13 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, return true; } +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_member_walk(env, btf, t, 0); +} + enum kfunc_ptr_arg_type { KF_ARG_CONST_MEM_SIZE, KF_ARG_MEM_SIZE, @@ -12066,7 +12072,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12082,7 +12088,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13138,7 +13144,7 @@ check_ok: break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13680,7 +13686,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14059,7 +14065,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, * otherwise a pointer field would be laundered into a scalar * and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + if (!btf_type_is_scalar_struct(env, desc_btf, t)) { verbose(env, "kernel function %s returns %s %s that is not composed of scalars\n", func_name, btf_type_str(t), -- cgit v1.2.3 From 3fc3e7331af6f60979fdf55e776bfe67105b584a Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:29 -0700 Subject: bpf: Add btf_type_is_arena_ptr() Add btf_type_is_arena_ptr() to test whether a BTF type is a pointer carrying the "arena" type tag, and use it to simplify both btf_scan_type_tags() and btf_validate_return_type(). The helper has external linkage because a later patch calls it from verifier.c to decide whether a struct member returned by value may be an arena pointer. btf_scan_type_tags() drove btf_type_tag_walk() with a one-entry match table, so the "multiple type tags" error it checked for could never fire: btf_type_tag_walk() only fails when the walk sets more than one flag. Dropping that error path, and with it the int return, is therefore no functional change. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061529.1693266-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/btf.h | 1 + kernel/bpf/btf.c | 62 +++++++++++++++++++---------------------------------- 2 files changed, 23 insertions(+), 40 deletions(-) (limited to 'include/linux') diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..ddd0f4f32d24 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 47d43eb983a5..280530d25886 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -8106,9 +8090,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) -- cgit v1.2.3 From 8d7ea1b752e5e49b00442f27f2a76228423aa74b Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Fri, 28 Aug 2026 23:15:34 -0700 Subject: bpf: Let the by-value struct walk take the kinds of member it accepts btf_struct_member_walk() hardcodes that every member of a struct returned by value must be a scalar. A later patch needs the same walk to also accept arena pointers, so give it a mask of the member kinds it allows. The mask reaches callers through btf_struct_is_composed_of(), which replaces btf_type_is_scalar_struct() as the entry point exported to btf.c. btf_type_is_scalar_struct() stays as a verifier-local wrapper for the call sites that only ever ask about scalars. There is no functional change: btf_member_kind_allowed() is exactly btf_type_is_scalar() when the mask is BTF_MEMBER_SCALAR alone, and the shared check is the last statement of the loop body, so dropping the array branch's continue neither skips a member nor checks one twice. Signed-off-by: Yonghong Song Link: https://lore.kernel.org/r/20260829061534.1693789-1-yonghong.song@linux.dev Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 11 +++++++++-- kernel/bpf/btf.c | 2 +- kernel/bpf/verifier.c | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 38 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 3eb61edc8c5e..ae9f606539f4 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 280530d25886..b1f4ef614d4c 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7979,7 +7979,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt */ bool local_func = subprog && !is_global; - if (local_func || btf_type_is_scalar_struct(env, btf, t)) + if (local_func || btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR)) return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed077a5422af..90139f1b78d1 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11646,9 +11646,23 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, 4 levels of nesting allowed. An array member counts as its + * element type. + */ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec) + const struct btf_type *t, u32 member_kinds, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11666,7 +11680,7 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!btf_struct_member_walk(env, btf, member_type, rec + 1)) + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, rec + 1)) return false; continue; } @@ -11675,21 +11689,25 @@ static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct bt if (!array->nelems) return false; member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; } - if (!btf_type_is_scalar(member_type)) + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) return false; } return true; } -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t) + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) { - return btf_struct_member_walk(env, btf, t, 0); + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); } enum kfunc_ptr_arg_type { -- cgit v1.2.3 From 264d8fd2794fabe76a89abd5d4f5f8c1ed5fad85 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:17 +0200 Subject: bpf, keys: Add a bpf keyring for program signature validation BPF program signatures can currently be verified against one of the system keyrings (builtin, secondary, platform) or against an arbitrary user/session caller-supplied keyring named through keyring_id. There is nothing in between: the system keyrings need a kernel rebuild or a vouched-for enrollment to rotate a key, while a caller-supplied keyring is fully controlled by the loader and therefore carries no trust on its own (unless explicitly combined with BPF LSM to protect against key tampering). Add a dedicated bpf keyring to fill that gap, modelled after the dm-verity keyring which was added in commit 033724b1c627 ("dm-verity: add dm-verity keyring") and which can eventually be used also via systemd [0] through the same enrollment method as in dm-verity's case. It is selected with the new KEY_SPEC_BPF_KEYRING special key id and gives an operator a place to enroll a BPF-only signing key at boot, specifically scoped to BPF program loading and nothing else in the kernel's trust hierarchy. The id is reserved from the KEY_SPEC space so that the latter is not linked into any process keyring, and lookup_user_key() resolves KEY_SPEC_BPF_KEYRING constant instead of having to look it up via /proc/keys first. By default the keyring is sealed empty at init. Systems that want to provision keys pass bpf.keyring_unsealed=1, which leaves the keyring open for the initrd to add keys to. The keyring is only ever consulted once it is both non-empty and restricted. An unrestricted keyring is ignored. Signed-off-by: Daniel Borkmann Link: https://github.com/systemd/systemd/pull/43549 [0] Link: https://lore.kernel.org/r/20260828175227.1537793-2-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 +++ include/linux/bpf.h | 7 +++ include/linux/key.h | 2 + include/uapi/linux/keyctl.h | 1 + kernel/bpf/Makefile | 3 ++ kernel/bpf/keys.c | 68 +++++++++++++++++++++++++ kernel/bpf/verifier.c | 9 ++++ security/keys/process_keys.c | 25 +++++++++ 8 files changed, 123 insertions(+) create mode 100644 kernel/bpf/keys.c (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index e4643634a9b1..2beb61092bb3 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -667,6 +667,14 @@ Kernel parameters See Documentation/admin-guide/bootconfig.rst + bpf.keyring_unsealed= + [BPF] When set to 1, leave the bpf keyring unsealed + after initialization so that userspace can provision + keys. Once the keyring is restricted it becomes active + and can be used for BPF program signature verification. + + See Documentation/bpf/signing.rst + bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) bttv.radio= Most important insmod options are available as kernel args too. diff --git a/include/linux/bpf.h b/include/linux/bpf.h index c2027688be3e..039674b2f875 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1735,6 +1735,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -3819,6 +3820,7 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3839,6 +3841,11 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..bd10fe45819d 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -440,6 +440,8 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index 4c8884eea808..fa85b9760391 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -24,6 +24,7 @@ #define KEY_SPEC_GROUP_KEYRING -6 /* - key ID for GID-specific keyring */ #define KEY_SPEC_REQKEY_AUTH_KEY -7 /* - key ID for assumed request_key auth key */ #define KEY_SPEC_REQUESTOR_KEYRING -8 /* - key ID for request_key() dest keyring */ +#define KEY_SPEC_BPF_KEYRING -9 /* - key ID for the BPF-specific keyring */ /* request-key default keyrings */ #define KEY_REQKEY_DEFL_NO_CHANGE -1 diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 90255d80e5be..9a92c348bbda 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o obj-$(CONFIG_BPF_SYSCALL) += tcx.o endif +ifeq ($(CONFIG_KEYS),y) +obj-$(CONFIG_BPF_SYSCALL) += keys.o +endif ifeq ($(CONFIG_PERF_EVENTS),y) obj-$(CONFIG_BPF_SYSCALL) += stackmap.o endif diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c new file mode 100644 index 000000000000..08549b3220c1 --- /dev/null +++ b/kernel/bpf/keys.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Isovalent */ + +#include +#include +#include +#include +#include +#include +#include + +#undef MODULE_PARAM_PREFIX +#define MODULE_PARAM_PREFIX "bpf." + +static struct key *bpf_keyring; + +static bool bpf_keyring_unsealed __ro_after_init; +module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); +MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); + +struct bpf_key *bpf_lookup_keyring(void) +{ + struct bpf_key *bkey; + + if (!bpf_keyring) + return NULL; + if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) || + !READ_ONCE(bpf_keyring->restrict_link)) + return NULL; + + bkey = kmalloc_obj(*bkey); + if (!bkey) + return NULL; + + bkey->key = bpf_keyring; + bkey->has_ref = false; + return bkey; +} + +static int __init bpf_keyring_init(void) +{ + struct key *keyring; + + keyring = keyring_alloc(".bpf", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | + KEY_USR_WRITE | KEY_USR_SEARCH | + KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(keyring)) { + pr_err("bpf: cannot allocate bpf keyring: %ld\n", + PTR_ERR(keyring)); + return 0; + } + if (!bpf_keyring_unsealed && + keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) { + pr_err("bpf: cannot seal bpf keyring\n"); + key_revoke(keyring); + key_put(keyring); + return 0; + } + + bpf_keyring = keyring; + key_register_bpf_keyring(keyring); + return 0; +} +late_initcall(bpf_keyring_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a725e3917976..0c229a226eab 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -21127,6 +21128,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id) return BPF_SIG_KEYRING_SECONDARY; case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING: return BPF_SIG_KEYRING_PLATFORM; + case KEY_SPEC_BPF_KEYRING: + return BPF_SIG_KEYRING_BPF; default: return BPF_SIG_KEYRING_USER; } @@ -21164,9 +21167,15 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, return -EINVAL; if (system_keyring_id_check(attr->keyring_id) == 0) key = bpf_lookup_system_key(attr->keyring_id); + else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + key = bpf_lookup_keyring(); else key = bpf_lookup_user_key(attr->keyring_id, 0); if (!key) { + if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { + verbose(env, "the bpf keyring is empty or has not been restricted\n"); + return -ENOKEY; + } verbose(env, "cannot resolve signing keyring with keyring_id %d\n", attr->keyring_id); return -EINVAL; diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index a63c46bb2d14..44358388e395 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -22,6 +22,9 @@ /* Session keyring create vs join semaphore */ static DEFINE_MUTEX(key_session_mutex); +/* BPF keyring reachable through KEY_SPEC_BPF_KEYRING */ +static struct key *bpf_keyring __ro_after_init; + /* The root user's tracking struct */ struct key_user root_key_user = { .usage = REFCOUNT_INIT(3), @@ -590,6 +593,20 @@ bool lookup_user_key_possessed(const struct key *key, return key == match_data->raw_data; } +/** + * key_register_bpf_keyring - Publish the BPF keyring for KEY_SPEC_BPF_KEYRING + * @keyring: The keyring to publish + * + * Make @keyring reachable by userspace through the KEY_SPEC_BPF_KEYRING + * special key ID, so that provisioning it does not require scraping its + * serial out of /proc/keys first. Called once, from an initcall, and never + * undone. + */ +void key_register_bpf_keyring(struct key *keyring) +{ + bpf_keyring = keyring; +} + /* * Look up a key ID given us by userspace with a given permissions mask to get * the key it refers to. @@ -741,6 +758,14 @@ try_again: key_ref = make_key_ref(key, 1); break; + case KEY_SPEC_BPF_KEYRING: + key = bpf_keyring; + if (!key) + goto error; + __key_get(key); + key_ref = make_key_ref(key, 0); + break; + default: key_ref = ERR_PTR(-EINVAL); if (id < 1) -- cgit v1.2.3 From abaa0835b4f381067cc04bfd58180679d813578b Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Fri, 28 Aug 2026 19:52:18 +0200 Subject: bpf: Refuse caller-supplied keyrings when the bpf one is active Nothing changes for systems that do not use the bpf keyring. Without bpf.keyring_unsealed=1 a caller-supplied keyring behaves exactly as before, which also lets it serve as the staging step for software installed onto a running system whose signing key is not enrolled anywhere yet. Passing bpf.keyring_unsealed=1 states that the bpf keyring is the trust anchor for this boot, so from the first program load onwards a caller- supplied keyring is refused with -EPERM. Deriving this from the boot flag rather than from the keyring's runtime state keeps the decision immutable from userspace. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260828175227.1537793-3-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/admin-guide/kernel-parameters.txt | 8 ++++++++ include/linux/bpf.h | 6 ++++++ kernel/bpf/keys.c | 5 +++++ kernel/bpf/verifier.c | 10 +++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 2beb61092bb3..e70fb15757d4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -673,6 +673,14 @@ Kernel parameters keys. Once the keyring is restricted it becomes active and can be used for BPF program signature verification. + Setting this also means that the bpf keyring is the + only non-system keyring a loader may select for the + rest of the boot: caller-supplied user/session + keyrings are refused with -EPERM, whether or not + provisioning actually completed. The system keyrings + stay selectable. Leaving it unset keeps the prior + behaviour, where a caller-supplied keyring is allowed. + See Documentation/bpf/signing.rst bttv.card= [HW,V4L] bttv (bt848 + bt878 based grabber cards) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 039674b2f875..3a7eb2185c35 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -3821,6 +3821,7 @@ struct bpf_key { struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3846,6 +3847,11 @@ static inline struct bpf_key *bpf_lookup_keyring(void) return NULL; } +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c index 08549b3220c1..bffc356839ac 100644 --- a/kernel/bpf/keys.c +++ b/kernel/bpf/keys.c @@ -18,6 +18,11 @@ static bool bpf_keyring_unsealed __ro_after_init; module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444); MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed"); +bool bpf_keyring_enforced(void) +{ + return bpf_keyring_unsealed; +} + struct bpf_key *bpf_lookup_keyring(void) { struct bpf_key *bkey; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0c229a226eab..dab618836299 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21165,12 +21165,16 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env, if (!attr->signature_size || attr->signature_size > KMALLOC_MAX_CACHE_SIZE) return -EINVAL; - if (system_keyring_id_check(attr->keyring_id) == 0) + if (system_keyring_id_check(attr->keyring_id) == 0) { key = bpf_lookup_system_key(attr->keyring_id); - else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) + } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { key = bpf_lookup_keyring(); - else + } else if (bpf_keyring_enforced()) { + verbose(env, "caller-supplied keyring refused, use bpf keyring\n"); + return -EPERM; + } else { key = bpf_lookup_user_key(attr->keyring_id, 0); + } if (!key) { if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) { verbose(env, "the bpf keyring is empty or has not been restricted\n"); -- cgit v1.2.3 From d02bfbc49c8ef978eac5f39a2de67281b8e00edb Mon Sep 17 00:00:00 2001 From: "Alexis LothorĂ© (eBPF Foundation)" Date: Thu, 3 Sep 2026 15:51:30 +0200 Subject: bpf: mark instructions accessing program stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to prepare to emit KASAN checks in JITed programs, JIT compilers need to be aware about whether some load/store instructions are targeting the bpf program stack, as those should not be monitored (we already have guard pages for that, and it is difficult anyway to correctly monitor any kind of data passed on stack). To support this need, make the BPF verifier mark the instructions depending on whether they could access or not memory other than stack. As different states in the verifier could lead to different memory types for the same access, just marking an instruction as accessing stack only is not enough (it could be some other memory type in another verifier state), so the algorithm rather sets by default any load/store instruction as stack only, and if _any_ state leads to any memory access type other than PTR_TO_STACK, it overrides this setting. It also takes care about shifting back the instruction marking in adjust_insn_aux_data if the verifier patches instructions. However, if the verifier generates new BPF_ST/BPF_STX/BPF_LDX while patching some instructions, those new ones are systematically marked as non-stack-accessing: this may over-instrument a few memory accessing instructions, but it allows making sure that we will not miss accidentally any. Signed-off-by: Alexis LothorĂ© (eBPF Foundation) Link: https://lore.kernel.org/r/20260903-kasan-v9-1-2407fe99255a@bootlin.com Signed-off-by: Alexei Starovoitov --- include/linux/bpf_verifier.h | 16 ++++++++++++++++ kernel/bpf/fixups.c | 33 ++++++++++++++++++++++++++++++--- kernel/bpf/verifier.c | 10 ++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ae9f606539f4..1a3c44ab06a1 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -706,6 +706,7 @@ struct bpf_insn_aux_data { */ u32 calls_callback:1; u32 indirect_target:1; /* if it is an indirect jump target */ + u32 non_stack_access:1; /* instruction can access non-stack memory */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -1671,6 +1672,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 65b441e4a351..73fb3ffc18e3 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -213,7 +213,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env, * [0, off) and [off, end) to new locations, so the patched range stays zero */ static void adjust_insn_aux_data(struct bpf_verifier_env *env, - struct bpf_prog *new_prog, u32 off, u32 cnt) + struct bpf_prog *new_prog, u32 off, u32 cnt, + struct bpf_insn *original_insn) { struct bpf_insn_aux_data *data = env->insn_aux_data; struct bpf_insn *insn = new_prog->insnsi; @@ -227,8 +228,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, */ data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0; - if (cnt == 1) + if (cnt == 1) { + /* + * A non-memory accessing insn could have been replaced by a + * memory accessing insn, systematically mark it for non-stack + * access + */ + data[off].non_stack_access = bpf_is_mem_insn(insn + off); return; + } prog_len = new_prog->len; env->insn_aux_data_len = prog_len; @@ -239,8 +247,25 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, /* Expand insni[off]'s seen count to the patched range. */ data[i].seen = old_seen; data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0; + if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) { + data[i].non_stack_access = + data[off + cnt - 1].non_stack_access; + data[off + cnt - 1].non_stack_access = false; + } else if (bpf_is_mem_insn(insn + i)) { + data[i].non_stack_access = true; + } } + /* + * Last slot instruction could be a newly generated + * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access + * if it is not the original instruction, otherwise keep the + * original marking + */ + if (bpf_is_mem_insn(insn + off + cnt - 1) && + memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn))) + data[off + cnt - 1].non_stack_access = true; + /* * The indirect_target flag of the original instruction was moved to the last of the * new instructions by the above memmove and memset, but the indirect jump target is @@ -306,6 +331,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, { struct bpf_prog *new_prog; struct bpf_insn_aux_data *new_data = NULL; + struct bpf_insn original_insn; if (len > 1) { new_data = vrealloc(env->insn_aux_data, @@ -318,6 +344,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data = new_data; } + memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn)); new_prog = bpf_patch_insn_single(env->prog, off, patch, len); if (IS_ERR(new_prog)) { if (PTR_ERR(new_prog) == -ERANGE) @@ -326,7 +353,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, env->insn_aux_data[off].orig_idx); return NULL; } - adjust_insn_aux_data(env, new_prog, off, len); + adjust_insn_aux_data(env, new_prog, off, len, &original_insn); adjust_subprog_starts(env, off, len); adjust_insn_arrays(env, off, len); adjust_poke_descs(new_prog, off, len); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 8f585ceb2cd5..5b51e7ee1a3f 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3273,6 +3273,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx) env->insn_aux_data[idx].indirect_target = true; } +static void mark_non_stack_access(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].non_stack_access = true; +} + #define LR_FRAMENO_BITS 4 #define LR_SPI_BITS 6 #define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1) @@ -6417,6 +6422,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b int value_regno, bool strict_alignment_once, bool is_ldsx) { struct bpf_reg_state *regs = cur_regs(env); + enum bpf_reg_type ptr_type = reg->type; int size, err = 0; size = bpf_size_to_bytes(bpf_size); @@ -6655,6 +6661,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b clear_scalar_id(®s[value_regno]); } } + + if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK) + mark_non_stack_access(env, insn_idx); + return err; } -- cgit v1.2.3