summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/bpf/Kconfig17
-rw-r--r--kernel/bpf/Makefile3
-rw-r--r--kernel/bpf/backtrack.c32
-rw-r--r--kernel/bpf/btf.c109
-rw-r--r--kernel/bpf/cnum_defs.h37
-rw-r--r--kernel/bpf/core.c5
-rw-r--r--kernel/bpf/fixups.c33
-rw-r--r--kernel/bpf/keys.c73
-rw-r--r--kernel/bpf/liveness.c20
-rw-r--r--kernel/bpf/queue_stack_maps.c3
-rw-r--r--kernel/bpf/stream.c57
-rw-r--r--kernel/bpf/verifier.c442
12 files changed, 669 insertions, 162 deletions
diff --git a/kernel/bpf/Kconfig b/kernel/bpf/Kconfig
index eb3de35734f0..d7d25477ef48 100644
--- a/kernel/bpf/Kconfig
+++ b/kernel/bpf/Kconfig
@@ -17,6 +17,10 @@ config HAVE_CBPF_JIT
config HAVE_EBPF_JIT
bool
+# KASAN support for JIT compiler
+config HAVE_EBPF_JIT_KASAN
+ bool
+
# Used by archs to tell that they want the BPF JIT compiler enabled by
# default for kernels that were compiled with BPF JIT support.
config ARCH_WANT_DEFAULT_BPF_JIT
@@ -101,4 +105,17 @@ config BPF_LSM
If you are unsure how to answer this question, answer N.
+config BPF_JIT_KASAN
+ bool
+ depends on HAVE_EBPF_JIT_KASAN
+ depends on KASAN_GENERIC
+ depends on KASAN_VMALLOC
+ depends on BPF_JIT
+ default y
+ help
+ Makes JIT compiler insert generic outline KASAN checks in BPF
+ programs when they are inserted in the kernel. This feature is
+ automatically enabled if the needed set of KASAN and BPF
+ configuration options is enabled.
+
endmenu # "BPF subsystem"
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 90255d80e5be..9a92c348bbda 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -27,6 +27,9 @@ obj-$(CONFIG_BPF_SYSCALL) += offload.o
obj-$(CONFIG_BPF_SYSCALL) += net_namespace.o
obj-$(CONFIG_BPF_SYSCALL) += tcx.o
endif
+ifeq ($(CONFIG_KEYS),y)
+obj-$(CONFIG_BPF_SYSCALL) += keys.o
+endif
ifeq ($(CONFIG_PERF_EVENTS),y)
obj-$(CONFIG_BPF_SYSCALL) += stackmap.o
endif
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index 47282ffeeaf9..507a366dffa4 100644
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -423,6 +423,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
*/
verifier_bug_if(idx + 1 != subseq_idx, env,
"extra insn from subprog");
+ /* global subprog always sets R0 */
+ bt_clear_reg(bt, BPF_REG_0);
+ /* and if it does not set R2, main pass would catch it */
+ bt_clear_reg(bt, BPF_REG_2);
/* r1-r5 are invalidated after subprog call,
* so for global func call it shouldn't be set
* anymore
@@ -432,8 +436,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
bt_reg_mask(bt));
return -EFAULT;
}
- /* global subprog always sets R0 */
- bt_clear_reg(bt, BPF_REG_0);
return 0;
} else {
/* static subprog call instruction, which
@@ -506,6 +508,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
return -ENOTSUPP;
/* regular helper call sets R0 */
bt_clear_reg(bt, BPF_REG_0);
+ /* kfunc might also set R2 */
+ bt_clear_reg(bt, BPF_REG_2);
if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) {
/* if backtracking was looking for registers R1-R5
* they should have been found already.
@@ -520,40 +524,50 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
return -EFAULT;
}
} else if (opcode == BPF_EXIT) {
- bool from_subprog_call, r0_precise;
+ bool from_subprog_call, r0_precise, r2_precise;
/* BPF_EXIT in subprog or callback always returns
* right after the call instruction, so by checking
* whether the instruction at subseq_idx-1 is subprog
* call or not we can distinguish actual exit from
* *subprog* from exit from *callback*. In the former
- * case, we need to propagate r0 precision, if
- * necessary. In the former we never do that.
+ * case, we need to propagate the precision of the
+ * return registers, if necessary. In the latter we
+ * never do that.
*/
from_subprog_call = subseq_idx - 1 >= 0 &&
bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]);
r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0);
+ r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2);
/* Backtracking to a nested function call, 'idx' is a part of
* the inner frame 'subseq_idx' is a part of the outer frame.
- * In case of a regular function call, instructions giving
- * precision to registers R1-R5 should have been found already.
+ * In case of a regular function call, the callee defines the
+ * return registers R0 and R2, so clear them before checking
+ * that instructions giving precision to registers R1-R5 have
+ * been found already. R2 is also an argument register, hence
+ * it has to be cleared before that check.
* In case of a callback from bpf_loop(), R{1,4} in the calling
- * frame would be set as precise and that is correct.
+ * frame would be set as precise and that is correct, and R2
+ * might be a precise helper argument as well, so leave it set.
*/
+ bt_clear_reg(bt, BPF_REG_0);
+ if (from_subprog_call)
+ bt_clear_reg(bt, BPF_REG_2);
if (from_subprog_call && (bt_reg_mask(bt) & BPF_REGMASK_ARGS)) {
verifier_bug(env, "backtracking exit unexpected regs %x",
bt_reg_mask(bt));
return -EFAULT;
}
- bt_clear_reg(bt, BPF_REG_0);
if (bt_subprog_enter(bt))
return -EFAULT;
if (r0_precise)
bt_set_reg(bt, BPF_REG_0);
+ if (r2_precise)
+ bt_set_reg(bt, BPF_REG_2);
/* r6-r9 and stack slots will stay set in caller frame
* bitmasks until we return back from callee(s)
*/
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index d870bc5e50bc..47cf7411f6ae 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3538,6 +3538,22 @@ static int btf_type_tag_walk(const struct btf *btf,
return 0;
}
+bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t)
+{
+ if (!btf_type_is_ptr(t))
+ return false;
+
+ for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t);
+ t = btf_type_by_id(btf, t->type)) {
+ if (!btf_type_is_type_tag(t) || btf_type_kflag(t))
+ continue;
+ if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena"))
+ return true;
+ }
+
+ return false;
+}
+
static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
u32 off, int sz, struct btf_field_info *info, u32 field_mask)
{
@@ -7644,7 +7660,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
return -EINVAL;
}
ret = __get_type_size(btf, func->type, &t);
- if (ret < 0 || btf_type_is_struct(t)) {
+ if (ret < 0 || ret > 16) {
bpf_log(log,
"The function %s return type %s is unsupported.\n",
tname, btf_type_str(t));
@@ -7737,6 +7753,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log,
btf_type_str(t2), fn2);
return -EINVAL;
}
+ if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) {
+ bpf_log(log,
+ "Return type of %s() has size %u and of %s() size %u, and a size above 8 bytes cannot be replaced\n",
+ fn1, t1->size, fn2, t2->size);
+ return -EINVAL;
+ }
for (i = 0; i < nargs1; i++) {
t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL);
@@ -7981,51 +8003,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env,
return 0;
}
-static int btf_scan_type_tags(struct bpf_verifier_env *env,
- const struct btf *btf, u32 type_id,
- u32 *tags)
+static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags)
{
- static const struct btf_type_tag_match func_type_tags[] = {
- { "arena", ARG_TAG_ARENA },
- };
- struct btf_type_tag_walk_ctx ctx;
- const struct btf_type *t;
- int err;
-
/* Find the first pointer type in the chain. */
- t = btf_type_skip_modifiers(btf, type_id, NULL);
-
- /*
- * We currently reject type tags on non-pointer types,
- * which neither LLVM nor GCC support anyway.
- */
- if (!t || !btf_type_is_ptr(t))
- return 0;
-
- ctx.t = t;
- err = btf_type_tag_walk(btf, &ctx, func_type_tags,
- ARRAY_SIZE(func_type_tags));
- if (err) {
- bpf_log(&env->log,
- "function signature member has multiple type tags\n");
- return err;
- }
- *tags |= ctx.res;
+ const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL);
- return 0;
+ if (btf_type_is_arena_ptr(btf, t))
+ *tags |= ARG_TAG_ARENA;
}
/* Check whether the type is a valid return type. */
static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf,
- const struct btf_type *t, int subprog)
+ const struct btf_type *t, int subprog, bool is_global)
{
- u32 tags = 0;
- int err;
-
- err = btf_scan_type_tags(env, btf, t->type, &tags);
- if (err)
- return err;
-
t = btf_type_skip_modifiers(btf, t->type, NULL);
/*
@@ -8033,13 +8023,36 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt
* General arena variables are not allowed, since it makes no sense to return by value
* a variable that's on the heap in the first place.
*/
- if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t))
+ if (subprog && btf_type_is_arena_ptr(btf, t))
return 0;
/* We always accept void or scalars. */
if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t))
return 0;
+ if (btf_type_is_struct(t) && t->size <= 16) {
+ /*
+ * A global function's caller models the return as an opaque
+ * scalar pair, so a pointer member would be laundered into a
+ * scalar and escape provenance and reference tracking. Only
+ * scalars and arena pointers are allowed: an arena pointer has
+ * no provenance to lose, since a program may already derive one
+ * from any scalar with addr_space_cast(), which confines the
+ * result to the arena. The main program is the exception: it
+ * returns to the kernel, which has no arena to cast the address
+ * back into. A local function is verified inline, so its caller
+ * receives the real register state and any member is fine.
+ */
+ bool local_func = subprog && !is_global;
+ u32 member_kinds = BTF_MEMBER_SCALAR;
+
+ if (subprog)
+ member_kinds |= BTF_MEMBER_ARENA_PTR;
+
+ if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds))
+ return 0;
+ }
+
return -EOPNOTSUPP;
}
@@ -8127,13 +8140,19 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
return -EINVAL;
}
- err = btf_validate_return_type(env, btf, t, subprog);
+ err = btf_validate_return_type(env, btf, t, subprog, is_global);
if (err) {
if (is_global) {
+ /* Only a subprogram may return arena pointers. */
+ const char *supported = subprog ?
+ "void, scalar, arena pointer, or a struct/union of "
+ "scalars and arena pointers" :
+ "void, scalar, or a scalar-only struct/union";
+
bpf_log(log,
- "Global function %s() return value not void or scalar. "
- "Only those are supported.\n",
- tname);
+ "Global function %s() has unsupported return type. "
+ "Only %s up to 16 bytes is supported.\n",
+ tname, supported);
}
return err;
}
@@ -8147,9 +8166,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
if (err)
return err;
- err = btf_scan_type_tags(env, btf, args[i].type, &tags);
- if (err)
- return err;
+ btf_scan_type_tags(btf, args[i].type, &tags);
t = btf_type_by_id(btf, args[i].type);
while (btf_type_is_modifier(t))
diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h
index a90e317e3578..30685e43de04 100644
--- a/kernel/bpf/cnum_defs.h
+++ b/kernel/bpf/cnum_defs.h
@@ -33,7 +33,12 @@ struct cnum_t FN(from_srange)(st min, st max)
return (struct cnum_t){ .base = base, .size = size };
}
-/* True if this cnum represents two unsigned ranges. */
+/*
+ * True if this cnum represents two unsigned ranges.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * this function.
+ */
static inline bool FN(urange_overflow)(struct cnum_t cnum)
{
/* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */
@@ -44,6 +49,9 @@ static inline bool FN(urange_overflow)(struct cnum_t cnum)
* cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum.
* If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range
* [0..UT_MAX] is returned.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * cnum{T}_umin / cnum{T}_umax.
*/
ut FN(umin)(struct cnum_t cnum)
{
@@ -57,7 +65,12 @@ ut FN(umax)(struct cnum_t cnum)
}
EXPORT_SYMBOL_GPL(FN(umax));
-/* True if this cnum represents two signed ranges. */
+/*
+ * True if this cnum represents two signed ranges.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * this function.
+ */
static inline bool FN(srange_overflow)(struct cnum_t cnum)
{
return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN);
@@ -67,19 +80,18 @@ static inline bool FN(srange_overflow)(struct cnum_t cnum)
* cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum.
* If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range
* [ST_MIN..ST_MAX] is returned.
+ *
+ * The caller should ensure !is_empty(cnum) holds when calling
+ * cnum{T}_smin / cnum{T}_smax.
*/
st FN(smin)(struct cnum_t cnum)
{
- return FN(srange_overflow)(cnum)
- ? ST_MIN
- : min((st)cnum.base, (st)(cnum.base + cnum.size));
+ return FN(srange_overflow)(cnum) ? ST_MIN : (st)cnum.base;
}
st FN(smax)(struct cnum_t cnum)
{
- return FN(srange_overflow)(cnum)
- ? ST_MAX
- : max((st)cnum.base, (st)(cnum.base + cnum.size));
+ return FN(srange_overflow)(cnum) ? ST_MAX : (st)(cnum.base + cnum.size);
}
/*
@@ -181,7 +193,7 @@ void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max)
static inline struct cnum_t FN(normalize)(struct cnum_t cnum)
{
- if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX)
+ if (cnum.size == UT_MAX)
cnum.base = 0;
return cnum;
}
@@ -210,12 +222,7 @@ bool FN(is_empty)(struct cnum_t cnum)
bool FN(contains)(struct cnum_t cnum, ut v)
{
- if (FN(is_empty)(cnum))
- return false;
- if (FN(urange_overflow)(cnum))
- return v >= cnum.base || v <= (ut)cnum.base + cnum.size;
- else
- return v >= cnum.base && v <= (ut)cnum.base + cnum.size;
+ return !FN(is_empty)(cnum) && v - cnum.base <= cnum.size;
}
bool FN(is_const)(struct cnum_t cnum)
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7f11555a5070..a70a527412a4 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -3301,6 +3301,11 @@ bool __weak bpf_jit_supports_kfunc_call(void)
return false;
}
+bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void)
+{
+ return false;
+}
+
bool __weak bpf_jit_supports_stack_args(void)
{
return false;
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index d6f83521fc78..f9f77634082b 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -219,7 +219,8 @@ static int get_callee_stack_depth(struct bpf_verifier_env *env,
* [0, off) and [off, end) to new locations, so the patched range stays zero
*/
static void adjust_insn_aux_data(struct bpf_verifier_env *env,
- struct bpf_prog *new_prog, u32 off, u32 cnt)
+ struct bpf_prog *new_prog, u32 off, u32 cnt,
+ struct bpf_insn *original_insn)
{
struct bpf_insn_aux_data *data = env->insn_aux_data;
struct bpf_insn *insn = new_prog->insnsi;
@@ -233,8 +234,15 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
*/
data[off].zext_dst = bpf_insn_def32(new_prog, insn + off + cnt - 1) >= 0;
- if (cnt == 1)
+ if (cnt == 1) {
+ /*
+ * A non-memory accessing insn could have been replaced by a
+ * memory accessing insn, systematically mark it for non-stack
+ * access
+ */
+ data[off].non_stack_access = bpf_is_mem_insn(insn + off);
return;
+ }
prog_len = new_prog->len;
env->insn_aux_data_len = prog_len;
@@ -245,9 +253,26 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env,
/* Expand insni[off]'s seen count to the patched range. */
data[i].seen = old_seen;
data[i].zext_dst = bpf_insn_def32(new_prog, insn + i) >= 0;
+ if (!memcmp(insn + i, original_insn, sizeof(struct bpf_insn))) {
+ data[i].non_stack_access =
+ data[off + cnt - 1].non_stack_access;
+ data[off + cnt - 1].non_stack_access = false;
+ } else if (bpf_is_mem_insn(insn + i)) {
+ data[i].non_stack_access = true;
+ }
}
/*
+ * Last slot instruction could be a newly generated
+ * BPF_ST/BPF_LDX/BPF_STX, systematically mark it for non-stack access
+ * if it is not the original instruction, otherwise keep the
+ * original marking
+ */
+ if (bpf_is_mem_insn(insn + off + cnt - 1) &&
+ memcmp(insn + off + cnt - 1, original_insn, sizeof(struct bpf_insn)))
+ data[off + cnt - 1].non_stack_access = true;
+
+ /*
* The indirect_target flag of the original instruction was moved to the last of the
* new instructions by the above memmove and memset, but the indirect jump target is
* actually the first instruction, so move it back. This also matches with the behavior
@@ -325,6 +350,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
{
struct bpf_prog *new_prog;
struct bpf_insn_aux_data *new_data = NULL;
+ struct bpf_insn original_insn;
if (bpf_rewrite_must_abort())
return NULL;
@@ -340,6 +366,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
env->insn_aux_data = new_data;
}
+ memcpy(&original_insn, env->prog->insnsi + off, sizeof(struct bpf_insn));
new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
if (IS_ERR(new_prog)) {
if (PTR_ERR(new_prog) == -ERANGE)
@@ -348,7 +375,7 @@ struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
env->insn_aux_data[off].orig_idx);
return NULL;
}
- adjust_insn_aux_data(env, new_prog, off, len);
+ adjust_insn_aux_data(env, new_prog, off, len, &original_insn);
adjust_subprog_starts(env, off, len);
adjust_insn_arrays(env, off, len);
adjust_poke_descs(new_prog, off, len);
diff --git a/kernel/bpf/keys.c b/kernel/bpf/keys.c
new file mode 100644
index 000000000000..bffc356839ac
--- /dev/null
+++ b/kernel/bpf/keys.c
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Isovalent */
+
+#include <linux/bpf.h>
+#include <linux/cred.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/key.h>
+#include <linux/moduleparam.h>
+#include <linux/slab.h>
+
+#undef MODULE_PARAM_PREFIX
+#define MODULE_PARAM_PREFIX "bpf."
+
+static struct key *bpf_keyring;
+
+static bool bpf_keyring_unsealed __ro_after_init;
+module_param_named(keyring_unsealed, bpf_keyring_unsealed, bool, 0444);
+MODULE_PARM_DESC(keyring_unsealed, "Leave the bpf keyring unsealed");
+
+bool bpf_keyring_enforced(void)
+{
+ return bpf_keyring_unsealed;
+}
+
+struct bpf_key *bpf_lookup_keyring(void)
+{
+ struct bpf_key *bkey;
+
+ if (!bpf_keyring)
+ return NULL;
+ if (!READ_ONCE(bpf_keyring->keys.nr_leaves_on_tree) ||
+ !READ_ONCE(bpf_keyring->restrict_link))
+ return NULL;
+
+ bkey = kmalloc_obj(*bkey);
+ if (!bkey)
+ return NULL;
+
+ bkey->key = bpf_keyring;
+ bkey->has_ref = false;
+ return bkey;
+}
+
+static int __init bpf_keyring_init(void)
+{
+ struct key *keyring;
+
+ keyring = keyring_alloc(".bpf",
+ GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
+ current_cred(), KEY_POS_SEARCH |
+ KEY_USR_VIEW | KEY_USR_READ |
+ KEY_USR_WRITE | KEY_USR_SEARCH |
+ KEY_USR_SETATTR, KEY_ALLOC_NOT_IN_QUOTA,
+ NULL, NULL);
+ if (IS_ERR(keyring)) {
+ pr_err("bpf: cannot allocate bpf keyring: %ld\n",
+ PTR_ERR(keyring));
+ return 0;
+ }
+ if (!bpf_keyring_unsealed &&
+ keyring_restrict(make_key_ref(keyring, true), NULL, NULL)) {
+ pr_err("bpf: cannot seal bpf keyring\n");
+ key_revoke(keyring);
+ key_put(keyring);
+ return 0;
+ }
+
+ bpf_keyring = keyring;
+ key_register_bpf_keyring(keyring);
+ return 0;
+}
+late_initcall(bpf_keyring_init);
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 301fc60bddc4..7165ea325961 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -2060,7 +2060,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); }
/* Compute info->{use,def} fields for the instruction */
static void compute_insn_live_regs(struct bpf_verifier_env *env,
struct bpf_insn *insn,
- struct insn_live_regs *info)
+ struct insn_live_regs *info,
+ bool ret_reg_pair)
{
struct bpf_call_summary cs;
const u8 class = BPF_CLASS(insn->code);
@@ -2072,6 +2073,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
const u32 src32 = mask_lo(src);
const u32 dst32 = mask_lo(dst);
const u32 r0 = reg64_mask(0);
+ const u32 r2 = reg64_mask(BPF_REG_2);
u32 def = 0;
u32 use = U32_MAX;
@@ -2191,7 +2193,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
break;
case BPF_EXIT:
def = 0;
- use = r0;
+ use = ret_reg_pair ? (r0 | r2) : r0;
break;
case BPF_CALL:
def = ALL_CALLER_SAVED_REGS;
@@ -2228,8 +2230,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
struct insn_live_regs *state;
int insn_cnt = env->prog->len;
u64 pos, insn_pos;
- int err = 0, i, j;
- bool changed;
+ int err = 0, i, j, subprog, start, end;
+ bool changed, ret_reg_pair;
/* Use the following algorithm:
* - define the following:
@@ -2256,8 +2258,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
goto out;
}
- for (i = 0; i < insn_cnt; ++i)
- compute_insn_live_regs(env, &insns[i], &state[i]);
+ for (subprog = 0; subprog < env->subprog_cnt; subprog++) {
+ start = env->subprog_info[subprog].start;
+ end = env->subprog_info[subprog + 1].start;
+ ret_reg_pair = bpf_ret_reg_pair(env, subprog);
+
+ for (i = start; i < end; ++i)
+ compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair);
+ }
/* Forward pass: resolve stack access through FP-derived pointers */
err = bpf_compute_subprog_arg_access(env);
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
index 6e8b18c32a10..1cee2ba6e332 100644
--- a/kernel/bpf/queue_stack_maps.c
+++ b/kernel/bpf/queue_stack_maps.c
@@ -59,6 +59,9 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr)
*/
return -E2BIG;
+ if (attr->max_entries >= U32_MAX / attr->value_size)
+ return -E2BIG;
+
return 0;
}
diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c
index be9ce98e9469..2b80a0599865 100644
--- a/kernel/bpf/stream.c
+++ b/kernel/bpf/stream.c
@@ -22,11 +22,11 @@ static struct bpf_stream_elem *bpf_stream_elem_alloc(int len)
size_t alloc_size;
/*
- * Length denotes the amount of data to be written as part of stream element,
- * thus includes '\0' byte. We're capped by how much bpf_bprintf_buffers can
- * accomodate, therefore deny allocations that won't fit into them.
+ * Length is the payload pushed into the stream, excluding the
+ * trailing NUL of the bprintf buffer. Reject anything that cannot
+ * fit without copying that NUL into the stream element.
*/
- if (len < 0 || len > max_len)
+ if (len < 0 || len >= max_len)
return NULL;
alloc_size = offsetof(struct bpf_stream_elem, str[len]);
@@ -68,10 +68,8 @@ static int bpf_stream_consume_capacity(struct bpf_stream *stream, int len)
return 0;
}
-static void bpf_stream_release_capacity(struct bpf_stream *stream, struct bpf_stream_elem *elem)
+static void bpf_stream_release_capacity(struct bpf_stream *stream, int len)
{
- int len = elem->total_len;
-
atomic_sub(len, &stream->capacity);
}
@@ -79,7 +77,14 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l
{
int ret = bpf_stream_consume_capacity(stream, len);
- return ret ?: __bpf_stream_push_str(&stream->log, str, len);
+ if (ret)
+ return ret;
+
+ ret = __bpf_stream_push_str(&stream->log, str, len);
+ if (ret)
+ bpf_stream_release_capacity(stream, len);
+
+ return ret;
}
static struct bpf_stream *bpf_stream_get(enum bpf_stream_id stream_id, struct bpf_prog_aux *aux)
@@ -162,6 +167,7 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
while (rem_len) {
int pos = len - rem_len;
+ int chunk, n;
bool cont;
node = bpf_stream_backlog_peek(stream);
@@ -175,20 +181,21 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
cons_len = elem->consumed_len;
cont = bpf_stream_consume_elem(elem, &rem_len) == false;
-
- ret = copy_to_user(buf + pos, elem->str + cons_len,
- elem->consumed_len - cons_len);
- /* Restore in case of error. */
- if (ret) {
- ret = -EFAULT;
- elem->consumed_len = cons_len;
+ chunk = elem->consumed_len - cons_len;
+
+ n = copy_to_user(buf + pos, elem->str + cons_len, chunk);
+ if (n) {
+ /* Keep any successfully copied bytes; -EFAULT only if none. */
+ elem->consumed_len -= n;
+ rem_len += n;
+ ret = (len == rem_len) ? -EFAULT : 0;
break;
}
if (cont)
continue;
bpf_stream_backlog_pop(stream);
- bpf_stream_release_capacity(stream, elem);
+ bpf_stream_release_capacity(stream, elem->total_len);
bpf_stream_free_elem(elem);
}
@@ -196,13 +203,15 @@ static int bpf_stream_read(struct bpf_stream *stream, void __user *buf, int len)
return ret ? ret : len - rem_len;
}
-int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len)
+int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len)
{
struct bpf_stream *stream;
stream = bpf_stream_get(stream_id, prog->aux);
if (!stream)
return -ENOENT;
+ if (len > INT_MAX)
+ return -EINVAL;
return bpf_stream_read(stream, buf, len);
}
@@ -238,6 +247,11 @@ __bpf_kfunc int bpf_stream_vprintk(int stream_id, const char *fmt__str, const vo
return ret;
ret = bstr_printf(data.buf, MAX_BPRINTF_BUF, fmt__str, data.bin_args);
+ /* Truncation: reject before capacity charge (not -ENOMEM). */
+ if (ret >= MAX_BPRINTF_BUF) {
+ bpf_bprintf_cleanup(&data);
+ return -E2BIG;
+ }
/* Exclude NULL byte during push. */
ret = bpf_stream_push_str(stream, data.buf, ret);
bpf_bprintf_cleanup(&data);
@@ -311,17 +325,18 @@ int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...)
{
struct bpf_bprintf_buffers *buf;
va_list args;
- int ret;
+ int len, ret;
if (bpf_try_get_buffers(&buf))
return -EBUSY;
va_start(args, fmt);
- ret = vsnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
+ len = vscnprintf(buf->buf, ARRAY_SIZE(buf->buf), fmt, args);
va_end(args);
- ss->len += ret;
/* Exclude NULL byte during push. */
- ret = __bpf_stream_push_str(&ss->log, buf->buf, ret);
+ ret = __bpf_stream_push_str(&ss->log, buf->buf, len);
+ if (!ret)
+ ss->len += len;
bpf_put_buffers();
return ret;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 5f874979b8d7..6540fee31122 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -24,6 +24,7 @@
#include <linux/bpf_lsm.h>
#include <linux/security.h>
#include <linux/verification.h>
+#include <linux/keyctl.h>
#include <linux/btf_ids.h>
#include <linux/poison.h>
#include <linux/module.h>
@@ -394,27 +395,69 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog)
return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL;
}
-static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog)
{
- const struct btf_type *type, *func, *func_proto;
+ const struct btf_type *func, *func_proto;
const struct btf *btf = env->prog->aux->btf;
u32 btf_id;
+ if (!btf || !env->prog->aux->func_info)
+ return NULL;
+
btf_id = env->prog->aux->func_info[subprog].type_id;
+ /* Both already validated by prepare_btf_func() at prog load. */
func = btf_type_by_id(btf, btf_id);
- if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id))
- return false;
-
func_proto = btf_type_by_id(btf, func->type);
- if (!func_proto)
- return false;
- type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
- if (!type)
- return false;
+ return btf_type_skip_modifiers(btf, func_proto->type, NULL);
+}
+
+static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog)
+{
+ const struct btf_type *type = subprog_ret_type(env, subprog);
+
+ return type && btf_type_is_void(type);
+}
- return btf_type_is_void(type);
+static u32 ret_regs_cnt(u32 size)
+{
+ return size > 8 && size <= 16 ? 2 : 1;
+}
+
+/* Registers holding a function return value, in order. See ret_regs_cnt(). */
+static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 };
+
+static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env)
+{
+ const struct btf *btf = env->prog->aux->btf;
+ const struct btf_type *type;
+ int subprog;
+ u32 size;
+
+ if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux))
+ return 0;
+
+ /*
+ * Skip the main program: its return value is the program's exit code,
+ * read out of R0, so it never uses the register pair. An extension does
+ * have a real prototype for subprog 0, but btf_check_func_type_match()
+ * refuses to replace a function returning more than 8 bytes.
+ */
+ for (subprog = 1; subprog < env->subprog_cnt; subprog++) {
+ type = subprog_ret_type(env, subprog);
+ if (!type || btf_type_is_void(type))
+ continue;
+ if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env,
+ "cannot size return type of subprog %d", subprog))
+ return -EFAULT;
+ if (ret_regs_cnt(size) > 1) {
+ subprog_info(env, subprog)->ret_reg_pair = true;
+ env->prog->jit_required = 1;
+ }
+ }
+
+ return 0;
}
const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog)
@@ -2888,6 +2931,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model);
if (err)
return err;
+ if (func_model.ret_size > 8) {
+ if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) {
+ verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n",
+ kfunc.name);
+ return -EOPNOTSUPP;
+ }
+ if (!bpf_jit_supports_kfunc_ret_reg_pair()) {
+ verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n",
+ kfunc.name);
+ return -EOPNOTSUPP;
+ }
+ }
memset(&meta, 0, sizeof(meta));
meta.btf = kfunc.btf;
@@ -3232,6 +3287,11 @@ static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
env->insn_aux_data[idx].indirect_target = true;
}
+static void mark_non_stack_access(struct bpf_verifier_env *env, int idx)
+{
+ env->insn_aux_data[idx].non_stack_access = true;
+}
+
#define LR_FRAMENO_BITS 4
#define LR_SPI_BITS 6
#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -5923,6 +5983,28 @@ BTF_TYPE_SAFE_TRUSTED(struct file) {
struct inode *f_inode;
};
+/*
+ * The pointer fields in the sched_ext ops argument containers are pinned by the
+ * callers for the duration of the ops calls and are never NULL.
+ */
+BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args) {
+#ifdef CONFIG_EXT_GROUP_SCHED
+ struct cgroup *cgroup;
+#endif
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args) {
+ struct task_struct *task;
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args) {
+ struct sched_ext_ops *ops;
+};
+
+BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args) {
+ struct sched_ext_ops *ops;
+};
+
BTF_TYPE_SAFE_TRUSTED_OR_NULL(struct dentry) {
struct inode *d_inode;
};
@@ -5967,6 +6049,10 @@ static bool type_is_trusted(struct bpf_verifier_env *env,
BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct bpf_iter__task));
BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct linux_binprm));
BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct file));
+ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_init_task_args));
+ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_cpu_release_args));
+ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_attach_args));
+ BTF_TYPE_EMIT(BTF_TYPE_SAFE_TRUSTED(struct scx_sub_detach_args));
return btf_nested_type_is_trusted(&env->log, reg, field_name, btf_id, "__safe_trusted");
}
@@ -6388,6 +6474,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
int value_regno, bool strict_alignment_once, bool is_ldsx)
{
struct bpf_reg_state *regs = cur_regs(env);
+ enum bpf_reg_type ptr_type = reg->type;
int size, err = 0;
size = bpf_size_to_bytes(bpf_size);
@@ -6635,6 +6722,10 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
clear_scalar_id(&regs[value_regno]);
}
}
+
+ if (!err && bpf_is_mem_insn(&env->prog->insnsi[insn_idx]) && ptr_type != PTR_TO_STACK)
+ mark_non_stack_access(env, insn_idx);
+
return err;
}
@@ -9931,6 +10022,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
u16 callee_incoming, stack_arg_cnt;
struct bpf_func_state *caller;
int err, subprog, target_insn;
+ u32 i, nregs;
target_insn = *insn_idx + insn->imm + 1;
subprog = bpf_find_subprog(env, target_insn);
@@ -9986,10 +10078,20 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
clear_caller_saved_regs(env, caller->regs);
invalidate_outgoing_stack_args(env, cur_func(env));
- /* All non-void global functions return a 64-bit SCALAR_VALUE. */
+ /*
+ * A non-void global function returns a 64-bit SCALAR_VALUE in
+ * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair.
+ */
if (!returns_void) {
- mark_reg_unknown(env, caller->regs, BPF_REG_0);
+ nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
+ mark_reg_unknown(env, caller->regs, ret_regs[0]);
bpf_diag_mod_end(env);
+ for (i = 1; i < nregs; i++) {
+ bpf_diag_mod_begin(env, &caller->regs[ret_regs[i]], NULL,
+ BPF_DIAG_MOD_WRITE);
+ mark_reg_unknown(env, caller->regs, ret_regs[i]);
+ bpf_diag_mod_end(env);
+ }
}
if (env->subprog_info[subprog].might_throw) {
@@ -10364,11 +10466,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
struct bpf_func_state *caller, *callee;
struct bpf_reg_state *r0;
bool in_callback_fn;
+ u32 i, nregs;
int err;
callee = state->frame[state->curframe];
r0 = &callee->regs[BPF_REG_0];
- if (r0->type == PTR_TO_STACK) {
+ nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1;
+ for (i = 0; i < nregs; i++) {
+ if (callee->regs[ret_regs[i]].type != PTR_TO_STACK)
+ continue;
/* technically it's ok to return caller's stack pointer
* (or caller's caller's pointer) back to the caller,
* since these pointers are valid. Only current stack
@@ -10403,10 +10509,18 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
return -EFAULT;
}
} else {
- /* return to the caller whatever r0 had in the callee */
- bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE);
- caller->regs[BPF_REG_0] = *r0;
- bpf_diag_mod_end(env);
+ /*
+ * return to the caller whatever the callee had in the
+ * return register(s)
+ */
+ for (i = 0; i < nregs; i++) {
+ u32 regno = ret_regs[i];
+
+ bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno],
+ BPF_DIAG_MOD_WRITE);
+ caller->regs[regno] = callee->regs[regno];
+ bpf_diag_mod_end(env);
+ }
}
/* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
@@ -11364,6 +11478,37 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
return 0;
}
+/*
+ * Mark the register(s) holding a @size byte kfunc return value as unknown
+ * scalars and return how many registers that took. Both halves of a register
+ * pair are treated the same way.
+ *
+ * R0 is written in the diagnostic modification scope the caller opened before
+ * the caller-saved scrub, since R0 gets no scrub record of its own and that
+ * scope is what carries its pre-call value into the history. A scope tracks a
+ * single register and does not nest, so close it before recording the rest of
+ * the return value. A value that fits in R0 leaves the caller's scope open for
+ * the caller to close, so a later write to R0 still lands in it.
+ */
+static u32 mark_kfunc_ret_regs(struct bpf_verifier_env *env,
+ struct bpf_reg_state *regs, u32 size)
+{
+ u32 i, nregs = ret_regs_cnt(size);
+
+ mark_reg_unknown(env, regs, ret_regs[0]);
+ if (nregs == 1)
+ return nregs;
+
+ bpf_diag_mod_end(env);
+ for (i = 1; i < nregs; i++) {
+ bpf_diag_mod_begin(env, &regs[ret_regs[i]], NULL, BPF_DIAG_MOD_WRITE);
+ mark_reg_unknown(env, regs, ret_regs[i]);
+ bpf_diag_mod_end(env);
+ }
+
+ return nregs;
+}
+
static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ACQUIRE;
@@ -11635,13 +11780,39 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_
return argn <= arg_idx;
}
-/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */
-static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
- const struct btf *btf,
- const struct btf_type *t, int rec)
+#define BTF_MEMBER_MAX_DEPTH 4
+#define BTF_MEMBER_PATH_LEN 64
+
+struct btf_member_path {
+ const struct btf_member *member[BTF_MEMBER_MAX_DEPTH];
+ bool in_array[BTF_MEMBER_MAX_DEPTH];
+ const struct btf_type *bad_type;
+ int depth;
+ bool too_deep;
+};
+
+static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t,
+ u32 member_kinds)
+{
+ if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t))
+ return true;
+ if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t))
+ return true;
+ return false;
+}
+
+/*
+ * Returns true if every member of struct @t is of a kind listed in
+ * @member_kinds, 4 levels of nesting allowed. An array member counts as its
+ * element type.
+ */
+static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf,
+ const struct btf_type *t, u32 member_kinds, int rec,
+ struct btf_member_path *path)
{
const struct btf_type *member_type;
const struct btf_member *member;
+ bool in_array;
u32 i;
if (!btf_type_is_struct(t))
@@ -11650,29 +11821,83 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env,
for_each_member(i, t, member) {
const struct btf_array *array;
+ in_array = false;
member_type = btf_type_skip_modifiers(btf, member->type, NULL);
+ /*
+ * Every element of an array is laid out in the value being
+ * returned, so an array counts as its element type however many
+ * dimensions deep that is.
+ */
+ while (btf_type_is_array(member_type)) {
+ array = btf_array(member_type);
+ if (!array->nelems) {
+ member_type = NULL;
+ goto bad_member;
+ }
+ member_type = btf_type_skip_modifiers(btf, array->type, NULL);
+ in_array = true;
+ }
if (btf_type_is_struct(member_type)) {
- if (rec >= 3) {
+ if (rec >= BTF_MEMBER_MAX_DEPTH - 1) {
verbose(env, "max struct nesting depth exceeded\n");
+ if (path)
+ path->too_deep = true;
return false;
}
- if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1))
- return false;
- continue;
- }
- if (btf_type_is_array(member_type)) {
- array = btf_array(member_type);
- if (!array->nelems)
- return false;
- member_type = btf_type_skip_modifiers(btf, array->type, NULL);
- if (!btf_type_is_scalar(member_type))
- return false;
+ if (!btf_struct_member_walk(env, btf, member_type, member_kinds,
+ rec + 1, path))
+ goto bad_path;
continue;
}
- if (!btf_type_is_scalar(member_type))
- return false;
+ if (!btf_member_kind_allowed(btf, member_type, member_kinds))
+ goto bad_member;
}
return true;
+
+bad_member:
+ if (path) {
+ path->depth = rec + 1;
+ path->bad_type = member_type;
+ }
+bad_path:
+ if (path && path->depth) {
+ path->member[rec] = member;
+ path->in_array[rec] = in_array;
+ }
+ return false;
+}
+
+bool btf_struct_is_composed_of(struct bpf_verifier_env *env,
+ const struct btf *btf,
+ const struct btf_type *t, u32 member_kinds)
+{
+ return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL);
+}
+
+static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env,
+ const struct btf *btf,
+ const struct btf_type *t)
+{
+ return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR);
+}
+
+static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path,
+ char *buf, size_t buf_sz)
+{
+ size_t len = 0;
+ int i;
+
+ buf[0] = '\0';
+ for (i = 0; i < path->depth; i++) {
+ const char *name = btf_name_by_offset(btf, path->member[i]->name_off);
+
+ /* An anonymous struct or union has no name to spell. */
+ if (!name || !name[0])
+ continue;
+ len += scnprintf(buf + len, buf_sz - len, "%s%s", len ? "." : "", name);
+ if (path->in_array[i])
+ len += scnprintf(buf + len, buf_sz - len, "[]");
+ }
}
enum kfunc_ptr_arg_type {
@@ -12055,7 +12280,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
(is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) ||
is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) {
if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) &&
- !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n",
reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
return -EINVAL;
@@ -12071,7 +12296,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
* scalars. The access size is derived from the pointed-to BTF type.
*/
if (!btf_type_is_scalar(ref_t) &&
- !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ !btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n",
reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname);
return -EINVAL;
@@ -13131,7 +13356,7 @@ check_ok:
break;
}
- if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) {
+ if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) {
enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id);
const char *expected_type;
@@ -13678,7 +13903,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg
struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id);
if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) {
- if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) {
+ if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) {
verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n");
return -EINVAL;
}
@@ -13792,7 +14017,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_insn_aux_data *insn_aux;
const char *operation;
int err, insn_idx = *insn_idx_p;
- u32 i, nargs, ptr_type_id;
+ u32 i, nargs, ptr_type_id, ret_nregs = 1;
struct bpf_kfunc_desc *desc;
struct btf *desc_btf;
int id;
@@ -14055,10 +14280,63 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
if (btf_type_is_scalar(t)) {
- mark_reg_unknown(env, regs, BPF_REG_0);
+ ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] ||
meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave]))
__mark_reg_const_zero(env, &regs[BPF_REG_0]);
+ } else if (btf_type_is_struct(t)) {
+ struct btf_member_path path = {};
+ const char *member_note = "";
+
+ /*
+ * The returned struct comes back as raw register bits modeled
+ * as an unknown scalar, so a pointer member would be laundered
+ * into a scalar and escape provenance and reference tracking.
+ * Only scalars and arena pointers are allowed; see
+ * btf_validate_return_type() in btf.c for why an arena pointer
+ * is safe here.
+ */
+ if (!btf_struct_member_walk(env, desc_btf, t,
+ BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) {
+ verbose(env,
+ "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n",
+ func_name, btf_type_str(t),
+ btf_name_by_offset(desc_btf, t->name_off));
+ if (path.too_deep) {
+ member_note = bpf_diag_fmt(
+ env, " It nests structs more than %d levels deep.",
+ BTF_MEMBER_MAX_DEPTH);
+ } else if (path.depth) {
+ char bad_name[BTF_MEMBER_PATH_LEN];
+
+ btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name));
+ if (!path.bad_type) {
+ verbose(env, "member '%s' is a zero-length array\n",
+ bad_name);
+ member_note = bpf_diag_fmt(
+ env, " Its member '%s' is a zero-length array.",
+ bad_name);
+ } else {
+ verbose(env, "member '%s' has type %s\n", bad_name,
+ btf_type_str(path.bad_type));
+ member_note = bpf_diag_fmt(
+ env,
+ " Its member '%s' is %s, not a scalar or an arena pointer.",
+ bad_name, btf_type_str(path.bad_type));
+ }
+ }
+ bpf_diag_program_structure(
+ env, insn_idx, "unsupported kernel function return type",
+ "Call a kernel function that returns only scalars or arena pointers by value.",
+ "%s() returns %s %s by value.%s "
+ "Only kfuncs returning scalar values or arena pointers, or "
+ "structures composed of scalar values or arena pointers are "
+ "supported.",
+ func_name, btf_type_str(t),
+ btf_name_by_offset(desc_btf, t->name_off), member_note);
+ return -EINVAL;
+ }
+ ret_nregs = mark_kfunc_ret_regs(env, regs, t->size);
} else if (btf_type_is_ptr(t)) {
ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id);
err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf);
@@ -14189,7 +14467,8 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
* Record R0 before process_iter_next_call() snapshots the alternate
* iterator path's diagnostic position.
*/
- bpf_diag_mod_end(env);
+ if (ret_nregs == 1)
+ bpf_diag_mod_end(env);
if (bpf_is_iter_next_kfunc(&meta)) {
err = process_iter_next_call(env, insn_idx, &meta);
@@ -17669,37 +17948,54 @@ enforce_retval:
return 0;
}
-static int check_global_subprog_return_code(struct bpf_verifier_env *env)
+static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno)
{
- struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
- struct bpf_func_state *cur_frame = cur_func(env);
+ struct bpf_reg_state *reg;
int err;
- if (subprog_returns_void(env, cur_frame->subprogno))
- return 0;
-
- err = check_reg_arg(env, BPF_REG_0, SRC_OP);
+ err = check_reg_arg(env, regno, SRC_OP);
if (err)
return err;
/* Pointers to arena are safe to pass between subprograms. */
- if (is_arena_reg(env, BPF_REG_0))
+ if (is_arena_reg(env, regno))
return 0;
- if (is_pointer_value(env, BPF_REG_0)) {
- verbose(env, "R%d leaks addr as return value\n", BPF_REG_0);
+ if (is_pointer_value(env, regno)) {
+ verbose(env, "R%d leaks addr as return value\n", regno);
return -EACCES;
}
+ reg = reg_state(env, regno);
if (reg->type != SCALAR_VALUE) {
- verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
- reg_type_str(env, reg->type));
+ verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n",
+ regno, reg_type_str(env, reg->type));
return -EINVAL;
}
return 0;
}
+static int check_global_subprog_return_code(struct bpf_verifier_env *env)
+{
+ struct bpf_func_state *cur_frame = cur_func(env);
+ u32 subprog = cur_frame->subprogno;
+ u32 i, nregs;
+ int err;
+
+ if (subprog_returns_void(env, subprog))
+ return 0;
+
+ nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1;
+ for (i = 0; i < nregs; i++) {
+ err = check_global_ret_scalar_reg(env, ret_regs[i]);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
/* Bitmask with 1s for all caller saved registers */
#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
@@ -21063,6 +21359,14 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
return 0;
}
+/*
+ * Upper bound on the PKCS#7 signature blob passed with a program. Comfortably
+ * above the largest signature the kernel can verify, and far below anything
+ * that would make rejecting a load expensive. Deliberately a fixed number so
+ * that what the syscall accepts does not depend on PAGE_SIZE.
+ */
+#define BPF_PROG_MAX_SIGNATURE_SIZE (64 * 1024)
+
static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
{
switch (keyring_id) {
@@ -21072,6 +21376,8 @@ static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
return BPF_SIG_KEYRING_SECONDARY;
case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
return BPF_SIG_KEYRING_PLATFORM;
+ case KEY_SPEC_BPF_KEYRING:
+ return BPF_SIG_KEYRING_BPF;
default:
return BPF_SIG_KEYRING_USER;
}
@@ -21100,18 +21406,25 @@ static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
u64 data_sz;
int err = 0;
- /*
- * Don't attempt to use kmalloc_large or vmalloc for signatures.
- * Practical signature for BPF program should be below this limit.
- */
if (!attr->signature_size ||
- attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
+ attr->signature_size > BPF_PROG_MAX_SIGNATURE_SIZE)
return -EINVAL;
- if (system_keyring_id_check(attr->keyring_id) == 0)
+
+ if (system_keyring_id_check(attr->keyring_id) == 0) {
key = bpf_lookup_system_key(attr->keyring_id);
- else
+ } else if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
+ key = bpf_lookup_keyring();
+ } else if (bpf_keyring_enforced()) {
+ verbose(env, "caller-supplied keyring refused, use bpf keyring\n");
+ return -EPERM;
+ } else {
key = bpf_lookup_user_key(attr->keyring_id, 0);
+ }
if (!key) {
+ if (attr->keyring_id == KEY_SPEC_BPF_KEYRING) {
+ verbose(env, "the bpf keyring is empty or has not been restricted\n");
+ return -ENOKEY;
+ }
verbose(env, "cannot resolve signing keyring with keyring_id %d\n",
attr->keyring_id);
return -EINVAL;
@@ -21391,6 +21704,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
if (ret < 0)
goto skip_full_check;
+ /* must precede the first bpf_ret_reg_pair() user below */
+ ret = bpf_compute_subprog_ret_regs(env);
+ if (ret < 0)
+ goto skip_full_check;
+
ret = bpf_compute_live_registers(env);
if (ret < 0)
goto skip_full_check;