summaryrefslogtreecommitdiff
path: root/kernel/bpf/verifier.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/bpf/verifier.c')
-rw-r--r--kernel/bpf/verifier.c953
1 files changed, 634 insertions, 319 deletions
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 7aa47342dc65..e6f35f4e715b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22,6 +22,8 @@
#include <linux/ctype.h>
#include <linux/error-injection.h>
#include <linux/bpf_lsm.h>
+#include <linux/security.h>
+#include <linux/verification.h>
#include <linux/btf_ids.h>
#include <linux/poison.h>
#include <linux/module.h>
@@ -250,27 +252,6 @@ static int validate_ref_obj(struct bpf_verifier_env *env, struct ref_obj_desc *r
return 0;
}
-struct bpf_call_arg_meta {
- struct bpf_map_desc map;
- struct bpf_dynptr_desc dynptr;
- struct ref_obj_desc ref_obj;
- bool raw_mode;
- bool pkt_access;
- u8 release_regno;
- int regno;
- int access_size;
- int mem_size;
- u64 msize_max_value;
- int func_id;
- struct btf *btf;
- u32 btf_id;
- struct btf *ret_btf;
- u32 ret_btf_id;
- u32 subprogno;
- struct btf_field *kptr_field;
- s64 const_map_key;
-};
-
struct bpf_kfunc_meta {
struct btf *btf;
const struct btf_type *proto;
@@ -322,6 +303,7 @@ static const char *btf_type_name(const struct btf *btf, u32 id)
}
static DEFINE_MUTEX(bpf_verifier_lock);
+static DEFINE_MUTEX(btf_vmlinux_lock);
static DEFINE_MUTEX(bpf_percpu_ma_lock);
__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
@@ -927,10 +909,10 @@ static void __mark_reg_known_zero(struct bpf_reg_state *reg);
static bool in_rcu_cs(struct bpf_verifier_env *env);
-static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta);
+static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta);
static int mark_stack_slots_iter(struct bpf_verifier_env *env,
- struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_call_arg_meta *meta,
struct bpf_reg_state *reg, int insn_idx,
struct btf *btf, u32 btf_id, int nr_slots)
{
@@ -1063,7 +1045,7 @@ static int acquire_irq_state(struct bpf_verifier_env *env, int insn_idx);
static int release_irq_state(struct bpf_verifier_state *state, int id);
static int mark_stack_slot_irq_flag(struct bpf_verifier_env *env,
- struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_call_arg_meta *meta,
struct bpf_reg_state *reg, int insn_idx,
int kfunc_class)
{
@@ -2490,6 +2472,83 @@ int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id,
return 0;
}
+#define BPF_FD_SLOT_BTF 1UL
+
+static void fd_slot_set_map(struct bpf_fd_array *slot, struct bpf_map *map)
+{
+ slot->val = (unsigned long)map;
+}
+
+static void fd_slot_set_btf(struct bpf_fd_array *slot, struct btf *btf)
+{
+ slot->val = (unsigned long)btf | BPF_FD_SLOT_BTF;
+}
+
+static struct bpf_map *fd_slot_map(struct bpf_fd_array slot)
+{
+ if (slot.val & BPF_FD_SLOT_BTF)
+ return NULL;
+ return (struct bpf_map *)slot.val;
+}
+
+static struct btf *fd_slot_btf(struct bpf_fd_array slot)
+{
+ if (!(slot.val & BPF_FD_SLOT_BTF))
+ return NULL;
+ return (struct btf *)(slot.val & ~BPF_FD_SLOT_BTF);
+}
+
+static struct btf *
+fd_array_get_btf_continuous(struct bpf_verifier_env *env, u32 idx)
+{
+ struct btf *btf;
+
+ if (idx >= env->fd_array_cnt) {
+ verbose(env, "kfunc fd_idx %u out of bounds, fd_array_cnt %u\n",
+ idx, env->fd_array_cnt);
+ return ERR_PTR(-EINVAL);
+ }
+ btf = fd_slot_btf(env->fd_array[idx]);
+ if (!btf) {
+ verbose(env, "kfunc fd_idx %u is not a module BTF\n", idx);
+ return ERR_PTR(-EINVAL);
+ }
+ btf_get(btf);
+ return btf;
+}
+
+static struct btf *
+fd_array_get_btf_sparse(struct bpf_verifier_env *env, u32 idx)
+{
+ struct btf *btf;
+ int btf_fd;
+
+ if (copy_from_bpfptr_offset(&btf_fd, env->fd_array_raw,
+ (size_t)idx * sizeof(btf_fd), sizeof(btf_fd)))
+ return ERR_PTR(-EFAULT);
+ btf = btf_get_by_fd(btf_fd);
+ if (IS_ERR(btf)) {
+ verbose(env, "invalid module BTF fd specified\n");
+ return btf;
+ }
+ return btf;
+}
+
+static struct btf *fd_array_get_btf(struct bpf_verifier_env *env, u32 idx)
+{
+ if (env->signature) {
+ verbose(env, "signed program cannot bind any BTF\n");
+ return ERR_PTR(-EACCES);
+ }
+ if (env->fd_array)
+ return fd_array_get_btf_continuous(env, idx);
+ if (!bpfptr_is_null(env->fd_array_raw))
+ return fd_array_get_btf_sparse(env, idx);
+
+ verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
+ return ERR_PTR(-EPROTO);
+}
+
static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
s16 offset)
{
@@ -2498,7 +2557,6 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
struct bpf_kfunc_btf *b;
struct module *mod;
struct btf *btf;
- int btf_fd;
tab = env->prog->aux->kfunc_btf_tab;
b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
@@ -2509,22 +2567,9 @@ static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
return ERR_PTR(-E2BIG);
}
- if (bpfptr_is_null(env->fd_array)) {
- verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
- return ERR_PTR(-EPROTO);
- }
-
- if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
- offset * sizeof(btf_fd),
- sizeof(btf_fd)))
- return ERR_PTR(-EFAULT);
-
- btf = btf_get_by_fd(btf_fd);
- if (IS_ERR(btf)) {
- verbose(env, "invalid module BTF fd specified\n");
+ btf = fd_array_get_btf(env, offset);
+ if (IS_ERR(btf))
return btf;
- }
-
if (!btf_is_module(btf)) {
verbose(env, "BTF fd for kfunc is not a module BTF\n");
btf_put(btf);
@@ -2715,6 +2760,8 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
prog_aux->kfunc_tab = tab;
}
+ env->prog->jit_required = 1;
+
/* func_id == 0 is always invalid, but instead of returning an error, be
* conservative and wait until the code elimination pass before returning
* error, so that invalid calls that get pruned out can be in BPF programs
@@ -2769,11 +2816,6 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset)
return 0;
}
-bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
-{
- return !!prog->aux->kfunc_tab;
-}
-
static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprog = env->subprog_info;
@@ -3305,34 +3347,6 @@ static int mark_chain_precision_batch(struct bpf_verifier_env *env,
return bpf_mark_chain_precision(env, starting_state, -1, NULL);
}
-static bool is_spillable_regtype(enum bpf_reg_type type)
-{
- switch (base_type(type)) {
- case PTR_TO_MAP_VALUE:
- case PTR_TO_STACK:
- case PTR_TO_CTX:
- case PTR_TO_PACKET:
- case PTR_TO_PACKET_META:
- case PTR_TO_PACKET_END:
- case PTR_TO_FLOW_KEYS:
- case CONST_PTR_TO_MAP:
- case PTR_TO_SOCKET:
- case PTR_TO_SOCK_COMMON:
- case PTR_TO_TCP_SOCK:
- case PTR_TO_XDP_SOCK:
- case PTR_TO_BTF_ID:
- case PTR_TO_BUF:
- case PTR_TO_MEM:
- case PTR_TO_FUNC:
- case PTR_TO_MAP_KEY:
- case PTR_TO_ARENA:
- return true;
- default:
- return false;
- }
-}
-
-
/* check if register is a constant scalar value */
static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32)
{
@@ -3346,13 +3360,18 @@ static u64 reg_const_value(struct bpf_reg_state *reg, bool subreg32)
return subreg32 ? tnum_subreg(reg->var_off).value : reg->var_off.value;
}
+static bool is_pointer_regtype(enum bpf_reg_type type)
+{
+ return type != SCALAR_VALUE && type != NOT_INIT;
+}
+
static bool __is_pointer_value(bool allow_ptr_leaks,
const struct bpf_reg_state *reg)
{
if (allow_ptr_leaks)
return false;
- return reg->type != SCALAR_VALUE;
+ return is_pointer_regtype(reg->type);
}
static void clear_scalar_id(struct bpf_reg_state *reg)
@@ -3477,7 +3496,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
if (value_regno >= 0)
reg = &cur->regs[value_regno];
if (!env->bypass_spec_v4) {
- bool sanitize = reg && is_spillable_regtype(reg->type);
+ bool sanitize = reg && is_pointer_regtype(reg->type);
for (i = 0; i < size; i++) {
u8 type = state->stack[spi].slot_type[(slot - i) %
@@ -3518,7 +3537,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
__mark_reg_known(tmp_reg, insn->imm);
tmp_reg->type = SCALAR_VALUE;
save_register_state(env, state, spi, tmp_reg, size);
- } else if (reg && is_spillable_regtype(reg->type)) {
+ } else if (reg && is_pointer_regtype(reg->type)) {
/* register containing pointer is being spilled into stack */
if (size != BPF_REG_SIZE) {
verbose_linfo(env, insn_idx, "; ");
@@ -3703,14 +3722,21 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
* SCALAR. This function does not deal with register filling; the caller must
* ensure that all spilled registers in the stack range have been marked as
* read.
+ *
+ * STACK_SPILL bytes backed by spilled scalar const zeroes are also considered
+ * zero bytes. In that case, mark the contributing stack slots precise so
+ * pruning cannot reuse a zero-spill state for a later non-zero spill state.
+ *
+ * Returns an error if precision backtracking fails.
*/
-static void mark_reg_stack_read(struct bpf_verifier_env *env,
- /* func where src register points to */
- struct bpf_func_state *ptr_state,
- int min_off, int max_off, int dst_regno)
+static int mark_reg_stack_read(struct bpf_verifier_env *env,
+ /* func where src register points to */
+ struct bpf_func_state *ptr_state,
+ int min_off, int max_off, int dst_regno)
{
struct bpf_verifier_state *vstate = env->cur_state;
struct bpf_func_state *state = vstate->frame[vstate->curframe];
+ u64 zero_spill_mask = 0;
int i, slot, spi;
u8 *stype;
int zeros = 0;
@@ -3720,19 +3746,33 @@ static void mark_reg_stack_read(struct bpf_verifier_env *env,
spi = slot / BPF_REG_SIZE;
mark_stack_slot_scratched(env, spi);
stype = ptr_state->stack[spi].slot_type;
- if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
- break;
- zeros++;
+ if (stype[slot % BPF_REG_SIZE] == STACK_ZERO) {
+ zeros++;
+ continue;
+ }
+ if (stype[slot % BPF_REG_SIZE] == STACK_SPILL &&
+ bpf_register_is_null(&ptr_state->stack[spi].spilled_ptr)) {
+ zero_spill_mask |= 1ull << spi;
+ zeros++;
+ continue;
+ }
+ break;
}
if (zeros == max_off - min_off) {
/* Any access_size read into register is zero extended,
* so the whole register == const_zero.
*/
__mark_reg_const_zero(env, &state->regs[dst_regno]);
+ if (zero_spill_mask) {
+ bpf_bt_set_frame_slot_mask(&env->bt, ptr_state->frameno, zero_spill_mask);
+ return mark_chain_precision_batch(env, env->cur_state);
+ }
} else {
/* have read misc data from the stack */
mark_reg_unknown(env, state->regs, dst_regno);
}
+
+ return 0;
}
/* Read the stack at 'off' and put the results into the register indicated by
@@ -3754,6 +3794,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
struct bpf_reg_state *reg;
u8 *stype, type;
+ int err;
int insn_flags = INSN_F_STACK_ACCESS;
int hist_spi = spi, hist_frame = reg_state->frameno;
@@ -3836,7 +3877,10 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
__mark_reg_const_zero(env, &state->regs[dst_regno]);
insn_flags = 0; /* not restoring original register state */
} else {
- mark_reg_unknown(env, state->regs, dst_regno);
+ err = mark_reg_stack_read(env, reg_state, off, off + size,
+ dst_regno);
+ if (err)
+ return err;
insn_flags = 0; /* not restoring original register state */
}
}
@@ -3881,8 +3925,11 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
}
return -EACCES;
}
- if (dst_regno >= 0)
- mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
+ if (dst_regno >= 0) {
+ err = mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
+ if (err)
+ return err;
+ }
insn_flags = 0; /* we are not restoring spilled register */
}
if (insn_flags)
@@ -3936,7 +3983,10 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg
min_off = reg_smin(reg) + off;
max_off = reg_smax(reg) + off;
- mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
+ err = mark_reg_stack_read(env, ptr_state, min_off, max_off + size,
+ dst_regno);
+ if (err)
+ return err;
check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
return 0;
}
@@ -4349,7 +4399,8 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
*/
if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->var_off.value,
kptr_field->kptr.btf, kptr_field->kptr.btf_id,
- kptr_field->type != BPF_KPTR_UNREF))
+ kptr_field->type != BPF_KPTR_UNREF,
+ !type_is_alloc(reg->type)))
goto bad_type;
return 0;
bad_type:
@@ -4604,7 +4655,7 @@ static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *
}
static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
- const struct bpf_call_arg_meta *meta,
+ const struct bpf_func_proto *fn,
enum bpf_access_type t)
{
enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
@@ -4628,8 +4679,8 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
case BPF_PROG_TYPE_LWT_XMIT:
case BPF_PROG_TYPE_SK_SKB:
case BPF_PROG_TYPE_SK_MSG:
- if (meta)
- return meta->pkt_access;
+ if (fn)
+ return fn->pkt_access;
env->seen_direct_write = true;
return true;
@@ -5187,10 +5238,6 @@ continue_func:
if (verifier_bug_if(sidx < 0, env, "callee not found at insn %d", next_insn))
return -EFAULT;
if (subprog[sidx].is_async_cb) {
- if (subprog[sidx].has_tail_call) {
- verifier_bug(env, "subprog has tail_call and async cb");
- return -EFAULT;
- }
/* async callbacks don't increase bpf prog stack size unless called directly */
if (!bpf_pseudo_call(insn + i))
continue;
@@ -5213,8 +5260,8 @@ continue_func:
if (!priv_stack_supported)
subprog[idx].priv_stack_mode = NO_PRIV_STACK;
- if (subprog[idx].has_tail_call)
- tail_call_reachable = true;
+ /* sync tail_call_reachable with callee state on entry */
+ tail_call_reachable = subprog[idx].has_tail_call;
frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1;
if (frame >= MAX_CALL_FRAMES) {
@@ -5231,8 +5278,8 @@ continue_func:
*/
if (tail_call_reachable) {
for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
- if (subprog[tmp].is_exception_cb) {
- verbose(env, "cannot tail call within exception cb\n");
+ if (subprog[tmp].is_cb) {
+ verbose(env, "cannot tail call within callback\n");
return -EINVAL;
}
if (subprog[tmp].stack_arg_cnt) {
@@ -5789,6 +5836,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
return -EACCES;
}
+ if (atype != BPF_READ && (type_flag(reg->type) & PTR_UNTRUSTED)) {
+ verbose(env, "only read is supported\n");
+ return -EACCES;
+ }
+
if (env->ops->btf_struct_access && !type_is_alloc(reg->type) && atype == BPF_WRITE) {
if (!btf_is_kernel(reg->btf)) {
verifier_bug(env, "reg->btf must be kernel btf");
@@ -5801,8 +5853,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
reg_arg_name(env, argno), tname, off, size);
} else {
/* Writes are permitted with default btf_struct_access for
- * program allocated objects (which always have id > 0),
- * but not for untrusted PTR_TO_BTF_ID | MEM_ALLOC.
+ * program allocated objects (which always have id > 0).
*/
if (atype != BPF_READ && !type_is_ptr_alloc_obj(reg->type)) {
verbose(env, "only read is supported\n");
@@ -6326,11 +6377,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct b
if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
regs[value_regno].type == SCALAR_VALUE) {
- if (!is_ldsx)
+ if (!is_ldsx) {
/* b/h/w load zero-extends, mark upper bits as known 0 */
coerce_reg_to_size(&regs[value_regno], size);
- else
+ } else {
+ /*
+ * Sign-extension can change the register value relative
+ * to a scalar it is linked with by id (e.g. a zero-
+ * extending fill of the same spilled stack slot), thus
+ * drop the shared id in that case.
+ */
+ bool no_sext = reg_umax(&regs[value_regno]) <
+ (1ULL << (size * BITS_PER_BYTE - 1));
+
coerce_reg_to_size_sx(&regs[value_regno], size);
+ if (!no_sext)
+ clear_scalar_id(&regs[value_regno]);
+ }
}
return err;
}
@@ -6610,6 +6673,8 @@ static int check_stack_range_initialized(
* but BTF based global subprog validation isn't accurate enough.
*/
bool allow_poison = access_size < 0 || clobber;
+ /* The call will initialize the memory; uninitialized stack allowed */
+ bool raw_mode = meta && meta->arg_raw_mem.regno == reg_from_argno(argno);
access_size = abs(access_size);
@@ -6645,41 +6710,14 @@ static int check_stack_range_initialized(
* helper return since specific bounds are unknown what may
* cause uninitialized stack leaking.
*/
- if (meta && meta->raw_mode)
- meta = NULL;
+ raw_mode = false;
min_off = reg_smin(reg) + off;
max_off = reg_smax(reg) + off;
}
- if (meta && meta->raw_mode) {
- /* Ensure we won't be overwriting dynptrs when simulating byte
- * by byte access in check_helper_call using meta.access_size.
- * This would be a problem if we have a helper in the future
- * which takes:
- *
- * helper(uninit_mem, len, dynptr)
- *
- * Now, uninint_mem may overlap with dynptr pointer. Hence, it
- * may end up writing to dynptr itself when touching memory from
- * arg 1. This can be relaxed on a case by case basis for known
- * safe cases, but reject due to the possibilitiy of aliasing by
- * default.
- */
- for (i = min_off; i < max_off + access_size; i++) {
- int stack_off = -i - 1;
-
- spi = bpf_get_spi(i);
- /* raw_mode may write past allocated_stack */
- if (state->allocated_stack <= stack_off)
- continue;
- if (state->stack[spi].slot_type[stack_off % BPF_REG_SIZE] == STACK_DYNPTR) {
- verbose(env, "potential write to dynptr at off=%d disallowed\n", i);
- return -EACCES;
- }
- }
- meta->access_size = access_size;
- meta->regno = reg_from_argno(argno);
+ if (raw_mode) {
+ meta->arg_raw_mem.size = access_size;
return 0;
}
@@ -6921,6 +6959,40 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
return err;
}
+static int process_const_alloc_mem_size(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+ argno_t argno, struct ret_mem_desc *ret_mem)
+{
+ int regno = reg_from_argno(argno);
+ int err;
+
+ if (ret_mem->found) {
+ verifier_bug(env, "only one allocation size argument permitted");
+ return -EFAULT;
+ }
+
+ if (!tnum_is_const(reg->var_off)) {
+ verbose(env, "%s is not a const\n", reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+
+ if (reg->var_off.value > U32_MAX) {
+ verbose(env, "%s allocation size exceeds u32 max\n", reg_arg_name(env, argno));
+ return -EINVAL;
+ }
+
+ if (regno >= 0)
+ err = mark_chain_precision(env, regno);
+ else
+ err = mark_stack_arg_precision(env, arg_idx_from_argno(argno));
+ if (err)
+ return err;
+
+ ret_mem->size = reg->var_off.value;
+ ret_mem->found = true;
+
+ return 0;
+}
+
static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg,
struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno)
{
@@ -7159,7 +7231,7 @@ static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_sta
}
static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
return process_timer_func(env, reg, argno, &meta->map);
}
@@ -7326,23 +7398,23 @@ static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_stat
return err;
}
-static bool is_iter_kfunc(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_iter_kfunc(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & (KF_ITER_NEW | KF_ITER_NEXT | KF_ITER_DESTROY);
}
-static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_iter_new_kfunc(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ITER_NEW;
}
-static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_iter_destroy_kfunc(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ITER_DESTROY;
}
-static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx,
+static bool is_kfunc_arg_iter(struct bpf_call_arg_meta *meta, int arg_idx,
const struct btf_param *arg)
{
/* btf_check_iter_kfuncs() guarantees that first argument of any iter
@@ -7356,7 +7428,7 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx,
}
static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
struct bpf_func_state *state = bpf_func(env, reg);
const struct btf_type *t;
@@ -7523,7 +7595,7 @@ static int widen_imprecise_scalars(struct bpf_verifier_env *env,
}
static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_st,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
int iter_frameno = meta->iter.frameno;
int iter_spi = meta->iter.spi;
@@ -7610,7 +7682,7 @@ static struct bpf_reg_state *get_iter_from_state(struct bpf_verifier_state *cur_
* bpf_iter_num_destroy(&it);
*/
static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
struct bpf_verifier_state *cur_st = env->cur_state, *queued_st, *prev_st;
struct bpf_func_state *cur_fr = cur_st->frame[cur_st->curframe], *queued_fr;
@@ -7672,7 +7744,13 @@ static bool arg_type_is_mem_size(enum bpf_arg_type type)
static bool arg_type_is_raw_mem(enum bpf_arg_type type)
{
- return base_type(type) == ARG_PTR_TO_MEM &&
+ /*
+ * A map value output buffer (e.g. bpf_map_pop_elem) is also a raw
+ * (uninitialized) memory argument, and like ARG_PTR_TO_MEM it may be
+ * passed as a PTR_TO_STACK that reaches check_stack_range_initialized().
+ */
+ return (base_type(type) == ARG_PTR_TO_MEM ||
+ base_type(type) == ARG_PTR_TO_MAP_VALUE) &&
type & MEM_UNINIT;
}
@@ -7947,7 +8025,7 @@ found:
if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,
reg->var_off.value, btf_vmlinux, *arg_btf_id,
- strict_type_match)) {
+ strict_type_match, !type_is_alloc(reg->type))) {
verbose(env, "%s is of type %s but %s is expected\n",
reg_arg_name(env, argno),
btf_type_name(reg->btf, reg->btf_id),
@@ -8216,7 +8294,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
u32 regno = BPF_REG_1 + arg;
struct bpf_reg_state *reg = reg_state(env, regno);
enum bpf_arg_type arg_type = fn->arg_type[arg];
- argno_t argno = argno_from_arg(arg + 1);
+ argno_t argno = argno_from_reg(regno);
enum bpf_reg_type type = reg->type;
u32 *arg_btf_id = NULL;
u32 key_size;
@@ -8239,7 +8317,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
}
if (type_is_pkt_pointer(type) &&
- !may_access_direct_pkt_data(env, meta, BPF_READ)) {
+ !may_access_direct_pkt_data(env, fn, BPF_READ)) {
verbose(env, "helper access to the packet is not allowed\n");
return -EACCES;
}
@@ -8261,11 +8339,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
arg_btf_id = fn->arg_btf_id[arg];
- err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta);
+ err = check_reg_type(env, reg, argno, arg_type, arg_btf_id, meta);
if (err)
return err;
- err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type);
+ err = check_func_arg_reg_off(env, reg, argno, arg_type);
if (err)
return err;
@@ -8322,7 +8400,7 @@ skip_type_check:
return -EFAULT;
}
key_size = meta->map.ptr->key_size;
- err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL);
+ err = check_helper_mem_access(env, reg, argno, key_size, BPF_READ, false, NULL);
if (err)
return err;
if (can_elide_value_nullness(meta->map.ptr)) {
@@ -8348,8 +8426,16 @@ skip_type_check:
verifier_bug(env, "invalid map_ptr to access map->value");
return -EFAULT;
}
- meta->raw_mode = arg_type & MEM_UNINIT;
- err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size,
+
+ /*
+ * Disable raw mode for bpf_map_peek_elem() on a bloom filter. The helper reads
+ * the value buffer as an input rather than filling it.
+ */
+ if (meta->func_id == BPF_FUNC_map_peek_elem &&
+ meta->map.ptr->map_type == BPF_MAP_TYPE_BLOOM_FILTER)
+ meta->arg_raw_mem.regno = 0;
+
+ err = check_helper_mem_access(env, reg, argno, meta->map.ptr->value_size,
arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
false, meta);
break;
@@ -8367,11 +8453,11 @@ skip_type_check:
return -EACCES;
}
if (meta->func_id == BPF_FUNC_spin_lock) {
- err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK);
+ err = process_spin_lock(env, reg, argno, PROCESS_SPIN_LOCK);
if (err)
return err;
} else if (meta->func_id == BPF_FUNC_spin_unlock) {
- err = process_spin_lock(env, reg, argno_from_reg(regno), 0);
+ err = process_spin_lock(env, reg, argno, 0);
if (err)
return err;
} else {
@@ -8380,7 +8466,7 @@ skip_type_check:
}
break;
case ARG_PTR_TO_TIMER:
- err = process_timer_helper(env, reg, argno_from_reg(regno), meta);
+ err = process_timer_helper(env, reg, argno, meta);
if (err)
return err;
break;
@@ -8391,9 +8477,8 @@ skip_type_check:
/* The access to this pointer is only checked when we hit the
* next is_mem_size argument below.
*/
- meta->raw_mode = arg_type & MEM_UNINIT;
if (arg_type & MEM_FIXED_SIZE) {
- err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg],
+ err = check_helper_mem_access(env, reg, argno, fn->arg_size[arg],
arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
false, meta);
if (err)
@@ -8403,39 +8488,31 @@ skip_type_check:
}
break;
case ARG_CONST_SIZE:
- err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),
- argno_from_reg(regno),
- fn->arg_type[arg - 1] & MEM_WRITE ?
- BPF_WRITE : BPF_READ,
+ err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
+ argno_from_reg(regno - 1), argno,
+ fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
false, meta);
break;
case ARG_CONST_SIZE_OR_ZERO:
- err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),
- argno_from_reg(regno),
- fn->arg_type[arg - 1] & MEM_WRITE ?
- BPF_WRITE : BPF_READ,
+ err = check_mem_size_reg(env, reg_state(env, regno - 1), reg,
+ argno_from_reg(regno - 1), argno,
+ fn->arg_type[arg - 1] & MEM_WRITE ? BPF_WRITE : BPF_READ,
true, meta);
break;
case ARG_PTR_TO_DYNPTR:
- err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, &meta->ref_obj,
+ err = process_dynptr_func(env, reg, argno, insn_idx, arg_type, &meta->ref_obj,
&meta->dynptr);
if (err)
return err;
break;
case ARG_CONST_ALLOC_SIZE_OR_ZERO:
- if (!tnum_is_const(reg->var_off)) {
- verbose(env, "R%d is not a known constant'\n",
- regno);
- return -EACCES;
- }
- meta->mem_size = reg->var_off.value;
- err = mark_chain_precision(env, regno);
+ err = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
if (err)
return err;
break;
case ARG_PTR_TO_CONST_STR:
{
- err = check_arg_const_str(env, reg, argno_from_reg(regno));
+ err = check_arg_const_str(env, reg, argno);
if (err)
return err;
break;
@@ -8472,12 +8549,7 @@ static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
if (func_id == BPF_FUNC_map_delete_elem)
return true;
break;
- case BPF_PROG_TYPE_SOCKET_FILTER:
- case BPF_PROG_TYPE_SCHED_CLS:
- case BPF_PROG_TYPE_SCHED_ACT:
- case BPF_PROG_TYPE_XDP:
case BPF_PROG_TYPE_SK_REUSEPORT:
- case BPF_PROG_TYPE_FLOW_DISSECTOR:
case BPF_PROG_TYPE_SK_LOOKUP:
return true;
default:
@@ -8747,26 +8819,19 @@ error:
return -EINVAL;
}
-static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
+static bool check_raw_mode_ok(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta)
{
- int count = 0;
+ int i;
- if (arg_type_is_raw_mem(fn->arg1_type))
- count++;
- if (arg_type_is_raw_mem(fn->arg2_type))
- count++;
- if (arg_type_is_raw_mem(fn->arg3_type))
- count++;
- if (arg_type_is_raw_mem(fn->arg4_type))
- count++;
- if (arg_type_is_raw_mem(fn->arg5_type))
- count++;
+ for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
+ if (!arg_type_is_raw_mem(fn->arg_type[i]))
+ continue;
+ if (meta->arg_raw_mem.regno)
+ return false;
+ meta->arg_raw_mem.regno = i + 1;
+ }
- /* We only support one arg being in raw mode at the moment,
- * which is sufficient for the helper functions we have
- * right now.
- */
- return count <= 1;
+ return true;
}
static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
@@ -8856,7 +8921,7 @@ static bool check_proto_release_reg(const struct bpf_func_proto *fn, struct bpf_
static int check_func_proto(const struct bpf_func_proto *fn, struct bpf_call_arg_meta *meta)
{
- return check_raw_mode_ok(fn) &&
+ return check_raw_mode_ok(fn, meta) &&
check_arg_pair_ok(fn) &&
check_mem_arg_rw_flag_ok(fn) &&
check_proto_release_reg(fn, meta) &&
@@ -10215,7 +10280,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
}
memset(&meta, 0, sizeof(meta));
- meta.pkt_access = fn->pkt_access;
err = check_func_proto(fn, &meta);
if (err) {
@@ -10254,8 +10318,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
/* Mark slots with STACK_MISC in case of raw mode, stack offset
* is inferred from register state.
*/
- for (i = 0; i < meta.access_size; i++) {
- err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B,
+ for (i = 0; i < meta.arg_raw_mem.size; i++) {
+ err = check_mem_access(env, insn_idx, regs + meta.arg_raw_mem.regno,
+ argno_from_reg(meta.arg_raw_mem.regno), i, BPF_B,
BPF_WRITE, -1, false, false);
if (err)
return err;
@@ -10473,7 +10538,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
case RET_PTR_TO_MEM:
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
- regs[BPF_REG_0].mem_size = meta.mem_size;
+ regs[BPF_REG_0].mem_size = meta.ret_mem.size;
break;
case RET_PTR_TO_MEM_OR_BTF_ID:
{
@@ -10681,27 +10746,27 @@ static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
return __mark_btf_func_reg_size(env, cur_regs(env), regno, reg_size);
}
-static bool is_kfunc_acquire(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_ACQUIRE;
}
-static bool is_kfunc_release(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_release(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_RELEASE;
}
-static bool is_kfunc_destructive(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_DESTRUCTIVE;
}
-static bool is_kfunc_rcu(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_RCU;
}
-static bool is_kfunc_rcu_protected(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_rcu_protected(struct bpf_call_arg_meta *meta)
{
return meta->kfunc_flags & KF_RCU_PROTECTED;
}
@@ -10922,7 +10987,7 @@ static bool is_kfunc_arg_prog_aux(const struct btf *btf, const struct btf_param
* To determine whether an argument is implicit, we compare its position
* against the number of arguments in the prototype w/o implicit args.
*/
-static bool is_kfunc_arg_implicit(const struct bpf_kfunc_call_arg_meta *meta, u32 arg_idx)
+static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_idx)
{
const struct btf_type *func, *func_proto;
u32 argn;
@@ -11221,7 +11286,7 @@ static bool is_task_work_add_kfunc(u32 func_id)
func_id == special_kfunc_list[KF_bpf_task_work_schedule_resume];
}
-static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_ret_null(struct bpf_call_arg_meta *meta)
{
if (is_bpf_refcount_acquire_kfunc(meta->func_id) && meta->arg_owning_ref)
return false;
@@ -11229,34 +11294,34 @@ static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
return meta->kfunc_flags & KF_RET_NULL;
}
-static bool is_kfunc_bpf_rcu_read_lock(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_bpf_rcu_read_lock(struct bpf_call_arg_meta *meta)
{
return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_lock];
}
-static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_bpf_rcu_read_unlock(struct bpf_call_arg_meta *meta)
{
return meta->func_id == special_kfunc_list[KF_bpf_rcu_read_unlock];
}
-static bool is_kfunc_bpf_preempt_disable(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_bpf_preempt_disable(struct bpf_call_arg_meta *meta)
{
return meta->func_id == special_kfunc_list[KF_bpf_preempt_disable];
}
-static bool is_kfunc_bpf_preempt_enable(struct bpf_kfunc_call_arg_meta *meta)
+static bool is_kfunc_bpf_preempt_enable(struct bpf_call_arg_meta *meta)
{
return meta->func_id == special_kfunc_list[KF_bpf_preempt_enable];
}
-bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta)
+bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta)
{
return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data];
}
static enum kfunc_ptr_arg_type
get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller,
- struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_reg_state *regs, struct bpf_call_arg_meta *meta,
const struct btf_type *t, const struct btf_type *ref_t,
const char *ref_tname, const struct btf_param *args,
int arg, int nargs, argno_t argno, struct bpf_reg_state *reg)
@@ -11362,7 +11427,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
struct bpf_reg_state *reg,
const struct btf_type *ref_t,
const char *ref_tname, u32 ref_id,
- struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_call_arg_meta *meta,
int arg, argno_t argno)
{
const struct btf_type *reg_ref_t;
@@ -11414,7 +11479,8 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
reg_ref_t = btf_type_skip_modifiers(reg_btf, reg_ref_id, &reg_ref_id);
reg_ref_tname = btf_name_by_offset(reg_btf, reg_ref_t->name_off);
struct_same = btf_struct_ids_match(&env->log, reg_btf, reg_ref_id, reg->var_off.value,
- meta->btf, ref_id, strict_type_match);
+ meta->btf, ref_id, strict_type_match,
+ !type_is_alloc(reg->type));
/* If kfunc is accepting a projection type (ie. __sk_buff), it cannot
* actually use it -- it must cast to the underlying type. So we allow
* caller to pass in the underlying type.
@@ -11431,7 +11497,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
}
static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
int err, spi, kfunc_class = IRQ_NATIVE_KFUNC;
bool irq_save;
@@ -11756,7 +11822,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
static int
__process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_call_arg_meta *meta,
enum btf_field_type head_field_type,
struct btf_field **head_field)
{
@@ -11806,7 +11872,7 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD,
&meta->arg_list_head.field);
@@ -11814,7 +11880,7 @@ static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT,
&meta->arg_rbtree_root.field);
@@ -11823,7 +11889,7 @@ static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
static int
__process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta,
+ struct bpf_call_arg_meta *meta,
enum btf_field_type head_field_type,
enum btf_field_type node_field_type,
struct btf_field **node_field)
@@ -11861,7 +11927,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
et = btf_type_by_id(field->graph_root.btf, field->graph_root.value_btf_id);
t = btf_type_by_id(reg->btf, reg->btf_id);
if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, 0, field->graph_root.btf,
- field->graph_root.value_btf_id, true)) {
+ field->graph_root.value_btf_id, true,
+ !type_is_alloc(reg->type))) {
verbose(env, "operation on %s expects arg#1 %s at offset=%d "
"in struct %s, but arg is at offset=%d in struct %s\n",
btf_field_type_name(head_field_type),
@@ -11887,7 +11954,7 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
BPF_LIST_HEAD, BPF_LIST_NODE,
@@ -11896,7 +11963,7 @@ static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
struct bpf_reg_state *reg, argno_t argno,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
BPF_RB_ROOT, BPF_RB_NODE,
@@ -11925,7 +11992,7 @@ static bool check_css_task_iter_allowlist(struct bpf_verifier_env *env)
}
}
-static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
+static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
int insn_idx)
{
const char *func_name = meta->func_name, *ref_tname;
@@ -12021,23 +12088,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
}
if (is_ret_buf_sz) {
- if (meta->r0_size) {
- verbose(env, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
- return -EINVAL;
- }
-
- if (!tnum_is_const(reg->var_off)) {
- verbose(env, "%s is not a const\n",
- reg_arg_name(env, argno));
- return -EINVAL;
- }
-
- meta->r0_size = reg->var_off.value;
- if (regno >= 0)
- ret = mark_chain_precision(env, regno);
- else
- ret = mark_stack_arg_precision(env, i);
- if (ret)
+ ret = process_const_alloc_mem_size(env, reg, argno, &meta->ret_mem);
+ if (ret < 0)
return ret;
}
continue;
@@ -12519,16 +12571,17 @@ check_ok:
int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env,
s32 func_id,
s16 offset,
- struct bpf_kfunc_call_arg_meta *meta)
+ struct bpf_call_arg_meta *meta)
{
struct bpf_kfunc_meta kfunc;
int err;
+ memset(meta, 0, sizeof(*meta));
+
err = fetch_kfunc_meta(env, func_id, offset, &kfunc);
if (err)
return err;
- memset(meta, 0, sizeof(*meta));
meta->btf = kfunc.btf;
meta->func_id = kfunc.id;
meta->func_proto = kfunc.proto;
@@ -12678,7 +12731,7 @@ s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, struct bpf_insn *
int arg, int insn_idx)
{
struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
- struct bpf_kfunc_call_arg_meta meta;
+ struct bpf_call_arg_meta meta;
const struct btf_param *args;
const struct btf_type *t, *ref_t;
const struct btf *btf;
@@ -12739,7 +12792,7 @@ out:
* 0 - fall-through to 'else' branch
* < 0 - not fall-through to 'else' branch, return error
*/
-static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta,
+static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
struct bpf_reg_state *regs, struct bpf_insn_aux_data *insn_aux,
const struct btf_type *ptr_type, struct btf *desc_btf)
{
@@ -12918,7 +12971,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
struct bpf_reg_state *regs = cur_regs(env);
const char *func_name, *ptr_type_name;
const struct btf_type *t, *ptr_type;
- struct bpf_kfunc_call_arg_meta meta;
+ struct bpf_call_arg_meta meta;
struct bpf_insn_aux_data *insn_aux;
int err, insn_idx = *insn_idx_p;
const struct btf_param *args;
@@ -13016,11 +13069,6 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
}
}
- if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie]) {
- meta.r0_size = sizeof(u64);
- meta.r0_rdonly = false;
- }
-
if (is_bpf_wq_set_callback_kfunc(meta.func_id)) {
err = push_callback_call(env, insn, insn_idx, meta.subprogno,
set_timer_callback_state);
@@ -13153,15 +13201,19 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
/* kfunc returning 'void *' is equivalent to returning scalar */
mark_reg_unknown(env, regs, BPF_REG_0);
} else if (!__btf_type_is_struct(ptr_type)) {
- if (!meta.r0_size) {
+ if (!meta.ret_mem.found) {
__u32 sz;
if (!IS_ERR(btf_resolve_size(desc_btf, ptr_type, &sz))) {
- meta.r0_size = sz;
+ meta.ret_mem.found = true;
+ meta.ret_mem.size = sz;
meta.r0_rdonly = true;
}
+
+ if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
+ meta.r0_rdonly = false;
}
- if (!meta.r0_size) {
+ if (!meta.ret_mem.found) {
ptr_type_name = btf_name_by_offset(desc_btf,
ptr_type->name_off);
verbose(env,
@@ -13174,7 +13226,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_MEM;
- regs[BPF_REG_0].mem_size = meta.r0_size;
+ regs[BPF_REG_0].mem_size = meta.ret_mem.size;
if (meta.r0_rdonly)
regs[BPF_REG_0].type |= MEM_RDONLY;
@@ -16673,7 +16725,7 @@ bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm)
bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call,
struct bpf_call_summary *cs)
{
- struct bpf_kfunc_call_arg_meta meta;
+ struct bpf_call_arg_meta meta;
const struct bpf_func_proto *fn;
int i;
@@ -17567,6 +17619,11 @@ static int __add_used_btf(struct bpf_verifier_env *env, struct btf *btf)
if (env->used_btfs[i].btf == btf)
goto ret_put;
+ if (env->signature) {
+ verbose(env, "signed program cannot bind any BTF\n");
+ ret = -EACCES;
+ goto ret_put;
+ }
if (env->used_btf_cnt >= MAX_USED_BTFS) {
verbose(env, "The total number of btfs per program has reached the limit of %u\n",
MAX_USED_BTFS);
@@ -17759,7 +17816,9 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
}
+ }
+ if (btf_record_has_field(map->record, BPF_SPIN_LOCK)) {
if (is_tracing_prog_type(prog_type)) {
verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
@@ -17831,6 +17890,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
return -EOPNOTSUPP;
}
env->prog->aux->arena = (void *)map;
+ env->prog->jit_required = true;
if (!bpf_arena_get_user_vm_start(env->prog->aux->arena)) {
verbose(env, "arena's user address must be set via map_extra or mmap()\n");
return -EINVAL;
@@ -17849,6 +17909,12 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
if (env->used_maps[i] == map)
return i;
+ if (env->signature &&
+ env->prog->aux->sig.verdict == BPF_SIG_VERIFIED) {
+ verbose(env, "signed program cannot bind map '%s' not covered by the signature\n",
+ map->name);
+ return -EACCES;
+ }
if (env->used_map_cnt >= MAX_USED_MAPS) {
verbose(env, "The total number of maps per program has reached the limit of %u\n",
MAX_USED_MAPS);
@@ -17878,6 +17944,7 @@ static int __add_used_map(struct bpf_verifier_env *env, struct bpf_map *map)
return err;
}
env->insn_array_maps[env->insn_array_map_cnt++] = map;
+ env->prog->jit_required = true;
}
return env->used_map_cnt - 1;
@@ -17901,6 +17968,48 @@ static int add_used_map(struct bpf_verifier_env *env, int fd)
return __add_used_map(env, map);
}
+static int fd_array_get_map_idx_continuous(struct bpf_verifier_env *env, u32 idx)
+{
+ struct bpf_map *map;
+
+ if (idx >= env->fd_array_cnt) {
+ verbose(env, "fd_idx %u out of bounds, fd_array_cnt %u\n",
+ idx, env->fd_array_cnt);
+ return -EINVAL;
+ }
+ map = fd_slot_map(env->fd_array[idx]);
+ if (!map) {
+ verbose(env, "fd_idx %u is not a map\n", idx);
+ return -EINVAL;
+ }
+ return __add_used_map(env, map);
+}
+
+static int fd_array_get_map_idx_sparse(struct bpf_verifier_env *env, u32 idx)
+{
+ int fd;
+
+ if (copy_from_bpfptr_offset(&fd, env->fd_array_raw,
+ (size_t)idx * sizeof(fd), sizeof(fd)))
+ return -EFAULT;
+ return add_used_map(env, fd);
+}
+
+static int fd_array_get_map_idx(struct bpf_verifier_env *env, u32 idx)
+{
+ if (env->fd_array)
+ return fd_array_get_map_idx_continuous(env, idx);
+ if (env->signature) {
+ verbose(env, "signed program must bind maps via a continuous fd_array (fd_array_cnt)\n");
+ return -EACCES;
+ }
+ if (!bpfptr_is_null(env->fd_array_raw))
+ return fd_array_get_map_idx_sparse(env, idx);
+
+ verbose(env, "fd_idx without fd_array is invalid\n");
+ return -EPROTO;
+}
+
static int check_alu_fields(struct bpf_verifier_env *env, struct bpf_insn *insn)
{
u8 class = BPF_CLASS(insn->code);
@@ -18118,7 +18227,6 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
struct bpf_map *map;
int map_idx;
u64 addr;
- u32 fd;
if (i == insn_cnt - 1 || insn[1].code != 0 ||
insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
@@ -18170,21 +18278,17 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
switch (insn[0].src_reg) {
case BPF_PSEUDO_MAP_IDX_VALUE:
case BPF_PSEUDO_MAP_IDX:
- if (bpfptr_is_null(env->fd_array)) {
- verbose(env, "fd_idx without fd_array is invalid\n");
- return -EPROTO;
- }
- if (copy_from_bpfptr_offset(&fd, env->fd_array,
- insn[0].imm * sizeof(fd),
- sizeof(fd)))
- return -EFAULT;
+ map_idx = fd_array_get_map_idx(env, insn[0].imm);
break;
default:
- fd = insn[0].imm;
+ if (env->signature) {
+ verbose(env, "signed program cannot reference a map by fd, only via fd_array index\n");
+ return -EINVAL;
+ }
+ map_idx = add_used_map(env, insn[0].imm);
break;
}
- map_idx = add_used_map(env, fd);
if (map_idx < 0)
return map_idx;
map = env->used_maps[map_idx];
@@ -19422,7 +19526,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
return -ENOMEM;
if (tgt_prog && tgt_prog->aux->tail_call_reachable)
- tr->flags = BPF_TRAMP_F_TAIL_CALL_CTX;
+ bpf_trampoline_set_flags(tr, BPF_TRAMP_F_TAIL_CALL_CTX);
prog->aux->dst_trampoline = tr;
return 0;
@@ -19484,13 +19588,25 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt
struct btf *bpf_get_btf_vmlinux(void)
{
- if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
- mutex_lock(&bpf_verifier_lock);
- if (!btf_vmlinux)
- btf_vmlinux = btf_parse_vmlinux();
- mutex_unlock(&bpf_verifier_lock);
+ /* Pairs with the smp_store_release() on the parse path below. */
+ struct btf *btf = smp_load_acquire(&btf_vmlinux);
+
+ if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
+ mutex_lock(&btf_vmlinux_lock);
+ btf = btf_vmlinux;
+ if (!btf) {
+ btf = btf_parse_vmlinux();
+ /*
+ * Order the parsed BTF contents and the globals the
+ * parse populated (e.g. bpf_ctx_convert.t) before
+ * the pointer publication. Pairs with the acquire
+ * on the lockless fast path above.
+ */
+ smp_store_release(&btf_vmlinux, btf);
+ }
+ mutex_unlock(&btf_vmlinux_lock);
}
- return btf_vmlinux;
+ return btf;
}
/*
@@ -19498,7 +19614,7 @@ struct btf *bpf_get_btf_vmlinux(void)
* this case expect that every file descriptor in the array is either a map or
* a BTF. Everything else is considered to be trash.
*/
-static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
+static int add_fd_from_fd_array(struct bpf_verifier_env *env, u32 idx, int fd)
{
struct bpf_map *map;
struct btf *btf;
@@ -19510,51 +19626,83 @@ static int add_fd_from_fd_array(struct bpf_verifier_env *env, int fd)
err = __add_used_map(env, map);
if (err < 0)
return err;
+ fd_slot_set_map(&env->fd_array[idx], map);
return 0;
}
btf = __btf_get_by_fd(f);
if (!IS_ERR(btf)) {
btf_get(btf);
- return __add_used_btf(env, btf);
+ err = __add_used_btf(env, btf);
+ if (err < 0)
+ return err;
+ fd_slot_set_btf(&env->fd_array[idx], btf);
+ return 0;
}
verbose(env, "fd %d is not pointing to valid bpf_map or btf\n", fd);
return PTR_ERR(map);
}
-static int process_fd_array(struct bpf_verifier_env *env, union bpf_attr *attr, bpfptr_t uattr)
+/*
+ * A continuous fd_array is resolved into an in-memory cache with one slot
+ * per entry. The bound here is deliberately generous and not derived from
+ * the per-program object limits: Duplicate entries /are/ permitted, and
+ * the number of distinct maps and BTFs a program can bind is enforced when
+ * each entry is resolved by __add_used_map() and __add_used_btf().
+ */
+#define MAX_FD_ARRAY_CNT 4096
+
+static int process_fd_array_continuous(struct bpf_verifier_env *env,
+ bpfptr_t fd_array, u32 cnt)
{
- size_t size = sizeof(int);
- int ret;
- int fd;
+ int fd, ret;
u32 i;
- env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
-
- /*
- * The only difference between old (no fd_array_cnt is given) and new
- * APIs is that in the latter case the fd_array is expected to be
- * continuous and is scanned for map fds right away
- */
- if (!attr->fd_array_cnt)
- return 0;
-
- /* Check for integer overflow */
- if (attr->fd_array_cnt >= (U32_MAX / size)) {
- verbose(env, "fd_array_cnt is too big (%u)\n", attr->fd_array_cnt);
- return -EINVAL;
+ if (cnt > MAX_FD_ARRAY_CNT) {
+ verbose(env, "fd_array has too many entries (%u, max %u)\n",
+ cnt, MAX_FD_ARRAY_CNT);
+ return -E2BIG;
}
- for (i = 0; i < attr->fd_array_cnt; i++) {
- if (copy_from_bpfptr_offset(&fd, env->fd_array, i * size, size))
+ env->fd_array = kvcalloc(cnt, sizeof(*env->fd_array),
+ GFP_KERNEL_ACCOUNT);
+ if (!env->fd_array)
+ return -ENOMEM;
+ env->fd_array_cnt = cnt;
+ for (i = 0; i < cnt; i++) {
+ if (copy_from_bpfptr_offset(&fd, fd_array,
+ (size_t)i * sizeof(fd), sizeof(fd)))
return -EFAULT;
-
- ret = add_fd_from_fd_array(env, fd);
+ ret = add_fd_from_fd_array(env, i, fd);
if (ret)
return ret;
}
+ return 0;
+}
+
+static int process_fd_array(struct bpf_verifier_env *env,
+ union bpf_attr *attr, bpfptr_t uattr)
+{
+ bpfptr_t fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
+ if (bpfptr_is_null(fd_array)) {
+ if (attr->fd_array_cnt) {
+ verbose(env, "fd_array_cnt %u without fd_array is invalid\n",
+ attr->fd_array_cnt);
+ return -EINVAL;
+ }
+ return 0;
+ }
+ /*
+ * New API: the caller passes fd_array_cnt and a continuous array that
+ * is resolved and bound up front. Legacy API (no fd_array_cnt): keep
+ * the caller's array and resolve entries on the spot at each reference.
+ */
+ if (attr->fd_array_cnt)
+ return process_fd_array_continuous(env, fd_array,
+ attr->fd_array_cnt);
+ env->fd_array_raw = fd_array;
return 0;
}
@@ -19769,6 +19917,146 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
return 0;
}
+static enum bpf_sig_keyring bpf_classify_keyring(s32 keyring_id)
+{
+ switch (keyring_id) {
+ case 0:
+ return BPF_SIG_KEYRING_BUILTIN;
+ case (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING:
+ return BPF_SIG_KEYRING_SECONDARY;
+ case (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING:
+ return BPF_SIG_KEYRING_PLATFORM;
+ default:
+ return BPF_SIG_KEYRING_USER;
+ }
+}
+
+/*
+ * Verify the PKCS#7 signature of a loaded program. Called from bpf_check()
+ * once the program's metadata maps have been resolved into used_maps, so
+ * the exact maps folded into the signature are the ones the program binds.
+ *
+ * The signature covers the instructions followed by the frozen contents of
+ * each map, in @maps order: insns || map_0 || map_1 || [...]. On success the
+ * verdict and keyring info are recorded on prog->aux.
+ */
+static int bpf_prog_verify_signature(struct bpf_verifier_env *env,
+ union bpf_attr *attr, bool is_kernel)
+{
+ bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
+ struct bpf_dynptr_kern sig_ptr, data_ptr;
+ struct bpf_prog *prog = env->prog;
+ struct bpf_map **maps = env->used_maps;
+ struct bpf_key *key = NULL;
+ void *sig, *data = NULL;
+ u32 map_cnt = env->used_map_cnt;
+ u32 i, off, insns_sz;
+ u64 data_sz;
+ int err = 0;
+
+ /*
+ * Don't attempt to use kmalloc_large or vmalloc for signatures.
+ * Practical signature for BPF program should be below this limit.
+ */
+ if (!attr->signature_size ||
+ attr->signature_size > KMALLOC_MAX_CACHE_SIZE)
+ return -EINVAL;
+ if (system_keyring_id_check(attr->keyring_id) == 0)
+ key = bpf_lookup_system_key(attr->keyring_id);
+ else
+ key = bpf_lookup_user_key(attr->keyring_id, 0);
+ if (!key) {
+ verbose(env, "cannot resolve signing keyring with keyring_id %d\n",
+ attr->keyring_id);
+ return -EINVAL;
+ }
+
+ sig = kvmemdup_bpfptr(usig, attr->signature_size);
+ if (IS_ERR(sig)) {
+ bpf_key_put(key);
+ return PTR_ERR(sig);
+ }
+
+ insns_sz = prog->len * sizeof(struct bpf_insn);
+ data_sz = insns_sz;
+ for (i = 0; i < map_cnt; i++) {
+ struct bpf_map *map = maps[i];
+
+ if (map->map_type != BPF_MAP_TYPE_ARRAY ||
+ !map->ops->map_direct_value_addr) {
+ verbose(env, "signed program metadata map '%s' must be an array\n",
+ map->name);
+ err = -EINVAL;
+ goto out;
+ }
+ if (!READ_ONCE(map->frozen)) {
+ verbose(env, "signed program metadata map '%s' must be frozen\n",
+ map->name);
+ err = -EPERM;
+ goto out;
+ }
+ if (bpf_map_write_active(map)) {
+ verbose(env, "signed program metadata map '%s' has active writers\n",
+ map->name);
+ err = -EBUSY;
+ goto out;
+ }
+ if (!map->excl_prog_sha) {
+ verbose(env, "signed program metadata map '%s' must be exclusive\n",
+ map->name);
+ err = -EPERM;
+ goto out;
+ }
+ data_sz += map->value_size;
+ }
+ if (bpf_dynptr_check_size(data_sz)) {
+ verbose(env, "signed payload too large: %llu bytes\n", data_sz);
+ err = -E2BIG;
+ goto out;
+ }
+ data = kvmalloc(data_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ if (!data) {
+ err = -ENOMEM;
+ goto out;
+ }
+ memcpy(data, prog->insnsi, insns_sz);
+ off = insns_sz;
+ for (i = 0; i < map_cnt; i++) {
+ struct bpf_map *map = maps[i];
+ u64 addr;
+
+ err = map->ops->map_direct_value_addr(map, &addr, 0);
+ if (err) {
+ verbose(env, "failed to read signed metadata map '%s': %d\n",
+ map->name, err);
+ goto out;
+ }
+ memcpy(data + off, (void *)(unsigned long)addr,
+ map->value_size);
+ off += map->value_size;
+ }
+
+ bpf_dynptr_init(&data_ptr, data, BPF_DYNPTR_TYPE_LOCAL, 0, data_sz);
+ bpf_dynptr_init(&sig_ptr, sig, BPF_DYNPTR_TYPE_LOCAL, 0,
+ attr->signature_size);
+
+ err = bpf_verify_pkcs7_signature((struct bpf_dynptr *)&data_ptr,
+ (struct bpf_dynptr *)&sig_ptr, key);
+ if (err) {
+ verbose(env, "signature verification failed: %d\n", err);
+ } else {
+ verbose(env, "signature verification passed\n");
+ prog->aux->sig.keyring_serial = bpf_key_serial(key);
+ prog->aux->sig.keyring_type = bpf_classify_keyring(attr->keyring_id);
+ prog->aux->sig.verdict = BPF_SIG_VERIFIED;
+ }
+out:
+ kvfree(data);
+ bpf_key_put(key);
+ kvfree(sig);
+ return err;
+}
+
int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
struct bpf_log_attr *attr_log)
{
@@ -19791,18 +20079,6 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
return -ENOMEM;
env->bt.env = env;
-
- len = (*prog)->len;
- env->insn_aux_data =
- vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
- ret = -ENOMEM;
- if (!env->insn_aux_data)
- goto err_free_env;
- for (i = 0; i < len; i++)
- env->insn_aux_data[i].orig_idx = i;
- env->succ = bpf_iarray_realloc(NULL, 2);
- if (!env->succ)
- goto err_free_env;
env->prog = *prog;
env->ops = bpf_verifier_ops[env->prog->type];
@@ -19811,22 +20087,52 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
env->bypass_spec_v1 = bpf_bypass_spec_v1(env->prog->aux->token);
env->bypass_spec_v4 = bpf_bypass_spec_v4(env->prog->aux->token);
env->bpf_capable = is_priv = bpf_token_capable(env->prog->aux->token, CAP_BPF);
-
- bpf_get_btf_vmlinux();
-
- /* grab the mutex to protect few globals used by verifier */
- if (!is_priv)
- mutex_lock(&bpf_verifier_lock);
+ env->signature = attr->signature;
/* user could have requested verbose verifier output
* and supplied buffer to store the verification trace
*/
ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
if (ret)
- goto err_unlock;
+ goto err_free_env;
+ if (env->signature) {
+ ret = bpf_prog_calc_tag(env->prog);
+ if (ret < 0)
+ goto err_prep;
+ }
ret = process_fd_array(env, attr, uattr);
if (ret)
+ goto err_prep;
+
+ if (env->signature) {
+ ret = bpf_prog_verify_signature(env, attr, uattr.is_kernel);
+ if (ret)
+ goto err_prep;
+ }
+
+ ret = security_bpf_prog_load(env->prog, attr, env->prog->aux->token,
+ uattr.is_kernel);
+ if (ret)
+ goto err_prep;
+
+ bpf_get_btf_vmlinux();
+
+ /* Serialize verification of unprivileged programs. */
+ if (!is_priv)
+ mutex_lock(&bpf_verifier_lock);
+
+ len = env->prog->len;
+ env->insn_aux_data =
+ __vmalloc(array_size(sizeof(struct bpf_insn_aux_data), len),
+ GFP_KERNEL_ACCOUNT | __GFP_ZERO);
+ ret = -ENOMEM;
+ if (!env->insn_aux_data)
+ goto skip_full_check;
+ for (i = 0; i < len; i++)
+ env->insn_aux_data[i].orig_idx = i;
+ env->succ = bpf_iarray_realloc(NULL, 2);
+ if (!env->succ)
goto skip_full_check;
mark_verifier_state_clean(env);
@@ -20050,17 +20356,26 @@ err_release_maps:
*prog = env->prog;
module_put(env->attach_btf_mod);
-err_unlock:
if (!is_priv)
mutex_unlock(&bpf_verifier_lock);
- bpf_clear_insn_aux_data(env, 0, env->prog->len);
+ goto err_free_env;
+err_prep:
+ err = bpf_log_attr_finalize(attr_log, &env->log);
+ if (err)
+ ret = err;
+ release_insn_arrays(env);
+ release_maps(env);
+ release_btfs(env);
err_free_env:
+ if (env->insn_aux_data)
+ bpf_clear_insn_aux_data(env, 0, env->prog->len);
+ vfree(env->insn_aux_data);
+ kvfree(env->fd_array);
bpf_stack_liveness_free(env);
kvfree(env->cfg.insn_postorder);
kvfree(env->scc_info);
kvfree(env->succ);
kvfree(env->gotox_tmp_buf);
- vfree(env->insn_aux_data);
kvfree(env);
return ret;
}