diff options
Diffstat (limited to 'include/linux/bpf_verifier.h')
| -rw-r--r-- | include/linux/bpf_verifier.h | 83 |
1 files changed, 54 insertions, 29 deletions
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 54cb6d3d0653..7baeb831281f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -379,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -527,6 +532,8 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; + /* The callee left with unwind_exit. Next insn is unwind_goto. */ + bool unwound; /* * First and last insn idx of this verifier state. @@ -708,9 +715,6 @@ struct bpf_loop_exit { int to; /* instruction outside the loop */ }; -/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ -#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) - struct bpf_loop { struct bpf_backedge backedges[MAX_BACKEDGES]; /* edges exiting from this loop, includes edges from nested loops */ @@ -770,44 +774,52 @@ struct bpf_insn_aux_data { u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - bool arena_scalar; /* ldx/stx/st/atomic through a number, it's an address in arena */ - bool bb_end; /* last instruction of a basic block */ - bool need_scev; - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter_<type>_next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ - u32 non_stack_access:1; /* instruction can access non-stack memory */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -946,6 +958,7 @@ struct bpf_subprog_info { s16 fastcall_stack_off; bool has_tail_call: 1; bool might_throw: 1; + bool can_unwind: 1; /* has unwind_exit */ bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; @@ -1060,7 +1073,6 @@ struct bpf_verifier_env { bool strict_alignment; /* perform strict pointer alignment checks */ bool test_state_freq; /* test verifier with different pruning frequency */ bool test_reg_invariants; /* fail verification on register invariants violations */ - bool arena_scalar; /* load and store through a number is arena access */ struct bpf_verifier_state *cur_state; /* current verifier state */ /* Search pruning optimization, array of list_heads for * lists of struct bpf_verifier_state_list. @@ -1087,6 +1099,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; @@ -1264,6 +1278,7 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1298,6 +1313,12 @@ static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int ins return env->insn_aux_data[insn_idx].force_checkpoint; } +static inline void clear_scalar_id(struct bpf_reg_state *reg) +{ + reg->id = 0; + reg->delta = 0; +} + static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].calls_callback = true; @@ -1376,6 +1397,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); @@ -1798,6 +1821,8 @@ struct bpf_call_arg_meta { u32 kfunc_flags; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later @@ -1876,7 +1901,7 @@ int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, const struct bpf_reg_state *reg_b); int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, - struct cnum64 range, u16 base, u16 step); + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, const struct bpf_reg_state *src); |