diff options
Diffstat (limited to 'arch')
| -rw-r--r-- | arch/arc/net/bpf_jit_core.c | 39 | ||||
| -rw-r--r-- | arch/arm/net/bpf_jit_32.c | 41 | ||||
| -rw-r--r-- | arch/arm64/net/bpf_jit_comp.c | 72 | ||||
| -rw-r--r-- | arch/loongarch/net/bpf_jit.c | 59 | ||||
| -rw-r--r-- | arch/mips/net/bpf_jit_comp.c | 20 | ||||
| -rw-r--r-- | arch/parisc/net/bpf_jit_core.c | 73 | ||||
| -rw-r--r-- | arch/powerpc/net/bpf_jit_comp.c | 72 | ||||
| -rw-r--r-- | arch/riscv/net/bpf_jit_core.c | 61 | ||||
| -rw-r--r-- | arch/s390/net/bpf_jit_comp.c | 59 | ||||
| -rw-r--r-- | arch/sparc/net/bpf_jit_comp_64.c | 61 | ||||
| -rw-r--r-- | arch/x86/net/bpf_jit_comp.c | 43 | ||||
| -rw-r--r-- | arch/x86/net/bpf_jit_comp32.c | 33 |
12 files changed, 188 insertions, 445 deletions
diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c index 1421eeced0f5..973ceae48675 100644 --- a/arch/arc/net/bpf_jit_core.c +++ b/arch/arc/net/bpf_jit_core.c @@ -79,7 +79,6 @@ struct arc_jit_data { * The JIT pertinent context that is used by different functions. * * prog: The current eBPF program being handled. - * orig_prog: The original eBPF program before any possible change. * jit: The JIT buffer and its length. * bpf_header: The JITed program header. "jit.buf" points inside it. * emit: If set, opcodes are written to memory; else, a dry-run. @@ -94,12 +93,10 @@ struct arc_jit_data { * need_extra_pass: A forecast if an "extra_pass" will occur. * is_extra_pass: Indicates if the current pass is an extra pass. * user_bpf_prog: True, if VM opcodes come from a real program. - * blinded: True if "constant blinding" step returned a new "prog". * success: Indicates if the whole JIT went OK. */ struct jit_context { struct bpf_prog *prog; - struct bpf_prog *orig_prog; struct jit_buffer jit; struct bpf_binary_header *bpf_header; bool emit; @@ -114,7 +111,6 @@ struct jit_context { bool need_extra_pass; bool is_extra_pass; bool user_bpf_prog; - bool blinded; bool success; }; @@ -161,13 +157,7 @@ static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog) { memset(ctx, 0, sizeof(*ctx)); - ctx->orig_prog = prog; - - /* If constant blinding was requested but failed, scram. */ - ctx->prog = bpf_jit_blind_constants(prog); - if (IS_ERR(ctx->prog)) - return PTR_ERR(ctx->prog); - ctx->blinded = (ctx->prog != ctx->orig_prog); + ctx->prog = prog; /* If the verifier doesn't zero-extend, then we have to do it. */ ctx->do_zext = !ctx->prog->aux->verifier_zext; @@ -214,14 +204,6 @@ static inline void maybe_free(struct jit_context *ctx, void **mem) */ static void jit_ctx_cleanup(struct jit_context *ctx) { - if (ctx->blinded) { - /* if all went well, release the orig_prog. */ - if (ctx->success) - bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog); - else - bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog); - } - maybe_free(ctx, (void **)&ctx->bpf2insn); maybe_free(ctx, (void **)&ctx->jit_data); @@ -229,12 +211,19 @@ static void jit_ctx_cleanup(struct jit_context *ctx) ctx->bpf2insn_valid = false; /* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */ - if (!ctx->success && ctx->bpf_header) { - bpf_jit_binary_free(ctx->bpf_header); - ctx->bpf_header = NULL; - ctx->jit.buf = NULL; - ctx->jit.index = 0; - ctx->jit.len = 0; + if (!ctx->success) { + if (ctx->bpf_header) { + bpf_jit_binary_free(ctx->bpf_header); + ctx->bpf_header = NULL; + ctx->jit.buf = NULL; + ctx->jit.index = 0; + ctx->jit.len = 0; + } + if (ctx->is_extra_pass) { + ctx->prog->bpf_func = NULL; + ctx->prog->jited = 0; + ctx->prog->jited_len = 0; + } } ctx->emit = false; diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c index deeb8f292454..e6b1bb2de627 100644 --- a/arch/arm/net/bpf_jit_32.c +++ b/arch/arm/net/bpf_jit_32.c @@ -2144,9 +2144,7 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - struct bpf_prog *tmp, *orig_prog = prog; struct bpf_binary_header *header; - bool tmp_blinded = false; struct jit_ctx ctx; unsigned int tmp_idx; unsigned int image_size; @@ -2156,20 +2154,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * the interpreter. */ if (!prog->jit_requested) - return orig_prog; - - /* If constant blinding was enabled and we failed during blinding - * then we must fall back to the interpreter. Otherwise, we save - * the new JITed code. - */ - tmp = bpf_jit_blind_constants(prog); - - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; memset(&ctx, 0, sizeof(ctx)); ctx.prog = prog; @@ -2179,10 +2164,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * we must fall back to the interpreter */ ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL); - if (ctx.offsets == NULL) { - prog = orig_prog; - goto out; - } + if (ctx.offsets == NULL) + return prog; /* 1) fake pass to find in the length of the JITed code, * to compute ctx->offsets and other context variables @@ -2194,10 +2177,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * being successful in the second pass, so just fall back * to the interpreter. */ - if (build_body(&ctx)) { - prog = orig_prog; + if (build_body(&ctx)) goto out_off; - } tmp_idx = ctx.idx; build_prologue(&ctx); @@ -2213,10 +2194,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.idx += ctx.imm_count; if (ctx.imm_count) { ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL); - if (ctx.imms == NULL) { - prog = orig_prog; + if (ctx.imms == NULL) goto out_off; - } } #else /* there's nothing about the epilogue on ARMv7 */ @@ -2238,10 +2217,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) /* Not able to allocate memory for the structure then * we must fall back to the interpretation */ - if (header == NULL) { - prog = orig_prog; + if (header == NULL) goto out_imms; - } /* 2.) Actual pass to generate final JIT code */ ctx.target = (u32 *) image_ptr; @@ -2278,16 +2255,12 @@ out_imms: #endif out_off: kfree(ctx.offsets); -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); + return prog; out_free: image_ptr = NULL; bpf_jit_binary_free(header); - prog = orig_prog; goto out_imms; } diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 524b67c0867e..d310d1c35192 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2003,14 +2003,12 @@ struct arm64_jit_data { struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { int image_size, prog_size, extable_size, extable_align, extable_offset; - struct bpf_prog *tmp, *orig_prog = prog; struct bpf_binary_header *header; struct bpf_binary_header *ro_header = NULL; struct arm64_jit_data *jit_data; void __percpu *priv_stack_ptr = NULL; bool was_classic = bpf_prog_was_classic(prog); int priv_stack_alloc_sz; - bool tmp_blinded = false; bool extra_pass = false; struct jit_ctx ctx; u8 *image_ptr; @@ -2019,26 +2017,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) int exentry_idx; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - /* If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - prog = orig_prog; - goto out; - } + if (!jit_data) + return prog; prog->aux->jit_data = jit_data; } priv_stack_ptr = prog->aux->priv_stack_ptr; @@ -2050,10 +2035,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) + 2 * PRIV_STACK_GUARD_SZ; priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL); - if (!priv_stack_ptr) { - prog = orig_prog; + if (!priv_stack_ptr) goto out_priv_stack; - } priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz); prog->aux->priv_stack_ptr = priv_stack_ptr; @@ -2073,10 +2056,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.prog = prog; ctx.offset = kvzalloc_objs(int, prog->len + 1); - if (ctx.offset == NULL) { - prog = orig_prog; + if (ctx.offset == NULL) goto out_off; - } ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena); @@ -2089,15 +2070,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * BPF line info needs ctx->offset[i] to be the offset of * instruction[i] in jited image, so build prologue first. */ - if (build_prologue(&ctx, was_classic)) { - prog = orig_prog; + if (build_prologue(&ctx, was_classic)) goto out_off; - } - if (build_body(&ctx, extra_pass)) { - prog = orig_prog; + if (build_body(&ctx, extra_pass)) goto out_off; - } ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx, was_classic); @@ -2115,10 +2092,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u64), &header, &image_ptr, jit_fill_hole); - if (!ro_header) { - prog = orig_prog; + if (!ro_header) goto out_off; - } /* Pass 2: Determine jited position and result for each instruction */ @@ -2146,10 +2121,8 @@ skip_init_ctx: /* Dont write body instructions to memory for now */ ctx.write = false; - if (build_body(&ctx, extra_pass)) { - prog = orig_prog; + if (build_body(&ctx, extra_pass)) goto out_free_hdr; - } ctx.epilogue_offset = ctx.idx; ctx.exentry_idx = exentry_idx; @@ -2158,19 +2131,15 @@ skip_init_ctx: /* Pass 3: Adjust jump offset and write final image */ if (build_body(&ctx, extra_pass) || - WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) { - prog = orig_prog; + WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) goto out_free_hdr; - } build_epilogue(&ctx, was_classic); build_plt(&ctx); /* Extra pass to validate JITed code. */ - if (validate_ctx(&ctx)) { - prog = orig_prog; + if (validate_ctx(&ctx)) goto out_free_hdr; - } /* update the real prog size */ prog_size = sizeof(u32) * ctx.idx; @@ -2187,16 +2156,13 @@ skip_init_ctx: if (extra_pass && ctx.idx > jit_data->ctx.idx) { pr_err_once("multi-func JIT bug %d > %d\n", ctx.idx, jit_data->ctx.idx); - prog->bpf_func = NULL; - prog->jited = 0; - prog->jited_len = 0; goto out_free_hdr; } if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { - /* ro_header has been freed */ + /* ro_header and header has been freed */ ro_header = NULL; - prog = orig_prog; - goto out_off; + header = NULL; + goto out_free_hdr; } } else { jit_data->ctx = ctx; @@ -2233,13 +2199,15 @@ out_priv_stack: kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); + return prog; out_free_hdr: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } if (header) { bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 9cb796e16379..fcc8c0c29fb0 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1922,43 +1922,26 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - bool tmp_blinded = false, extra_pass = false; + bool extra_pass = false; u8 *image_ptr, *ro_image_ptr; int image_size, prog_size, extable_size; struct jit_ctx ctx; struct jit_data *jit_data; struct bpf_binary_header *header; struct bpf_binary_header *ro_header; - struct bpf_prog *tmp, *orig_prog = prog; /* * If BPF JIT was not enabled then we must fall back to * the interpreter. */ if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. Otherwise, we save - * the new JITed code. - */ - if (IS_ERR(tmp)) - return orig_prog; - - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - prog = orig_prog; - goto out; - } + if (!jit_data) + return prog; prog->aux->jit_data = jit_data; } if (jit_data->ctx.offset) { @@ -1978,17 +1961,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL); - if (ctx.offset == NULL) { - prog = orig_prog; + if (ctx.offset == NULL) goto out_offset; - } /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */ build_prologue(&ctx); - if (build_body(&ctx, extra_pass)) { - prog = orig_prog; + if (build_body(&ctx, extra_pass)) goto out_offset; - } ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx); @@ -2004,10 +1983,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) /* Now we know the size of the structure to make */ ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32), &header, &image_ptr, jit_fill_hole); - if (!ro_header) { - prog = orig_prog; + if (!ro_header) goto out_offset; - } /* 2. Now, the actual pass to generate final JIT code */ /* @@ -2027,17 +2004,13 @@ skip_init_ctx: ctx.num_exentries = 0; build_prologue(&ctx); - if (build_body(&ctx, extra_pass)) { - prog = orig_prog; + if (build_body(&ctx, extra_pass)) goto out_free; - } build_epilogue(&ctx); /* 3. Extra pass to validate JITed code */ - if (validate_ctx(&ctx)) { - prog = orig_prog; + if (validate_ctx(&ctx)) goto out_free; - } /* And we're done */ if (bpf_jit_enable > 1) @@ -2050,9 +2023,9 @@ skip_init_ctx: goto out_free; } if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) { - /* ro_header has been freed */ + /* ro_header and header have been freed */ ro_header = NULL; - prog = orig_prog; + header = NULL; goto out_free; } /* @@ -2084,13 +2057,15 @@ out_offset: prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); - return prog; out_free: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } + if (header) { bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size)); bpf_jit_binary_pack_free(ro_header, header); diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c index e355dfca4400..d2b6c955f18e 100644 --- a/arch/mips/net/bpf_jit_comp.c +++ b/arch/mips/net/bpf_jit_comp.c @@ -911,10 +911,8 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - struct bpf_prog *tmp, *orig_prog = prog; struct bpf_binary_header *header = NULL; struct jit_context ctx; - bool tmp_blinded = false; unsigned int tmp_idx; unsigned int image_size; u8 *image_ptr; @@ -925,19 +923,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) * the interpreter. */ if (!prog->jit_requested) - return orig_prog; - /* - * If constant blinding was enabled and we failed during blinding - * then we must fall back to the interpreter. Otherwise, we save - * the new JITed code. - */ - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; memset(&ctx, 0, sizeof(ctx)); ctx.program = prog; @@ -1025,14 +1011,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) prog->jited_len = image_size; out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); kfree(ctx.descriptors); return prog; out_err: - prog = orig_prog; if (header) bpf_jit_binary_free(header); goto out; diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c index a5eb6b51e27a..35dca372b5df 100644 --- a/arch/parisc/net/bpf_jit_core.c +++ b/arch/parisc/net/bpf_jit_core.c @@ -44,30 +44,19 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; - bool tmp_blinded = false, extra_pass = false; - struct bpf_prog *tmp, *orig_prog = prog; + bool extra_pass = false; int pass = 0, prev_ninsns = 0, prologue_len, i; struct hppa_jit_data *jit_data; struct hppa_jit_context *ctx; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - prog = orig_prog; - goto out; - } + if (!jit_data) + return prog; prog->aux->jit_data = jit_data; } @@ -81,10 +70,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx->prog = prog; ctx->offset = kzalloc_objs(int, prog->len); - if (!ctx->offset) { - prog = orig_prog; - goto out_offset; - } + if (!ctx->offset) + goto out_err; for (i = 0; i < prog->len; i++) { prev_ninsns += 20; ctx->offset[i] = prev_ninsns; @@ -93,10 +80,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) for (i = 0; i < NR_JIT_ITERATIONS; i++) { pass++; ctx->ninsns = 0; - if (build_body(ctx, extra_pass, ctx->offset)) { - prog = orig_prog; - goto out_offset; - } + if (build_body(ctx, extra_pass, ctx->offset)) + goto out_err; ctx->body_len = ctx->ninsns; bpf_jit_build_prologue(ctx); ctx->prologue_len = ctx->ninsns - ctx->body_len; @@ -116,10 +101,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) &jit_data->image, sizeof(long), bpf_fill_ill_insns); - if (!jit_data->header) { - prog = orig_prog; - goto out_offset; - } + if (!jit_data->header) + goto out_err; ctx->insns = (u32 *)jit_data->image; /* @@ -134,8 +117,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) pr_err("bpf-jit: image did not converge in <%d passes!\n", i); if (jit_data->header) bpf_jit_binary_free(jit_data->header); - prog = orig_prog; - goto out_offset; + goto out_err; } if (extable_size) @@ -148,8 +130,7 @@ skip_init_ctx: bpf_jit_build_prologue(ctx); if (build_body(ctx, extra_pass, NULL)) { bpf_jit_binary_free(jit_data->header); - prog = orig_prog; - goto out_offset; + goto out_err; } bpf_jit_build_epilogue(ctx); @@ -160,20 +141,19 @@ skip_init_ctx: { extern int machine_restart(char *); machine_restart(""); } } + if (!prog->is_func || extra_pass) { + if (bpf_jit_binary_lock_ro(jit_data->header)) { + bpf_jit_binary_free(jit_data->header); + goto out_err; + } + bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); + } + prog->bpf_func = (void *)ctx->insns; prog->jited = 1; prog->jited_len = prog_size; - bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns); - if (!prog->is_func || extra_pass) { - if (bpf_jit_binary_lock_ro(jit_data->header)) { - bpf_jit_binary_free(jit_data->header); - prog->bpf_func = NULL; - prog->jited = 0; - prog->jited_len = 0; - goto out_offset; - } prologue_len = ctx->epilogue_offset - ctx->body_len; for (i = 0; i < prog->len; i++) ctx->offset[i] += prologue_len; @@ -183,14 +163,19 @@ out_offset: kfree(jit_data); prog->aux->jit_data = NULL; } -out: + if (HPPA_JIT_REBOOT) { extern int machine_restart(char *); machine_restart(""); } - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); return prog; + +out_err: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } + goto out_offset; } u64 hppa_div64(u64 div, u64 divisor) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 50103b3794fb..2bae4699e78f 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -177,9 +177,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) void __percpu *priv_stack_ptr = NULL; struct bpf_binary_header *fhdr = NULL; struct bpf_binary_header *hdr = NULL; - struct bpf_prog *org_fp = fp; - struct bpf_prog *tmp_fp = NULL; - bool bpf_blinded = false; bool extra_pass = false; u8 *fimage = NULL; u32 *fcode_base = NULL; @@ -187,24 +184,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) u32 fixup_len; if (!fp->jit_requested) - return org_fp; - - tmp_fp = bpf_jit_blind_constants(org_fp); - if (IS_ERR(tmp_fp)) - return org_fp; - - if (tmp_fp != org_fp) { - bpf_blinded = true; - fp = tmp_fp; - } + return fp; jit_data = fp->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - fp = org_fp; - goto out; - } + if (!jit_data) + return fp; fp->aux->jit_data = jit_data; } @@ -219,10 +205,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) + 2 * PRIV_STACK_GUARD_SZ; priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL); - if (!priv_stack_ptr) { - fp = org_fp; + if (!priv_stack_ptr) goto out_priv_stack; - } priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size); fp->aux->priv_stack_ptr = priv_stack_ptr; @@ -249,10 +233,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) } addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); - if (addrs == NULL) { - fp = org_fp; - goto out_addrs; - } + if (addrs == NULL) + goto out_err; memset(&cgctx, 0, sizeof(struct codegen_context)); bpf_jit_init_reg_mapping(&cgctx); @@ -279,11 +261,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) } /* Scouting faux-generate pass 0 */ - if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) /* We hit something illegal or unsupported. */ - fp = org_fp; - goto out_addrs; - } + goto out_err; /* * If we have seen a tail call, we need a second pass. @@ -294,10 +274,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) */ if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { cgctx.idx = 0; - if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { - fp = org_fp; - goto out_addrs; - } + if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) + goto out_err; } bpf_jit_realloc_regs(&cgctx); @@ -318,10 +296,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image, bpf_jit_fill_ill_insns); - if (!fhdr) { - fp = org_fp; - goto out_addrs; - } + if (!fhdr) + goto out_err; if (extable_len) fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; @@ -340,8 +316,7 @@ skip_init_ctx: extra_pass)) { bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); bpf_jit_binary_pack_free(fhdr, hdr); - fp = org_fp; - goto out_addrs; + goto out_err; } bpf_jit_build_epilogue(code_base, &cgctx); @@ -363,15 +338,16 @@ skip_init_ctx: ((u64 *)image)[1] = local_paca->kernel_toc; #endif + if (!fp->is_func || extra_pass) { + if (bpf_jit_binary_pack_finalize(fhdr, hdr)) + goto out_err; + } + fp->bpf_func = (void *)fimage; fp->jited = 1; fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE; if (!fp->is_func || extra_pass) { - if (bpf_jit_binary_pack_finalize(fhdr, hdr)) { - fp = org_fp; - goto out_addrs; - } bpf_prog_fill_jited_linfo(fp, addrs); /* * On ABI V1, executable code starts after the function @@ -398,11 +374,15 @@ out_priv_stack: jit_data->hdr = hdr; } -out: - if (bpf_blinded) - bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); - return fp; + +out_err: + if (extra_pass) { + fp->bpf_func = NULL; + fp->jited = 0; + fp->jited_len = 0; + } + goto out_addrs; } /* diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index f7fd4afc3ca3..36f0aea8096d 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -44,29 +44,19 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { unsigned int prog_size = 0, extable_size = 0; - bool tmp_blinded = false, extra_pass = false; - struct bpf_prog *tmp, *orig_prog = prog; + bool extra_pass = false; int pass = 0, prev_ninsns = 0, i; struct rv_jit_data *jit_data; struct rv_jit_context *ctx; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); if (!jit_data) { - prog = orig_prog; - goto out; + return prog; } prog->aux->jit_data = jit_data; } @@ -83,15 +73,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena); ctx->prog = prog; ctx->offset = kzalloc_objs(int, prog->len); - if (!ctx->offset) { - prog = orig_prog; + if (!ctx->offset) goto out_offset; - } - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_offset; - } for (i = 0; i < prog->len; i++) { prev_ninsns += 32; @@ -105,10 +91,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); ctx->prologue_len = ctx->ninsns; - if (build_body(ctx, extra_pass, ctx->offset)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, ctx->offset)) goto out_offset; - } ctx->epilogue_offset = ctx->ninsns; bpf_jit_build_epilogue(ctx); @@ -126,10 +110,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) &jit_data->ro_image, sizeof(u32), &jit_data->header, &jit_data->image, bpf_fill_ill_insns); - if (!jit_data->ro_header) { - prog = orig_prog; + if (!jit_data->ro_header) goto out_offset; - } /* * Use the image(RW) for writing the JITed instructions. But also save @@ -150,7 +132,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) if (i == NR_JIT_ITERATIONS) { pr_err("bpf-jit: image did not converge in <%d passes!\n", i); - prog = orig_prog; goto out_free_hdr; } @@ -163,26 +144,27 @@ skip_init_ctx: ctx->nexentries = 0; bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); - if (build_body(ctx, extra_pass, NULL)) { - prog = orig_prog; + if (build_body(ctx, extra_pass, NULL)) goto out_free_hdr; - } bpf_jit_build_epilogue(ctx); if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); - prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); - prog->jited = 1; - prog->jited_len = prog_size - cfi_get_offset(); - if (!prog->is_func || extra_pass) { if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) { /* ro_header has been freed */ jit_data->ro_header = NULL; - prog = orig_prog; - goto out_offset; + jit_data->header = NULL; + goto out_free_hdr; } + } + + prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); + prog->jited = 1; + prog->jited_len = prog_size - cfi_get_offset(); + + if (!prog->is_func || extra_pass) { for (i = 0; i < prog->len; i++) ctx->offset[i] = ninsns_rvoff(ctx->offset[i]); bpf_prog_fill_jited_linfo(prog, ctx->offset); @@ -191,14 +173,15 @@ out_offset: kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); return prog; out_free_hdr: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } if (jit_data->header) { bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size, sizeof(jit_data->header->size)); diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index d08d159b6319..2dfc279b1be2 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2314,36 +2314,20 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, */ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) { - struct bpf_prog *tmp, *orig_fp = fp; struct bpf_binary_header *header; struct s390_jit_data *jit_data; - bool tmp_blinded = false; bool extra_pass = false; struct bpf_jit jit; int pass; if (!fp->jit_requested) - return orig_fp; - - tmp = bpf_jit_blind_constants(fp); - /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_fp; - if (tmp != fp) { - tmp_blinded = true; - fp = tmp; - } + return fp; jit_data = fp->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - fp = orig_fp; - goto out; - } + if (!jit_data) + return fp; fp->aux->jit_data = jit_data; } if (jit_data->ctx.addrs) { @@ -2356,34 +2340,27 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) memset(&jit, 0, sizeof(jit)); jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); - if (jit.addrs == NULL) { - fp = orig_fp; - goto free_addrs; - } + if (jit.addrs == NULL) + goto out_err; /* * Three initial passes: * - 1/2: Determine clobbered registers * - 3: Calculate program size and addrs array */ for (pass = 1; pass <= 3; pass++) { - if (bpf_jit_prog(&jit, fp, extra_pass)) { - fp = orig_fp; - goto free_addrs; - } + if (bpf_jit_prog(&jit, fp, extra_pass)) + goto out_err; } /* * Final pass: Allocate and generate program */ header = bpf_jit_alloc(&jit, fp); - if (!header) { - fp = orig_fp; - goto free_addrs; - } + if (!header) + goto out_err; skip_init_ctx: if (bpf_jit_prog(&jit, fp, extra_pass)) { bpf_jit_binary_free(header); - fp = orig_fp; - goto free_addrs; + goto out_err; } if (bpf_jit_enable > 1) { bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); @@ -2392,8 +2369,7 @@ skip_init_ctx: if (!fp->is_func || extra_pass) { if (bpf_jit_binary_lock_ro(header)) { bpf_jit_binary_free(header); - fp = orig_fp; - goto free_addrs; + goto out_err; } } else { jit_data->header = header; @@ -2411,11 +2387,16 @@ free_addrs: kfree(jit_data); fp->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(fp, fp == orig_fp ? - tmp : orig_fp); + return fp; + +out_err: + if (extra_pass) { + fp->bpf_func = NULL; + fp->jited = 0; + fp->jited_len = 0; + } + goto free_addrs; } bool bpf_jit_supports_kfunc_call(void) diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c index b23d1c645ae5..e83e29137566 100644 --- a/arch/sparc/net/bpf_jit_comp_64.c +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -1479,37 +1479,22 @@ struct sparc64_jit_data { struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { - struct bpf_prog *tmp, *orig_prog = prog; struct sparc64_jit_data *jit_data; struct bpf_binary_header *header; u32 prev_image_size, image_size; - bool tmp_blinded = false; bool extra_pass = false; struct jit_ctx ctx; u8 *image_ptr; int pass, i; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - /* If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - prog = orig_prog; - goto out; - } + if (!jit_data) + return prog; prog->aux->jit_data = jit_data; } if (jit_data->ctx.offset) { @@ -1527,10 +1512,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.prog = prog; ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL); - if (ctx.offset == NULL) { - prog = orig_prog; - goto out_off; - } + if (ctx.offset == NULL) + goto out_err; /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook * the offset array so that we converge faster. @@ -1543,10 +1526,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) ctx.idx = 0; build_prologue(&ctx); - if (build_body(&ctx)) { - prog = orig_prog; - goto out_off; - } + if (build_body(&ctx)) + goto out_err; build_epilogue(&ctx); if (bpf_jit_enable > 1) @@ -1569,10 +1550,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) image_size = sizeof(u32) * ctx.idx; header = bpf_jit_binary_alloc(image_size, &image_ptr, sizeof(u32), jit_fill_hole); - if (header == NULL) { - prog = orig_prog; - goto out_off; - } + if (header == NULL) + goto out_err; ctx.image = (u32 *)image_ptr; skip_init_ctx: @@ -1582,8 +1561,7 @@ skip_init_ctx: if (build_body(&ctx)) { bpf_jit_binary_free(header); - prog = orig_prog; - goto out_off; + goto out_err; } build_epilogue(&ctx); @@ -1592,8 +1570,7 @@ skip_init_ctx: pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n", prev_image_size, ctx.idx * 4); bpf_jit_binary_free(header); - prog = orig_prog; - goto out_off; + goto out_err; } if (bpf_jit_enable > 1) @@ -1604,8 +1581,7 @@ skip_init_ctx: if (!prog->is_func || extra_pass) { if (bpf_jit_binary_lock_ro(header)) { bpf_jit_binary_free(header); - prog = orig_prog; - goto out_off; + goto out_err; } } else { jit_data->ctx = ctx; @@ -1624,9 +1600,14 @@ out_off: kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); + return prog; + +out_err: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } + goto out_off; } diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index e9b78040d703..77d00a8dec87 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3717,13 +3717,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_binary_header *rw_header = NULL; struct bpf_binary_header *header = NULL; - struct bpf_prog *tmp, *orig_prog = prog; void __percpu *priv_stack_ptr = NULL; struct x64_jit_data *jit_data; int priv_stack_alloc_sz; int proglen, oldproglen = 0; struct jit_context ctx = {}; - bool tmp_blinded = false; bool extra_pass = false; bool padding = false; u8 *rw_image = NULL; @@ -3733,27 +3731,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) int i; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc_obj(*jit_data); - if (!jit_data) { - prog = orig_prog; - goto out; - } + if (!jit_data) + return prog; prog->aux->jit_data = jit_data; } priv_stack_ptr = prog->aux->priv_stack_ptr; @@ -3765,10 +3749,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) + 2 * PRIV_STACK_GUARD_SZ; priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL); - if (!priv_stack_ptr) { - prog = orig_prog; + if (!priv_stack_ptr) goto out_priv_stack; - } priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz); prog->aux->priv_stack_ptr = priv_stack_ptr; @@ -3786,10 +3768,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) goto skip_init_addrs; } addrs = kvmalloc_objs(*addrs, prog->len + 1); - if (!addrs) { - prog = orig_prog; + if (!addrs) goto out_addrs; - } /* * Before first pass, make a rough estimation of addrs[] @@ -3820,8 +3800,6 @@ out_image: sizeof(rw_header->size)); bpf_jit_binary_pack_free(header, rw_header); } - /* Fall back to interpreter mode */ - prog = orig_prog; if (extra_pass) { prog->bpf_func = NULL; prog->jited = 0; @@ -3852,10 +3830,8 @@ out_image: header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size, &image, align, &rw_header, &rw_image, jit_fill_hole); - if (!header) { - prog = orig_prog; + if (!header) goto out_addrs; - } prog->aux->extable = (void *) image + roundup(proglen, align); } oldproglen = proglen; @@ -3908,8 +3884,6 @@ out_image: prog->bpf_func = (void *)image + cfi_get_offset(); prog->jited = 1; prog->jited_len = proglen - cfi_get_offset(); - } else { - prog = orig_prog; } if (!image || !prog->is_func || extra_pass) { @@ -3925,10 +3899,7 @@ out_priv_stack: kfree(jit_data); prog->aux->jit_data = NULL; } -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); + return prog; } diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index dda423025c3d..5f259577614a 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -2521,35 +2521,19 @@ bool bpf_jit_needs_zext(void) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_binary_header *header = NULL; - struct bpf_prog *tmp, *orig_prog = prog; int proglen, oldproglen = 0; struct jit_context ctx = {}; - bool tmp_blinded = false; u8 *image = NULL; int *addrs; int pass; int i; if (!prog->jit_requested) - return orig_prog; - - tmp = bpf_jit_blind_constants(prog); - /* - * If blinding was requested and we failed during blinding, - * we must fall back to the interpreter. - */ - if (IS_ERR(tmp)) - return orig_prog; - if (tmp != prog) { - tmp_blinded = true; - prog = tmp; - } + return prog; addrs = kmalloc_objs(*addrs, prog->len); - if (!addrs) { - prog = orig_prog; - goto out; - } + if (!addrs) + return prog; /* * Before first pass, make a rough estimation of addrs[] @@ -2574,7 +2558,6 @@ out_image: image = NULL; if (header) bpf_jit_binary_free(header); - prog = orig_prog; goto out_addrs; } if (image) { @@ -2588,10 +2571,8 @@ out_image: if (proglen == oldproglen) { header = bpf_jit_binary_alloc(proglen, &image, 1, jit_fill_hole); - if (!header) { - prog = orig_prog; + if (!header) goto out_addrs; - } } oldproglen = proglen; cond_resched(); @@ -2604,16 +2585,10 @@ out_image: prog->bpf_func = (void *)image; prog->jited = 1; prog->jited_len = proglen; - } else { - prog = orig_prog; } out_addrs: kfree(addrs); -out: - if (tmp_blinded) - bpf_jit_prog_release_other(prog, prog == orig_prog ? - tmp : orig_prog); return prog; } |