diff options
| author | Daniel Borkmann <daniel@iogearbox.net> | 2026-10-13 19:12:24 +0200 |
|---|---|---|
| committer | Kumar Kartikeya Dwivedi <memxor@gmail.com> | 2026-10-15 02:40:10 +0200 |
| commit | 181576ebd87d90aafeedbecf7a2dcc109dcff391 (patch) | |
| tree | dc976e2a940712e25a7a905f0ee0b90f9026c588 | |
| parent | 74309a9294a74975fee75e4d6259eef026495325 (diff) | |
| download | linux-next-181576ebd87d90aafeedbecf7a2dcc109dcff391.tar.gz linux-next-181576ebd87d90aafeedbecf7a2dcc109dcff391.zip | |
bpf, x86: Compare the PROBE_MEM address range with a shift
The PROBE_MEM check rejects src_reg + off - VSYSCALL_ADDR up to a limit
of TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR, which does not fit into a
32-bit immediate and therefore needed a movabs into r10 and a register
compare. Compare against the next power of two above the limit instead,
which is a shift by fls64(limit) followed by a jne on the result. The
addresses between the limit and that power of two are non-canonical on
every x86-64 layout, since user space ends at TASK_SIZE_MAX and the
kernel half starts far above 2^57, so they faulted and had the register
zeroed by the exception handler before. Now they are zeroed by the check
without taking the fault. The compare and branch stay two uops on Intel
and AMD, while the sequence shrinks from 15 to 6 bytes.
Before, for the compare of a 4-level paging kernel:
49 ba 00 00 a0 00 00 80 00 00 movabsq $0x800000a00000,%r10
4d 39 d3 cmpq %r10,%r11
77 05 ja 0x...
After:
49 c1 eb 30 shrq $0x30,%r11
75 05 jne 0x...
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20261013171230.523677-13-daniel@iogearbox.net
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
| -rw-r--r-- | arch/x86/net/bpf_jit_comp.c | 20 |
1 files changed, 10 insertions, 10 deletions
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 75194f9caa5a..564cead6eab7 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -2848,6 +2848,11 @@ populate_extable: * src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE * and * src_reg + insn->off < VSYSCALL_ADDR + * + * The check is done on src_reg + insn->off - VSYSCALL_ADDR + * against the next power of two above the limit. Addresses + * between the limit and that power of two are non-canonical + * and would fault anyway, so they get zeroed a bit earlier. */ u64 limit = TASK_SIZE_MAX + PAGE_SIZE - VSYSCALL_ADDR; @@ -2864,16 +2869,11 @@ populate_extable: emit_insn_suffix(&prog, src_reg, AUX_REG, insn->off - (s32)VSYSCALL_ADDR); - /* movabsq r10, limit */ - emit_mov_imm64(&prog, BPF_REG_AX, (long)limit >> 32, - (u32)(long)limit); - - /* cmp r10, r11 */ - maybe_emit_mod(&prog, AUX_REG, BPF_REG_AX, true); - EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); + /* shr r11, fls64(limit) */ + emit_shift_imm(&prog, BPF_RSH, AUX_REG, fls64(limit), true); - /* if unsigned '>', goto load */ - EMIT2(X86_JA, 0); + /* if not zero, goto load */ + EMIT2(X86_JNE, 0); end_of_jmp = prog; /* xor dst_reg, dst_reg */ @@ -2881,7 +2881,7 @@ populate_extable: /* jmp byte_after_ldx */ EMIT2(0xEB, 0); - /* populate jmp_offset for JAE above to jump to start_of_ldx */ + /* populate jmp_offset for JNE above to jump to start_of_ldx */ start_of_ldx = prog; end_of_jmp[-1] = start_of_ldx - end_of_jmp; } else if (!accesses_stack_only) { |