diff options
| author | Thomas Zimmermann <tzimmermann@suse.de> | 2026-09-07 13:44:24 +0200 |
|---|---|---|
| committer | Thomas Zimmermann <tzimmermann@suse.de> | 2026-09-07 13:44:24 +0200 |
| commit | 3ab04dcce43cd4fdaed94f08f4e6289be4921ea4 (patch) | |
| tree | 61229f2644f48301f9d09722e390295cb55a2cf4 /tools | |
| parent | a9f09b5ea0c3db1e2d4c0f8d3ebdd612d8aa0366 (diff) | |
| parent | df2908090cda368b01ff43709f51890076c56157 (diff) | |
| download | linux-3ab04dcce43cd4fdaed94f08f4e6289be4921ea4.tar.gz linux-3ab04dcce43cd4fdaed94f08f4e6289be4921ea4.zip | |
Merge drm/drm-next into drm-misc-next
Backmerging to get drm-misc-next up to v7.3-rc2. Requested for
commit 3a2c4d55e32a ("treewide: refresh kmalloc_obj() conversions").
Signed-off-by: Thomas Zimmermann <tzimmermann@suse.de>
Diffstat (limited to 'tools')
67 files changed, 3548 insertions, 118 deletions
diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 0ba307e78fe1..4d5993da9f94 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -27,6 +27,7 @@ CFLAGS += -fno-strict-aliasing ifeq ($(CC_NO_CLANG), 0) CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu + CLANG_TARGET_FLAGS_loongarch := loongarch64-linux-gnu CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu @@ -1142,6 +1143,7 @@ ifndef NO_RUST ifneq ($(CROSS_COMPILE),) RUST_TARGET_FLAGS_arm := arm-unknown-linux-gnueabi RUST_TARGET_FLAGS_arm64 := aarch64-unknown-linux-gnu + RUST_TARGET_FLAGS_loongarch := loongarch64-unknown-linux-gnu RUST_TARGET_FLAGS_m68k := m68k-unknown-linux-gnu RUST_TARGET_FLAGS_mips := mipsel-unknown-linux-gnu RUST_TARGET_FLAGS_powerpc := powerpc64le-unknown-linux-gnu diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h index 979d4cabfaf9..76f5e025e107 100644 --- a/tools/sched_ext/include/scx/common.bpf.h +++ b/tools/sched_ext/include/scx/common.bpf.h @@ -48,6 +48,7 @@ extern int LINUX_KERNEL_VERSION __kconfig; extern const char CONFIG_CC_VERSION_TEXT[64] __kconfig __weak; extern const char CONFIG_LOCALVERSION[64] __kconfig __weak; +extern bool CONFIG_PREEMPT_RCU __kconfig __weak; /* * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can @@ -97,6 +98,7 @@ s32 scx_bpf_pick_any_cpu_node(const cpumask_t *cpus_allowed, int node, u64 flags s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym; bool scx_bpf_task_running(const struct task_struct *p) __ksym; s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym; +struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym __weak; struct rq *scx_bpf_locked_rq(void) __ksym; struct task_struct *scx_bpf_cpu_curr(s32 cpu) __ksym __weak; struct task_struct *scx_bpf_tid_to_task(u64 tid) __ksym __weak; @@ -528,31 +530,102 @@ static __always_inline const struct cpumask *cast_mask(struct bpf_cpumask *mask) } /* + * True if the non-sleepable BPF trampoline prolog (__bpf_prog_enter) calls + * migrate_disable() for the current task. Recorded once by + * scx_lib_init_probe, an fentry program on bpf_scx_reg() that fires during + * the natural scheduler-attach call chain (auto-attached by scx_ops_attach!). + * + * Defaults to true (conservative). Over-reporting in is_migration_disabled() + * causes local-only dispatch, which is safe. Under-reporting can crash the + * scheduler, so we err high if the probe somehow fails to run. + */ +bool __scx_prolog_disables_migration __weak = true; + +/* + * scx_lib_init_probe - non-sleepable prolog probe. + * + * Attached to bpf_scx_reg(), the .reg callback in bpf_sched_ext_ops + * (kernel/sched/ext.c). The kernel's struct_ops machinery invokes + * bpf_scx_reg when userspace creates the scheduler link, before + * ops.init() fires. Its address is taken in the vtable, so the symbol + * is non-inlinable and has been stable since introduction. + * + * Entering via fentry runs us through __bpf_prog_enter -- the + * non-sleepable prolog that consumers of is_migration_disabled() live + * under. + * + * Loud warning: the prolog adds at most 1 to migration_disabled. + * Reading > 1 means something upstream in the + * bpf_struct_ops_link_create -> bpf_scx_reg path disabled migration + * before the prolog ran, invalidating the probe; audit and adjust. + */ +SEC("fentry/bpf_scx_reg") __weak +int scx_lib_init_probe(void *ctx) +{ + if (bpf_core_field_exists(((struct task_struct *)0)->migration_disabled)) { + const struct task_struct *p = bpf_get_current_task_btf(); + unsigned int md = p->migration_disabled; + + if (md > 1) + bpf_printk("scx_lib_init_probe: unexpected migration_disabled=%u " + "upstream of BPF prolog; probe result unreliable", + md); + + __scx_prolog_disables_migration = md > 0; + } + return 0; +} + +/* * Return true if task @p cannot migrate to a different CPU, false * otherwise. + * + * IMPORTANT: designed for NON-SLEEPABLE BPF contexts only. Sleepable + * contexts (BPF_STRUCT_OPS_SLEEPABLE, SEC("syscall"), + * SEC("fentry.s/...")) enter via __bpf_prog_enter_sleepable() or + * __bpf_prog_enter_sleepable_recur(), both of which unconditionally + * call migrate_disable(); this helper can yield a false negative for + * p == current there, which can crash the scheduler. */ static inline bool is_migration_disabled(const struct task_struct *p) { /* - * Testing p->migration_disabled in a BPF code is tricky because the - * migration is _always_ disabled while running the BPF code. - * The prolog (__bpf_prog_enter) and epilog (__bpf_prog_exit) for BPF - * code execution disable and re-enable the migration of the current - * task, respectively. So, the _current_ task of the sched_ext ops is - * always migration-disabled. Moreover, p->migration_disabled could be - * two or greater when a sched_ext ops BPF code (e.g., ops.tick) is - * executed in the middle of the other BPF code execution. + * Testing p->migration_disabled in BPF is tricky because the BPF prolog + * (__bpf_prog_enter) may call migrate_disable() for the current task, + * making migration_disabled == 1 even for tasks that are not truly + * migration-disabled. + * + * Since commit 8e4f0b1ebcf2 ("bpf: use rcu_read_lock_dont_migrate() for + * trampoline.c"), the BPF prolog calls migrate_disable() only when + * CONFIG_PREEMPT_RCU is enabled. Two fast paths cover the common cases: + * + * 1) CONFIG_PREEMPT_RCU: prolog always calls migrate_disable(), so + * migration_disabled == 1 for the current task is ambiguous. + * Disambiguate by checking p == current. + * + * 2) v6.18+ without CONFIG_PREEMPT_RCU: prolog never calls + * migrate_disable(), so migration_disabled == 1 is unambiguously + * a real migrate_disable() call. * - * Therefore, we should decide that the _current_ task is - * migration-disabled only when its migration_disabled count is greater - * than one. In other words, when p->migration_disabled == 1, there is - * an ambiguity, so we should check if @p is the current task or not. + * A slow path handles pre-v6.18 kernels without CONFIG_PREEMPT_RCU, + * where the prolog historically called migrate_disable() unconditionally + * but a cherry-picked downstream kernel may not. The runtime-probed flag + * __scx_prolog_disables_migration (set by scx_lib_init_probe) distinguishes + * the two cases without relying on the kernel version alone. */ if (bpf_core_field_exists(p->migration_disabled)) { - if (p->migration_disabled == 1) - return bpf_get_current_task_btf() != p; - else - return p->migration_disabled; + if (p->migration_disabled == 1) { + /* Fast path: prolog always disables migration */ + if (CONFIG_PREEMPT_RCU) + return bpf_get_current_task_btf() != p; + /* Fast path: prolog never disables migration */ + if (LINUX_KERNEL_VERSION >= KERNEL_VERSION(6, 18, 0)) + return true; + /* Slow path: pre-v6.18, !PREEMPT_RCU - use runtime flag */ + return __scx_prolog_disables_migration ? + bpf_get_current_task_btf() != p : true; + } + return p->migration_disabled; } return false; } @@ -1021,7 +1094,20 @@ static inline u64 scx_clock_task(u32 cpu) { struct rq___local *rq = get_current_rq(cpu); - /* Equivalent to the kernel's rq_clock_task(). */ + /* + * Equivalent to the kernel's rq_clock_task(): wall-clock time minus + * cumulative IRQ time (CONFIG_IRQ_TIME_ACCOUNTING) and hypervisor + * steal time (CONFIG_PARAVIRT_TIME_ACCOUNTING). Without those configs, + * it equals rq->clock. + * + * Conceptually this clock advances during idle (the idle task counts + * as a running task), but rq->clock_task is only updated on scheduling + * events. With NO_HZ_IDLE (the default), the periodic tick is stopped + * on idle CPUs, so rq->clock_task is not refreshed while a CPU is + * idle. Reading this clock for a remote idle CPU from a BPF timer + * callback returns the value from when the CPU last went idle, making + * the delta over an idle interval effectively zero. + */ return rq ? rq->clock_task : 0; } @@ -1032,9 +1118,23 @@ static inline u64 scx_clock_pelt(u32 cpu) /* * Equivalent to the kernel's rq_clock_pelt(): subtracts * lost_idle_time from clock_pelt to absorb the jump that occurs - * when clock_pelt resyncs with clock_task at idle exit. The result - * is a continuous, capacity-invariant clock safe for both task - * execution time stamping and cross-idle measurements. + * when clock_pelt resyncs with clock_task at idle exit. The intent + * is a continuous, capacity- and frequency-invariant clock that is + * frozen during idle, IRQ, and hypervisor steal. + * + * However, like scx_clock_task(), this clock has a stale-read issue + * for remote idle CPUs with NO_HZ_IDLE (the default). clock_pelt + * itself advances at wall-clock rate (hardware-clock based), but + * lost_idle_time is only updated via update_rq_clock_pelt(), which + * requires update_rq_clock() to be called. With NO_HZ_IDLE, the + * periodic tick is stopped on idle CPUs, so lost_idle_time is not + * refreshed during idle. Reading this clock for a remote idle CPU + * from a BPF timer callback therefore returns a value that drifts + * at wall-clock rate -- the same stale behaviour as scx_clock_task(). + * + * Without NO_HZ_IDLE, periodic ticks keep lost_idle_time nearly in + * sync (stale by at most one tick period, ~1 ms), so the result is + * accurate. */ return rq ? (rq->clock_pelt - rq->lost_idle_time) : 0; } diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index 3ab642f92c8a..6944221f96cc 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -92,15 +92,20 @@ int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __k /* * v6.19: Introduce lockless peek API for user DSQs. + * v7.1: Fix scx_bpf_dsq_peek() spuriously returning NULL on non-empty + * FIFO DSQs (2f2ea7709266). * - * Preserve the following macro until v6.21. + * The kfunc exists from v6.19 but can return NULL for a non-empty FIFO DSQ + * before the v7.1 fix. Require kernel version >= 7.1.0 before calling it; + * otherwise fall through to the bpf_iter_scx_dsq fallback below. */ static inline struct task_struct *__COMPAT_scx_bpf_dsq_peek(u64 dsq_id) { struct task_struct *p = NULL; struct bpf_iter_scx_dsq it; - if (bpf_ksym_exists(scx_bpf_dsq_peek)) + if (bpf_ksym_exists(scx_bpf_dsq_peek) && + LINUX_KERNEL_VERSION >= KERNEL_VERSION(7, 1, 0)) return scx_bpf_dsq_peek(dsq_id); if (!bpf_iter_scx_dsq_new(&it, dsq_id, 0)) p = bpf_iter_scx_dsq_next(&it); @@ -239,6 +244,26 @@ static inline bool __COMPAT_is_enq_cpu_selected(u64 enq_flags) scx_bpf_pick_any_cpu(cpus_allowed, flags)) /* + * v6.18: Add a helper to retrieve the current task running on a CPU. + * + * The kernel tree dropped this helper and scx_bpf_cpu_rq(), but schedulers in + * this tree still support pre-v6.18 kernels where scx_bpf_cpu_curr() doesn't + * resolve and the scx_bpf_cpu_rq() fallback still exists. Keep it until + * pre-v6.18 kernels fall out of the support window. + */ +static inline struct task_struct *__COMPAT_scx_bpf_cpu_curr(int cpu) +{ + struct rq *rq; + + if (bpf_ksym_exists(scx_bpf_cpu_curr)) + return scx_bpf_cpu_curr(cpu); + + rq = scx_bpf_cpu_rq(cpu); + + return rq ? rq->curr : NULL; +} + +/* * v6.19: To work around BPF maximum parameter limit, the following kfuncs are * replaced with variants that pack scalar arguments in a struct. Wrappers are * provided to maintain source compatibility. @@ -379,6 +404,17 @@ static inline void scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime) } /* + * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This + * will eventually deprecate scx_bpf_reenqueue_local(). + */ +void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; + +static inline bool __COMPAT_has_generic_reenq(void) +{ + return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); +} + +/* * v6.19: The new void variant can be called from anywhere while the older v1 * variant can only be called from ops.cpu_release(). The double ___ prefixes on * the v2 variant need to be removed once libbpf is updated to ignore ___ prefix @@ -395,21 +431,31 @@ static inline bool __COMPAT_scx_bpf_reenqueue_local_from_anywhere(void) static inline void scx_bpf_reenqueue_local(void) { - if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) + if (__COMPAT_has_generic_reenq()) + scx_bpf_dsq_reenq___compat(SCX_DSQ_LOCAL, 0); + else if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) scx_bpf_reenqueue_local___v2___compat(); else scx_bpf_reenqueue_local___v1(); } -/* - * v7.1: New scx_bpf_dsq_reenq() that allows re-enqueues on more DSQs. This - * will eventually deprecate scx_bpf_reenqueue_local(). - */ -void scx_bpf_dsq_reenq___compat(u64 dsq_id, u64 reenq_flags) __ksym __weak; - -static inline bool __COMPAT_has_generic_reenq(void) +static inline int scx_bpf_reenqueue_local_from_anywhere(void) { - return bpf_ksym_exists(scx_bpf_dsq_reenq___compat); + /* + * The generic reenq kfunc and the v2 reenqueue-local variant can both be + * called from anywhere; v1 cannot. Test each ksym in its own branch with a + * distinct call: combining them with || would fold into a bitwise OR of the + * two ksym addresses, which the verifier rejects. + */ + if (__COMPAT_has_generic_reenq()) { + scx_bpf_dsq_reenq___compat(SCX_DSQ_LOCAL, 0); + return 0; + } + if (__COMPAT_scx_bpf_reenqueue_local_from_anywhere()) { + scx_bpf_reenqueue_local___v2___compat(); + return 0; + } + return -EOPNOTSUPP; } static inline void scx_bpf_dsq_reenq(u64 dsq_id, u64 reenq_flags) diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h index d2e4384df5af..7c12df45fdba 100644 --- a/tools/sched_ext/include/scx/compat.h +++ b/tools/sched_ext/include/scx/compat.h @@ -10,9 +10,14 @@ #include <bpf/btf.h> #include <bpf/libbpf.h> #include <fcntl.h> +#include <stdint.h> +#include <stdio.h> #include <stdlib.h> +#include <string.h> #include <unistd.h> +#include "enums_abi.autogen.h" + struct btf *__COMPAT_vmlinux_btf __attribute__((weak)); static inline void __COMPAT_load_vmlinux_btf(void) @@ -23,6 +28,85 @@ static inline void __COMPAT_load_vmlinux_btf(void) } } +/* + * Recover the true value of a 64-bit enum enumerator whose kernel BTF entry + * was truncated to its low 32 bits. + * + * Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode + * 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only + * carry the low 32 bits. This happens with pahole < 1.24, which predates + * ENUM64, and with pahole passing --skip_encoding_btf_enum64 (e.g. Google's + * Container-Optimized OS / GKE kernels deliberately pass it for backward + * compatibility with older BTF consumers). The high bits + * can't be recovered from kernel BTF, so substitute the value from the + * vmlinux.h this tree was built against, cross-checked against the low 32 + * bits the kernel did provide. + * + * Note that this is a best-effort recovery, not a ground truth. The + * substitution assumes the running kernel agrees with this tree's vmlinux.h + * on the high 32 bits, but only the low 32 bits can actually be verified. + * The cross-check is vacuous for enumerators whose value has no low bits + * set (e.g. SCX_DSQ_FLAG_BUILTIN, __SCX_ENQ_INTERNAL_MASK, + * SCX_ENQ_CLEAR_OPSS, SCX_ECODE_*): their lo32 is 0 and matches anything, + * so those substitutions rest entirely on the high bits never moving. An + * enumerator missing from the table (a kernel newer than this tree's + * vmlinux.h, or a stale autogen table) can't be recovered at all. If a + * substitution is ever wrong, the scheduler operates on bogus values (e.g. + * dispatching to nonexistent DSQ ids or silently dropping flags) and can + * wildly malfunction, which is why the mismatch and table-miss paths refuse + * instead of guessing. + */ +static inline bool __COMPAT_recover_truncated_enum64(const char *type, + const char *name, + u32 lo32, u64 *v) +{ + static bool warned; + size_t i; + + for (i = 0; i < sizeof(__scx_enum_abi_vals) / sizeof(__scx_enum_abi_vals[0]); i++) { + const struct __scx_enum_abi_val *e = &__scx_enum_abi_vals[i]; + + if (strcmp(e->type, type) || strcmp(e->name, name)) + continue; + + if (e->val <= (u64)UINT32_MAX) { + *v = lo32; + return true; + } + + if ((u32)e->val != lo32) { + fprintf(stderr, "ERROR: kernel BTF value of %s::%s (0x%x) doesn't match the low 32 bits of the vmlinux.h value (0x%llx); refusing to substitute\n", + type, name, lo32, (unsigned long long)e->val); + return false; + } + + if (!warned) { + fprintf(stderr, + "WARNING: kernel BTF lacks BTF_KIND_ENUM64 encoding (generated by\n" + "WARNING: pahole < 1.24 or with --skip_encoding_btf_enum64), so 64-bit\n" + "WARNING: scx enum values are truncated to their low 32 bits in kernel\n" + "WARNING: BTF. Substituting the full 64-bit values from the vmlinux.h\n" + "WARNING: this binary was built against, cross-checked against the low\n" + "WARNING: 32 bits the kernel does provide. The high 32 bits cannot be\n" + "WARNING: verified: if the running kernel's actual values differ from\n" + "WARNING: the build-time vmlinux.h (e.g. an enum that moved in a newer\n" + "WARNING: kernel), the scheduler will operate on bogus values, such as\n" + "WARNING: dispatching to nonexistent DSQ ids, and can wildly malfunction.\n"); + warned = true; + } + *v = e->val; + return true; + } + + /* + * Unknown enumerator (likely a stale autogen table). Fail + * pessimistically to avoid returning an invalid value. + */ + fprintf(stderr, "ERROR: kernel BTF truncates 64-bit enum %s::%s to 0x%x; 64-bit variant not found in vmlinux.h\n", + type, name, lo32); + return false; +} + static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v) { const struct btf_type *t; @@ -46,6 +130,19 @@ static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off); SCX_BUG_ON(!n, "btf__name_by_offset()"); if (!strcmp(n, name)) { + /* + * Try to recover a 64-bit enum from an 8-byte + * BTF_KIND_ENUM that was encoded without ENUM64 + * support (old pahole or + * --skip_encoding_btf_enum64). Only scx_* + * types are covered by the substitution table; + * non-scx types fall through to the raw value + * so this generic utility keeps working for + * them. + */ + if (t->size == 8 && !strncmp(type, "scx_", 4)) + return __COMPAT_recover_truncated_enum64(type, name, + (u32)e[i].val, v); *v = e[i].val; return true; } diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h index 19aa1de3e700..63b6b14b19bd 100644 --- a/tools/sched_ext/include/scx/enum_defs.autogen.h +++ b/tools/sched_ext/include/scx/enum_defs.autogen.h @@ -56,6 +56,10 @@ #define HAVE_SCX_DEQ_SLEEP #define HAVE_SCX_DEQ_CORE_SCHED_EXEC #define HAVE_SCX_DEQ_SCHED_CHANGE +#define HAVE_SCX_DSP_NONE +#define HAVE_SCX_DSP_LOCAL +#define HAVE_SCX_DSP_PREV +#define HAVE_SCX_DSP_RETRY #define HAVE_SCX_DSQ_FLAG_BUILTIN #define HAVE_SCX_DSQ_FLAG_LOCAL_ON #define HAVE_SCX_DSQ_INVALID @@ -188,7 +192,6 @@ #define HAVE_SCX_RQ_SUB_IDLE_RENOTIFY #define HAVE_SCX_RQ_ROOT_IDLE_RENOTIFY #define HAVE_SCX_RQ_IN_WAKEUP -#define HAVE_SCX_RQ_IN_BALANCE #define HAVE_SCX_RQ_IN_DISPATCH #define HAVE_SCX_SCHED_PCPU_BYPASSING #define HAVE_SCX_SLICE_OOB_DUR_BITS diff --git a/tools/sched_ext/include/scx/enums_abi.autogen.h b/tools/sched_ext/include/scx/enums_abi.autogen.h new file mode 100644 index 000000000000..d53899764f5a --- /dev/null +++ b/tools/sched_ext/include/scx/enums_abi.autogen.h @@ -0,0 +1,223 @@ +/* + * WARNING: This file is autogenerated from gen_enum_defs.py [1]. + * + * scx enumerator values from the vmlinux.h this tree is built against. + * Used as the substitution source when the running kernel's BTF lacks + * BTF_KIND_ENUM64 encoding and 64-bit enum values are truncated. + * + * [1] https://github.com/sched-ext/scx/blob/main/scripts/gen_enum_defs.py + */ + +#ifndef __ENUMS_ABI_AUTOGEN_H__ +#define __ENUMS_ABI_AUTOGEN_H__ + +struct __scx_enum_abi_val { + const char *type; + const char *name; + u64 val; +}; + +static const struct __scx_enum_abi_val __scx_enum_abi_vals[] + __attribute__((unused)) = { + { "scx_arena_consts", "SCX_ARENA_MIN_ORDER", 0x3LLU }, + { "scx_arena_consts", "SCX_ARENA_GROW_PAGES", 0x4LLU }, + { "scx_cap_flags", "__SCX_CAP_ENQ_IMMED", 0x0LLU }, + { "scx_cap_flags", "__SCX_CAP_ENQ", 0x1LLU }, + { "scx_cap_flags", "__SCX_CAP_PREEMPT", 0x2LLU }, + { "scx_cap_flags", "__SCX_CAP_PERF", 0x3LLU }, + { "scx_cap_flags", "__SCX_NR_CAPS", 0x4LLU }, + { "scx_cap_flags", "__SCX_CAP_ALL", 0xfLLU }, + { "scx_cap_flags", "SCX_CAP_ENQ_IMMED", 0x1LLU }, + { "scx_cap_flags", "SCX_CAP_ENQ", 0x2LLU }, + { "scx_cap_flags", "SCX_CAP_PREEMPT", 0x4LLU }, + { "scx_cap_flags", "SCX_CAP_PERF", 0x8LLU }, + { "scx_cap_flags", "SCX_CAP_BASE", 0x1LLU }, + { "scx_cap_flags", "SCX_CAPS_REENQ_ON_LOSS", 0x3LLU }, + { "scx_cid_consts", "SCX_CID_SHARD_SIZE_DFL", 0x18LLU }, + { "scx_cid_consts", "SCX_CID_SHARD_MAX_CPUS", 0x200LLU }, + { "scx_consts", "SCX_DSP_DFL_MAX_BATCH", 0x20LLU }, + { "scx_consts", "SCX_DSP_MAX_LOOPS", 0x20LLU }, + { "scx_consts", "SCX_WATCHDOG_MAX_TIMEOUT", 0x7530LLU }, + { "scx_consts", "SCX_RESCUE_DFL_BW_PPT", 0x14LLU }, + { "scx_consts", "SCX_RESCUE_MAX_BW_PPT", 0xfaLLU }, + { "scx_consts", "SCX_RESCUE_DISABLE", 0xffffffffLLU }, + { "scx_consts", "SCX_RESCUE_DFL_QUANTUM_US", 0x1388LLU }, + { "scx_consts", "SCX_RESCUE_MIN_QUANTUM_US", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_MAX_QUANTUM_US", 0x186a0LLU }, + { "scx_consts", "SCX_RESCUE_MIN_SLICE_US", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_OVERLOAD_MULT", 0x10LLU }, + { "scx_consts", "SCX_RESCUE_MIN_OVERLOAD_MS", 0x3e8LLU }, + { "scx_consts", "SCX_RESCUE_MAX_OVERLOAD_MS", 0x3a98LLU }, + { "scx_consts", "SCX_TID_CHUNK", 0x400LLU }, + { "scx_consts", "SCX_EXIT_BT_LEN", 0x40LLU }, + { "scx_consts", "SCX_EXIT_MSG_LEN", 0x400LLU }, + { "scx_consts", "SCX_EXIT_DUMP_DFL_LEN", 0x8000LLU }, + { "scx_consts", "SCX_CPUPERF_ONE", 0x400LLU }, + { "scx_consts", "SCX_TASK_ITER_BATCH", 0x20LLU }, + { "scx_consts", "SCX_BYPASS_HOST_NTH", 0x2LLU }, + { "scx_consts", "SCX_BYPASS_LB_DFL_INTV_US", 0x7a120LLU }, + { "scx_consts", "SCX_BYPASS_LB_DONOR_PCT", 0x7dLLU }, + { "scx_consts", "SCX_BYPASS_LB_MIN_DELTA_DIV", 0x4LLU }, + { "scx_consts", "SCX_BYPASS_LB_BATCH", 0x100LLU }, + { "scx_consts", "SCX_REENQ_MAX_REPEAT", 0x100LLU }, + { "scx_consts", "SCX_SUB_MAX_DEPTH", 0x4LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_RT", 0x0LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_DL", 0x1LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_STOP", 0x2LLU }, + { "scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_UNKNOWN", 0x3LLU }, + { "scx_deq_flags", "SCX_DEQ_SLEEP", 0x1LLU }, + { "scx_deq_flags", "SCX_DEQ_CORE_SCHED_EXEC", 0x100000000LLU }, + { "scx_deq_flags", "SCX_DEQ_SCHED_CHANGE", 0x200000000LLU }, + { "scx_dsp_verdict", "SCX_DSP_NONE", 0x0LLU }, + { "scx_dsp_verdict", "SCX_DSP_LOCAL", 0x1LLU }, + { "scx_dsp_verdict", "SCX_DSP_PREV", 0x2LLU }, + { "scx_dsp_verdict", "SCX_DSP_RETRY", 0x3LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_FLAG_BUILTIN", 0x8000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_FLAG_LOCAL_ON", 0x4000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_INVALID", 0x8000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_GLOBAL", 0x8000000000000001LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL", 0x8000000000000002LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_BYPASS", 0x8000000000000003LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_REJECT", 0x8000000000000004LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_RESCUE", 0x8000000000000005LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL_ON", 0xc000000000000000LLU }, + { "scx_dsq_id_flags", "SCX_DSQ_LOCAL_CPU_MASK", 0xffffffffLLU }, + { "scx_dsq_iter_flags", "SCX_DSQ_ITER_REV", 0x10000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_SLICE", 0x40000000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_VTIME", 0x80000000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_USER_FLAGS", 0x10000LLU }, + { "scx_dsq_iter_flags", "__SCX_DSQ_ITER_ALL_FLAGS", 0xc0010000LLU }, + { "scx_dsq_lnode_flags", "SCX_DSQ_LNODE_ITER_CURSOR", 0x1LLU }, + { "scx_dsq_lnode_flags", "__SCX_DSQ_LNODE_PRIV_SHIFT", 0x10LLU }, + { "scx_enable_state", "SCX_ENABLING", 0x0LLU }, + { "scx_enable_state", "SCX_ENABLED", 0x1LLU }, + { "scx_enable_state", "SCX_DISABLING", 0x2LLU }, + { "scx_enable_state", "SCX_DISABLED", 0x3LLU }, + { "scx_enq_flags", "SCX_ENQ_WAKEUP", 0x1LLU }, + { "scx_enq_flags", "SCX_ENQ_HEAD", 0x10000LLU }, + { "scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000LLU }, + { "scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000LLU }, + { "scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000LLU }, + { "scx_enq_flags", "SCX_ENQ_RESCUE", 0x400000000LLU }, + { "scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000LLU }, + { "scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_NESTED", 0x400000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_GDSQ_FALLBACK", 0x800000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_IGNORE_CAPS", 0x1000000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_APPLY_SLICE", 0x2000000000000000LLU }, + { "scx_enq_flags", "SCX_ENQ_SLICE_DFL", 0x4000000000000000LLU }, + { "scx_ent_dsq_flags", "SCX_TASK_DSQ_ON_PRIQ", 0x1LLU }, + { "scx_ent_flags", "SCX_TASK_QUEUED", 0x1LLU }, + { "scx_ent_flags", "SCX_TASK_IN_CUSTODY", 0x2LLU }, + { "scx_ent_flags", "SCX_TASK_RESET_RUNNABLE_AT", 0x4LLU }, + { "scx_ent_flags", "SCX_TASK_DEQD_FOR_SLEEP", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10LLU }, + { "scx_ent_flags", "SCX_TASK_IMMED", 0x20LLU }, + { "scx_ent_flags", "SCX_TASK_PROTECTED", 0x40LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_BITS", 0x3LLU }, + { "scx_ent_flags", "SCX_TASK_STATE_MASK", 0x700LLU }, + { "scx_ent_flags", "SCX_TASK_NONE", 0x0LLU }, + { "scx_ent_flags", "SCX_TASK_INIT_BEGIN", 0x100LLU }, + { "scx_ent_flags", "SCX_TASK_INIT", 0x200LLU }, + { "scx_ent_flags", "SCX_TASK_READY", 0x300LLU }, + { "scx_ent_flags", "SCX_TASK_ENABLED", 0x400LLU }, + { "scx_ent_flags", "SCX_TASK_DEAD", 0x500LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xcLLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x3LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x7000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000LLU }, + { "scx_ent_flags", "SCX_TASK_REENQ_CAP", 0x4000LLU }, + { "scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000LLU }, + { "scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000LLU }, + { "scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000LLU }, + { "scx_exit_code", "SCX_ECODE_ACT_RESTART", 0x1000000000000LLU }, + { "scx_exit_flags", "SCX_EFLAG_INITIALIZED", 0x1LLU }, + { "scx_exit_kind", "SCX_EXIT_NONE", 0x0LLU }, + { "scx_exit_kind", "SCX_EXIT_DONE", 0x1LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG", 0x40LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG_BPF", 0x41LLU }, + { "scx_exit_kind", "SCX_EXIT_UNREG_KERN", 0x42LLU }, + { "scx_exit_kind", "SCX_EXIT_SYSRQ", 0x43LLU }, + { "scx_exit_kind", "SCX_EXIT_PARENT", 0x44LLU }, + { "scx_exit_kind", "SCX_EXIT_PARENT_KILL", 0x45LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR", 0x400LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_BPF", 0x401LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_STALL", 0x402LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_REENQ", 0x403LLU }, + { "scx_exit_kind", "SCX_EXIT_ERROR_RESCUE", 0x404LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_UNLOCKED", 0x1LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_INIT_CIDS", 0x2LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_CPU_RELEASE", 0x4LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_DISPATCH", 0x8LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_ENQUEUE", 0x10LLU }, + { "scx_kf_allow_flags", "SCX_KF_ALLOW_SELECT_CPU", 0x20LLU }, + { "scx_kick_flags", "SCX_KICK_IDLE", 0x1LLU }, + { "scx_kick_flags", "SCX_KICK_PREEMPT", 0x2LLU }, + { "scx_kick_flags", "SCX_KICK_WAIT", 0x4LLU }, + { "scx_opi", "SCX_OPI_BEGIN", 0x0LLU }, + { "scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0LLU }, + { "scx_opi", "SCX_OPI_NORMAL_END", 0x21LLU }, + { "scx_opi", "SCX_OPI_CPU_HOTPLUG_BEGIN", 0x21LLU }, + { "scx_opi", "SCX_OPI_CPU_HOTPLUG_END", 0x23LLU }, + { "scx_opi", "SCX_OPI_END", 0x23LLU }, + { "scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE", 0x1LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_LAST", 0x2LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_EXITING", 0x4LLU }, + { "scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL", 0x8LLU }, + { "scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED", 0x10LLU }, + { "scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP", 0x20LLU }, + { "scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40LLU }, + { "scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80LLU }, + { "scx_ops_flags", "SCX_OPS_TID_TO_TASK", 0x100LLU }, + { "scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0x1ffLLU }, + { "scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000LLU }, + { "scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000LLU }, + { "scx_ops_state", "SCX_OPSS_NONE", 0x0LLU }, + { "scx_ops_state", "SCX_OPSS_QUEUEING", 0x1LLU }, + { "scx_ops_state", "SCX_OPSS_QUEUED", 0x2LLU }, + { "scx_ops_state", "SCX_OPSS_DISPATCHING", 0x3LLU }, + { "scx_ops_state", "SCX_OPSS_QSEQ_SHIFT", 0x2LLU }, + { "scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE", 0x1LLU }, + { "scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE", 0x2LLU }, + { "scx_public_consts", "SCX_OPS_NAME_LEN", 0x80LLU }, + { "scx_public_consts", "SCX_SLICE_DFL", 0x1312d00LLU }, + { "scx_public_consts", "SCX_SLICE_BYPASS", 0x4c4b40LLU }, + { "scx_public_consts", "SCX_SLICE_INF", 0xffffffffffffffffLLU }, + { "scx_reenq_flags", "SCX_REENQ_ANY", 0x1LLU }, + { "scx_reenq_flags", "SCX_REENQ_CAP_REVOKE", 0x2LLU }, + { "scx_reenq_flags", "__SCX_REENQ_FILTER_MASK", 0xffffLLU }, + { "scx_reenq_flags", "__SCX_REENQ_USER_MASK", 0x1LLU }, + { "scx_reenq_flags", "SCX_REENQ_TSR_RQ_OPEN", 0x100000000LLU }, + { "scx_reenq_flags", "SCX_REENQ_TSR_NOT_FIRST", 0x200000000LLU }, + { "scx_reenq_flags", "__SCX_REENQ_TSR_MASK", 0xf00000000LLU }, + { "scx_rq_flags", "SCX_RQ_ONLINE", 0x1LLU }, + { "scx_rq_flags", "SCX_RQ_CAN_STOP_TICK", 0x2LLU }, + { "scx_rq_flags", "SCX_RQ_CLK_VALID", 0x20LLU }, + { "scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40LLU }, + { "scx_rq_flags", "SCX_RQ_SUB_IDLE_RENOTIFY", 0x80LLU }, + { "scx_rq_flags", "SCX_RQ_ROOT_IDLE_RENOTIFY", 0x100LLU }, + { "scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000LLU }, + { "scx_rq_flags", "SCX_RQ_IN_DISPATCH", 0x20000LLU }, + { "scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1LLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_DUR_BITS", 0x2bLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_BITS", 0x14LLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_DUR_MASK", 0x7ffffffffffLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_SHIFT", 0x2bLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_ID_MASK", 0xfffffLLU }, + { "scx_slice_oob_consts", "SCX_SLICE_OOB_PENDING", 0x8000000000000000LLU }, + { "scx_tg_flags", "SCX_TG_ONLINE", 0x1LLU }, + { "scx_tg_flags", "SCX_TG_INITED", 0x2LLU }, + { "scx_tg_flags", "SCX_TG_SUB_INIT", 0x4LLU }, + { "scx_wake_flags", "SCX_WAKE_FORK", 0x4LLU }, + { "scx_wake_flags", "SCX_WAKE_TTWU", 0x8LLU }, + { "scx_wake_flags", "SCX_WAKE_SYNC", 0x10LLU }, +}; + +#endif /* __ENUMS_ABI_AUTOGEN_H__ */ diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c index 64dd60b3e922..65dae9e45400 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -299,6 +299,7 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) u64 now = scx_bpf_now(); u64 nr_to_kick = nr_queued; s32 i, curr_cpu; + int ret; curr_cpu = bpf_get_smp_processor_id(); if (timer_pinned && (curr_cpu != central_cpu)) { @@ -332,7 +333,10 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); } - bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN); + ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, + timer_pinned ? BPF_F_TIMER_CPU_PIN : 0); + if (ret) + scx_bpf_error("bpf_timer_start failed (%d)", ret); __sync_fetch_and_add(&nr_timers, 1); return 0; } diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 64cf4dd964d6..454ebb820c5e 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -937,7 +937,7 @@ void BPF_STRUCT_OPS(fcg_cgroup_move, struct task_struct *p, if (!(from_cgc = find_cgrp_ctx(from)) || !(to_cgc = find_cgrp_ctx(to))) return; - delta = time_delta(p->scx.dsq_vtime, from_cgc->tvtime_now); + delta = (s64)(p->scx.dsq_vtime - from_cgc->tvtime_now); scx_bpf_task_set_dsq_vtime(p, to_cgc->tvtime_now + delta); } diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 5bb8b90a275a..9f6e61d7ca07 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -1246,7 +1246,8 @@ static int monitor_timerfn(void *map, int *key, struct bpf_timer *timer) scx_read_event(&events, SCX_EV_BYPASS_ACTIVATE)); } - bpf_timer_start(timer, ONE_SEC_IN_NS, 0); + if (bpf_timer_start(timer, ONE_SEC_IN_NS, 0)) + scx_bpf_error("failed to re-arm stats timer"); return 0; } @@ -1268,7 +1269,8 @@ struct { static int lowpri_timerfn(void *map, int *key, struct bpf_timer *timer) { scx_bpf_dsq_reenq(LOWPRI_DSQ, 0); - bpf_timer_start(timer, LOWPRI_INTV_NS, 0); + if (bpf_timer_start(timer, LOWPRI_INTV_NS, 0)) + scx_bpf_error("failed to re-arm lowpri timer"); return 0; } @@ -1747,7 +1749,8 @@ static void rr_advance(void) static int round_robin_timerfn(void *map, int *key, struct bpf_timer *timer) { rr_advance(); - bpf_timer_start(timer, round_robin_ns, 0); + if (bpf_timer_start(timer, round_robin_ns, 0)) + scx_bpf_error("failed to re-arm round-robin timer"); return 0; } diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c index a329f901c5ed..0857d64c322a 100644 --- a/tools/testing/selftests/alsa/mixer-test.c +++ b/tools/testing/selftests/alsa/mixer-test.c @@ -319,8 +319,8 @@ static bool ctl_value_index_valid(struct ctl_data *ctl, /* Only check step size if there is one and we're in bounds */ if (snd_ctl_elem_info_get_step(ctl->info) && - (int_val - snd_ctl_elem_info_get_min(ctl->info) % - snd_ctl_elem_info_get_step(ctl->info))) { + (int_val - snd_ctl_elem_info_get_min(ctl->info)) % + snd_ctl_elem_info_get_step(ctl->info)) { ksft_print_msg("%s.%d value %ld invalid for step %ld minimum %ld\n", ctl->name, index, int_val, snd_ctl_elem_info_get_step(ctl->info), diff --git a/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c new file mode 100644 index 000000000000..3248bccc3557 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/btf_map_keyless.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <bpf/btf.h> + +/* + * A hash map with a key-less BTF (btf_key_type_id == 0) used to be accepted + * and then NULL-deref in btf_type_show() when dumped through bpffs. A fixed + * kernel rejects it at creation; verify that rejection, with a keyed positive + * control so the -EINVAL is about the missing key type and not some unrelated + * failure. + */ +static void check_keyless(int map_type, __u32 map_flags, int btf_fd, int val_id) +{ + LIBBPF_OPTS(bpf_map_create_opts, opts); + int map_fd; + + opts.map_flags = map_flags; + opts.btf_fd = btf_fd; + opts.btf_value_type_id = val_id; + + /* Positive control: the same map with a real key type is accepted. */ + opts.btf_key_type_id = val_id; + map_fd = bpf_map_create(map_type, "keyed_map", 4, 4, 8, &opts); + if (!ASSERT_GE(map_fd, 0, "keyed create is accepted")) + return; + close(map_fd); + + /* A key-less BTF must be rejected. */ + opts.btf_key_type_id = 0; + map_fd = bpf_map_create(map_type, "keyless_map", 4, 4, 8, &opts); + ASSERT_EQ(map_fd, -EINVAL, "key-less create is rejected"); + if (map_fd >= 0) + close(map_fd); +} + +void test_btf_map_keyless(void) +{ + int btf_fd, val_id; + struct btf *btf; + + btf = btf__new_empty(); + if (!ASSERT_OK_PTR(btf, "btf__new_empty")) + return; + + val_id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED); + if (!ASSERT_GT(val_id, 0, "btf__add_int")) + goto out; + + if (!ASSERT_OK(btf__load_into_kernel(btf), "btf__load_into_kernel")) + goto out; + btf_fd = btf__fd(btf); + + if (test__start_subtest("hash")) + check_keyless(BPF_MAP_TYPE_HASH, 0, btf_fd, val_id); + if (test__start_subtest("rhash")) + check_keyless(BPF_MAP_TYPE_RHASH, BPF_F_NO_PREALLOC, btf_fd, val_id); +out: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/rhash.c b/tools/testing/selftests/bpf/prog_tests/rhash.c index 98bb66907b7f..0641bd5b0a9e 100644 --- a/tools/testing/selftests/bpf/prog_tests/rhash.c +++ b/tools/testing/selftests/bpf/prog_tests/rhash.c @@ -172,6 +172,12 @@ void test_rhash(void) if (test__start_subtest("test_rhash_delete_nonexistent")) rhash_run("test_rhash_delete_nonexistent"); + if (test__start_subtest("test_rhash_kptr_update")) + rhash_run("test_rhash_kptr_update"); + + if (test__start_subtest("test_rhash_kptr_delete")) + rhash_run("test_rhash_kptr_delete"); + if (test__start_subtest("test_rhash_map_extra_presize")) rhash_map_extra_presize(); diff --git a/tools/testing/selftests/bpf/prog_tests/rhash_timer.c b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c new file mode 100644 index 000000000000..3aad9fc02e06 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rhash_timer.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include <sched.h> + +#include <test_progs.h> +#include "rhash_timer.skel.h" + +#define MAX_ATTEMPTS 256 +#define RCU_SYNC_INTERVAL 64 + +static int pin_to_first_cpu(cpu_set_t *old_mask) +{ + cpu_set_t new_mask; + int cpu; + + if (sched_getaffinity(0, sizeof(*old_mask), old_mask)) + return -errno; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) + if (CPU_ISSET(cpu, old_mask)) + break; + if (cpu == CPU_SETSIZE) + return -EINVAL; + + CPU_ZERO(&new_mask); + CPU_SET(cpu, &new_mask); + if (sched_setaffinity(0, sizeof(new_mask), &new_mask)) + return -errno; + return 0; +} + +static int update_timer_map(int map_fd, __u64 key) +{ + __u64 value[3] = {}; + + return bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST); +} + +static int run_prog(int prog_fd, struct bpf_test_run_opts *opts) +{ + int err; + + err = bpf_prog_test_run_opts(prog_fd, opts); + if (err) + return err; + return opts->retval; +} + +void test_rhash_timer(void) +{ + LIBBPF_OPTS(bpf_test_run_opts, opts); + struct rhash_timer *skel = NULL; + cpu_set_t old_mask; + int map_fd = -1, arm_fd, cancel_fd; + bool affinity_set = false; + __u64 key = 1; + int attempt, err; + + err = pin_to_first_cpu(&old_mask); + if (!ASSERT_OK(err, "pin_to_first_cpu")) + return; + affinity_set = true; + + skel = rhash_timer__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto out; + + map_fd = bpf_map__fd(skel->maps.timer_map); + if (!ASSERT_GE(map_fd, 0, "timer_map fd")) + goto out; + arm_fd = bpf_program__fd(skel->progs.arm_deleted_timer); + if (!ASSERT_GE(arm_fd, 0, "arm_deleted_timer fd")) + goto out; + cancel_fd = bpf_program__fd(skel->progs.cancel_recycled_timer); + if (!ASSERT_GE(cancel_fd, 0, "cancel_recycled_timer fd")) + goto out; + + err = update_timer_map(map_fd, key); + if (!ASSERT_OK(err, "seed_timer_map")) + goto out; + + for (attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + err = run_prog(arm_fd, &opts); + if (err) { + ASSERT_OK(err, "arm_deleted_timer"); + goto out; + } + if (skel->bss->armed != attempt + 1) { + ASSERT_EQ(skel->bss->armed, attempt + 1, "armed"); + goto out; + } + if (skel->bss->timer_init_err) { + ASSERT_OK(skel->bss->timer_init_err, "timer_init_err"); + goto out; + } + if (skel->bss->timer_set_callback_err) { + ASSERT_OK(skel->bss->timer_set_callback_err, + "timer_set_callback_err"); + goto out; + } + if (skel->bss->timer_start_err) { + ASSERT_OK(skel->bss->timer_start_err, "timer_start_err"); + goto out; + } + + if ((attempt + 1) % RCU_SYNC_INTERVAL == 0) { + err = kern_sync_rcu(); + if (err) { + ASSERT_OK(err, "kern_sync_rcu"); + goto out; + } + } + + err = update_timer_map(map_fd, ++key); + if (err) { + ASSERT_OK(err, "replace_timer_map"); + goto out; + } + + err = run_prog(cancel_fd, &opts); + if (err) { + ASSERT_OK(err, "cancel_recycled_timer"); + goto out; + } + if (skel->bss->timer_cancel_err) { + ASSERT_OK(skel->bss->timer_cancel_err, "timer_cancel_err"); + goto out; + } + if (skel->bss->cancelled) + break; + } + + ASSERT_GT(skel->bss->cancelled, 0, "preserved timer"); +out: + if (map_fd >= 0) + bpf_map_delete_elem(map_fd, &key); + rhash_timer__destroy(skel); + if (affinity_set) + sched_setaffinity(0, sizeof(old_mask), &old_mask); +} diff --git a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c index dd41b826be30..edce9c1b54fb 100644 --- a/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c +++ b/tools/testing/selftests/bpf/prog_tests/snprintf_btf.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 #include <test_progs.h> #include <linux/btf.h> +#include <bpf/btf.h> #include "netif_receive_skb.skel.h" +#include "snprintf_btf_void.skel.h" /* Demonstrate that bpf_snprintf_btf succeeds and that various data types * are formatted correctly. @@ -58,3 +60,80 @@ void serial_test_snprintf_btf(void) cleanup: netif_receive_skb__destroy(skel); } + +/* + * bpf_snprintf_btf() renders a type_id taken straight from the vmlinux BTF. + * Two such type_ids used to NULL-deref in the BTF show path: + * - a "const void" (a modifier resolving to void) in btf_modifier_show() + * - a BTF_KIND_VAR in btf_var_show() (base BTF has no resolved_ids) + * A fixed kernel renders both without crashing. + */ +static long run(struct snprintf_btf_void *skel, __u32 type_id) +{ + LIBBPF_OPTS(bpf_test_run_opts, topts); + char ctx[8] = {}; + + skel->bss->type_id = type_id; + topts.ctx_in = ctx; + topts.ctx_size_in = sizeof(ctx); + if (!ASSERT_OK(bpf_prog_test_run_opts(bpf_program__fd(skel->progs.dump_type), + &topts), "test_run")) + return -1; + return skel->bss->ret; +} + +void test_snprintf_btf_void(void) +{ + const struct btf_type *t; + struct snprintf_btf_void *skel; + int i, n, cv = 0, var = 0; + struct btf *btf; + + btf = btf__parse("/sys/kernel/btf/vmlinux", NULL); + if (!btf) { + test__skip(); + return; + } + + skel = snprintf_btf_void__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + goto out_btf; + + n = btf__type_cnt(btf); + for (i = 1; i < n && !(cv && var); i++) { + t = btf__type_by_id(btf, i); + if (!cv && btf_kind(t) == BTF_KIND_CONST && t->type == 0) + cv = i; + /* Pick a VAR small enough to render from the program's buffer. */ + if (!var && btf_kind(t) == BTF_KIND_VAR) { + long sz = btf__resolve_size(btf, t->type); + + if (sz > 0 && sz <= (long)sizeof(skel->bss->obj)) + var = i; + } + } + + /* "const void" renders the "<unsupported kind:0>" placeholder. */ + if (test__start_subtest("const_void")) { + if (cv) { + ASSERT_EQ(run(skel, cv), + sizeof("<unsupported kind:0>") - 1, "ret"); + ASSERT_STREQ(skel->bss->out, "<unsupported kind:0>", + "placeholder"); + } else { + test__skip(); + } + } + + /* A BTF_KIND_VAR must resolve and render without error. */ + if (test__start_subtest("var")) { + if (var) + ASSERT_GT(run(skel, var), 0, "ret"); + else + test__skip(); + } + + snprintf_btf_void__destroy(skel); +out_btf: + btf__free(btf); +} diff --git a/tools/testing/selftests/bpf/prog_tests/timer_mim.c b/tools/testing/selftests/bpf/prog_tests/timer_mim.c index c930c7d7105b..fa7bb769ca31 100644 --- a/tools/testing/selftests/bpf/prog_tests/timer_mim.c +++ b/tools/testing/selftests/bpf/prog_tests/timer_mim.c @@ -59,10 +59,32 @@ void serial_test_timer_mim(void) int err; old_print_fn = libbpf_set_print(NULL); - timer_reject_skel = timer_mim_reject__open_and_load(); - libbpf_set_print(old_print_fn); - if (!ASSERT_ERR_PTR(timer_reject_skel, "timer_reject_skel_load")) + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "timer_reject_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.test1, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "timer_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_reject_skel_open")) goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_mismatch, true); + err = timer_mim_reject__load(timer_reject_skel); + ASSERT_ERR(err, "callback_reject_skel_load"); + timer_mim_reject__destroy(timer_reject_skel); + + timer_reject_skel = timer_mim_reject__open(); + if (!ASSERT_OK_PTR(timer_reject_skel, "callback_accept_skel_open")) + goto cleanup; + bpf_program__set_autoload(timer_reject_skel->progs.callback_map_uid_match, true); + err = timer_mim_reject__load(timer_reject_skel); + if (!ASSERT_OK(err, "callback_accept_skel_load")) + goto cleanup; + timer_mim_reject__destroy(timer_reject_skel); + timer_reject_skel = NULL; + libbpf_set_print(old_print_fn); timer_skel = timer_mim__open_and_load(); if (!timer_skel && errno == EOPNOTSUPP) { @@ -75,6 +97,7 @@ void serial_test_timer_mim(void) err = timer_mim(timer_skel); ASSERT_OK(err, "timer_mim"); cleanup: + libbpf_set_print(old_print_fn); timer_mim__destroy(timer_skel); timer_mim_reject__destroy(timer_reject_skel); } diff --git a/tools/testing/selftests/bpf/progs/iters.c b/tools/testing/selftests/bpf/progs/iters.c index 62d7df9e80be..c6699159dacd 100644 --- a/tools/testing/selftests/bpf/progs/iters.c +++ b/tools/testing/selftests/bpf/progs/iters.c @@ -2149,4 +2149,43 @@ __naked int stack_misc_vs_scalar_in_a_loop(void) ); } +__used +static int loop_cb5(int i, __u64 *ctx) +{ + /* unsafe on a second iteration */ + small_arr[*ctx] = i; + *ctx = 100500; + return 0; +} + +SEC("raw_tp") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("memory access is {{.*}} and is outside of the object of size 64") +__naked void loop_counter_precision_2nd_iter(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "*(u64 *)(r10 - 8) = 0;" + "r1 = 2;" + "if r0 == 42 goto +1;" + "r1 = 1;" + "r2 = loop_cb5 ll;" + "r3 = r10;" + "r3 += -8;" + "r4 = 0;" + /* + * Explore with nr_loops=1 on a first path and nr_loops=2 on a second path. + * Buggy verifier did not propagate r1 precision properly, + * and thus checkpoints created for nr_loops=1 case matched nr_loops=2 case. + */ + "call %[bpf_loop];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_loop), + __imm(bpf_get_prandom_u32) + : __clobber_all + ); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/map_kptr_fail.c b/tools/testing/selftests/bpf/progs/map_kptr_fail.c index 5e25ca806060..eee35d203b66 100644 --- a/tools/testing/selftests/bpf/progs/map_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/map_kptr_fail.c @@ -409,4 +409,41 @@ int reject_scalar_store_to_kptr(struct __sk_buff *ctx) return 0; } +SEC("?tc") +__description("reject imprecise scalar store to kptr after state pruning") +__failure __msg("invalid kptr access, R7 type=scalar") +__naked void reject_imprecise_scalar_store_to_kptr(void) +{ + asm volatile ( + "r0 = 0;" + "*(u32 *)(r10 - 4) = r0;" + "r2 = r10;" + "r2 += -4;" + "r1 = %[array_map] ll;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto l2_%=;" + "r6 = r0;" + "r9 = *(u64 *)(r6 + 0);" + "if r9 != 0 goto l0_%=;" + "r7 = 0;" + ".rept 10;" + "r5 = 1;" + ".endr;" + "goto l1_%=;" + "l0_%=:" + "r7 = 0x4141414141414141 ll;" + ".rept 10;" + "r5 = 1;" + ".endr;" + "l1_%=:" + "*(u64 *)(r6 + 8) = r7;" + "l2_%=:" + "r0 = 0;" + "exit;" + : + : __imm(bpf_map_lookup_elem), + __imm_addr(array_map) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c index 08379c3b6a03..3701f4ea58c7 100644 --- a/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c +++ b/tools/testing/selftests/bpf/progs/percpu_alloc_fail.c @@ -33,6 +33,20 @@ struct { __type(value, struct elem); } array SEC(".maps"); +struct kernel_percpu_elem { + struct task_struct __percpu_kptr *task; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct kernel_percpu_elem); +} kernel_percpu_array SEC(".maps"); + +struct task_struct *bpf_task_from_pid(s32 pid) __ksym; +void bpf_task_release(struct task_struct *p) __ksym; + long ret; SEC("?fentry/bpf_fentry_test1") @@ -137,6 +151,51 @@ int BPF_PROG(test_array_map_5) return 0; } +SEC("?syscall") +__failure __msg("invalid kptr access, R2 type=trusted_ptr_ expected=ptr_task_struct") +int reject_kernel_ptr_into_percpu_kptr(void *ctx) +{ + struct kernel_percpu_elem *e; + struct task_struct *p, *old; + int index = 0; + + e = bpf_map_lookup_elem(&kernel_percpu_array, &index); + if (!e) + return 0; + + p = bpf_task_from_pid(1); + if (!p) + return 0; + + old = bpf_kptr_xchg(&e->task, p); + if (old) + bpf_task_release(old); + return 0; +} + +SEC("?fentry.s/bpf_fentry_test1") +__failure __msg("invalid kptr access, R2 type=ptr_ expected=ptr_val_t") +int BPF_PROG(reject_plain_alloc_into_percpu_kptr) +{ + struct val_t __percpu_kptr *old; + struct val_t *p; + struct elem *e; + int index = 0; + + e = bpf_map_lookup_elem(&array, &index); + if (!e) + return 0; + + p = bpf_obj_new(struct val_t); + if (!p) + return 0; + + old = bpf_kptr_xchg(&e->pc, p); + if (old) + bpf_percpu_obj_drop(old); + return 0; +} + SEC("?fentry.s/bpf_fentry_test1") __failure __msg("bpf_percpu_obj_new type ID argument must be of a struct of scalars") int BPF_PROG(test_array_map_6) diff --git a/tools/testing/selftests/bpf/progs/preempt_lock.c b/tools/testing/selftests/bpf/progs/preempt_lock.c index 6d5fce7e6ffc..81c459435680 100644 --- a/tools/testing/selftests/bpf/progs/preempt_lock.c +++ b/tools/testing/selftests/bpf/progs/preempt_lock.c @@ -115,6 +115,58 @@ int preempt_sleepable_helper(void *ctx) return 0; } +SEC("?uprobe.s") +__failure __msg("sleepable helper bpf_get_stack#") +int preempt_sleepable_get_stack(struct pt_regs *ctx) +{ + struct bpf_stack_build_id stack; + + bpf_preempt_disable(); + bpf_get_stack(ctx, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + bpf_preempt_enable(); + return 0; +} + +SEC("?uprobe.s") +__failure __msg("sleepable helper bpf_get_task_stack#") +int preempt_sleepable_get_task_stack(void *ctx) +{ + struct bpf_stack_build_id stack; + struct task_struct *task; + + task = bpf_get_current_task_btf(); + bpf_preempt_disable(); + bpf_get_task_stack(task, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + bpf_preempt_enable(); + return 0; +} + +SEC("?uprobe.s") +__success +int sleepable_get_stack(struct pt_regs *ctx) +{ + struct bpf_stack_build_id stack; + + bpf_get_stack(ctx, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + return 0; +} + +SEC("?uprobe.s") +__success +int sleepable_get_task_stack(void *ctx) +{ + struct bpf_stack_build_id stack; + struct task_struct *task; + + task = bpf_get_current_task_btf(); + bpf_get_task_stack(task, &stack, sizeof(stack), + BPF_F_USER_STACK | BPF_F_USER_BUILD_ID); + return 0; +} + SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") __failure __msg("kernel func bpf_copy_from_user_str is sleepable within non-preemptible region") int preempt_sleepable_kfunc(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c index 0d58114a4955..725d73c9ffe1 100644 --- a/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c +++ b/tools/testing/selftests/bpf/progs/raw_tp_null_fail.c @@ -22,3 +22,56 @@ int test_raw_tp_null_sched_pi_setprio_arg_2(void *ctx) { asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u64 *)(r1 +0);" ::: __clobber_all); return 0; } + +/* Plain raw tracepoint arguments remain scalar values. */ +SEC("raw_tp/signal_generate") +__success +int test_raw_tp_signal_generate_info_scalar(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all); + return 0; +} + +/* tp_btf programs may inspect the sentinel as a scalar value. */ +SEC("tp_btf/signal_generate") +__success +int test_tp_btf_signal_generate_info_scalar(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 != 1 goto +0;" ::: __clobber_all); + return 0; +} + +/* SEND_SIG_PRIV is non-NULL, so a NULL check cannot make info safe. */ +SEC("tp_btf/signal_generate") +__failure __msg("R1 invalid mem access 'scalar'") +int test_tp_btf_signal_generate_info_no_deref(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); if r1 == 0 goto +1; " + "r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/signal_deliver") +__failure __msg("R1 invalid mem access 'scalar'") +int test_tp_btf_signal_deliver_info_no_deref(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +8); r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/sched_process_wait") +__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'") +int test_raw_tp_null_sched_process_wait_arg_1(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +0); r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} + +SEC("tp_btf/sched_process_wait") +__success +int test_raw_tp_null_sched_process_wait_arg_1_checked(void *ctx) +{ + asm volatile("r1 = *(u64 *)(r1 +0); if r1 == 0 goto +1; " + "r1 = *(u32 *)(r1 +0);" ::: __clobber_all); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/rbtree_fail.c b/tools/testing/selftests/bpf/progs/rbtree_fail.c index 555379952dcc..4504608196ab 100644 --- a/tools/testing/selftests/bpf/progs/rbtree_fail.c +++ b/tools/testing/selftests/bpf/progs/rbtree_fail.c @@ -16,6 +16,7 @@ struct node_data { private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_data, node); private(A) struct bpf_rb_root groot2 __contains(node_data, node); +private(B) struct bpf_res_spin_lock res_glock; static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -265,6 +266,53 @@ static bool less__bad_fn_call_first_unlock_after(struct bpf_rb_node *a, const st return node_a->key < node_b->key; } +static bool less__bad_res_spin_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + bpf_res_spin_unlock(&res_glock); + return false; +} + +static __noinline void rbtree_cb_unlock_relock(void) +{ + bpf_spin_unlock(&glock); + bpf_spin_lock(&glock); +} + +static __noinline void rbtree_cb_nested_unlock(void) +{ + rbtree_cb_unlock_relock(); + asm volatile (""); +} + +static bool less__bad_subprog_unlock(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct node_data *node_a; + struct node_data *node_b; + + node_a = container_of(a, struct node_data, node); + node_b = container_of(b, struct node_data, node); + rbtree_cb_nested_unlock(); + + return node_a->key < node_b->key; +} + +static __noinline void rbtree_cb_noop(void) +{ + asm volatile (""); +} + +static bool less__subprog_allowed(struct bpf_rb_node *a, const struct bpf_rb_node *b) +{ + struct node_data *node_a; + struct node_data *node_b; + + node_a = container_of(a, struct node_data, node); + node_b = container_of(b, struct node_data, node); + rbtree_cb_noop(); + + return node_a->key < node_b->key; +} + static __always_inline long add_with_cb(bool (cb)(struct bpf_rb_node *a, const struct bpf_rb_node *b)) { @@ -301,4 +349,40 @@ long rbtree_api_add_bad_cb_bad_fn_call_first_unlock_after(void *ctx) return add_with_cb(less__bad_fn_call_first_unlock_after); } +SEC("?tc") +__failure __msg("can't res_spin_{lock,unlock} in rbtree cb") +long rbtree_api_add_bad_cb_res_spin_unlock(void *ctx) +{ + struct node_data *n; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return 1; + + bpf_spin_lock(&glock); + if (bpf_res_spin_lock(&res_glock)) { + bpf_spin_unlock(&glock); + bpf_obj_drop(n); + return 1; + } + bpf_rbtree_add(&groot, &n->node, less__bad_res_spin_unlock); + bpf_res_spin_unlock(&res_glock); + bpf_spin_unlock(&glock); + return 0; +} + +SEC("?tc") +__failure __msg("can't spin_{lock,unlock} in rbtree cb") +long rbtree_api_add_bad_cb_subprog_unlock(void *ctx) +{ + return add_with_cb(less__bad_subprog_unlock); +} + +SEC("?tc") +__success +long rbtree_api_add_cb_subprog_allowed(void *ctx) +{ + return add_with_cb(less__subprog_allowed); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/rcu_read_lock.c b/tools/testing/selftests/bpf/progs/rcu_read_lock.c index 31d4081c3a9f..cdb255addbc3 100644 --- a/tools/testing/selftests/bpf/progs/rcu_read_lock.c +++ b/tools/testing/selftests/bpf/progs/rcu_read_lock.c @@ -592,9 +592,9 @@ int non_own_ref_untrusted_ld(void *ctx) } bpf_rcu_read_unlock(); /* - * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED - * | NON_OWN_REF, and the load below has to get the BPF_PROBE_MEM - * rewrite for it, otherwise a bad address panics the kernel. + * The unlock leaves node as PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED, + * and the load below has to get the BPF_PROBE_MEM rewrite for it, + * otherwise a bad address panics the kernel. */ non_own_ref_key = node->key; return 0; diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr.c b/tools/testing/selftests/bpf/progs/refcounted_kptr.c index 61906f48025c..cae00f7b0a24 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr.c @@ -23,6 +23,15 @@ struct map_value { struct node_data __kptr *node; }; +struct node_refcount_only { + long key; + struct bpf_refcount refcount; +}; + +struct map_value_refcount_only { + struct node_refcount_only __kptr *node; +}; + struct { __uint(type, BPF_MAP_TYPE_ARRAY); __type(key, int); @@ -30,6 +39,13 @@ struct { __uint(max_entries, 2); } stashed_nodes SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_refcount_only); + __uint(max_entries, 1); +} stashed_refcount_only SEC(".maps"); + struct node_acquire { long key; long data; @@ -832,6 +848,51 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) return 0; } +SEC("tc") +__success +long refcount_acquire_owning_input_no_null_check(void *ctx) +{ + struct node_refcount_only *n, *m; + + n = bpf_obj_new(typeof(*n)); + if (!n) + return 1; + + m = bpf_refcount_acquire(n); + bpf_obj_drop(m); + bpf_obj_drop(n); + + return 0; +} + +SEC("?syscall") +__success +long refcount_acquire_rcu_map_kptr_null_checked(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *n, *m; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 1; + + bpf_rcu_read_lock(); + n = mapval->node; + if (!n) { + bpf_rcu_read_unlock(); + return 2; + } + m = bpf_refcount_acquire(n); + bpf_rcu_read_unlock(); + + if (!m) + return 3; + bpf_obj_drop(m); + + return 0; +} + static long __stash_map_empty_xchg(struct node_data *n, int idx) { struct map_value *mapval = bpf_map_lookup_elem(&stashed_nodes, &idx); diff --git a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c index eaaed0859f94..338e43822ffe 100644 --- a/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +++ b/tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c @@ -19,6 +19,26 @@ struct node_refcounted { struct bpf_refcount refcount; }; +struct node_refcount_only { + long key; + struct bpf_refcount refcount; +}; + +struct map_value_refcount_only { + struct node_refcount_only __kptr *node; +}; + +struct rcu_graph_node { + struct bpf_rb_node node; + long data; +}; + +struct rcu_graph_node *just_here_because_btf_bug; + +struct map_value_rcu_graph { + struct rcu_graph_node __kptr *node; +}; + extern void bpf_rcu_read_lock(void) __ksym; extern void bpf_rcu_read_unlock(void) __ksym; @@ -27,6 +47,22 @@ private(A) struct bpf_spin_lock glock; private(A) struct bpf_rb_root groot __contains(node_acquire, node); private(B) struct bpf_spin_lock lock; private(B) struct bpf_list_head head __contains(node_refcounted, list); +private(C) struct bpf_spin_lock graph_lock; +private(C) struct bpf_rb_root graph_root __contains(rcu_graph_node, node); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_refcount_only); + __uint(max_entries, 1); +} stashed_refcount_only SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, struct map_value_rcu_graph); + __uint(max_entries, 1); +} stashed_rcu_graph SEC(".maps"); static bool less(struct bpf_rb_node *a, const struct bpf_rb_node *b) { @@ -89,6 +125,120 @@ long refcount_acquire_non_object(void *ctx) return bpf_refcount_acquire(ctx) != NULL; } +SEC("?syscall") +__failure __msg("Possibly NULL pointer passed to trusted R1") +long refcount_acquire_rcu_map_kptr_unchecked_drop(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *tmp, *n, *m; + int idx = 0; + + /* Force Clang to emit complete BTF for struct node_refcount_only. */ + tmp = bpf_obj_new(typeof(*tmp)); + if (!tmp) + return 3; + bpf_obj_drop(tmp); + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 1; + + bpf_rcu_read_lock(); + n = mapval->node; + if (!n) { + bpf_rcu_read_unlock(); + return 2; + } + m = bpf_refcount_acquire(n); + bpf_rcu_read_unlock(); + + bpf_obj_drop(m); + + return 0; +} + +SEC("?syscall") +__failure +__msg("bpf_rbtree_remove can only take non-owning or refcounted " + "bpf_rb_node pointer") +long rbtree_remove_after_rcu_unlock(void *ctx) +{ + struct map_value_rcu_graph *mapval; + struct bpf_rb_node *rb_node; + struct rcu_graph_node *node; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx); + if (!mapval) + return 0; + + bpf_rcu_read_lock(); + node = mapval->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + + bpf_spin_lock(&graph_lock); + rb_node = bpf_rbtree_remove(&graph_root, &node->node); + bpf_spin_unlock(&graph_lock); + if (rb_node) + bpf_obj_drop(container_of(rb_node, struct rcu_graph_node, node)); + + return 0; +} + +SEC("?syscall") +__failure __msg("R1 is neither owning or non-owning ref") +long refcount_acquire_after_rcu_unlock(void *ctx) +{ + struct map_value_refcount_only *mapval; + struct node_refcount_only *node, *ref; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_refcount_only, &idx); + if (!mapval) + return 0; + + bpf_rcu_read_lock(); + node = mapval->node; + if (!node) { + bpf_rcu_read_unlock(); + return 0; + } + bpf_rcu_read_unlock(); + + ref = bpf_refcount_acquire(node); + if (ref) + bpf_obj_drop(ref); + + return 0; +} + +SEC("?syscall") +__failure __msg("invalid mem access 'scalar'") +long graph_kptr_after_spin_unlock(void *ctx) +{ + struct map_value_rcu_graph *mapval; + struct rcu_graph_node *node; + int idx = 0; + + mapval = bpf_map_lookup_elem(&stashed_rcu_graph, &idx); + if (!mapval) + return 0; + + bpf_spin_lock(&graph_lock); + node = mapval->node; + if (!node) { + bpf_spin_unlock(&graph_lock); + return 0; + } + bpf_spin_unlock(&graph_lock); + + return node->data; +} + SEC("?tc") __failure __msg("Unreleased reference id=3 alloc_insn={{[0-9]+}}") long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) diff --git a/tools/testing/selftests/bpf/progs/rhash.c b/tools/testing/selftests/bpf/progs/rhash.c index fc2dac3a719e..aea4de8dc781 100644 --- a/tools/testing/selftests/bpf/progs/rhash.c +++ b/tools/testing/selftests/bpf/progs/rhash.c @@ -19,6 +19,11 @@ struct elem { int val; }; +struct special_elem { + struct task_struct __kptr *task; + int val; +}; + struct { __uint(type, BPF_MAP_TYPE_RHASH); __uint(map_flags, BPF_F_NO_PREALLOC); @@ -27,6 +32,17 @@ struct { __type(value, struct elem); } rhmap SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct special_elem); +} special_fields SEC(".maps"); + +extern struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym; +extern void bpf_task_release(struct task_struct *p) __ksym; + SEC("syscall") int test_rhash_lookup_update(void *ctx) { @@ -246,3 +262,99 @@ int test_rhash_delete_nonexistent(void *ctx) err = 0; return 0; } + +SEC("syscall") +int test_rhash_kptr_update(void *ctx) +{ + struct special_elem val1 = { .val = 1 }; + struct special_elem val2 = { .val = 2 }; + struct task_struct *task, *old; + struct special_elem *elem; + int key = 0; + + err = 1; + if (bpf_map_update_elem(&special_fields, &key, &val1, BPF_NOEXIST)) + return 1; + + err = 2; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem) + return 2; + + err = 3; + task = bpf_task_acquire(bpf_get_current_task_btf()); + if (!task) + return 3; + + err = 4; + old = bpf_kptr_xchg(&elem->task, task); + if (old) { + bpf_task_release(old); + return 4; + } + + err = 5; + if (bpf_map_update_elem(&special_fields, &key, &val2, BPF_EXIST)) + return 5; + + err = 6; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem || elem->val != 2) + return 6; + + err = 7; + old = bpf_kptr_xchg(&elem->task, NULL); + if (!old) + return 7; + bpf_task_release(old); + + err = 8; + if (bpf_map_delete_elem(&special_fields, &key)) + return 8; + + err = 0; + return 0; +} + +SEC("syscall") +int test_rhash_kptr_delete(void *ctx) +{ + struct special_elem val = {}; + struct task_struct *task, *old; + struct special_elem *elem; + int key = 0; + + err = 1; + if (bpf_map_update_elem(&special_fields, &key, &val, BPF_NOEXIST)) + return 1; + + err = 2; + elem = bpf_map_lookup_elem(&special_fields, &key); + if (!elem) + return 2; + + err = 3; + task = bpf_task_acquire(bpf_get_current_task_btf()); + if (!task) + return 3; + + err = 4; + old = bpf_kptr_xchg(&elem->task, task); + if (old) { + bpf_task_release(old); + return 4; + } + + err = 5; + if (bpf_map_delete_elem(&special_fields, &key)) + return 5; + + err = 6; + old = bpf_kptr_xchg(&elem->task, NULL); + if (!old) + return 6; + bpf_task_release(old); + + err = 0; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/rhash_timer.c b/tools/testing/selftests/bpf/progs/rhash_timer.c new file mode 100644 index 000000000000..2e06a463c605 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rhash_timer.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <vmlinux.h> +#include <errno.h> +#include <bpf/bpf_helpers.h> + +#define CLOCK_MONOTONIC 1 +#define TIMER_NSEC (60ULL * 1000 * 1000 * 1000) + +struct timer_value { + struct bpf_timer timer; + u64 data; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RHASH); + __uint(map_flags, BPF_F_NO_PREALLOC); + __uint(max_entries, 1); + __type(key, u64); + __type(value, struct timer_value); +} timer_map SEC(".maps"); + +u64 armed; +u64 cancelled; +long timer_init_err; +long timer_set_callback_err; +long timer_start_err; +long timer_cancel_err; + +static int timer_cb(void *map, u64 *key, struct timer_value *value) +{ + return 0; +} + +static long arm_timer_cb(struct bpf_map *map, u64 *key, + struct timer_value *value, void *ctx) +{ + u64 key_copy = *key; + long err; + + err = bpf_map_delete_elem(map, &key_copy); + if (err) + return 1; + + err = bpf_timer_init(&value->timer, map, CLOCK_MONOTONIC); + if (err) { + timer_init_err = err; + return 1; + } + + err = bpf_timer_set_callback(&value->timer, timer_cb); + if (err) { + timer_set_callback_err = err; + return 1; + } + + err = bpf_timer_start(&value->timer, TIMER_NSEC, BPF_F_TIMER_CPU_PIN); + if (err) { + timer_start_err = err; + return 1; + } + + __sync_fetch_and_add(&armed, 1); + return 1; +} + +static long cancel_timer_cb(struct bpf_map *map, u64 *key, + struct timer_value *value, void *ctx) +{ + long err; + + err = bpf_timer_cancel(&value->timer); + if (err == -EINVAL) + return 1; + if (err < 0) { + timer_cancel_err = err; + return 1; + } + + __sync_fetch_and_add(&cancelled, 1); + return 1; +} + +SEC("syscall") +int arm_deleted_timer(void *ctx) +{ + bpf_for_each_map_elem(&timer_map, arm_timer_cb, NULL, 0); + return 0; +} + +SEC("syscall") +int cancel_recycled_timer(void *ctx) +{ + bpf_for_each_map_elem(&timer_map, cancel_timer_cb, NULL, 0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/snprintf_btf_void.c b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c new file mode 100644 index 000000000000..44af80fbbb80 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/snprintf_btf_void.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "btf_ptr.h" +#include <bpf/bpf_helpers.h> + +__u32 type_id; +/* A buffer we own to render the selected type from, kept in bounds. */ +char obj[256]; +char out[64]; +long ret; + +SEC("raw_tp/sys_enter") +int dump_type(void *ctx) +{ + struct btf_ptr ptr = { + .ptr = obj, + .type_id = type_id, + .flags = 0, + }; + + ret = bpf_snprintf_btf(out, sizeof(out), &ptr, sizeof(ptr), 0); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/tailcall_callback.c b/tools/testing/selftests/bpf/progs/tailcall_callback.c index c41632cf423b..14fa7a87028e 100644 --- a/tools/testing/selftests/bpf/progs/tailcall_callback.c +++ b/tools/testing/selftests/bpf/progs/tailcall_callback.c @@ -45,6 +45,13 @@ int callback_loop(int index, void **cb_ctx) } static __noinline +int callback_tail(int index, void **cb_ctx) +{ + bpf_tail_call_static(*cb_ctx, &jmp_table, 0); + return 0; +} + +static __noinline int callback_empty(int index, void *data) { return 0; @@ -78,4 +85,13 @@ int tailcall_callback_2(struct __sk_buff *skb) return 0; } +/* callback with a direct tail call is rejected without a verifier bug */ +SEC("tc") +__failure __msg("cannot tail call within callback") +int tailcall_callback_3(struct __sk_buff *skb) +{ + bpf_loop(1, callback_tail, &skb, 0); + return 0; +} + char __license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h index 0df8a12fd61e..a31a399870be 100644 --- a/tools/testing/selftests/bpf/progs/task_local_data.bpf.h +++ b/tools/testing/selftests/bpf/progs/task_local_data.bpf.h @@ -61,6 +61,7 @@ #define TLD_ROUND_UP(x, y) ((((x) - 1) | TLD_ROUND_MASK(x, y)) + 1) #define TLD_MAX_DATA_CNT (__PAGE_SIZE / sizeof(struct tld_metadata) - 1) +#define TLD_DATA_SIZE (__PAGE_SIZE - sizeof(__u64)) #ifndef TLD_NAME_LEN #define TLD_NAME_LEN 62 @@ -189,6 +190,8 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s return start + off; off += TLD_ROUND_UP(metadata[i].size, 8); + if (off > TLD_DATA_SIZE) + break; } return -cnt; diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index df43649ecb78..eda9b7bbab75 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -190,8 +190,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; @@ -293,8 +293,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple * ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; diff --git a/tools/testing/selftests/bpf/progs/timer_mim_reject.c b/tools/testing/selftests/bpf/progs/timer_mim_reject.c index dd3f1ed6d6e6..83f31138336b 100644 --- a/tools/testing/selftests/bpf/progs/timer_mim_reject.c +++ b/tools/testing/selftests/bpf/progs/timer_mim_reject.c @@ -43,7 +43,7 @@ static int timer_cb(void *map, int *key, struct hmap_elem *val) return 0; } -SEC("fentry/bpf_fentry_test1") +SEC("?fentry/bpf_fentry_test1") int BPF_PROG(test1, int a) { struct hmap_elem init = {}; @@ -72,3 +72,85 @@ int BPF_PROG(test1, int a) err |= 8; return 0; } + +struct callback_ctx { + void *map; +}; + +static int mismatch_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx) +{ + bpf_timer_init(&val->timer, ctx->map, CLOCK_MONOTONIC); + return 0; +} + +static int timer_mismatch_cb(void *map, int *key, struct hmap_elem *val) +{ + struct callback_ctx ctx = { .map = map }; + struct bpf_map *inner_map2; + int array_key2 = ARRAY_KEY2; + + inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2); + if (!inner_map2) + return 0; + bpf_for_each_map_elem(inner_map2, mismatch_iter_cb, &ctx, 0); + return 0; +} + +static int match_iter_cb(void *map, int *key, struct hmap_elem *val, struct callback_ctx *ctx) +{ + bpf_timer_init(&val->timer, map, CLOCK_MONOTONIC); + return 0; +} + +static int timer_match_cb(void *map, int *key, struct hmap_elem *val) +{ + struct callback_ctx ctx = {}; + struct bpf_map *inner_map2; + int array_key2 = ARRAY_KEY2; + + inner_map2 = bpf_map_lookup_elem(&outer_arr, &array_key2); + if (!inner_map2) + return 0; + bpf_for_each_map_elem(inner_map2, match_iter_cb, &ctx, 0); + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(callback_map_uid_mismatch, int a) +{ + struct hmap_elem *val; + struct bpf_map *inner_map; + int array_key = ARRAY_KEY; + int hash_key = HASH_KEY; + + inner_map = bpf_map_lookup_elem(&outer_arr, &array_key); + if (!inner_map) + return 0; + val = bpf_map_lookup_elem(inner_map, &hash_key); + if (!val) + return 0; + + bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC); + bpf_timer_set_callback(&val->timer, timer_mismatch_cb); + return 0; +} + +SEC("?fentry/bpf_fentry_test1") +int BPF_PROG(callback_map_uid_match, int a) +{ + struct hmap_elem *val; + struct bpf_map *inner_map; + int array_key = ARRAY_KEY; + int hash_key = HASH_KEY; + + inner_map = bpf_map_lookup_elem(&outer_arr, &array_key); + if (!inner_map) + return 0; + val = bpf_map_lookup_elem(inner_map, &hash_key); + if (!val) + return 0; + + bpf_timer_init(&val->timer, inner_map, CLOCK_MONOTONIC); + bpf_timer_set_callback(&val->timer, timer_match_cb); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c index 6bf95550a024..e0926767bbd3 100644 --- a/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c +++ b/tools/testing/selftests/bpf/progs/verifier_async_cb_context.c @@ -62,6 +62,104 @@ int timer_sleepable_prog(void *ctx) return 0; } +static int timer_sys_bpf_cb(void *map, int *key, struct bpf_timer *timer) +{ + __u64 attr = 0; + + bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr)); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_sys_bpf#{{[0-9]+}} in non-sleepable prog") +int timer_sys_bpf_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_sys_bpf_cb); + return 0; +} + +static int timer_sys_close_cb(void *map, int *key, struct bpf_timer *timer) +{ + bpf_sys_close(0); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_sys_close#{{[0-9]+}} in non-sleepable prog") +int timer_sys_close_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_sys_close_cb); + return 0; +} + +static int timer_btf_find_cb(void *map, int *key, struct bpf_timer *timer) +{ + char name[] = "task_struct"; + + bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0); + return 0; +} + +SEC("syscall") +__failure __msg("sleepable helper bpf_btf_find_by_name_kind#{{[0-9]+}} in non-sleepable prog") +int timer_btf_find_prog(void *ctx) +{ + struct timer_elem *val; + int key = 0; + + val = bpf_map_lookup_elem(&timer_map, &key); + if (!val) + return 0; + + bpf_timer_init(&val->t, &timer_map, 0); + bpf_timer_set_callback(&val->t, timer_btf_find_cb); + return 0; +} + +SEC("syscall") +__success +int syscall_sys_bpf_prog(void *ctx) +{ + __u64 attr = 0; + + bpf_sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr)); + return 0; +} + +SEC("syscall") +__success +int syscall_sys_close_prog(void *ctx) +{ + bpf_sys_close(0); + return 0; +} + +SEC("syscall") +__success +int syscall_btf_find_prog(void *ctx) +{ + char name[] = "task_struct"; + + bpf_btf_find_by_name_kind(name, sizeof(name), BTF_KIND_STRUCT, 0); + return 0; +} + /* Workqueue tests */ struct wq_elem { diff --git a/tools/testing/selftests/bpf/progs/verifier_bounds.c b/tools/testing/selftests/bpf/progs/verifier_bounds.c index 1a273e416fed..df8d5309657e 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bounds.c +++ b/tools/testing/selftests/bpf/progs/verifier_bounds.c @@ -2267,6 +2267,47 @@ __naked void deduce64_from_32_wrapping_32bit(void) : __clobber_all); } +/* + * Unprivileged variable pointer arithmetic on a PTR_TO_MAP_VALUE whose + * offset collapses to a constant. The Spectre-v1 speculative path snapshots + * the pointer while its r32 has just been blanked but its offset not yet + * synced; the following register move used to trip reg_bounds_sanity_check() + * ("const subreg tnum out of sync with range bounds"). With + * BPF_F_TEST_REG_INVARIANTS that violation turns into a load failure, so the + * unprivileged program must still load. + */ +SEC("socket") +__success __success_unpriv +__flag(BPF_F_TEST_REG_INVARIANTS) +__naked void spec_ptr_alu_const_offset(void) +{ + asm volatile (" \ + call %[bpf_ktime_get_ns]; \ + *(u64*)(r10 - 16) = r0; \ + r1 = 0; \ + *(u64*)(r10 - 8) = r1; \ + r2 = r10; \ + r2 += -8; \ + r1 = %[map_hash_8b] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto l0_%=; \ + r1 = *(u64*)(r10 - 16); \ + r2 = 0x40000000; \ + if r1 > r2 goto l0_%=; \ + if r1 s> 1 goto l0_%=; /* r1 in [0, 1] */ \ + r0 += r1; /* ptr += bounded scalar */ \ + r9 = r0; /* used to trip the warning */ \ + *(u8*)(r0 + 0) = r1; \ +l0_%=: r0 = 0; \ + exit; \ + " + : + : __imm(bpf_ktime_get_ns), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + /* Check that range_within() compares cnum ranges, not min/max projections. */ SEC("socket") __failure __msg("div by zero") diff --git a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c index 328cf630210a..a73b837553fb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c +++ b/tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c @@ -621,6 +621,116 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) : __clobber_all); } +/* A jump to the first spill executes the whole pattern, rewrite is safe. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (jump_to_first_spill) main {{.*}} stack 0") +__xlated("2: if r0 == 0x2a goto pc+0") +__xlated("3: r0 = ") +__xlated("4: r0 = &(void __percpu *)(r0)") +__success +__naked void jump_to_first_spill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" +"l0_%=:" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A jump to the call skips the spill, the pattern must be kept. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (jump_to_call) main {{.*}} stack 8") +__xlated("2: if r0 == 0x2a goto pc+1") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__success +__naked void jump_to_call(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" + "*(u64 *)(r10 - 8) = r1;" +"l0_%=:" + "call %[bpf_get_smp_processor_id];" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* A jump to the fill skips the spill, the pattern must be kept. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (jump_to_fill) main {{.*}} stack 8") +__xlated("2: if r0 == 0x2a goto pc+4") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__success +__naked void jump_to_fill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l0_%=;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" +"l0_%=:" + "r1 = *(u64 *)(r10 - 8);" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + +/* Same as above, but the fill is entered by an unconditional jump. */ +SEC("raw_tp") +__arch_x86_64 +__log_level(4) +__msg("subprog 0 (unconditional_jump_to_fill) main {{.*}} stack 8") +__xlated("3: *(u64 *)(r10 -8) = r1") +__xlated("...") +__xlated("7: r1 = *(u64 *)(r10 -8)") +__xlated("8: exit") +__xlated("9: goto pc-3") +__success +__naked void unconditional_jump_to_fill(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 1;" + "if r0 == 42 goto l1_%=;" + "*(u64 *)(r10 - 8) = r1;" + "call %[bpf_get_smp_processor_id];" +"l0_%=:" + "r1 = *(u64 *)(r10 - 8);" + "exit;" +"l1_%=:" + "goto l0_%=;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_smp_processor_id) + : __clobber_all); +} + SEC("raw_tp") __arch_x86_64 __log_level(4) diff --git a/tools/testing/selftests/bpf/progs/verifier_cfg.c b/tools/testing/selftests/bpf/progs/verifier_cfg.c index c1f55e1d80a4..3c3bb03e8217 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cfg.c +++ b/tools/testing/selftests/bpf/progs/verifier_cfg.c @@ -3,6 +3,7 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include "../../../include/linux/filter.h" #include "bpf_misc.h" SEC("socket") @@ -56,6 +57,19 @@ __naked void out_of_range_jump2(void) } SEC("socket") +__description("invalid DW LDSX instruction in diagnostics") +__failure __msg("BUG_ldx_99") +__log_level(2) +__naked void invalid_dw_ldsx(void) +{ + asm volatile (" \ + .8byte %[ldsx_dw]; \ +" : + : __imm_insn(ldsx_dw, BPF_RAW_INSN(BPF_LDX | BPF_MEMSX | BPF_DW, BPF_REG_0, BPF_REG_0, 0, 0)) + : __clobber_all); +} + +SEC("socket") __description("loop (back-edge)") __failure __msg("unreachable insn 1") __msg_unpriv("back-edge") diff --git a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c index 9a13f5c11ac7..884080a5bffc 100644 --- a/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c +++ b/tools/testing/selftests/bpf/progs/verifier_cgroup_storage.c @@ -305,4 +305,33 @@ __naked void cpu_cgroup_storage_access_6(void) : __clobber_all); } +/* + * Verification takes two paths: with r2 being scalar zero on path (1) + * and with r2 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("cgroup/skb") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("get_local_storage() doesn't support non-zero flags") +__naked void non_zero_flags_on_a_pruned_path(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + /* r2 is 0 on the path explored first, 1 on the other */\ + r2 = 1; \ + if r0 == 0 goto 1f; \ + r2 = 0; \ +1: r1 = %[cgroup_storage] ll; \ + call %[bpf_get_local_storage]; \ + r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_get_local_storage), + __imm_addr(cgroup_storage) + : __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c index 0bdeb7bc4687..a3d2af8dc839 100644 --- a/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c +++ b/tools/testing/selftests/bpf/progs/verifier_global_ptr_args.c @@ -56,6 +56,30 @@ int trusted_task_arg_nullable(void *ctx) return res; } +/* + * Check that the verifier does not use checkpoints created + * on path with r1 == 0 to prune path with r1 != 0. + */ +SEC("?tp_btf/task_newtask") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("R1 type=scalar expected=ptr_, trusted_ptr_, rcu_ptr_") +__naked int null_btf_id_arg_global_subprog(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "call subprog_trusted_task_nullable;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + __weak int subprog_trusted_task_nonnull(struct task_struct *task __arg_trusted) { return task->pid + task->tgid; diff --git a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c index 75dd922e4e9f..1fbcc5228306 100644 --- a/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c +++ b/tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c @@ -169,6 +169,23 @@ static int iter_limit_cb(__u32 idx, struct num_context *ctx) } SEC("?raw_tp") +__failure __msg("R1 type=ctx expected=scalar") +__naked void bpf_loop_reject_pointer(void) +{ + asm volatile ( + "r2 = %[iter_limit_cb];" + "r3 = 0;" + "r4 = 0;" + "call %[bpf_loop];" + "exit;" + : + : __imm_ptr(iter_limit_cb), + __imm(bpf_loop) + : __clobber_common + ); +} + +SEC("?raw_tp") __success int bpf_loop_iter_limit_ok(void *unused) { diff --git a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c index 3d1e8de4390c..3c789c565b18 100644 --- a/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c +++ b/tools/testing/selftests/bpf/progs/verifier_jeq_infer_not_null.c @@ -3,7 +3,9 @@ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <stdbool.h> #include "bpf_misc.h" +#include "bpf_kfuncs.h" struct { __uint(type, BPF_MAP_TYPE_XSKMAP); @@ -12,6 +14,13 @@ struct { __type(value, int); } map_xskmap SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1); + __type(key, int); + __type(value, int); +} map_hash SEC(".maps"); + /* This is equivalent to the following program: * * r6 = skb->sk; @@ -264,4 +273,188 @@ __naked void jne_reg_reg_null_check(void) : __clobber_all); } +/* + * A comparison between PTR_TO_MEM | MEM_RDONLY | PTR_UNTRUSTED and + * PTR_TO_MAP_VALUE_OR_NULL should not infer that map pointer is not null. + * A bug in check_cond_jmp_op() made such inference possible. + */ +SEC("raw_tp") +__failure +__msg("error: invalid dereference of R0 (a nullable map value pointer)") +__msg(">>> 11 | (61) r0 = *(u32 *)(r0 +0)") +__naked void untrusted_mem_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_rdonly_cast(0, 0); */ \ + r1 = 0; \ + r2 = 0; \ + call %[bpf_rdonly_cast]; \ + r6 = r0; \ + /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + /* \ + * buggy verifier assumed that r6 can't be null \ + * and marked r0 non-null as well. \ + */ \ + if r6 != r0 goto 1f; \ + r0 = *(u32 *)(r0 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_rdonly_cast), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * A pointer with an offset that is not bounded from above may be null at + * runtime, hence it is not a witness for the pointer it is compared with. + */ +SEC("socket") +__failure +__msg("error: invalid dereference of R7 (a nullable map value pointer)") +__naked void unbounded_offset_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + r6 = r0; \ + /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \ + *(u64 *)(r10 - 8) = 1; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r7 = r0; \ + /* pointer - pointer is an unknown scalar */ \ + r8 = r7; \ + r8 -= r6; \ + /* r8 is in [0, S64_MAX] */ \ + r8 <<= 1; \ + r8 >>= 1; \ + /* r6 may wrap to zero at runtime */ \ + r6 += r8; \ + if r7 != r6 goto 1f; \ + r0 = *(u8 *)(r7 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* Same, but the offset is bounded, so the inference is still done. */ +SEC("socket") +__success +__naked void bounded_offset_infers_map_value_non_null(void) +{ + asm volatile (" \ + /* r6 = bpf_map_lookup_elem(map_hash, &0); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + r6 = r0; \ + /* r7 = bpf_map_lookup_elem(map_hash, &1); */ \ + *(u64 *)(r10 - 8) = 1; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + r7 = r0; \ + /* pointer - pointer is an unknown scalar */ \ + r8 = r7; \ + r8 -= r6; \ + /* r8 is in [0, 3] */ \ + r8 &= 3; \ + r6 += r8; \ + if r7 != r6 goto 1f; \ + r0 = *(u8 *)(r7 + 0); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * The low 32 bits of a map value pointer may be zero, hence a 32-bit + * compare with zero cannot be predicted from the pointer being non-NULL + * and both successors of such a jump have to be verified. + */ +SEC("socket") +__failure __msg("invalid access to map value, value_size=4 off=32 size=4") +__naked void jmp32_ptr_vs_zero_jne(void) +{ + asm volatile (" \ + /* r0 = bpf_map_lookup_elem(map_hash, &key); */ \ + *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + if w0 != 0 goto 1f; \ + r0 = *(u32 *)(r0 + 32); \ +1: r0 = 0; \ + exit; \ +" : + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +/* + * The below program is explored in two paths: r6 == 0 and r6 == 1. + * On the first path comparison "if r0 == r6 goto 2f" should mark r6 as precise, + * otherwise unsafe path with r6 == 1 would be incorrectly pruned. + */ +SEC("socket") +__failure +__flag(BPF_F_TEST_STATE_FREQ) +__msg("error: invalid dereference of R0 (a nullable map value pointer)") +__naked void imprecise_zero_does_not_infer_map_value_non_null(void) +{ + asm volatile (" \ + call %[bpf_get_prandom_u32]; \ + /* r6 is 0 on the path explored first, 1 on the other */\ + r6 = 1; \ + if r0 == 0 goto 1f; \ + r6 = 0; \ + /* r0 = bpf_map_lookup_elem(map_hash, &0); */ \ +1: *(u64 *)(r10 - 8) = 0; \ + r1 = %[map_hash] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 == r6 goto 2f; \ + r0 = *(u8 *)(r0 + 0); \ +2: r0 = 0; \ + exit; \ +" : + : __imm(bpf_get_prandom_u32), + __imm(bpf_map_lookup_elem), + __imm_addr(map_hash) + : __clobber_all); +} + +void kfunc_root(void) +{ + bpf_rdonly_cast(0, 0); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c index 09e81b99eecb..32989f981fb6 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c +++ b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c @@ -194,6 +194,102 @@ __naked void ld_ind_subprog_both_paths_safe(void) ::: __clobber_all); } +__naked __noinline __used +static int ld_abs_callback(void) +{ + asm volatile ( + "r6 = *(u64 *)(r2 + 0);" + ".8byte %[ld_abs];" + "r0 = 0;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: reject in callback") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_abs_callback_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_abs_callback, &skb, 0); + return 0; +} + +__naked __noinline __used +static int ld_ind_callback_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 0;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +__naked __noinline __used +static int ld_ind_callback(void) +{ + asm volatile ( + "r1 = *(u64 *)(r2 + 0);" + "call ld_ind_callback_subprog;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: reject in callback subprog") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_ind_callback_subprog_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_ind_callback, &skb, 0); + return 0; +} + +static __noinline int ld_ind_global_static(struct __sk_buff *skb) +{ + asm volatile ( + "r6 = %[skb];" + "r7 = 0;" + ".8byte %[ld_ind];" + : + : [skb] "r"(skb), + __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_common, "r6", "r7"); + return skb->mark; +} + +__noinline int ld_ind_global(struct __sk_buff *skb) +{ + return ld_ind_global_static(skb); +} + +static int ld_ind_global_callback(__u32 index, struct __sk_buff **ctx) +{ + ld_ind_global(*ctx); + return 0; +} + +SEC("socket") +__description("ld_ind: reject in callback global subprog") +__failure __msg("cannot use BPF_LD_[ABS|IND] within callback") +int ld_ind_global_callback_reject(struct __sk_buff *skb) +{ + bpf_loop(1, ld_ind_global_callback, &skb, 0); + return 0; +} + +SEC("socket") +__description("ld_ind: allow in non-callback global subprog") +__success +int ld_ind_global_subprog_ok(struct __sk_buff *skb) +{ + return ld_ind_global(skb); +} + /* * ld_{abs,ind} in subprogs require scalar (int) return type in BTF. * A test with void return must be rejected. diff --git a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c index e2cbc5bda65e..b5d7f567d0d4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c +++ b/tools/testing/selftests/bpf/progs/verifier_netfilter_ctx.c @@ -113,4 +113,82 @@ int with_valid_ctx_access_test6(struct bpf_nf_ctx *ctx) return th->dest == bpf_htons(22) ? NF_ACCEPT : NF_DROP; } +SEC("netfilter") +__description("netfilter test prog with skb write access") +__failure __msg("only read is supported") +int skb_len_write(struct bpf_nf_ctx *ctx) +{ + ctx->skb->len = 1; + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with skb data write access") +__failure __msg("cannot write into rdonly_untrusted_mem") +int skb_data_write(struct bpf_nf_ctx *ctx) +{ + ctx->skb->data[0] = 0; + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_write") +__success __failure_unpriv +__retval(0) +int with_dynptr_write(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + struct bpf_dynptr ptr; + u8 buffer[1] = {}; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + if (bpf_dynptr_write(&ptr, 0, buffer, sizeof(buffer), 0)) + return 0; /* must always fail */ + + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_slice_rdwr") +__failure __msg("the prog does not allow writes to packet data") +int with_dynptr_rdwr(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + u8 buffer_iph[20] = {}; + struct bpf_dynptr ptr; + struct iphdr *iph; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + iph = bpf_dynptr_slice_rdwr(&ptr, 0, buffer_iph, sizeof(buffer_iph)); + if (!iph) + return 0; + + return 1; +} + +SEC("netfilter") +__description("netfilter test prog with bpf_dynptr_slice + write") +__failure __msg("cannot write into rdonly_mem") +int with_dynptr_store(struct bpf_nf_ctx *ctx) +{ + struct __sk_buff *skb = (struct __sk_buff *)ctx->skb; + u8 buffer_iph[20] = {}; + struct bpf_dynptr ptr; + struct iphdr *iph; + + if (bpf_dynptr_from_skb(skb, 0, &ptr)) + return 1; + + iph = bpf_dynptr_slice(&ptr, 0, buffer_iph, sizeof(buffer_iph)); + if (!iph) + return 0; + iph->protocol = 42; + + return 1; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_precision.c b/tools/testing/selftests/bpf/progs/verifier_precision.c index 6f325876efdd..f4459561bf39 100644 --- a/tools/testing/selftests/bpf/progs/verifier_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_precision.c @@ -2,8 +2,10 @@ /* Copyright (C) 2023 SUSE LLC */ #include <linux/bpf.h> #include <bpf/bpf_helpers.h> +#include <stdbool.h> #include "../../../include/linux/filter.h" #include "bpf_misc.h" +#include "bpf_kfuncs.h" struct { __uint(type, BPF_MAP_TYPE_ARRAY); @@ -642,4 +644,102 @@ __naked int bpf_atomic_cmpxchg_32bit_precision(void) : __clobber_all); } +/* + * Verification takes two paths: with r1 being scalar zero on path (1) + * and with r1 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("?tc") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 type=scalar expected=fp") +__naked int null_mem_arg_zero_size(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "r2 = 0;" + "r3 = 0;" + "r4 = 0;" + "r5 = 0;" + /* + * ARG_PTR_TO_MEM | PTR_MAYBE_NULL parameter can be NULL, + * but can't be some other scalar value. + */ + "call %[bpf_csum_diff];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32), + __imm(bpf_csum_diff) + : __clobber_all); +} + +__weak int subprog_mem_arg(int *p) +{ + if (p) + return *p; + return 0; +} + +/* + * Verification takes two paths: with r1 being scalar zero on path (1) + * and with r1 being some other scalar on path (2). + * Check that the verifier does not use checkpoints created + * on path (1) to prune path (2). + */ +SEC("?raw_tp") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R1 type=scalar expected=fp") +__naked int null_mem_arg_global_subprog(void) +{ + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r1 = 42;" + "if r0 > 42 goto 1f;" + "r1 = 0;" + "1:" + "call subprog_mem_arg;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_get_prandom_u32) + : __clobber_all); +} + +/* Same as above, check that path with r3 == 0 does not prune the path with r3 != 0 */ +SEC("?tc") +__flag(BPF_F_TEST_STATE_FREQ) +__failure __msg("R3 type=scalar expected=fp") +int null_kfunc_arg_dynptr_slice(struct __sk_buff *skb) +{ + struct bpf_dynptr ptr; + + bpf_dynptr_from_skb(skb, 0, &ptr); + asm volatile ( + "call %[bpf_get_prandom_u32];" + "r3 = 42;" + "if r0 > 42 goto 1f;" + "r3 = 0;" + "1:" + "r1 = %[ptr];" + "r2 = 0;" + "r4 = 8;" + "call %[bpf_dynptr_slice];" + : + : __imm_ptr(ptr), + __imm(bpf_get_prandom_u32), + __imm(bpf_dynptr_slice) + : __clobber_common); + return 0; +} + +void __kfunc_btf_root(void) +{ + bpf_dynptr_slice(0, 0, 0, 0); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c index 663d15fc5fd2..256547048cc4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c +++ b/tools/testing/selftests/bpf/progs/verifier_scalar_ids.c @@ -380,13 +380,14 @@ SEC("socket") __success __log_level(2) __flag(BPF_F_TEST_STATE_FREQ) /* - * check that r0 and r5 have different IDs after 'if', - * collect_linked_regs() can't tie more than 5 registers for a single insn. + * check that r5 is unlinked after 'if', collect_linked_regs() can't tie + * more than 5 registers for a single insn and the register compared by + * the jump is not exempt from that. */ -__msg("7: (25) if r0 > 0x7 goto pc+0 ; R0=scalar(id=1") +__msg("7: (25) if r5 > 0x7 goto pc+0 ; R5=scalar(smin=") __msg("12: (bf) r5 = r5 ; R5=scalar(id=2") /* check that r{0-4} are marked precise after 'if' */ -__msg("frame0: regs=r0 stack= before 7: (25) if r0 > 0x7 goto pc+0") +__msg("frame0: regs=r0 stack= before 7: (25) if r5 > 0x7 goto pc+0") __msg("frame0: parent state regs=r0,r1,r2,r3,r4 stack=:") __naked void linked_regs_too_many_regs(void) { @@ -400,8 +401,8 @@ __naked void linked_regs_too_many_regs(void) "r3 = r0;" "r4 = r0;" "r5 = r0;" - /* propagate range for r{0-5} */ - "if r0 > 7 goto +0;" + /* r{0-4} fill the record, r5 does not fit and is unlinked */ + "if r5 > 7 goto +0;" /* keep r{1-4} live */ "r1 = r1;" "r2 = r2;" diff --git a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c index 8b166c42c4e0..39a1766dae3f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_spill_fill.c +++ b/tools/testing/selftests/bpf/progs/verifier_spill_fill.c @@ -1403,6 +1403,46 @@ __naked void partial_fill_from_cleaned_pointer_spill(void) ::: __clobber_all); } +SEC("raw_tp") +__failure +__msg("access may be outside object bounds") +__flag(BPF_F_TEST_STATE_FREQ) +__naked void imprecise_scalar_spill_half_dead(void) +{ + asm volatile ( + /* + * Fork two paths: the one explored first spills an imprecise zero, + * the one explored second, an imprecise non-zero scalar. + */ + "call %[bpf_get_prandom_u32];" + "if r0 > 42 goto 1f;" + "r6 = 0;" + "goto 2f;" +"1:" + /* causes out of bounds access on a second path. */ + "r6 = 100500;" +"2:" + /* Force a checkpoint before the spill. */ + "goto +0;" + "*(u64 *)(r10 - 8) = r6;" + /* + * Force stack cleanup, only the low half of the spill is alive, + * so the dead high half is degraded to raw stack bytes. + * Buggy verifier converted it to STACK_ZERO w/o proper precision propagation. + */ + "goto +0;" + "r7 = *(u32 *)(r10 - 4);" + /* Use r7 as an offset into a one-byte buffer. */ + "r1 = %[single_byte_buf] ll;" + "r1 += r7;" + "r0 = *(u8 *)(r1 + 0);" + "exit;" +: +: __imm(bpf_get_prandom_u32), + __imm_addr(single_byte_buf) +: __clobber_all); +} + /* check valid spill/fill, ptr to tp buffer */ SEC("raw_tracepoint.w") __success diff --git a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c index d21d32f6a676..dc0c7034c04f 100644 --- a/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c +++ b/tools/testing/selftests/bpf/progs/verifier_subprog_precision.c @@ -287,9 +287,9 @@ __msg("17: (b7) r0 = 0") __msg("18: (95) exit") __msg("returning from callee:") __msg("to caller at 9:") -__msg("frame 0: propagating r1,r4") +__msg("frame 0: propagating r1,r3,r4") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack= before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack= before 18: (95) exit") __msg("from 18 to 9: safe") __naked int callback_result_precise(void) { @@ -419,9 +419,9 @@ __msg("to caller at 9:") /* r1, r4 are always precise for bpf_loop(), * r6 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,r6") +__msg("frame 0: propagating r1,r3,r4,r6") __msg("mark_precise: frame0: last_idx 9 first_idx 9 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4,r6 stack= before 16: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4,r6 stack= before 16: (95) exit") __msg("mark_precise: frame1: regs= stack= before 15: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 9: (85) call bpf_loop") __msg("mark_precise: frame0: parent state regs= stack=:") @@ -575,9 +575,9 @@ __msg("to caller at 10:") /* r1, r4 are always precise for bpf_loop(), * fp-8 was marked before backtracking to callback body. */ -__msg("frame 0: propagating r1,r4,fp-8") +__msg("frame 0: propagating r1,r3,r4,fp-8") __msg("mark_precise: frame0: last_idx 10 first_idx 10 subseq_idx -1") -__msg("mark_precise: frame0: regs=r1,r4 stack=-8 before 18: (95) exit") +__msg("mark_precise: frame0: regs=r1,r3,r4 stack=-8 before 18: (95) exit") __msg("mark_precise: frame1: regs= stack= before 17: (b7) r0 = 0") __msg("mark_precise: frame1: regs= stack= before 10: (85) call bpf_loop#181") __msg("mark_precise: frame0: parent state regs= stack=:") @@ -846,4 +846,55 @@ __naked int subprog_result_tail_call(void) ); } +__naked __noinline __used +static int ld_abs_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = r1;" + ".8byte %[ld_abs];" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +/* + * Buggy verifier did not properly backtrack early subprogram exit + * modelled for BPF_LD | BPF_ABS instruction, causing a segfault. + */ +SEC("socket") +__success +__log_level(2) +/* early exit path */ +__msg("3: (0f) r1 += r7") +__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r7 stack= before 9: (20) r0 = *(u32 *)skb[0]") +__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1") +__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1") +__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5") +__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8") +/* fallthrough path */ +__msg("3: (0f) r1 += r7") +__msg("mark_precise: frame0: regs=r7 stack= before 2: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r7 stack= before 10: (95) exit") +__msg("mark_precise: frame1: regs= stack= before 9: (20) r0 = *(u32 *)skb[0]") +__msg("mark_precise: frame1: regs= stack= before 8: (bf) r7 = r1") +__msg("mark_precise: frame1: regs= stack= before 7: (bf) r6 = r1") +__msg("mark_precise: frame1: regs= stack= before 1: (85) call pc+5") +__msg("mark_precise: frame0: regs=r7 stack= before 0: (b7) r7 = -8") +__naked int ld_abs_backtrack_both_paths(void) +{ + asm volatile ( + "r7 = -8;" + "call ld_abs_subprog;" + "r1 = r10;" + "r1 += r7;" /* mark r7 as precise */ + "*(u64 *)(r1 + 0) = 0;" + "r0 = 0;" + "exit;" + ::: __clobber_all + ); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_zext.c b/tools/testing/selftests/bpf/progs/verifier_zext.c index 8f2362da91d6..572017fe28fb 100644 --- a/tools/testing/selftests/bpf/progs/verifier_zext.c +++ b/tools/testing/selftests/bpf/progs/verifier_zext.c @@ -356,6 +356,32 @@ __naked void arena_ptr(void) : __clobber_all); } +/* + * Result of a 32-bit cmpxchg is always explicitly zero extended. + * Check that this holds for arenas (BPF_PROBE_ATOMIC instruction flavor). + */ +SEC("socket") +__success +__xlated("probe r0 = atomic_cmpxchg((u32 *)(r1 +0), r0, r2)") +__xlated("w0 = w0") +__naked void zext_arena_cmpxchg32(void) +{ + asm volatile (" \ + r9 = %[arena] ll; /* associate the arena with the program */ \ + r1 = 0; \ + r1 = addr_space_cast(r1, 0, 1); \ + r0 = 0; \ + r2 = 0; \ + .8byte %[cmpxchg32]; \ + r0 >>= 32; /* make the upper half live */ \ + exit; \ +" : + : __imm_addr(arena), + __imm_insn(cmpxchg32, + BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_1, BPF_REG_2, 0)) + : __clobber_all); +} + #endif /* Check if probe mem loads keep their zero extension. */ diff --git a/tools/testing/selftests/bpf/verifier/pseudo_func.c b/tools/testing/selftests/bpf/verifier/pseudo_func.c new file mode 100644 index 000000000000..63c5c67d51de --- /dev/null +++ b/tools/testing/selftests/bpf/verifier/pseudo_func.c @@ -0,0 +1,45 @@ +/* + * Buggy verifier accepted the program below while not patching BPF_PSEUDO_FUNC + * load instruction to contain a real address. Which resulted in a function call + * to a bogus address. + */ +{ + "BPF_PSEUDO_FUNC reference to the main program", + .insns = { + /* r6 = bpf_map_lookup_elem(&timer_map, &(int){0}); */ + BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0), + BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10), + BPF_MOV64_REG(BPF_REG_6, BPF_REG_0), + /* bpf_timer_init(r6, &timer_map, 0); */ + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_LD_MAP_FD(BPF_REG_2, 0), + BPF_MOV64_IMM(BPF_REG_3, 0), + BPF_EMIT_CALL(BPF_FUNC_timer_init), + /* bpf_timer_set_callback(r6, <insn #0>); */ + BPF_MOV64_REG(BPF_REG_1, BPF_REG_6), + BPF_RAW_INSN(BPF_LD | BPF_IMM | BPF_DW, BPF_REG_2, BPF_PSEUDO_FUNC, 0, -15), + BPF_RAW_INSN(0, 0, 0, 0, 0), + BPF_EMIT_CALL(BPF_FUNC_timer_set_callback), + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .fixup_map_timer = { 3, 9 }, + .result = REJECT, + .errstr = "callback function cannot be the main program", + .func_info = { { 0, 4 /* main_prog */ } }, + .func_info_cnt = 1, + .btf_strings = "\0int\0ctx\0main_prog", + .btf_types = { + /* 1: int */ BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), + /* 2: void* */ BTF_PTR_ENC(0), + /* 3: int __(void *) */ BTF_FUNC_PROTO_ENC(1, 1), + BTF_FUNC_PROTO_ARG_ENC(5, 2), + /* 4: main_prog */ BTF_FUNC_ENC(9, 3), + BTF_END_RAW + } +}, diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index e9bee164bb70..20d2b63774c3 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -919,7 +919,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), &nsdelegate)) { if (setup_named_v1_root(root, sizeof(root), CG_NAMED_NAME)) ksft_exit_skip("cgroup v2 isn't mounted and could not setup named v1 hierarchy\n"); @@ -932,6 +931,7 @@ int main(int argc, char *argv[]) ksft_exit_skip("Failed to set memory controller\n"); post_v2_setup: + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index f9f7017d9299..735a53bb222b 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -832,7 +832,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -840,6 +839,7 @@ int main(int argc, char *argv[]) if (cg_write(root, "cgroup.subtree_control", "+cpu")) ksft_exit_skip("Failed to set cpu controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index 8c2d4d4ef1fc..3dfadd280c1c 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -497,7 +497,6 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -505,6 +504,7 @@ int main(int argc, char *argv[]) if (cg_write(root, "cgroup.subtree_control", "+cpuset")) ksft_exit_skip("Failed to set cpuset controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index da8f7b920178..131d8b4551ef 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -797,7 +797,6 @@ check_isolcpus() EXPECTED_ISOLCPUS=$1 ISCPUS=${CGROUP2}/cpuset.cpus.isolated ISOLCPUS=$(cat $ISCPUS) - HKICPUS=$(cat /sys/devices/system/cpu/isolated) LASTISOLCPU= SCHED_DOMAINS=/sys/kernel/debug/sched/domains if [[ $EXPECTED_ISOLCPUS = . ]] @@ -836,11 +835,6 @@ check_isolcpus() EXPECTED_ISOLCPUS=$EXPECTED_SDOMAIN # - # The inverse of HK_TYPE_DOMAIN cpumask in $HKICPUS should match $ISOLCPUS - # - [[ "$ISOLCPUS" != "$HKICPUS" ]] && return 1 - - # # Use the sched domain in debugfs to check isolated CPUs, if available # [[ -d $SCHED_DOMAINS ]] || return 0 @@ -1162,6 +1156,63 @@ test_isolated() } # +# Select an online CPU isolated from scheduler domains at boot. +# $1: test name used in the skip message +# +get_boot_isolated_cpu() +{ + TEST_NAME=$1 + BOOT_ISOLATED_FILE=/sys/devices/system/cpu/isolated + + [[ -r $BOOT_ISOLATED_FILE ]] || { + echo "$TEST_NAME test SKIPPED: boot isolation state unavailable" + return 1 + } + BOOT_CPUS=$(cat $BOOT_ISOLATED_FILE) + [[ -n "$BOOT_CPUS" ]] || { + echo "$TEST_NAME test SKIPPED: no boot-isolated CPU" + return 1 + } + + BOOT_CPU=$(echo "$BOOT_CPUS" | sed -e 's/[,-].*//') + CPU_ONLINE=/sys/devices/system/cpu/cpu${BOOT_CPU}/online + [[ ! -e $CPU_ONLINE || $(cat $CPU_ONLINE) -eq 1 ]] || { + echo "$TEST_NAME test SKIPPED: CPU $BOOT_CPU is offline" + return 1 + } +} + +# +# A CPU isolated at boot must stay isolated after it is released by a dynamic +# isolated partition. +# +test_boot_isolated() +{ + TEST_NAME="Boot-isolated CPU partition release" + get_boot_isolated_cpu "$TEST_NAME" || return 0 + echo "Running $TEST_NAME test ..." + + cd $CGROUP2/test + echo member > cpuset.cpus.partition + echo $BOOT_CPU > cpuset.cpus + [[ $(cat cpuset.cpus.effective) = "$BOOT_CPU" ]] || { + echo "$TEST_NAME test SKIPPED: CPU $BOOT_CPU is unavailable" + echo "" > cpuset.cpus + cd $CGROUP2 + return 0 + } + test_partition isolated + test_partition member + check_isolcpus "." || { + echo "Boot-isolated CPU $BOOT_CPU was lost after partition release" + exit 1 + } + echo "" > cpuset.cpus + cd $CGROUP2 + echo "$TEST_NAME test PASSED." +} + +# # Wait for inotify event for the given file and read it # $1: cgroup file to wait for # $2: file to store the read result @@ -1232,5 +1283,6 @@ trap cleanup 0 2 3 6 run_state_test TEST_MATRIX run_remote_state_test REMOTE_TEST_MATRIX test_isolated +test_boot_isolated test_inotify echo "All tests PASSED." diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c index 0569e93fa6b0..f28bb02e9783 100644 --- a/tools/testing/selftests/cgroup/test_freezer.c +++ b/tools/testing/selftests/cgroup/test_freezer.c @@ -1491,9 +1491,9 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_kill.c b/tools/testing/selftests/cgroup/test_kill.c index f6cd23a8ecc7..bac1ddd8cb94 100644 --- a/tools/testing/selftests/cgroup/test_kill.c +++ b/tools/testing/selftests/cgroup/test_kill.c @@ -7,6 +7,7 @@ #include <stdlib.h> #include <string.h> #include <sys/types.h> +#include <sys/wait.h> #include <unistd.h> #include "kselftest.h" @@ -261,6 +262,59 @@ cleanup: return ret; } +/* + * Test that a cgroup that was killed in the past can still be the target + * of clone3(CLONE_INTO_CGROUP): writing cgroup.kill must only kill the + * tasks in the cgroup at the time of the write, not tasks cloned into + * it afterwards. + */ +static int test_cgkill_clone_into_killed(const char *root) +{ + pid_t pid; + int cgroup_fd = -EBADF; + int ret = KSFT_FAIL; + char *cgroup = NULL; + + cgroup = cg_name(root, "cg_test_clone_into_killed"); + if (!cgroup) + goto cleanup; + + if (cg_create(cgroup)) + goto cleanup; + + /* Kill the cgroup while it is still empty. */ + if (cg_write(cgroup, "cgroup.kill", "1")) + goto cleanup; + + cgroup_fd = dirfd_open_opath(cgroup); + if (cgroup_fd < 0) + goto cleanup; + + pid = clone_into_cgroup(cgroup_fd); + if (pid < 0) { + if (errno == ENOSYS) + ret = KSFT_SKIP; + goto cleanup; + } + + if (pid == 0) + exit(EXIT_SUCCESS); + + /* The child must not be SIGKILLed; it has to exit cleanly. */ + if (clone_reap(pid, WEXITED) != EXIT_SUCCESS) + goto cleanup; + + ret = KSFT_PASS; + +cleanup: + if (cgroup_fd >= 0) + close(cgroup_fd); + if (cgroup) + cg_destroy(cgroup); + free(cgroup); + return ret; +} + #define T(x) { x, #x } struct cgkill_test { int (*fn)(const char *root); @@ -269,6 +323,7 @@ struct cgkill_test { T(test_cgkill_simple), T(test_cgkill_tree), T(test_cgkill_forkbomb), + T(test_cgkill_clone_into_killed), }; #undef T @@ -278,9 +333,9 @@ int main(int argc, char *argv[]) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c index 1db0ba1226b9..437f2d35f205 100644 --- a/tools/testing/selftests/cgroup/test_kmem.c +++ b/tools/testing/selftests/cgroup/test_kmem.c @@ -145,7 +145,7 @@ static int cg_run_in_subcgroups(const char *parent, return -1; } - if (cg_run(child, fn, NULL)) { + if (cg_run(child, fn, arg)) { cg_destroy(child); free(child); return -1; @@ -426,7 +426,6 @@ int main(int argc, char **argv) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -441,6 +440,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+memory")) ksft_exit_skip("Failed to set memory controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index 0ebf796f3cff..3a84d068fbf3 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -1798,7 +1798,6 @@ int main(int argc, char **argv) page_size = BUF_SIZE; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -1823,6 +1822,7 @@ int main(int argc, char **argv) ksft_exit_skip("Failed to query cgroup mount option\n"); has_localevents = proc_status; + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_pids.c b/tools/testing/selftests/cgroup/test_pids.c index 9a387c815d2c..710109b53dfe 100644 --- a/tools/testing/selftests/cgroup/test_pids.c +++ b/tools/testing/selftests/cgroup/test_pids.c @@ -148,7 +148,6 @@ int main(int argc, char **argv) char root[PATH_MAX]; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -163,6 +162,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+pids")) ksft_exit_skip("Failed to set pids controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (int i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index f7b4c4370db6..609c48f38524 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -819,7 +819,6 @@ int main(int argc, char **argv) page_size = BUF_SIZE; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(tests)); if (cg_find_unified_root(root, sizeof(root), NULL)) ksft_exit_skip("cgroup v2 isn't mounted\n"); @@ -836,6 +835,7 @@ int main(int argc, char **argv) if (cg_write(root, "cgroup.subtree_control", "+memory")) ksft_exit_skip("Failed to set memory controller\n"); + ksft_set_plan(ARRAY_SIZE(tests)); for (i = 0; i < ARRAY_SIZE(tests); i++) { switch (tests[i].fn(root)) { case KSFT_PASS: diff --git a/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c b/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c index fc9694fc4e89..1d49df671919 100644 --- a/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c +++ b/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c @@ -390,6 +390,116 @@ static void test_alloc_errors(char *heap_name) close(heap_fd); } +/* + * count_open_fds - return the number of open file descriptors. + * + * The fd opened by opendir() itself is counted, but since it is opened + * and closed within each call, it cancels out when comparing two counts. + * Returns -1 on error. + */ +static int count_open_fds(void) +{ + DIR *d = opendir("/proc/self/fd"); + struct dirent *de; + int count = 0; + + if (!d) + return -1; + + while ((de = readdir(d))) + if (de->d_name[0] != '.') + count++; + closedir(d); + return count; +} + +/* + * test_alloc_no_fd_leak_on_efault - verify no fd is leaked when + * copy_to_user() fails during DMA_HEAP_IOCTL_ALLOC. + * + * The bug: dma_buf_fd() called fd_install() before copy_to_user(). + * If copy_to_user() then failed (e.g. via mprotect), the fd was + * silently installed in the fd table but never returned to userspace. + * + * The fix: reserve the fd with get_unused_fd_flags() first, attempt + * copy_to_user(), and only call fd_install() on success. + * + * We trigger the failure by placing the ioctl argument in a private + * anonymous page and flipping it to PROT_READ before the ioctl. + * Inside the kernel, copy_from_user() reads from the page (reads are + * allowed under PROT_READ, so it succeeds), but copy_to_user() that + * writes the fd number back faults, returning -EFAULT. We then + * count open file descriptors before and after; with the bug an extra + * fd is left in the table. + */ +static void test_alloc_no_fd_leak_on_efault(char *heap_name) +{ + int heap_fd = -1; + int fd_before, fd_after; + int ret; + long page_size; + struct dma_heap_allocation_data *req; + + ksft_print_msg("Testing fd leak when copy_to_user() fails:\n"); + + heap_fd = dmabuf_heap_open(heap_name); + + page_size = sysconf(_SC_PAGESIZE); + + /* + * Place the ioctl argument in its own private anonymous page so + * we can flip its protection independently. + */ + req = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (req == MAP_FAILED) { + ksft_test_result_fail("mmap failed: %s\n", strerror(errno)); + goto out; + } + + memset(req, 0, sizeof(*req)); + req->len = page_size; + req->fd_flags = O_RDWR | O_CLOEXEC; + + fd_before = count_open_fds(); + if (fd_before < 0) { + ksft_test_result_fail("count_open_fds: %s\n", strerror(errno)); + munmap(req, page_size); + goto out; + } + + /* + * Make the page read-only so copy_to_user() will fault. The + * ioctl must fail with -1; if it returns success the test setup + * is broken (mprotect is synchronous, so there is no race). + */ + mprotect(req, page_size, PROT_READ); + + ret = ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, req); + + /* Re-allow writes so munmap can clean up */ + mprotect(req, page_size, PROT_READ | PROT_WRITE); + munmap(req, page_size); + + if (ret != -1) { + ksft_test_result_fail("ioctl returned %d, expected -1 EFAULT\n", + ret); + goto out; + } + + fd_after = count_open_fds(); + if (fd_after < 0) { + ksft_test_result_fail("count_open_fds: %s\n", strerror(errno)); + goto out; + } + + ksft_test_result(fd_before == fd_after, + "fd leak on EFAULT: before=%d after=%d\n", + fd_before, fd_after); +out: + close(heap_fd); +} + static int numer_of_heaps(void) { DIR *d = opendir(DEVPATH); @@ -420,7 +530,7 @@ int main(void) return KSFT_SKIP; } - ksft_set_plan(11 * numer_of_heaps()); + ksft_set_plan(12 * numer_of_heaps()); while ((dir = readdir(d))) { if (!strncmp(dir->d_name, ".", 2)) @@ -435,6 +545,7 @@ int main(void) test_alloc_zeroed(dir->d_name, ONE_MEG); test_alloc_compat(dir->d_name); test_alloc_errors(dir->d_name); + test_alloc_no_fd_leak_on_efault(dir->d_name); } closedir(d); diff --git a/tools/testing/selftests/hid/hid_bpf.c b/tools/testing/selftests/hid/hid_bpf.c index b851339308c2..7ab86296ff23 100644 --- a/tools/testing/selftests/hid/hid_bpf.c +++ b/tools/testing/selftests/hid/hid_bpf.c @@ -67,14 +67,17 @@ struct test_program { int insert_head; }; #define LOAD_PROGRAMS(progs) \ - load_programs(progs, ARRAY_SIZE(progs), _metadata, self, variant) + load_programs(progs, ARRAY_SIZE(progs), false, _metadata, self, variant) +#define LOAD_PROGRAMS_MAY_FAIL(progs) \ + load_programs(progs, ARRAY_SIZE(progs), true, _metadata, self, variant) #define LOAD_BPF \ - load_programs(NULL, 0, _metadata, self, variant) -static void load_programs(const struct test_program programs[], - const size_t progs_count, - struct __test_metadata *_metadata, - FIXTURE_DATA(hid_bpf) * self, - const FIXTURE_VARIANT(hid_bpf) * variant) + load_programs(NULL, 0, false, _metadata, self, variant) +static int load_programs(const struct test_program programs[], + const size_t progs_count, + bool load_may_fail, + struct __test_metadata *_metadata, + FIXTURE_DATA(hid_bpf) * self, + const FIXTURE_VARIANT(hid_bpf) * variant) { struct bpf_map *iter_map; int err = -EINVAL; @@ -128,6 +131,9 @@ static void load_programs(const struct test_program programs[], } err = hid__load(self->skel); + if (err && load_may_fail) + return err; + ASSERT_OK(err) TH_LOG("hid_skel_load failed: %d", err); for (int i = 0; i < progs_count; i++) { @@ -147,6 +153,7 @@ static void load_programs(const struct test_program programs[], self->hidraw_fd = open_hidraw(&self->hid); ASSERT_GE(self->hidraw_fd, 0) TH_LOG("open_hidraw"); + return 0; } /* @@ -904,11 +911,39 @@ TEST_F(hid_bpf, test_rdesc_fixup_get_data_overflow) { .name = "hid_rdesc_fixup_get_data_overflow" }, }; - LOAD_PROGRAMS(progs); + /* newer verifier can detect the overflow at load time */ + if (LOAD_PROGRAMS_MAY_FAIL(progs)) + return; ASSERT_EQ(self->skel->bss->get_data_overflow_check, 1); } +TEST_F(hid_bpf, test_rdesc_fixup_change_uniq_name_phys) +{ + const struct test_program progs[] = { + { .name = "hid_rdesc_fixup_change_uniq_name_phys" }, + }; + char expected[256], buf[256] = {}; + int err; + + LOAD_PROGRAMS(progs); + + err = ioctl(self->hidraw_fd, HIDIOCGRAWNAME(sizeof(buf)), buf); + ASSERT_GE(err, 0) TH_LOG("HIDIOCGRAWNAME"); + ASSERT_STREQ("name coming from bpf", buf); + + snprintf(expected, sizeof(expected), "%d phys:coming:from:bpf", self->hid.dev_id); + + err = ioctl(self->hidraw_fd, HIDIOCGRAWPHYS(sizeof(buf)), buf); + ASSERT_GE(err, 0) TH_LOG("HIDIOCGRAWPHYS"); + ASSERT_STREQ(expected, buf); + + err = ioctl(self->hidraw_fd, HIDIOCGRAWUNIQ(sizeof(buf)), buf); + ASSERT_GE(err, 0) TH_LOG("HIDIOCGRAWUNIQ"); + ASSERT_STREQ("uniq:coming:from:bpf", buf); + +} + static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) { diff --git a/tools/testing/selftests/hid/progs/hid.c b/tools/testing/selftests/hid/progs/hid.c index b21fbb13c926..361dc7eaad22 100644 --- a/tools/testing/selftests/hid/progs/hid.c +++ b/tools/testing/selftests/hid/progs/hid.c @@ -255,6 +255,32 @@ struct hid_bpf_ops rdesc_fixup_get_data_overflow = { .hid_rdesc_fixup = (void *)hid_rdesc_fixup_get_data_overflow, }; +SEC("?struct_ops.s/hid_rdesc_fixup") +int BPF_PROG(hid_rdesc_fixup_change_uniq_name_phys, struct hid_bpf_ctx *hid_ctx) +{ +#define HID_BPF_MEMCPY(target, str) \ + __builtin_memcpy(target, str, sizeof(str)) + + HID_BPF_MEMCPY(hid_ctx->hid->name, "name coming from bpf"); + HID_BPF_MEMCPY(hid_ctx->hid->uniq, "uniq:coming:from:bpf"); + /* hid_bpf relies on a phys being a rand % 1024 */ + for (int i = 0; i < 5; i++) { + if (!hid_ctx->hid->phys[i]) { + HID_BPF_MEMCPY(hid_ctx->hid->phys + i, " phys:coming:from:bpf"); + break; + } + } + +#undef HID_BPF_MEMCPY + + return 0; +} + +SEC(".struct_ops.link") +struct hid_bpf_ops rdesc_fixup_change_uniq_name_phys = { + .hid_rdesc_fixup = (void *)hid_rdesc_fixup_change_uniq_name_phys, +}; + SEC("?struct_ops/hid_device_event") int BPF_PROG(hid_test_insert1, struct hid_bpf_ctx *hid_ctx, enum hid_report_type type) { diff --git a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h index cdca912f3afd..05698793762a 100644 --- a/tools/testing/selftests/hid/progs/hid_bpf_helpers.h +++ b/tools/testing/selftests/hid/progs/hid_bpf_helpers.h @@ -61,6 +61,9 @@ enum hid_report_type { struct hid_device { unsigned int id; + char name[128]; + char phys[64]; + char uniq[64]; } __attribute__((preserve_access_index)); struct bpf_wq { diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index aac4f795c327..c55d84c5e613 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -57,33 +57,6 @@ static void test_file_apis(int fd) pass("file IO is blocked as expected\n"); } -static void test_mlock_limit(int fd) -{ - size_t len; - char *mem; - - len = mlock_limit_cur; - if (len % page_size != 0) - len = (len/page_size) * page_size; - - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem == MAP_FAILED) { - fail("unable to mmap secret memory\n"); - return; - } - munmap(mem, len); - - len = mlock_limit_max * 2; - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem != MAP_FAILED) { - fail("unexpected mlock limit violation\n"); - munmap(mem, len); - return; - } - - pass("mlock limit is respected\n"); -} - static void test_vmsplice(int fd, const char *desc) { ssize_t transferred; @@ -297,7 +270,7 @@ static void prepare(void) strerror(errno)); } -#define NUM_TESTS 6 +#define NUM_TESTS 5 int main(int argc, char *argv[]) { @@ -319,7 +292,6 @@ int main(int argc, char *argv[]) if (ftruncate(fd, page_size)) ksft_exit_fail_msg("ftruncate failed: %s\n", strerror(errno)); - test_mlock_limit(fd); test_file_apis(fd); /* * We have to run the first vmsplice test before any secretmem page was diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile index 0f5c178bc224..517c09d60bef 100644 --- a/tools/testing/selftests/net/Makefile +++ b/tools/testing/selftests/net/Makefile @@ -28,6 +28,7 @@ TEST_PROGS := \ double_udp_encap.sh \ drop_monitor_tests.sh \ ecmp_rehash.sh \ + exception_cache.sh \ fcnal-ipv4.sh \ fcnal-ipv6.sh \ fcnal-other.sh \ diff --git a/tools/testing/selftests/net/exception_cache.sh b/tools/testing/selftests/net/exception_cache.sh new file mode 100755 index 000000000000..8d3eed5c532a --- /dev/null +++ b/tools/testing/selftests/net/exception_cache.sh @@ -0,0 +1,521 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Test that the state of the route exception cache after an ICMP error is +# processed does not depend on whether the quoted packet was matched to a +# socket. Otherwise, an off-path attacker can probe the cache to discover the +# ephemeral port used by a connected UDP socket. +# +# When the quoted packet is not matched to a socket, the same exception is +# created as when it is matched, so that neither its presence nor its contents +# reveal the result of socket matching. +# +# +----+ +# +---------| r1 | +# | +----+ +# +----+ +--------+ | .1 +# | h1 |---| bridge | | 198.51.100.0/30 +# +----+ +--------+ | 2001:db8:2::/64 +# .1 | | .2 +# | +----+ +----+ +# +---------| r2 |-----------| h2 | +# .2 .3 +----+ .1 .2 +----+ +# 203.0.113.0/24 +# 2001:db8:3::/64 +# 192.0.2.0/24 +# 2001:db8:1::/64 +# +# Traffic from h1 to h2 is routed via r1, which reaches h2's network via r2 +# over the point-to-point link. The MTU of the r2 - h2 link is lowered so that +# r2 emits ICMP errors towards h1. +# +# For the redirect tests r1's route to h2's network is replaced with one via r2 +# on the shared segment, so that r1 forwards the packet back to the segment it +# arrived from and emits a redirect towards h1. +# +# The packets that provoke the ICMP errors are injected with a packet socket so +# that no socket is ever associated with them. A socket is created separately, +# with socat, when a test needs the ICMP error to be matched. + +# shellcheck disable=SC1091,SC2034,SC2154,SC2329 +source lib.sh + +require_command jq +require_command mausezahn +require_command nstat +require_command socat + +ALL_TESTS=" + pmtu_no_socket_ipv4 + pmtu_no_socket_ipv6 + pmtu_socket_ipv4 + pmtu_socket_ipv6 + pmtu_omit_ipv4 + pmtu_omit_ipv6 + redirect_no_socket_ipv4 + redirect_no_socket_ipv6 + redirect_socket_ipv4 + redirect_socket_ipv6 +" + +# Shared segment. +H1_ADDR4=192.0.2.1 +R1_ADDR4=192.0.2.2 +R2_ADDR4=192.0.2.3 +H1_ADDR6=2001:db8:1::1 +R1_ADDR6=2001:db8:1::2 +R2_ADDR6=2001:db8:1::3 + +# r1 - r2 link. +R2_R1_ADDR4=198.51.100.2 +R2_R1_ADDR6=2001:db8:2::2 + +# r2 - h2 link. +H2_ADDR4=203.0.113.2 +H2_NET4=203.0.113.0/24 +H2_ADDR6=2001:db8:3::2 +H2_NET6=2001:db8:3::/64 + +SPORT=12345 +DPORT=54321 + +# The MTU of the shared segment and of the r1 - r2 link. Large enough for the +# injected packets to reach r2 intact. +SEGMENT_MTU=2000 +# Size of the injected packets. The PMTU tests need a size that exceeds every +# MTU used for the r2 - h2 link, so that r2 responds with an ICMP error. The +# redirect tests need a size that does not, otherwise r2 would respond with an +# ICMP error in addition to the redirect emitted by r1. +PMTU_PACKET_SIZE=1800 +REDIRECT_PACKET_SIZE=100 + +# The MTUs used for the r2 - h2 link. All of them must be at least +# IPV6_MIN_MTU, otherwise IPv6 silently ignores the error instead of creating +# an exception. +MTU_MID=1400 +MTU_LOW=1300 + +# Values for the IP{,V6}_MTU_DISCOVER socket option. +PMTUDISC_DONT=0 +PMTUDISC_OMIT=5 + +SOCAT_PID= + +linklocal_get() +{ + local ns=$1; shift + local dev=$1; shift + + ip -n "$ns" -j -6 addr show dev "$dev" | \ + jq -r '.[]["addr_info"][] | select(.scope == "link") | .local' +} + +linklocal_exists() +{ + local ns=$1; shift + local dev=$1; shift + + [ -n "$(linklocal_get "$ns" "$dev")" ] +} + +family_vars_set() +{ + local family=$1; shift + + FAMILY=$family + + if [ "$family" -eq 4 ]; then + H1_ADDR=$H1_ADDR4 + H2_ADDR=$H2_ADDR4 + MZ_FAMILY_OPT=() + # Without the Don't Fragment bit set r2 fragments the packet + # instead of reporting the MTU of the next hop. + MZ_IP_OPTS="df," + SOCAT_DST="UDP4-CONNECT:$H2_ADDR4:$DPORT" + SOCAT_BIND="bind=$H1_ADDR4:$SPORT" + SOCAT_PMTUDISC="ip-mtu-discover" + else + H1_ADDR=$H1_ADDR6 + H2_ADDR=$H2_ADDR6 + MZ_FAMILY_OPT=(-6) + MZ_IP_OPTS= + SOCAT_DST="UDP6-CONNECT:[$H2_ADDR6]:$DPORT" + SOCAT_BIND="bind=[$H1_ADDR6]:$SPORT" + SOCAT_PMTUDISC="ipv6-mtu-discover" + fi +} + +topology_setup() +{ + local ns + + setup_ns h1 r1 r2 h2 sw + defer cleanup_all_ns + + # Link-local addresses are generated from the MAC address and read + # back during setup, so request that generation mode explicitly and + # make the addresses available as soon as the devices are brought up. + for ns in "$h1" "$r1" "$r2" "$h2" "$sw"; do + ip netns exec "$ns" sysctl -qw \ + net.ipv6.conf.default.addr_gen_mode=0 \ + net.ipv6.conf.default.accept_dad=0 \ + net.ipv6.conf.all.accept_dad=0 + done + + ip -n "$sw" link add name br0 type bridge + ip -n "$sw" link set dev br0 mtu "$SEGMENT_MTU" up + + ip -n "$h1" link add name eth0 mtu "$SEGMENT_MTU" type veth \ + peer name swp1 mtu "$SEGMENT_MTU" netns "$sw" + ip -n "$r1" link add name eth0 mtu "$SEGMENT_MTU" type veth \ + peer name swp2 mtu "$SEGMENT_MTU" netns "$sw" + ip -n "$r2" link add name eth0 mtu "$SEGMENT_MTU" type veth \ + peer name swp3 mtu "$SEGMENT_MTU" netns "$sw" + ip -n "$r1" link add name eth1 mtu "$SEGMENT_MTU" type veth \ + peer name eth1 mtu "$SEGMENT_MTU" netns "$r2" + ip -n "$r2" link add name eth2 type veth peer name eth0 netns "$h2" + + ip -n "$sw" link set dev swp1 master br0 up + ip -n "$sw" link set dev swp2 master br0 up + ip -n "$sw" link set dev swp3 master br0 up + + ip -n "$h1" link set dev eth0 up + ip -n "$r1" link set dev eth0 up + ip -n "$r1" link set dev eth1 up + ip -n "$r2" link set dev eth0 up + ip -n "$r2" link set dev eth1 up + ip -n "$r2" link set dev eth2 up + ip -n "$h2" link set dev eth0 up + + ip -n "$h1" address add "$H1_ADDR4/24" dev eth0 + ip -n "$r1" address add "$R1_ADDR4/24" dev eth0 + ip -n "$r2" address add "$R2_ADDR4/24" dev eth0 + ip -n "$r1" address add 198.51.100.1/30 dev eth1 + ip -n "$r2" address add "$R2_R1_ADDR4/30" dev eth1 + ip -n "$r2" address add 203.0.113.1/24 dev eth2 + ip -n "$h2" address add "$H2_ADDR4/24" dev eth0 + + ip -n "$h1" -6 address add "$H1_ADDR6/64" dev eth0 nodad + ip -n "$r1" -6 address add "$R1_ADDR6/64" dev eth0 nodad + ip -n "$r2" -6 address add "$R2_ADDR6/64" dev eth0 nodad + ip -n "$r1" -6 address add 2001:db8:2::1/64 dev eth1 nodad + ip -n "$r2" -6 address add "$R2_R1_ADDR6/64" dev eth1 nodad + ip -n "$r2" -6 address add 2001:db8:3::1/64 dev eth2 nodad + ip -n "$h2" -6 address add "$H2_ADDR6/64" dev eth0 nodad + + ip netns exec "$r1" sysctl -qw net.ipv4.ip_forward=1 + ip netns exec "$r1" sysctl -qw net.ipv4.conf.all.send_redirects=1 + ip netns exec "$r1" sysctl -qw net.ipv6.conf.all.forwarding=1 + ip netns exec "$r2" sysctl -qw net.ipv4.ip_forward=1 + ip netns exec "$r2" sysctl -qw net.ipv6.conf.all.forwarding=1 + + ip netns exec "$h1" sysctl -qw net.ipv4.conf.all.accept_redirects=1 + ip netns exec "$h1" sysctl -qw net.ipv4.conf.eth0.accept_redirects=1 + ip netns exec "$h1" sysctl -qw net.ipv6.conf.all.accept_redirects=1 + ip netns exec "$h1" sysctl -qw net.ipv6.conf.eth0.accept_redirects=1 + + slowwait 5 linklocal_exists "$r1" eth0 + check_err $? "r1: link-local address was not generated" + slowwait 5 linklocal_exists "$r2" eth0 + check_err $? "r2: link-local address was not generated" + + R1_LLADDR=$(linklocal_get "$r1" eth0) + R2_LLADDR=$(linklocal_get "$r2" eth0) + R1_MAC=$(ip -n "$r1" -j link show dev eth0 | jq -r '.[]["address"]') + R2_MAC=$(ip -n "$r2" -j link show dev eth0 | jq -r '.[]["address"]') + + ip -n "$h1" route add "$H2_NET4" via "$R1_ADDR4" dev eth0 + ip -n "$h1" -6 route add "$H2_NET6" via "$R1_LLADDR" dev eth0 + ip -n "$r1" route add "$H2_NET4" via "$R2_R1_ADDR4" dev eth1 + ip -n "$r1" -6 route add "$H2_NET6" via "$R2_R1_ADDR6" dev eth1 + ip -n "$h2" route add default via 203.0.113.1 dev eth0 + ip -n "$h2" -6 route add default via 2001:db8:3::1 dev eth0 + + far_mtu_set "$MTU_MID" +} + +# Make r1 forward towards h2's network over the segment it receives the packet +# from, so that it emits a redirect towards h1. +redirect_route_set() +{ + ip -n "$r1" route replace "$H2_NET4" via "$R2_ADDR4" dev eth0 + ip -n "$r1" -6 route replace "$H2_NET6" via "$R2_LLADDR" dev eth0 + + # __ip_do_redirect() only creates an exception if the new gateway is + # already a valid neighbour. Otherwise it merely triggers address + # resolution. IPv6 resolves the target itself, in rt6_do_redirect(). + ip -n "$h1" neigh replace "$R2_ADDR4" lladdr "$R2_MAC" dev eth0 \ + nud permanent +} + +far_mtu_set() +{ + local mtu=$1; shift + + ip -n "$r2" link set dev eth2 mtu "$mtu" + ip -n "$h2" link set dev eth0 mtu "$mtu" +} + +socket_is_open() +{ + ip netns exec "$h1" ss -uHn "sport = :$SPORT" | grep -q . +} + +socket_start() +{ + # Disable PMTU discovery by default so that ICMP errors are not + # reported to the socket. Otherwise socat would exit when the first one + # arrives and later packets in the same test would not be matched to a + # socket. The exception is still created, as ip{,6}_sk_accept_pmtu() + # only rejects IP{,V6}_PMTUDISC_{INTERFACE,OMIT}. + local pmtudisc=${1:-$PMTUDISC_DONT} + + # Send socat's diagnostics to /dev/null. It reports the ICMP errors + # that reach the socket, which is exactly what the tests provoke. + ip netns exec "$h1" socat -u -lf/dev/null \ + "$SOCAT_DST,$SOCAT_BIND,$SOCAT_PMTUDISC=$pmtudisc" \ + OPEN:/dev/null,wronly=1 & + SOCAT_PID=$! + defer socket_stop + + slowwait 5 socket_is_open + check_err $? "socket did not open" +} + +socket_stop() +{ + [ -z "$SOCAT_PID" ] && return 0 + + kill "$SOCAT_PID" &> /dev/null + wait "$SOCAT_PID" 2> /dev/null + SOCAT_PID= +} + +# Inject a packet towards h2 with a packet socket. No socket is associated with +# it, so an ICMP error quoting it is matched to a socket only if one was +# created separately with the same source port. +packet_send() +{ + local size=$1; shift + + ip netns exec "$h1" mausezahn "${MZ_FAMILY_OPT[@]}" eth0 \ + -a own -b "$R1_MAC" -A "$H1_ADDR" -B "$H2_ADDR" \ + -t udp "${MZ_IP_OPTS}sp=$SPORT,dp=$DPORT" \ + -p "$size" -c 1 -q +} + +exception_show() +{ + if [ "$FAMILY" -eq 4 ]; then + # IPv4 exceptions without a bound route are not dumped, but + # "route get" reports the exception and binds a route to it. + ip -n "$h1" route get "$H2_ADDR" + else + # IPv6 does not report a cache indication in "route get" + # output, so dump the exceptions instead. + ip -n "$h1" -6 route show cache | grep -F "$H2_ADDR" || true + fi +} + +exception_mtu_get() +{ + exception_show | grep -o "mtu [0-9]*" | cut -d ' ' -f 2 +} + +exception_gw_get() +{ + exception_show | grep -o "via [0-9a-f.:]*" | cut -d ' ' -f 2 +} + +exception_mtu_check() +{ + local expected=$1; shift + + [ "$(exception_mtu_get)" = "$expected" ] +} + +icmp_errors_get() +{ + local ctr=IcmpInDestUnreachs + + [ "$FAMILY" -eq 6 ] && ctr=Icmp6InPktTooBigs + + ip netns exec "$h1" nstat -asz "$ctr" | \ + awk -v ctr="$ctr" '$1 == ctr { print $2 }' +} + +exception_pmtu_check() +{ + local mtu=$1; shift + local desc=$1; shift + + busywait "$BUSYWAIT_TIMEOUT" exception_mtu_check "$mtu" + check_err $? "$desc: exception does not carry an MTU of $mtu" +} + +pmtu_no_socket() +{ + local family=$1; shift + + RET=0 + family_vars_set "$family" + topology_setup + + packet_send "$PMTU_PACKET_SIZE" + exception_pmtu_check "$MTU_MID" "No socket" + + log_test "IPv$family: PMTU: exception without a matching socket" +} + +pmtu_no_socket_ipv4() +{ + pmtu_no_socket 4 +} + +pmtu_no_socket_ipv6() +{ + pmtu_no_socket 6 +} + +pmtu_socket() +{ + local family=$1; shift + local t0 + + RET=0 + family_vars_set "$family" + topology_setup + socket_start + + packet_send "$PMTU_PACKET_SIZE" + exception_pmtu_check "$MTU_MID" "Matching socket" + + # A lower PMTU replaces the one currently stored in the exception. + far_mtu_set "$MTU_LOW" + packet_send "$PMTU_PACKET_SIZE" + exception_pmtu_check "$MTU_LOW" "Lower PMTU" + + # A higher PMTU is ignored, so the exception is left as it is. Wait + # for the error to be received, as otherwise the check below would + # pass even if it never was. + far_mtu_set "$MTU_MID" + t0=$(icmp_errors_get) + packet_send "$PMTU_PACKET_SIZE" + busywait "$BUSYWAIT_TIMEOUT" until_counter_is ">= $((t0 + 1))" \ + icmp_errors_get > /dev/null + check_err $? "Higher PMTU: ICMP error was not received" + + exception_mtu_check "$MTU_LOW" + check_err $? "Higher PMTU: exception does not carry an MTU of $MTU_LOW" + + log_test "IPv$family: PMTU: exception with a matching socket" +} + +pmtu_socket_ipv4() +{ + pmtu_socket 4 +} + +pmtu_socket_ipv6() +{ + pmtu_socket 6 +} + +pmtu_omit() +{ + local family=$1; shift + + RET=0 + family_vars_set "$family" + topology_setup + socket_start "$PMTUDISC_OMIT" + + packet_send "$PMTU_PACKET_SIZE" + exception_pmtu_check "$MTU_MID" "PMTU discovery disabled" + + log_test "IPv$family: PMTU: exception with a socket ignoring it" +} + +pmtu_omit_ipv4() +{ + pmtu_omit 4 +} + +pmtu_omit_ipv6() +{ + pmtu_omit 6 +} + +exception_gw_check() +{ + local expected=$1; shift + + [ -n "$expected" ] && [ "$(exception_gw_get)" = "$expected" ] +} + +redirect_gw_new() +{ + if [ "$FAMILY" -eq 4 ]; then + echo "$R2_ADDR4" + else + echo "$R2_LLADDR" + fi +} + +redirect_no_socket() +{ + local family=$1; shift + + RET=0 + family_vars_set "$family" + topology_setup + redirect_route_set + + packet_send "$REDIRECT_PACKET_SIZE" + busywait "$BUSYWAIT_TIMEOUT" exception_gw_check "$(redirect_gw_new)" + check_err $? "No socket: exception does not carry the new gateway" + + log_test "IPv$family: Redirect: exception without a matching socket" +} + +redirect_no_socket_ipv4() +{ + redirect_no_socket 4 +} + +redirect_no_socket_ipv6() +{ + redirect_no_socket 6 +} + +redirect_socket() +{ + local family=$1; shift + + RET=0 + family_vars_set "$family" + topology_setup + redirect_route_set + socket_start + + packet_send "$REDIRECT_PACKET_SIZE" + busywait "$BUSYWAIT_TIMEOUT" exception_gw_check "$(redirect_gw_new)" + check_err $? "Matching socket: exception does not carry the new gateway" + + log_test "IPv$family: Redirect: exception with a matching socket" +} + +redirect_socket_ipv4() +{ + redirect_socket 4 +} + +redirect_socket_ipv6() +{ + redirect_socket 6 +} + +trap defer_scopes_cleanup EXIT +tests_run + +exit "$EXIT_STATUS" diff --git a/tools/testing/selftests/net/test_vxlan_mdb.sh b/tools/testing/selftests/net/test_vxlan_mdb.sh index 58da5de99ac4..f9600aabd4a2 100755 --- a/tools/testing/selftests/net/test_vxlan_mdb.sh +++ b/tools/testing/selftests/net/test_vxlan_mdb.sh @@ -685,6 +685,9 @@ star_g_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $grp dst $vtep_ip src_vni 10010" log_test $? 255 "Invalid source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $all_zeros_grp dst $vtep_ip src_vni 10010" + log_test $? 255 "All-zeros source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent source_list $src1 dst $vtep_ip src_vni 10010" log_test $? 255 "Source list without filter mode" } @@ -784,6 +787,9 @@ sg_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $grp permanent dst $vtep_ip src_vni 10010" log_test $? 255 "(S, G) with an invalid source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $all_zeros_grp permanent dst $vtep_ip src_vni 10010" + log_test $? 255 "(S, G) with an all-zeros source" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $all_zeros_grp src $src permanent dst $vtep_ip src_vni 10010" log_test $? 255 "All-zeros group with source" } diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json index b2ca9d4e991b..e2b03f2b5e89 100644 --- a/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json +++ b/tools/testing/selftests/tc-testing/tc-tests/filters/u32.json @@ -353,5 +353,28 @@ "teardown": [ "$TC qdisc del dev $DEV1 parent root drr" ] + }, + { + "id": "70fd", + "name": "Add u32 filter when node ID pool is exhausted (4096th filter rejected)", + "category": [ + "filter", + "u32" + ], + "plugins": { + "requires": "nsPlugin" + }, + "setup": [ + "$TC qdisc add dev $DUMMY clsact", + "bash -c 'for i in {1..4095}; do echo filter add dev $DUMMY ingress prio 1 protocol ip u32 match u8 0 0 at 0; done | $TC -b -'" + ], + "cmdUnderTest": "$TC filter add dev $DUMMY ingress prio 1 protocol ip u32 match u8 0 0 at 0", + "expExitCode": "2", + "verifyCmd": "$TC -d filter show dev $DUMMY ingress", + "matchPattern": "fh 800::", + "matchCount": "4095", + "teardown": [ + "$TC qdisc del dev $DUMMY clsact" + ] } ] diff --git a/tools/workqueue/wq_dump.py b/tools/workqueue/wq_dump.py index 31afc24ef17b..9313ebe0c525 100644 --- a/tools/workqueue/wq_dump.py +++ b/tools/workqueue/wq_dump.py @@ -78,6 +78,12 @@ def cpumask_str(cpumask): wq_type_len = 9 +def wq_attrs(wq): + try: + return wq.attrs + except AttributeError: + return wq.unbound_attrs + def wq_type_str(wq): if wq.flags & WQ_BH: return f'{"bh":{wq_type_len}}' @@ -85,7 +91,7 @@ def wq_type_str(wq): if wq.flags & WQ_ORDERED: return f'{"ordered":{wq_type_len}}' else: - if wq.attrs.affn_strict: + if wq_attrs(wq).affn_strict: return f'{"unbound,S":{wq_type_len}}' else: return f'{"unbound":{wq_type_len}}' @@ -206,7 +212,7 @@ for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_( print(f'{wq.name.string_().decode():{WQ_NAME_LEN}}', end='') if wq.flags & WQ_UNBOUND: - print(f' {cpumask_str(wq.attrs.cpumask):{ucpus_len}}', end='') + print(f' {cpumask_str(wq_attrs(wq).cpumask):{ucpus_len}}', end='') else: print(f' {"":{ucpus_len}}', end='') |
