From 96ff396e6e0eb754b73f1870ef7e72f2f80cdbe0 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Fri, 15 May 2026 10:28:40 +0300 Subject: thunderbolt: Use separate lock class for each ring When connected to another host and then unplugging cable lockdep triggers following: ====================================================== WARNING: possible circular locking dependency detected 7.1.0-rc2+ #1775 Tainted: G U ------------------------------------------------------ kworker/u16:6/312 is trying to acquire lock: ffff8881179c70a8 ((work_completion)(&ring->work)){+.+.}-{0:0}, at: __flush_work+0x3cf/0xd10 but task is already holding lock: ffff8881a8b810b0 (&net->connection_lock){+.+.}-{4:4}, at: tbnet_tear_down+0x110/0x720 [thunderbolt_net] which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #1 (&net->connection_lock){+.+.}-{4:4}: __mutex_lock+0x19a/0x2490 mutex_lock_nested+0x1b/0x30 tbnet_handle_packet+0x74c/0xd70 [thunderbolt_net] tb_xdomain_handle_request+0x37c/0x4b0 [thunderbolt] tb_domain_event_cb+0xc9/0x140 [thunderbolt] tb_ctl_handle_event+0xd6/0x2c0 [thunderbolt] tb_ctl_rx_callback+0x22c/0xa10 [thunderbolt] ring_work+0x715/0xcb0 [thunderbolt] process_one_work+0x902/0x1790 worker_thread+0x5cd/0xfe0 kthread+0x339/0x420 ret_from_fork+0x79a/0x9d0 ret_from_fork_asm+0x1a/0x30 -> #0 ((work_completion)(&ring->work)){+.+.}-{0:0}: __lock_acquire+0x1592/0x2640 lock_acquire+0x1a3/0x300 __flush_work+0x3e9/0xd10 flush_work+0x21/0x30 tb_ring_stop+0x240/0x840 [thunderbolt] tbnet_tear_down+0x2ff/0x720 [thunderbolt_net] tbnet_stop+0x47/0x1a0 [thunderbolt_net] __dev_close_many+0x19e/0x4e0 netif_close_many+0x1e8/0x640 unregister_netdevice_many_notify+0x6d3/0x22d0 unregister_netdevice_queue+0x2b9/0x3a0 unregister_netdev+0x1c/0x70 tbnet_remove+0x52/0xb0 [thunderbolt_net] tb_service_remove+0x8a/0xe0 [thunderbolt] device_remove+0xc5/0x190 device_release_driver_internal+0x3db/0x590 device_release_driver+0x12/0x20 bus_remove_device+0x2c1/0x580 device_del+0x3d9/0x9f0 device_unregister+0x17/0xc0 unregister_service+0x46/0x60 [thunderbolt] device_for_each_child_reverse+0xfa/0x180 tb_xdomain_unregister+0x57/0xe0 [thunderbolt] unregister_unplugged_xdomain+0x101/0x1a0 [thunderbolt] bus_for_each_dev+0x111/0x1a0 tb_domain_unregister_unplugged_xdomains+0x98/0xe0 [thunderbolt] tb_handle_hotplug+0xc3/0x2bb0 [thunderbolt] process_one_work+0x902/0x1790 worker_thread+0x5cd/0xfe0 kthread+0x339/0x420 ret_from_fork+0x79a/0x9d0 ret_from_fork_asm+0x1a/0x30 other info that might help us debug this: Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(&net->connection_lock); lock((work_completion)(&ring->work)); lock(&net->connection_lock); lock((work_completion)(&ring->work)); This in fact is false positive because they involve unrelated rings (and unrelated work structures). In the first one it is ring 0 which is used for control traffic and in the second it is dealing with another ring used for the high-speed traffic. Fix this by using separate lock class for each ring worker. Signed-off-by: Mika Westerberg --- include/linux/thunderbolt.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index d48623fda79b..b62dfa52b149 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -22,6 +22,7 @@ struct device; #include #include #include +#include #include #include #include @@ -565,6 +566,7 @@ struct tb_nhi { * @interval_nsec: Interval counter if interrupt throttling is to be * used with this ring (in ns) * @wait: Used to signal that the ring may be empty now + * @lock_key: Lock validator class key per-ring */ struct tb_ring { spinlock_t lock; @@ -590,6 +592,7 @@ struct tb_ring { void *poll_data; unsigned int interval_nsec; wait_queue_head_t wait; + struct lock_class_key lock_key; }; /* Leave ring interrupt enabled on suspend */ -- cgit v1.2.3 From e6031f02269c0f51cf67886d177f02bc300b47cd Mon Sep 17 00:00:00 2001 From: Ivan Pustogarov Date: Thu, 3 Sep 2026 17:26:16 +0200 Subject: wifi: mac80211: avoid out-of-bounds read for empty PREQ elements ieee80211_mesh_preq_size_ok() derives the location of the PREQ bottom fields before checking whether the element contains even the fixed header. ieee80211_mesh_hwmp_preq_get_bottom() reads the flags byte to account for the optional Address Extension field. Consequently, an empty PREQ element causes a one-byte read beyond its declared payload. Move the helper call after both size checks, so the bottom fields are only accessed when they are present. Fixes: 8b40b1d24a60 ("wifi: mac80211: Fix overread in PREQ frame processing") Cc: stable@vger.kernel.org Signed-off-by: Ivan Pustogarov Link: https://patch.msgid.link/20260903152616.1646637-1-ivan@ipust.net Signed-off-by: Johannes Berg --- include/linux/ieee80211-mesh.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/ieee80211-mesh.h b/include/linux/ieee80211-mesh.h index 7eb15834531c..9e548b9173df 100644 --- a/include/linux/ieee80211-mesh.h +++ b/include/linux/ieee80211-mesh.h @@ -361,8 +361,7 @@ ieee80211_mesh_hwmp_perr_get_rcode(const u8 *ie, u8 dst_idx) /* IEEE Std 802.11-2016 9.4.2.113 PREQ element */ static inline bool ieee80211_mesh_preq_size_ok(const u8 *pos, u8 elen) { - struct ieee80211_mesh_hwmp_preq_bottom *preq_elem_bottom = - ieee80211_mesh_hwmp_preq_get_bottom(pos); + struct ieee80211_mesh_hwmp_preq_bottom *preq_elem_bottom; u8 target_count; int needed; @@ -378,6 +377,7 @@ static inline bool ieee80211_mesh_preq_size_ok(const u8 *pos, u8 elen) if (elen < needed) return false; + preq_elem_bottom = ieee80211_mesh_hwmp_preq_get_bottom(pos); target_count = preq_elem_bottom->target_count; /* IEEE Std 802.11-2016 Table 14-10 to 14-16 */ if (target_count < 1) -- cgit v1.2.3 From b890e6163761ddb580cc74f43e15f8bbde15647e Mon Sep 17 00:00:00 2001 From: Nuno Sá Date: Mon, 31 Aug 2026 16:07:17 +0100 Subject: mtd: spinand: fix NULL pointer dereference with no ECC engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When "nand-no-ecc-engine" is set in DT, nanddev_get_ecc_engine() takes the NAND_ECC_ENGINE_TYPE_NONE path and returns success while leaving nand->ecc.engine NULL. The SPI-NAND code nevertheless dereferences it unconditionally to test for a pipelined engine, so probing such a device oopses immediately. Rather than open-coding the test three times, add a nand_ecc_is_pipelined() helper to the NAND core that folds the NULL check into the integration comparison, and use it everywhere. Future callers then cannot reintroduce the problem. Fixes: f9d7c7265bcf ("mtd: spinand: Create direct mapping descriptors for ECC operations") Cc: stable@vger.kernel.org Signed-off-by: Nuno Sá Signed-off-by: Miquel Raynal --- drivers/mtd/nand/ecc.c | 7 +++++++ drivers/mtd/nand/spi/core.c | 8 +++----- include/linux/mtd/nand.h | 2 ++ 3 files changed, 12 insertions(+), 5 deletions(-) (limited to 'include/linux') diff --git a/drivers/mtd/nand/ecc.c b/drivers/mtd/nand/ecc.c index 6ccdff3fc913..1d809cbd47e6 100644 --- a/drivers/mtd/nand/ecc.c +++ b/drivers/mtd/nand/ecc.c @@ -159,6 +159,13 @@ int nand_ecc_finish_io_req(struct nand_device *nand, } EXPORT_SYMBOL(nand_ecc_finish_io_req); +bool nand_ecc_is_pipelined(const struct nand_device *nand) +{ + return nand->ecc.engine && + nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED; +} +EXPORT_SYMBOL(nand_ecc_is_pipelined); + /* Define default OOB placement schemes for large and small page devices */ static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, struct mtd_oob_region *oobregion) diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c index bea352dd82d7..4fb769581595 100644 --- a/drivers/mtd/nand/spi/core.c +++ b/drivers/mtd/nand/spi/core.c @@ -526,8 +526,7 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand, else rdesc->info.op_tmpl = &rdesc->info.primary_op_tmpl; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED && - req->mode != MTD_OPS_RAW) + if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW) rdesc->info.op_tmpl->data.ecc = true; else rdesc->info.op_tmpl->data.ecc = false; @@ -621,8 +620,7 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand, wdesc = spinand->dirmaps[req->pos.plane].wdesc; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED && - req->mode != MTD_OPS_RAW) + if (nand_ecc_is_pipelined(nand) && req->mode != MTD_OPS_RAW) wdesc->info.op_tmpl->data.ecc = true; else wdesc->info.op_tmpl->data.ecc = false; @@ -1279,7 +1277,7 @@ static int spinand_create_dirmap(struct spinand_device *spinand, struct spi_mem_dirmap_desc *desc; bool enable_ecc = false, secondary_op = false; - if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED) + if (nand_ecc_is_pipelined(nand)) enable_ecc = true; if (spinand->cont_read_possible && spinand->op_templates->cont_read_cache) diff --git a/include/linux/mtd/nand.h b/include/linux/mtd/nand.h index 09c8c93e4dba..6936180b6ea5 100644 --- a/include/linux/mtd/nand.h +++ b/include/linux/mtd/nand.h @@ -305,6 +305,8 @@ int nand_ecc_prepare_io_req(struct nand_device *nand, struct nand_page_io_req *req); int nand_ecc_finish_io_req(struct nand_device *nand, struct nand_page_io_req *req); +bool nand_ecc_is_pipelined(const struct nand_device *nand); + bool nand_ecc_is_strong_enough(struct nand_device *nand); #if IS_REACHABLE(CONFIG_MTD_NAND_CORE) -- cgit v1.2.3 From 646e5b28b4fdbc1b828a2398d6ced2c1dd308a46 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Fri, 21 Aug 2026 17:45:45 +0200 Subject: USB: serial: fix driver deregistration order USB serial driver modules register one driver for the USB bus and one or more drivers for the ports on the USB serial bus. When unloading a driver module, the USB driver must be deregistered before the USB serial bus drivers so that I/O is stopped before unbinding the ports to avoid use-after-free in completion handlers accessing port data. Note that the "new_id" attributes must first be removed to prevent new ids from being added and triggering a probe of the USB driver after it has been deregistered. Fixes: 765e0ba62613 ("usb-serial: new API for driver registration") Cc: stable@vger.kernel.org # 3.4 Cc: Alan Stern Signed-off-by: Johan Hovold --- drivers/usb/serial/bus.c | 4 ++++ drivers/usb/serial/usb-serial.c | 21 ++++++++++++++++++--- include/linux/usb/serial.h | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/drivers/usb/serial/bus.c b/drivers/usb/serial/bus.c index 295c61afa44c..ea1fe5d1e449 100644 --- a/drivers/usb/serial/bus.c +++ b/drivers/usb/serial/bus.c @@ -169,3 +169,7 @@ void usb_serial_bus_deregister(struct usb_serial_driver *driver) free_dynids(driver); } +void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver) +{ + driver_remove_file(&driver->driver, &driver_attr_new_id); +} diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c index a4fbc849c0fa..11a0ed6d6546 100644 --- a/drivers/usb/serial/usb-serial.c +++ b/drivers/usb/serial/usb-serial.c @@ -1469,7 +1469,7 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers { int rc; struct usb_driver *udriver; - struct usb_serial_driver * const *sd; + struct usb_serial_driver * const *sd, * const *s; /* * udriver must be registered before any of the serial drivers, @@ -1522,9 +1522,11 @@ int __usb_serial_register_drivers(struct usb_serial_driver *const serial_drivers return 0; err_deregister_drivers: + for (s = serial_drivers; s < sd; ++s) + usb_serial_bus_remove_new_id(*s); + usb_deregister(udriver); while (sd-- > serial_drivers) usb_serial_deregister(*sd); - usb_deregister(udriver); err_free_driver: kfree(udriver); return rc; @@ -1542,10 +1544,23 @@ EXPORT_SYMBOL_GPL(__usb_serial_register_drivers); void usb_serial_deregister_drivers(struct usb_serial_driver *const serial_drivers[]) { struct usb_driver *udriver = (*serial_drivers)->usb_driver; + struct usb_serial_driver * const *sd; + + /* + * udriver must be deregistered before the serial drivers so that + * I/O is stopped before unbinding the ports. + * + * Remove the new_id attributes to prevent ids from being added and + * triggering a probe of udriver after it has been deregistered. + */ + for (sd = serial_drivers; *sd; ++sd) + usb_serial_bus_remove_new_id(*sd); + + usb_deregister(udriver); for (; *serial_drivers; ++serial_drivers) usb_serial_deregister(*serial_drivers); - usb_deregister(udriver); + kfree(udriver); } EXPORT_SYMBOL_GPL(usb_serial_deregister_drivers); diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 534e6650e2aa..ab0a32a90629 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -381,6 +381,7 @@ void usb_serial_handle_dcd_change(struct usb_serial_port *usb_port, int usb_serial_bus_register(struct usb_serial_driver *device); void usb_serial_bus_deregister(struct usb_serial_driver *device); +void usb_serial_bus_remove_new_id(struct usb_serial_driver *driver); extern const struct bus_type usb_serial_bus_type; extern struct tty_driver *usb_serial_tty_driver; -- cgit v1.2.3 From a1c7570cedd03372812a5b693732880867babbca Mon Sep 17 00:00:00 2001 From: "Lorenzo Stoakes (ARM)" Date: Thu, 13 Aug 2026 12:01:24 +0300 Subject: x86/mm/pat: Acquire init_mm write lock on collapse to avoid UAF x86 implements page attribute modification using its Change Page Attributes (CPA) mechanism. This tracks properties of ranges such as cache mode through x86 page attributes, and as part of that logic manipulates kernel page tables. Since commit: 41d88484c71c ("x86/mm/pat: restore large ROX pages after fragmentation") ranges of kernel page table entries can be collapsed into huge page table entries as part of this logic. As part of this collapse, it frees the page tables which the collapsed entries previously pointed to, and it does so without any relevant locks being held to preclude concurrent kernel page table walkers. The only way this code can be reached is if CPA_COLLAPSE is specified, and this is only set in set_memory_rox() via: set_memory_rox() -> change_page_attr_set_clr() -> cpa_flush() -> cpa_collapse_large_pages() Notable users of this are execmem and BPF when manipulating executable mappings. However, this is problematic for ptdump as it walks ranges it does not own and thus runs the risk of a use-after-free on page tables freed underneath it. In addition, concurrent CPA collapse operations are possible which can also cause races. Resolve the issue by acquiring the mmap write lock on init_mm across the whole operation. It is safe to acquire a sleeping lock as all the callers invoke set_memory_rox() from process context and in any case, change_page_attr_set_clr() calls vm_unmap_alias() which ultimately takes a mutex, disallowing atomic context here. Fixes: 41d88484c71c ("x86/mm/pat: restore large ROX pages after fragmentation") Signed-off-by: Lorenzo Stoakes (ARM) Signed-off-by: Mike Rapoport (Microsoft) Signed-off-by: Dave Hansen Signed-off-by: Ingo Molnar Reviewed-by: Mike Rapoport (Microsoft) Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Dave Hansen Reviewed-by: Will Deacon Reviewed-by: David Carlier Tested-by: Atish Patra Tested-by: Nikunj A Dadhania Cc:stable@vger.kernel.org Link: https://patch.msgid.link/20260813-cpa-fixes-v2-1-39b4ff90f91d@kernel.org --- arch/x86/mm/pat/set_memory.c | 15 ++++++++++++++- include/linux/mmap_lock.h | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index c38faf39ce15..4abddd763882 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -409,7 +410,7 @@ static void __cpa_flush_tlb(void *data) static int collapse_large_pages(unsigned long addr, struct list_head *pgtables); -static void cpa_collapse_large_pages(struct cpa_data *cpa) +static void __cpa_collapse_large_pages(struct cpa_data *cpa) { unsigned long start, addr, end; struct ptdesc *ptdesc, *tmp; @@ -443,6 +444,18 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) } } +static void cpa_collapse_large_pages(struct cpa_data *cpa) +{ + /* + * Take the mmap write lock on init_mm to: + * - Avoid a use-after-free if raced by ptdump (which takes its own + * write lock on init_mm). + * - Serialise concurrent CPA walkers. + */ + scoped_guard(mmap_write_lock, &init_mm) + __cpa_collapse_large_pages(cpa); +} + static void cpa_flush(struct cpa_data *cpa, int cache) { unsigned int i; diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index bec0eab6ef03..b8a13b8d36a4 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -630,6 +630,8 @@ static inline void mmap_read_unlock(struct mm_struct *mm) DEFINE_GUARD(mmap_read_lock, struct mm_struct *, mmap_read_lock(_T), mmap_read_unlock(_T)) DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T)) +DEFINE_GUARD(mmap_write_lock, struct mm_struct *, + mmap_write_lock(_T), mmap_write_unlock(_T)) static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) { -- cgit v1.2.3 From 88ce88e933e40f4ab4b81a5e9d0a10328583f593 Mon Sep 17 00:00:00 2001 From: Daniel Borkmann Date: Thu, 10 Sep 2026 23:35:07 +0200 Subject: bpf: Add KF_PERFMON kfunc flag Tracing related BPF helpers e.g. under bpf_base_func_proto() are gated behind CAP_PERFMON. However, the same is currently not true for kfuncs and they are accessible via plain CAP_BPF. Add a new KF_PERFMON flag which can be used such that check_kfunc_call() ensures env->allow_ptr_leaks is permitted. This follows similar pattern to existing KF_DESTRUCTIVE flag. The rejection returns -EPERM to match the other CAP_PERFMON gates in the verifier, that is, check_ptr_to_btf_access() and check_ptr_to_map_access(), which report the very same policy to user space. Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/r/20260910213510.49358-1-daniel@iogearbox.net Signed-off-by: Alexei Starovoitov --- Documentation/bpf/kfuncs.rst | 10 ++++++++++ include/linux/btf.h | 1 + kernel/bpf/verifier.c | 14 ++++++++++++++ 3 files changed, 25 insertions(+) (limited to 'include/linux') diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 85f73e0bbd0f..6691fe8a32c3 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -486,6 +486,16 @@ Example usage in BPF program: /* note that the last argument is omitted */ bpf_task_work_schedule_signal(task, &work->tw, &arrmap, task_work_callback); +2.5.10 KF_PERFMON flag +---------------------- + +The KF_PERFMON flag is used for kfuncs that can expose kernel memory or kernel +addresses to the BPF program, for example by reading through a pointer that the +verifier does not check. Calling such a kfunc requires CAP_PERFMON, or +CAP_SYS_ADMIN, in the same way that the equivalent BPF helpers are gated in +bpf_base_func_proto(). A program loaded with CAP_BPF alone is rejected at load +time. + 2.6 Registering the kfuncs -------------------------- diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f117..7c62ea17b116 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -80,6 +80,7 @@ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ +#define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 72a3f5998dd2..939e535a3442 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -11356,6 +11356,11 @@ static bool is_kfunc_destructive(struct bpf_call_arg_meta *meta) return meta->kfunc_flags & KF_DESTRUCTIVE; } +static bool is_kfunc_perfmon(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_PERFMON; +} + static bool is_kfunc_rcu(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_RCU; @@ -13834,6 +13839,15 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EACCES; } + if (is_kfunc_perfmon(&meta) && !env->allow_ptr_leaks) { + verbose(env, "%s is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", + func_name); + operation = bpf_diag_fmt(env, "kfunc %s", func_name); + bpf_diag_policy(env, insn_idx, operation, "the kfunc requires CAP_PERFMON", + "Load the program with CAP_PERFMON, or avoid the kfunc."); + return -EPERM; + } + sleepable = bpf_is_kfunc_sleepable(&meta); if (sleepable && !in_sleepable(env)) { verbose(env, "program must be sleepable to call sleepable kfunc %s\n", func_name); -- cgit v1.2.3 From 8cd280282d1239bca68f8c3632ef1ac8556a396b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Aug 2026 14:46:18 -0700 Subject: KVM: Never clear KVM_REQ_VM_DEAD from a vCPU's requests Use kvm_test_request() instead of kvm_check_request() when querying KVM_REQ_VM_DEAD, i.e. don't clear KVM_REQ_VM_DEAD, as the entire purpose of KVM_REQ_VM_DEAD is to prevent the vCPU from enterring the guest ever again, even if userspace insists on redoing KVM_RUN. Ensuring KVM_REQ_VM_DEAD is never cleared will allow relaxing KVM's rule that ioctls can't be invoked on dead VMs, to only disallow ioctls if the VM is bugged, i.e. if KVM hit a KVM_BUG_ON(). Opportunistically add compile-time assertions to guard against clearing KVM_REQ_VM_DEAD through the standard APIs. Reviewed-by: Kai Huang Acked-by: Marc Zyngier Link: https://patch.msgid.link/20260806214618.82180-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/arm64/kvm/arm.c | 2 +- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/vmx/tdx.c | 2 +- arch/x86/kvm/x86.c | 2 +- include/linux/kvm_host.h | 9 +++++++-- 5 files changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 8b080804bc90..62c81d6ae8ee 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1129,7 +1129,7 @@ static int kvm_vcpu_suspend(struct kvm_vcpu *vcpu) static int check_vcpu_requests(struct kvm_vcpu *vcpu) { if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 064ecc33b926..8e62476e477b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5058,7 +5058,7 @@ static int kvm_tdp_page_prefault(struct kvm_vcpu *vcpu, gpa_t gpa, if (signal_pending(current)) return -EINTR; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) return -EIO; cond_resched(); diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index b272c20586a7..6c842e9191a5 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -1995,7 +1995,7 @@ static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu) if (kvm_vcpu_has_events(vcpu) || signal_pending(current)) break; - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { ret = -EIO; break; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 79468ddfe473..a137dc6dd8c6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8060,7 +8060,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) bool req_immediate_exit = false; if (kvm_request_pending(vcpu)) { - if (kvm_check_request(KVM_REQ_VM_DEAD, vcpu)) { + if (kvm_test_request(KVM_REQ_VM_DEAD, vcpu)) { r = -EIO; goto out; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 03bfc92864b6..cf7fe835c4ad 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2324,13 +2324,18 @@ static inline bool kvm_test_request(int req, struct kvm_vcpu *vcpu) return test_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) +static __always_inline void kvm_clear_request(int req, struct kvm_vcpu *vcpu) { + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + clear_bit(req & KVM_REQUEST_MASK, (void *)&vcpu->requests); } -static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) +static __always_inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) { + /* Once a VM is dead, it needs to stay dead. */ + BUILD_BUG_ON(req == KVM_REQ_VM_DEAD); + if (kvm_test_request(req, vcpu)) { kvm_clear_request(req, vcpu); -- cgit v1.2.3 From 3ed11c671ff7ec58c8fd96410233c677df23f407 Mon Sep 17 00:00:00 2001 From: Christian König Date: Fri, 4 Sep 2026 10:26:13 +0200 Subject: dma-buf/dma-fence: fix checking signaling bit for timeline and driver name v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The patch "dma-buf: dma-fence: Fix potential NULL pointer dereference" changed the check to test for the ops pointer instead of the signaled bit to avoid a potential NULL dereference when the ops pointer has been cleared. The problem is now that the ops pointer is cleared only when neither the release nor the wait callback is implemented and this isn't true for a lot of dma_fence implementations yet. So those implementations lost the RCU protection after signaling of the returned string resulting in potential use after free. Add the signaling check additional to the ops pointer check so that we have both the protection against NULL dereference as well as the RCU protection after signaling for the returned string. v2: improve comments to note RCU protection and explain why we check both signaling state and ops pointer v3: some comment improvements suggested by Philip Signed-off-by: Christian König Fixes: 035219a760ed ("dma-buf: dma-fence: Fix potential NULL pointer dereference") CC: stable@vger.kernel.org # 7.2+ Reported-by: Jonghyuk Kim(MalHyuk) Tested-by: Jonghyuk Kim(MalHyuk) Reviewed-by: Philipp Stanner Link: https://lore.kernel.org/r/20260914182740.1587-1-christian.koenig@amd.com --- drivers/dma-buf/dma-fence.c | 14 ++++++++++++-- include/linux/dma-fence.h | 6 ++++++ 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 05090fb0fd5a..bd58688b81a7 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -1170,7 +1170,12 @@ const char __rcu *dma_fence_driver_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (ops) + + /* + * Make load ordering irrelevant by checking both signaled state and ops + * pointer and ops pointer is only set to NULL on newer implementations. + */ + if (!dma_fence_test_signaled_flag(fence) && ops) return (const char __rcu *)ops->get_driver_name(fence); else return (const char __rcu *)"detached-driver"; @@ -1203,7 +1208,12 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) /* RCU protection is required for safe access to returned string */ ops = rcu_dereference(fence->ops); - if (ops) + + /* + * Make load ordering irrelevant by checking both signaled state and ops + * pointer and ops pointer is only set to NULL on newer implementations. + */ + if (!dma_fence_test_signaled_flag(fence) && ops) return (const char __rcu *)ops->get_timeline_name(fence); else return (const char __rcu *)"signaled-timeline"; diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index 158cd609f103..ffa99b930843 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -141,6 +141,9 @@ struct dma_fence_ops { * compute the name at runtime, without having it to store permanently * for each fence, or build a cache of some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_driver_name)(struct dma_fence *fence); @@ -153,6 +156,9 @@ struct dma_fence_ops { * having it to store permanently for each fence, or build a cache of * some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_timeline_name)(struct dma_fence *fence); -- cgit v1.2.3 From 9a7d3340cd0f615a05c81c8f972cbffc8eb96230 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sun, 30 Aug 2026 22:52:36 -0700 Subject: siphash: clean up kernel-doc comments Use the correct function parameter names and add function return value descriptions to avoid kernel-doc warnings: Warning: include/linux/siphash.h:82 function parameter 'len' not described in 'siphash' Warning: include/linux/siphash.h:82 No description found for return value of 'siphash' Warning: include/linux/siphash.h:132 function parameter 'len' not described in 'hsiphash' Warning: include/linux/siphash.h:132 No description found for return value of 'hsiphash' Signed-off-by: Randy Dunlap Signed-off-by: Jason A. Donenfeld --- include/linux/siphash.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/siphash.h b/include/linux/siphash.h index 9153e77382e1..00c58bbe549a 100644 --- a/include/linux/siphash.h +++ b/include/linux/siphash.h @@ -75,8 +75,10 @@ static inline u64 ___siphash_aligned(const __le64 *data, size_t len, /** * siphash - compute 64-bit siphash PRF value * @data: buffer to hash - * @size: size of @data + * @len: size of @data * @key: the siphash key + * + * Returns: siphash PRF value */ static inline u64 siphash(const void *data, size_t len, const siphash_key_t *key) @@ -125,8 +127,10 @@ static inline u32 ___hsiphash_aligned(const __le32 *data, size_t len, /** * hsiphash - compute 32-bit hsiphash PRF value * @data: buffer to hash - * @size: size of @data + * @len: size of @data * @key: the hsiphash key + * + * Returns: hsiphash PRF value */ static inline u32 hsiphash(const void *data, size_t len, const hsiphash_key_t *key) -- cgit v1.2.3 From 40c2096961b4e8f48d1fc17406a90c5a36ac0a8d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Mon, 14 Sep 2026 15:19:20 +0200 Subject: bpf: Verify global subprogs in each sleepability context Global subprograms are verified independently with a fresh verifier root. do_check_common() currently seeds that root's in_sleepable state from the program, even though a global subprogram can also run from callbacks whose execution context differs from the program's main entry point. In particular, workqueue and task-work callbacks are sleepable even when the containing program is not. A global subprogram of that program is therefore verified as non-sleepable, making in_rcu_cs() true and allowing loads of RCU-protected kptrs to produce trusted MEM_RCU pointers. The same subprogram can then be called from a sleepable callback without a classic RCU reader. It can retain such a pointer while the object is freed and use it after free. The verifier's execution-context predicates are complementary. A state is sleepable only when in_sleepable is set and no RCU, preemption, IRQ, or lock region is active. Each condition which prevents sleeping also provides RCU protection, while in_rcu_cs() treats a non-sleepable state as implicitly protected. Use this relationship to represent a global subprogram caller with only the result of in_sleepable_context(). A protected sleepable caller is normalized to in_sleepable=false at the independent verification root. This both prevents sleepable operations and makes in_rcu_cs() true without copying caller-owned lock state. Track only the contexts in which each global subprogram is actually reached. Verify it once if all reachable calls use the same context, and twice only if both sleepable and non-sleepable calls reach it. Calls found while verifying globals or asynchronous callbacks mark further contexts for checking. Repeat the existing subprogram walk until all called contexts have been verified; unreachable global calls remain unchecked. Accumulate instruction counts over those verification passes. Preserve the total recorded before each pass, since path accounting has already added this pass's synchronous instructions and its root total must also include asynchronous subprograms. This makes an unprotected callback verify the global subprogram as sleepable, turning its RCU-protected kptr load into an untrusted pointer. Protected callers and global subprograms which do not depend on implicit RCU protection remain valid. Fixes: 81f1d7a583fa ("bpf: wq: add bpf_wq_set_callback_impl") Fixes: 38aa7003e369 ("bpf: task work scheduling kfuncs") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260914131923.2544250-2-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf.h | 5 ++-- kernel/bpf/verifier.c | 71 +++++++++++++++++++++++++++------------------------ 2 files changed, 40 insertions(+), 36 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e57af902560c..1d2676782d70 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1651,8 +1651,9 @@ static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags struct bpf_func_info_aux { u16 linkage; bool unreliable; - bool called : 1; - bool verified : 1; + /* Indexed by in_sleepable. */ + bool called[2]; + bool verified[2]; }; enum bpf_jit_poke_reason { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ddba53eaa333..5d7080c260d8 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -9931,6 +9931,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (err == -EFAULT) return err; if (bpf_subprog_is_global(env, subprog)) { + struct bpf_func_info_aux *sub_aux = subprog_aux(env, subprog); const char *sub_name = bpf_subprog_name(env, subprog); const char *operation; bool returns_void; @@ -9962,11 +9963,10 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, if (env->log.level & BPF_LOG_LEVEL) verbose(env, "Func#%d ('%s') is global and assumed valid.\n", subprog, sub_name); + sub_aux->called[in_sleepable_context(env)] = true; returns_void = subprog_returns_void(env, subprog); if (env->subprog_info[subprog].changes_pkt_data) clear_all_pkt_pointers(env); - /* mark global subprog for verifying after main prog */ - subprog_aux(env, subprog)->called = true; if (returns_void) bpf_diag_record_scrub(env, &caller->regs[BPF_REG_0], BPF_DIAG_MOD_CALLER_SAVED); else @@ -10784,11 +10784,7 @@ int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, /* Check if we're in a sleepable context. */ static inline bool in_sleepable_context(struct bpf_verifier_env *env) { - return !env->cur_state->active_rcu_locks && - !env->cur_state->active_preempt_locks && - !env->cur_state->active_locks && - !env->cur_state->active_irq_id && - in_sleepable(env); + return !in_rcu_cs(env); } static const char *non_sleepable_context_description(struct bpf_verifier_env *env) @@ -19447,13 +19443,14 @@ static void free_states(struct bpf_verifier_env *env) } } -static int do_check_common(struct bpf_verifier_env *env, int subprog) +static int do_check_common(struct bpf_verifier_env *env, int subprog, bool is_sleepable) { bool pop_log = !(env->log.level & BPF_LOG_LEVEL2); struct bpf_subprog_info *sub = subprog_info(env, subprog); struct bpf_prog_aux *aux = env->prog->aux; struct bpf_verifier_state *state; struct bpf_reg_state *regs; + u32 old_insns_total = sub->insns_total; u32 insn_processed = env->insn_processed; int ret, i; @@ -19466,7 +19463,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) state->curframe = 0; state->speculative = false; state->branches = 1; - state->in_sleepable = env->prog->sleepable; + state->in_sleepable = is_sleepable; state->frame[0] = kzalloc_obj(struct bpf_func_state, GFP_KERNEL_ACCOUNT); if (!state->frame[0]) { kfree(state); @@ -19607,8 +19604,10 @@ out: * not accounted as callees by account_current_path(). * Accumulate their total counts as total counts of the main or * global subprog hosting the async call. + * Start from the saved total of earlier contexts: adding to the current + * total would count this pass's synchronous paths twice. */ - env->subprog_info[subprog].insns_total = env->insn_processed - insn_processed; + sub->insns_total = old_insns_total + (env->insn_processed - insn_processed); return ret; } @@ -19636,14 +19635,19 @@ static int do_check_subprogs(struct bpf_verifier_env *env) { struct bpf_prog_aux *aux = env->prog->aux; struct bpf_func_info_aux *sub_aux; - int i, ret, new_cnt; + int context, i, ret, new_cnt; if (!aux->func_info) return 0; - /* exception callback is presumed to be always called */ - if (env->exception_callback_subprog) - subprog_aux(env, env->exception_callback_subprog)->called = true; + /* + * Callbacks cannot throw, so the exception callback always runs in the + * main program's context. It is presumed to be always called. + */ + if (env->exception_callback_subprog) { + sub_aux = subprog_aux(env, env->exception_callback_subprog); + sub_aux->called[env->prog->sleepable] = true; + } again: new_cnt = 0; @@ -19652,29 +19656,28 @@ again: continue; sub_aux = subprog_aux(env, i); - if (!sub_aux->called || sub_aux->verified) - continue; + for (context = 0; context < ARRAY_SIZE(sub_aux->called); context++) { + if (!sub_aux->called[context] || sub_aux->verified[context]) + continue; - env->insn_idx = env->subprog_info[i].start; - WARN_ON_ONCE(env->insn_idx == 0); - ret = do_check_common(env, i); - if (ret) { - return ret; - } else if (env->log.level & BPF_LOG_LEVEL) { - verbose(env, "Func#%d ('%s') is safe for any args that match its prototype\n", - i, bpf_subprog_name(env, i)); - } + env->insn_idx = env->subprog_info[i].start; + WARN_ON_ONCE(env->insn_idx == 0); + ret = do_check_common(env, i, context); + if (ret) + return ret; + if (env->log.level & BPF_LOG_LEVEL) + verbose(env, "Func#%d ('%s') is safe for any args " + "that match its prototype\n", + i, bpf_subprog_name(env, i)); - /* We verified new global subprog, it might have called some - * more global subprogs that we haven't verified yet, so we - * need to do another pass over subprogs to verify those. - */ - sub_aux->verified = true; - new_cnt++; + sub_aux->verified[context] = true; + new_cnt++; + } } - /* We can't loop forever as we verify at least one global subprog on - * each pass. + /* + * We can't loop forever as each pass verifies at least one new context, + * and there are only two contexts per global subprog. */ if (new_cnt) goto again; @@ -19687,7 +19690,7 @@ static int do_check_main(struct bpf_verifier_env *env) int ret; env->insn_idx = 0; - ret = do_check_common(env, 0); + ret = do_check_common(env, 0, env->prog->sleepable); if (!ret) env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; return ret; -- cgit v1.2.3 From c26e97721b172163042b98572fead234797f2c3c Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:12 +0200 Subject: bpf: Apply CO-RE relocations before subprogram validation check_subprogs() verifies that each subprogram ends in an exit or an unconditional jump before in-kernel CO-RE relocations are applied. An unresolved relocation can then replace that terminal instruction with an invalid helper call. The resulting fall-through into another subprogram breaks the CFG invariant used by postorder and stack liveness analysis, which can write past their per-subprogram arrays. Apply CO-RE relocations immediately after preparing the program BTF, before subprogram discovery and validation. Keep func_info and line_info validation after subprogram discovery because those records depend on the complete subprogram layout. Reject an ldimm64 first slot at the end of the instruction stream before CO-RE can inspect its missing second slot. check_subprogs() previously rejected this form before relocation processing because it is not a valid subprogram terminator. Moving CO-RE ahead of check_subprogs() removes that implicit protection, so perform an explicit check before applying relocations. Include core_relo_cnt when deciding whether to prepare program BTF. A load that supplied only CO-RE relocation metadata previously skipped both BTF setup and relocation processing. Fixes: fbd94c7afcf9 ("bpf: Pass a set of bpf_core_relo-s to prog_load command.") Suggested-by: Andrii Nakryiko Suggested-by: Alexei Starovoitov Suggested-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-5-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 2 ++ kernel/bpf/check_btf.c | 12 ++++-------- kernel/bpf/verifier.c | 12 +++++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 36b65797877d..bba5a727c651 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1197,6 +1197,8 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) int bpf_prepare_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); int bpf_check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); diff --git a/kernel/bpf/check_btf.c b/kernel/bpf/check_btf.c index 0e8b3ccc7a5b..4c1ed842f661 100644 --- a/kernel/bpf/check_btf.c +++ b/kernel/bpf/check_btf.c @@ -338,9 +338,9 @@ err_free: #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo) #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE -static int check_core_relo(struct bpf_verifier_env *env, - const union bpf_attr *attr, - bpfptr_t uattr) +int bpf_check_core_relo(struct bpf_verifier_env *env, + const union bpf_attr *attr, + bpfptr_t uattr) { u32 i, nr_core_relo, ncopy, expected_size, rec_size; struct bpf_core_relo core_relo = {}; @@ -414,7 +414,7 @@ int bpf_prepare_btf_info(struct bpf_verifier_env *env, struct btf *btf; int err; - if (!attr->func_info_cnt && !attr->line_info_cnt) { + if (!attr->func_info_cnt && !attr->line_info_cnt && !attr->core_relo_cnt) { if (check_abnormal_return(env)) return -EINVAL; return 0; @@ -455,9 +455,5 @@ int bpf_check_btf_info(struct bpf_verifier_env *env, if (err) return err; - err = check_core_relo(env, attr, uattr); - if (err) - return err; - return 0; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5d7080c260d8..33161dc64568 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21199,6 +21199,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, ret = bpf_diag_init(env); if (ret) goto err_prep; + if (env->prog->insnsi[env->prog->len - 1].code == (BPF_LD | BPF_IMM | BPF_DW)) { + verbose(env, "invalid bpf_ld_imm64 insn\n"); + ret = -EINVAL; + goto err_prep; + } if (env->signature) { ret = bpf_prog_calc_tag(env->prog); if (ret < 0) @@ -21274,6 +21279,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* Apply CO-RE before validating the program's instruction layout. */ + ret = bpf_check_core_relo(env, attr, uattr); + if (ret < 0) + goto skip_full_check; + /* Discover all subprograms before validating their layout and BTF. */ ret = add_subprogs(env); if (ret < 0) @@ -21283,7 +21293,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; - /* Validate BTF against the complete subprogram layout and apply CO-RE. */ + /* Validate BTF against the complete subprogram layout. */ ret = bpf_check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; -- cgit v1.2.3 From 71919742c83c32afcccf06a7a28e28f3f55f21a2 Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Fri, 18 Sep 2026 01:32:16 +0200 Subject: bpf: Assign lock identity to callback map values A nested bpf_for_each_map_elem() callback can unlock a different element of the same map: static long inner(void *map, int *key, struct value *v, struct value **outer_value) { bpf_spin_lock(&v->lock); bpf_spin_unlock(&(*outer_value)->lock); return 0; } static long outer(void *map, int *key, struct value *v, void *ctx) { bpf_for_each_map_elem(map, inner, &v, 0); return 0; } Both callback values currently have ID zero and the same map_ptr. process_spin_lock() compares those two fields, so it accepts the unlock even though the two callbacks can receive different map elements. Assign a fresh ID to every callback map value in the for-each, timer/workqueue, and task-work constructors. Copies of one callback argument retain its ID, so locking and unlocking through that argument continues to work. Distinct callbacks also get distinct IDs for single-element arrays, including inner arrays sharing inner_map_meta. Preserve map_uid for every inner-map lookup and compare it through check_ids() during state pruning. This preserves relationships between maps, keys, and values while allowing equivalent states with different lookup IDs to match. It avoids field-specific rules for when an inner map needs an identity. Move map_uid out of the metadata union and next to the other IDs, so register comparisons can use the existing memcmp() ranges and remap the IDs separately. Clear it when resetting a register or converting a map lookup result to a socket pointer. Shrink frameno to u8, which is enough for MAX_CALL_FRAMES, to make room without growing bpf_reg_state. Fixes: d0d78c1df9b1 ("bpf: Allow locking bpf_spin_lock global variables") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Kumar Kartikeya Dwivedi Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260917233222.2542500-9-memxor@gmail.com Signed-off-by: Eduard Zingerman --- include/linux/bpf_verifier.h | 25 +++++++++++++------------ kernel/bpf/states.c | 6 ++++-- kernel/bpf/verifier.c | 13 +++++++++---- 3 files changed, 26 insertions(+), 18 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bba5a727c651..a7202b44ab10 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -46,17 +46,11 @@ struct bpf_reg_state { /* valid when type == PTR_TO_PACKET */ int range; - /* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE | - * PTR_TO_MAP_VALUE_OR_NULL + /* + * For CONST_PTR_TO_MAP, PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE and + * PTR_TO_INSN. */ - struct { - struct bpf_map *map_ptr; - /* To distinguish map lookups from outer map - * the map_uid is non-zero for registers - * pointing to inner maps. - */ - u32 map_uid; - }; + struct bpf_map *map_ptr; /* for PTR_TO_BTF_ID */ struct { @@ -155,13 +149,20 @@ struct bpf_reg_state { * gets parent_id set to the dynptr's id. */ u32 parent_id; - /* Inside the callee two registers can be both PTR_TO_STACK like + /* + * Distinguishes inner-map lookups and their keys and values. Zero for + * other registers. Kept outside the metadata union for ID remapping + * during state comparisons. + */ + u32 map_uid; + /* + * Inside the callee two registers can be both PTR_TO_STACK like * R1=fp-8 and R2=fp-8, but one of them points to this function stack * while another to the caller's stack. To differentiate them 'frameno' * is used which is an index in bpf_verifier_state->frame[] array * pointing to bpf_func_state. */ - u32 frameno; + u8 frameno; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index 6c88ad95b63b..e4ec007f7fa6 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -491,7 +491,8 @@ static bool regs_exact(const struct bpf_reg_state *rold, { return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); } enum exact_level { @@ -616,7 +617,8 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off) && check_ids(rold->id, rcur->id, idmap) && - check_ids(rold->parent_id, rcur->parent_id, idmap); + check_ids(rold->parent_id, rcur->parent_id, idmap) && + check_ids(rold->map_uid, rcur->map_uid, idmap); case PTR_TO_PACKET_META: case PTR_TO_PACKET: /* We must have at least as much range as the old ptr diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 33161dc64568..02be326f235c 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1864,6 +1864,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); reg->id = 0; reg->parent_id = 0; + reg->map_uid = 0; ___mark_reg_known(reg, imm); } @@ -1925,17 +1926,18 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg) if (map->inner_map_meta) { reg->type = CONST_PTR_TO_MAP | maybe_null; reg->map_ptr = map->inner_map_meta; - /* transfer reg's id which is unique for every map_lookup_elem + /* + * transfer reg's id which is unique for every map_lookup_elem * as UID of the inner map. */ - if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK)) - reg->map_uid = reg->id; + reg->map_uid = reg->id; } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK | maybe_null; + reg->map_uid = 0; } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP || map->map_type == BPF_MAP_TYPE_SOCKHASH) { reg->type = PTR_TO_SOCKET | maybe_null; + reg->map_uid = 0; } } @@ -10048,6 +10050,7 @@ int map_set_for_each_callback_args(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr; callee->regs[BPF_REG_3].map_uid = caller->regs[BPF_REG_1].map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* pointer to stack or null */ callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3]; @@ -10144,6 +10147,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); @@ -10262,6 +10266,7 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, __mark_reg_known_zero(&callee->regs[BPF_REG_3]); callee->regs[BPF_REG_3].map_ptr = map_ptr; callee->regs[BPF_REG_3].map_uid = map_uid; + callee->regs[BPF_REG_3].id = ++env->id_gen; /* unused */ bpf_mark_reg_not_init(env, &callee->regs[BPF_REG_4]); -- cgit v1.2.3 From 1b82958f3f035df5ccaab5430a2302f08a5d5351 Mon Sep 17 00:00:00 2001 From: Alexander Duyck Date: Mon, 14 Sep 2026 14:09:57 -0700 Subject: net: ethtool: keep rtnl_lock for the ioctl self test An offline self test that brings the interface down and back up with netif_close() / netif_open() requires rtnl_lock for both. Since the ethtool IOCTL path became rtnl-optional for ops-locked drivers, the ETHTOOL_TEST ioctl runs holding only the netdev instance lock, so on an ops-locked driver the self test now tears the device down without rtnl_lock. With lockdep this reproduces deterministically on every offline self test on such a driver; note the sole lock held is the instance lock, not rtnl: WARNING: suspicious RCU usage net/core/netpoll.c:207 suspicious rcu_dereference_protected() usage! 1 lock held by ethtool/107: #0: (&dev->lock){+.+.}, at: dev_ethtool Call Trace: netpoll_poll_disable __dev_close_many netif_close_many netif_close fbnic_self_test dev_ethtool_locked dev_ethtool dev_ioctl sock_ioctl __x64_sys_ioctl Without lockdep the same condition trips ASSERT_RTNL() in __dev_close_many() / __dev_open(); that check only samples the global rtnl state, so it can be masked by a concurrent rtnl holder, but the device is still being reconfigured without the lock it requires. The ethtool self_test is a legacy ioctl-only command, so an ETHTOOL_TEST case is only needed on the ioctl path. Add an opt-in bit for drivers whose self test needs rtnl_lock and set it on the ops-locked drivers whose offline self test tears the interface down and up: - fbnic (ops-locked via queue_mgmt_ops): fbnic_self_test() offline path uses netif_close() / netif_open(). - bnxt (ops-locked via queue_mgmt_ops): bnxt_self_test() offline path goes through bnxt_close_nic() / bnxt_half_open_nic() / bnxt_half_close_nic() / bnxt_open_nic(), which close and reopen the device. Fixes: f994752b1127 ("net: ethtool: optionally skip rtnl_lock on IOCTL path") Signed-off-by: Alexander Duyck Reviewed-by: Simon Horman Link: https://patch.msgid.link/178942019771.7700.338431553546884773.stgit@ahduyck-xeon-server.home.arpa Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 3 ++- drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c | 3 ++- include/linux/ethtool.h | 2 ++ net/ethtool/common.h | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 62bc9cae613c..622e89587e5d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -5733,7 +5733,8 @@ const struct ethtool_ops bnxt_ethtool_ops = { .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | ETHTOOL_OP_NEEDS_RTNL_SCOALESCE | - ETHTOOL_OP_NEEDS_RTNL_RSS, + ETHTOOL_OP_NEEDS_RTNL_RSS | + ETHTOOL_OP_NEEDS_RTNL_TEST, .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USECS_IRQ | diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 0e47088ec44b..423f179c9d47 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -2025,7 +2025,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = { ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM | ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | - ETHTOOL_OP_NEEDS_RTNL_GLINK, + ETHTOOL_OP_NEEDS_RTNL_GLINK | + ETHTOOL_OP_NEEDS_RTNL_TEST, .get_drvinfo = fbnic_get_drvinfo, .get_regs_len = fbnic_get_regs_len, .get_regs = fbnic_get_regs, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 253600c0eccd..c4c9ce038611 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -944,6 +944,7 @@ struct kernel_ethtool_ts_info { #define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) #define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) #define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) +#define ETHTOOL_OP_NEEDS_RTNL_TEST BIT(9) /** * struct ethtool_ops - optional netdev operations @@ -981,6 +982,7 @@ struct kernel_ethtool_ts_info { * - netdev_update_features() * - netif_set_real_num_tx_queues() * - ethtool_op_get_link() (syncs link watch under rtnl_lock) + * - netif_open() / netif_close() (used by @self_test) * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are diff --git a/net/ethtool/common.h b/net/ethtool/common.h index 4e5356e26f40..ae32e7fdb563 100644 --- a/net/ethtool/common.h +++ b/net/ethtool/common.h @@ -163,6 +163,8 @@ ethtool_ioctl_needs_rtnl(const struct net_device *dev, u32 ethcmd) return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_RSS; case ETHTOOL_GLINK: return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_GLINK; + case ETHTOOL_TEST: + return ops->op_needs_rtnl & ETHTOOL_OP_NEEDS_RTNL_TEST; } return false; } -- cgit v1.2.3 From 8901cee9316d53a5a97398f026c2d59a3043159d Mon Sep 17 00:00:00 2001 From: Kumar Kartikeya Dwivedi Date: Sat, 19 Sep 2026 03:42:10 +0200 Subject: bpf: Compare stack frames in regs_exact() regs_exact() compares the register state up to id, followed by the ID mappings, but does not compare frameno. The PTR_TO_STACK case in regsafe() checks frameno separately, which is bypassed when exact comparison is requested. Consequently, infinite-loop detection can treat pointers to different stack frames as the same pointer and reject a finite loop. For example, initialize fp-8 to zero in the caller and to one in the callee, then pass the caller's fp-8 to the callee as r1: loop: r0 = *(u64 *)(r1 + 0); if r0 != 0 goto done; r1 = r10; r1 += -8; goto loop; done: exit; The loop terminates after reading the callee's slot on its second iteration. At the loop header, however, the only relevant difference is r1's frameno, so exact comparison incorrectly reports an infinite loop. The same problem occurs when the pointer is spilled to the stack. Move frameno into the type-specific metadata union, ahead of id, so the existing prefix comparison in regs_exact() covers it. Ordinary stack pointers do not use another union member. Iterator and IRQ stack-slot states use their dedicated union views and do not need a frame lookup. This also keeps bpf_reg_state at 80 bytes. Since frameno now shares storage with other pointer metadata, it is only meaningful for PTR_TO_STACK registers. Return NULL from bpf_func() for other register types. process_iter_arg(), get_constant_map_key() and is_dynptr_reg_valid_init() look up the frame before checking the register type and would otherwise index frame[] with a byte of the register's map or BTF pointer. They dereference the frame only after their type check. Move the states_maybe_looping() boundary from frameno to precise after the field relocation. Its prefix comparison continues to cover the complete value state and now includes frameno. Continue to ignore precise. Precision marks control whether pruning may ignore scalar ranges; they do not change the represented values, and exact comparison already compares those ranges unconditionally. Marks can also change through backtracking while an ancestor state is still being explored. Fixes: d5b892fd607a ("bpf: make infinite loop detection in is_state_visited() exact") Reported-by: Eduard Zingerman Signed-off-by: Kumar Kartikeya Dwivedi Signed-off-by: Alexei Starovoitov Acked-by: Eduard Zingerman Link: https://patch.msgid.link/20260919014213.1840880-2-memxor@gmail.com --- include/linux/bpf_verifier.h | 23 +++++++++++++++-------- kernel/bpf/states.c | 7 ++----- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index a7202b44ab10..6fe8e5dc57aa 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -45,6 +45,14 @@ struct bpf_reg_state { union { /* valid when type == PTR_TO_PACKET */ int range; + /* + * Valid when type == PTR_TO_STACK. Inside the callee two registers + * can be both PTR_TO_STACK like R1=fp-8 and R2=fp-8, but one of them + * points to this function stack while another to the caller's stack. + * To differentiate them 'frameno' is used which is an index in + * bpf_verifier_state->frame[] array pointing to bpf_func_state. + */ + u8 frameno; /* * For CONST_PTR_TO_MAP, PTR_TO_MAP_KEY, PTR_TO_MAP_VALUE and @@ -155,14 +163,6 @@ struct bpf_reg_state { * during state comparisons. */ u32 map_uid; - /* - * Inside the callee two registers can be both PTR_TO_STACK like - * R1=fp-8 and R2=fp-8, but one of them points to this function stack - * while another to the caller's stack. To differentiate them 'frameno' - * is used which is an index in bpf_verifier_state->frame[] array - * pointing to bpf_func_state. - */ - u8 frameno; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; @@ -1239,11 +1239,18 @@ static inline int bpf_get_spi(s32 off) return (-off - 1) / BPF_REG_SIZE; } +/* + * Return the function state a stack pointer register refers to. frameno + * shares storage with other pointer metadata, so return NULL for any + * other register type instead of indexing frame[] with aliased bytes. + */ static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env, const struct bpf_reg_state *reg) { struct bpf_verifier_state *cur = env->cur_state; + if (reg->type != PTR_TO_STACK) + return NULL; return cur->frame[reg->frameno]; } diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c index e4ec007f7fa6..012b82513a3b 100644 --- a/kernel/bpf/states.c +++ b/kernel/bpf/states.c @@ -644,10 +644,7 @@ static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, return range_within(rold, rcur) && tnum_in(rold->var_off, rcur->var_off); case PTR_TO_STACK: - /* two stack pointers are equal only if they're pointing to - * the same stack frame, since fp-8 in foo != fp-8 in bar - */ - return regs_exact(rold, rcur, idmap) && rold->frameno == rcur->frameno; + return regs_exact(rold, rcur, idmap); case PTR_TO_ARENA: return true; case PTR_TO_INSN: @@ -1126,7 +1123,7 @@ static bool states_maybe_looping(struct bpf_verifier_state *old, fcur = cur->frame[fr]; for (i = 0; i < MAX_BPF_REG; i++) if (memcmp(&fold->regs[i], &fcur->regs[i], - offsetof(struct bpf_reg_state, frameno))) + offsetof(struct bpf_reg_state, precise))) return false; return true; } -- cgit v1.2.3 From ab888242fce4f16f6c4d4c6ec53939ad36aa3b3a Mon Sep 17 00:00:00 2001 From: Xiang Mei Date: Tue, 15 Sep 2026 01:31:52 -0700 Subject: vlan: require the MAC header to be present in __vlan_insert_inner_tag() __vlan_insert_inner_tag() only guarantees head room via skb_cow_head(), never that mac_len bytes of MAC header are present. Its ETH_HLEN wrappers - __vlan_insert_tag() under skb_vlan_push(), and vlan_insert_tag() under validate_xmit_vlan() on the generic transmit path - therefore rewrite the first 16 bytes at skb->data: a 12-byte memmove plus two 2-byte stores at +12 and +14. No caller supplies the bound, while the pop helpers use skb_ensure_writable()/pskb_may_pull(). An IFF_TUN device has hard_header_len == 0, so packet_snd() accepts a one-byte AF_PACKET/SOCK_RAW frame. The first vlan push only sets a hwaccel tag; the next - clsact "action vlan push" or bpf_skb_vlan_push() - enters the helper with skb->len still 1. The head comes from skbuff_small_head without __GFP_ZERO, so each push drags bytes from beyond skb->tail into the frame. After three the one-byte send leaves as 13 bytes carrying 11 bytes of uninitialised slab: 0000: 5a b3 62 12 80 88 ff ff 00 b3 62 12 81 `------------------------------' only 0x5a was sent; the rest is slab, here the top 56 bits of a linear-map address Require the MAC header the helper rewrites to be present, so such a frame is dropped rather than transmitted. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Cc: stable@vger.kernel.org Reported-by: co+0ea1ac045375cf05@bugs.sh Signed-off-by: Xiang Mei Reviewed-by: Simon Horman Link: https://patch.msgid.link/20260915083152.705309-1-xmei5@asu.edu Signed-off-by: Jakub Kicinski --- include/linux/if_vlan.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/linux') diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 20cc16ea4e5a..4846032bf4ff 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -365,6 +365,9 @@ static inline int __vlan_insert_inner_tag(struct sk_buff *skb, const u8 meta_len = mac_len > ETH_TLEN ? skb_metadata_len(skb) : 0; struct vlan_ethhdr *veth; + if (unlikely(!pskb_may_pull(skb, mac_len))) + return -EINVAL; + if (skb_cow_head(skb, meta_len + VLAN_HLEN) < 0) return -ENOMEM; -- cgit v1.2.3 From e6bae5034ef4a61f3f062b18140d4f58c8b9a149 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Fri, 18 Sep 2026 14:40:32 +0200 Subject: ata: libata-core: Extend Samsung LPM quirk to AMD controllers A Samsung SSD 870 QVO 8TB connected to an AMD 600 Series chipset SATA controller is reported to time out on STANDBY IMMEDIATE during system suspend with med_power_with_dipm enabled. The command completes when using max_performance instead. The existing Samsung LPM quirk only matches ATI controllers, leaving AMD controllers unaffected. Rename it to ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD and extend the vendor check to AMD for the same Samsung SSD model patterns. Keep LPM behavior unchanged for other controller vendors, including Intel. Leave ATA_QUIRK_NO_NCQ_ON_ATI restricted to ATI, since the reported AMD issue concerns LPM rather than NCQ. Link: https://bugzilla.kernel.org/show_bug.cgi?id=221986 Reviewed-by: Damien Le Moal Reviewed-by: Mario Limonciello (AMD) > --- Link: https://lore.kernel.org/r/20260918124030.1962773-5-cassel@kernel.org Signed-off-by: Niklas Cassel --- drivers/ata/libata-core.c | 15 ++++++++------- include/linux/libata.h | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index f482c0a6d7e9..92233fb25051 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2972,9 +2972,10 @@ static void ata_dev_config_lpm(struct ata_device *dev) (dev->id[ATA_ID_SATA_CAPABILITY] & 0xe) == 0x2) dev->quirks |= ATA_QUIRK_NOLPM; - /* ATI specific quirk */ - if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI) && - ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI)) + /* ATI and AMD specific quirk */ + if ((dev->quirks & ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD) && + (ata_dev_check_adapter(dev, PCI_VENDOR_ID_ATI) || + ata_dev_check_adapter(dev, PCI_VENDOR_ID_AMD))) dev->quirks |= ATA_QUIRK_NOLPM; } @@ -4136,7 +4137,7 @@ static const char * const ata_quirk_names[] = { [__ATA_QUIRK_MAX_SEC] = "maxsec", [__ATA_QUIRK_MAX_TRIM_128M] = "maxtrim128m", [__ATA_QUIRK_NO_NCQ_ON_ATI] = "noncqonati", - [__ATA_QUIRK_NO_LPM_ON_ATI] = "nolpmonati", + [__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD] = "nolpmonatiandamd", [__ATA_QUIRK_NO_ID_DEV_LOG] = "noiddevlog", [__ATA_QUIRK_NO_LOG_DIR] = "nologdir", [__ATA_QUIRK_NO_FUA] = "nofua", @@ -4420,15 +4421,15 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "Samsung SSD 860*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "Samsung SSD 870*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "SAMSUNG*MZ7LH*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM | ATA_QUIRK_NO_NCQ_ON_ATI | - ATA_QUIRK_NO_LPM_ON_ATI }, + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD }, { "FCCT*M500*", NULL, ATA_QUIRK_NO_NCQ_TRIM | ATA_QUIRK_ZERO_AFTER_TRIM }, diff --git a/include/linux/libata.h b/include/linux/libata.h index 313e96173b19..48bde275968d 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -76,7 +76,7 @@ enum ata_quirks { __ATA_QUIRK_MAX_SEC, /* Limit max sectors */ __ATA_QUIRK_MAX_TRIM_128M, /* Limit max trim size to 128M */ __ATA_QUIRK_NO_NCQ_ON_ATI, /* Disable NCQ on ATI chipset */ - __ATA_QUIRK_NO_LPM_ON_ATI, /* Disable LPM on ATI chipset */ + __ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD, /* Disable LPM on ATI and AMD chipsets */ __ATA_QUIRK_NO_ID_DEV_LOG, /* Identify device log missing */ __ATA_QUIRK_NO_LOG_DIR, /* Do not read log directory */ __ATA_QUIRK_NO_FUA, /* Do not use FUA */ @@ -115,7 +115,7 @@ enum { ATA_QUIRK_MAX_SEC = BIT_ULL(__ATA_QUIRK_MAX_SEC), ATA_QUIRK_MAX_TRIM_128M = BIT_ULL(__ATA_QUIRK_MAX_TRIM_128M), ATA_QUIRK_NO_NCQ_ON_ATI = BIT_ULL(__ATA_QUIRK_NO_NCQ_ON_ATI), - ATA_QUIRK_NO_LPM_ON_ATI = BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI), + ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD = BIT_ULL(__ATA_QUIRK_NO_LPM_ON_ATI_AND_AMD), ATA_QUIRK_NO_ID_DEV_LOG = BIT_ULL(__ATA_QUIRK_NO_ID_DEV_LOG), ATA_QUIRK_NO_LOG_DIR = BIT_ULL(__ATA_QUIRK_NO_LOG_DIR), ATA_QUIRK_NO_FUA = BIT_ULL(__ATA_QUIRK_NO_FUA), -- cgit v1.2.3 From 28f9c0e0a0b94c5d3e1b634db545f6e1f94858c5 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:24 -0700 Subject: sched/cache: Decouple sched_cache_group from mm to fix UAF Currently the sched cache grouping is by mm and the scheduling statistics sched_cache_stat lives in the mm structure. This ties the life cycle of scheduling stats with mm. In account_mm_sched(), the scheduling stats are accessed by task->mm->sc_stat. However, a task may be switching mm on one CPU when another CPU is running account_mm_sched(), and possibly accessing the old mm that was freed. This problem was found when running tests with KASAN by Hyunwoo: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Instead of serializing the mm access by introducing extra acquisition of rq lock in the mm free path, extract sched_cache_stat from mm_struct, rename it as sched_cache_group and manage its life cycle apart from mm_struct with its own ref counting. This allows us in the next patch access sched_cache_group directly from task, and add a refcount on sched_cache_group when a task links to it. This prevents the use after free issue when accessing stale and released old mm and its sched cache stat a task switches to a new mm while account_mm_sched() is done elsewhere. The other benefit of this restructure is in the future, the grouping of tasks to a LLC would have the flexibility to be associated with a user defined grouping, or cgroup, cookie group, numa_group or others instead of just with a single mm address space. Rename sched_cache_stat to sched_cache_group and turn it into a refcounted object allocated from mm_struct. The mm_struct now holds a pointer (sched_cache_grp) to this object instead of embedding it. Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Closes: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Closes: https://lore.kernel.org/all/343a7e07-7fad-4979-9c9b-82ec038c293c@linux.dev/ Reported-by: Hyunwoo Kim Reported-by: Zenghui Yu (Huawei) Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: #7.2.x Link: https://patch.msgid.link/91fd1e3266707c865bc9abecfb3e17bc676712df.1790035273.git.tim.c.chen@linux.intel.com --- include/linux/mm_types.h | 15 ++-- include/linux/sched.h | 6 +- kernel/exit.c | 11 ++- kernel/sched/fair.c | 173 ++++++++++++++++++++++++++++++++++------------- 4 files changed, 144 insertions(+), 61 deletions(-) (limited to 'include/linux') diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 6d815f6440c9..f3e5a2fadbe5 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1226,7 +1226,7 @@ struct mm_struct { struct mm_mm_cid mm_cid; /* sched_cache related statistics */ - struct sched_cache_stat sc_stat; + struct sched_cache_group *sched_cache_grp; #ifdef CONFIG_MMU atomic_long_t pgtables_bytes; /* size of all page tables */ #endif @@ -1624,8 +1624,9 @@ static inline unsigned int mm_cid_size(void) #endif /* CONFIG_SCHED_MM_CID */ #ifdef CONFIG_SCHED_CACHE -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *pcpu_sched); +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *pcpu_sched); +void mm_destroy_sched(struct mm_struct *mm); static inline int mm_alloc_sched_noprof(struct mm_struct *mm) { @@ -1635,17 +1636,11 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm) if (!pcpu_sched) return -ENOMEM; - mm_init_sched(mm, pcpu_sched); - return 0; + return mm_init_sched(mm, pcpu_sched); } #define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__)) -static inline void mm_destroy_sched(struct mm_struct *mm) -{ - free_percpu(mm->sc_stat.pcpu_sched); - mm->sc_stat.pcpu_sched = NULL; -} #else /* !CONFIG_SCHED_CACHE */ static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 705970d07614..e14ad43522c8 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2405,7 +2405,7 @@ struct sched_cache_time { unsigned long epoch; }; -struct sched_cache_stat { +struct sched_cache_group { struct sched_cache_time __percpu *pcpu_sched; raw_spinlock_t lock; unsigned long epoch; @@ -2413,11 +2413,13 @@ struct sched_cache_stat { unsigned long next_scan; unsigned long footprint; int cpu; + refcount_t refcnt; + struct rcu_head rcu; } ____cacheline_aligned_in_smp; #else -struct sched_cache_stat { }; +struct sched_cache_group { }; #endif diff --git a/kernel/exit.c b/kernel/exit.c index 424c44a42a4d..024350e9b48c 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -558,18 +558,23 @@ void mm_update_next_owner(struct mm_struct *mm) */ static void exit_mm_sched_cache(struct mm_struct *mm) { + struct sched_cache_group *grp; unsigned long fp, sub; if (!current->total_numa_faults) return; /* * No lock protection due to performance considerations. - * Make sure mm->sc_stat.footprint does not become + * Make sure the group footprint does not become * negative. */ - fp = READ_ONCE(mm->sc_stat.footprint); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + fp = READ_ONCE(grp->footprint); sub = min(fp, current->total_numa_faults); - WRITE_ONCE(mm->sc_stat.footprint, fp - sub); + WRITE_ONCE(grp->footprint, fp - sub); } #else static inline void exit_mm_sched_cache(struct mm_struct *mm) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 514bd54ccd56..f0a9586332fd 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1492,12 +1492,17 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return true; if (static_branch_likely(&sched_numa_balancing)) { + struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp); + + if (!grp) + return true; + /* * TBD: RDT exclusive LLC ways reserved should be * excluded. */ llc = sd->llc_bytes; - footprint = READ_ONCE(mm->sc_stat.footprint); + footprint = READ_ONCE(grp->footprint); /* * Scale the LLC size by 256*llc_aggr_tolerance @@ -1529,6 +1534,7 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, int cpu) { + struct sched_cache_group *grp; int scale; if (get_nr_threads(p) <= 1) @@ -1542,7 +1548,11 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads), + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return true; + + return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } @@ -1648,12 +1658,20 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p) } } -void mm_init_sched(struct mm_struct *mm, - struct sched_cache_time __percpu *_pcpu_sched) +int mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *_pcpu_sched) { + struct sched_cache_group *grp; unsigned long epoch = 0; int i; + grp = kzalloc_obj(*grp); + if (!grp) { + free_percpu(_pcpu_sched); + mm->sched_cache_grp = NULL; + return -ENOMEM; + } + for_each_possible_cpu(i) { struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i); struct rq *rq = cpu_rq(i); @@ -1664,18 +1682,51 @@ void mm_init_sched(struct mm_struct *mm, epoch = rq->cpu_epoch; } - raw_spin_lock_init(&mm->sc_stat.lock); - mm->sc_stat.epoch = epoch; - mm->sc_stat.cpu = -1; - mm->sc_stat.next_scan = jiffies; - mm->sc_stat.nr_running_avg = 0; - mm->sc_stat.footprint = 0; + raw_spin_lock_init(&grp->lock); + grp->epoch = epoch; + grp->cpu = -1; + grp->next_scan = jiffies; + grp->nr_running_avg = 0; + grp->footprint = 0; + refcount_set(&grp->refcnt, 1); /* - * The update to mm->sc_stat should not be reordered - * before initialization to mm's other fields, in case + * The update to grp->pcpu_sched should not be reordered + * before initialization to grp's other fields, in case * the readers may get invalid mm_sched_epoch, etc. */ - smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched); + smp_store_release(&grp->pcpu_sched, _pcpu_sched); + /* + * Publish the group last. Not every reader qualifies it by + * grp->pcpu_sched - can_migrate_llc_task() only checks that the + * pointer is non-NULL before reading grp->footprint and + * grp->nr_running_avg - so a reachable group must already be + * fully initialized. + */ + smp_store_release(&mm->sched_cache_grp, grp); + return 0; +} + +static void sched_cache_group_free_rcu(struct rcu_head *rcu) +{ + struct sched_cache_group *grp = + container_of(rcu, struct sched_cache_group, rcu); + + free_percpu(grp->pcpu_sched); + kfree(grp); +} + +static void sched_cache_group_put(struct sched_cache_group *grp) +{ + if (!grp || !refcount_dec_and_test(&grp->refcnt)) + return; + + call_rcu(&grp->rcu, sched_cache_group_free_rcu); +} + +void mm_destroy_sched(struct mm_struct *mm) +{ + sched_cache_group_put(mm->sched_cache_grp); + mm->sched_cache_grp = NULL; } /* because why would C be fully specified */ @@ -1729,11 +1780,16 @@ static unsigned long fraction_mm_sched(struct rq *rq, static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) { int mm_sched_llc = -1, mm_sched_cpu; + struct sched_cache_group *grp; if (!mm) return -1; - mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return -1; + + mm_sched_cpu = READ_ONCE(grp->cpu); if (mm_sched_cpu != -1) { mm_sched_llc = llc_id(mm_sched_cpu); @@ -1764,6 +1820,7 @@ static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { struct sched_cache_time *pcpu_sched; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1777,10 +1834,14 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * init_task, kthreads and user thread created * by user_mode_thread() don't have mm. */ - if (!mm || !mm->sc_stat.pcpu_sched) + if (!mm) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; - pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq)); + pcpu_sched = per_cpu_ptr(grp->pcpu_sched, cpu_of(rq)); scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) { __update_mm_sched(rq, pcpu_sched); @@ -1793,11 +1854,11 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * If this process hasn't hit task_cache_work() for a while invalidate * its preferred state. */ - if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout || + if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || invalid_llc_nr(mm, p, cpu_of(rq)) || exceed_llc_capacity(mm, cpu_of(rq))) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); } mm_sched_llc = get_pref_llc(p, mm); @@ -1814,30 +1875,35 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { struct callback_head *work = &p->cache_work; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD || - !mm->sc_stat.pcpu_sched) + if (!mm || p->flags & PF_KTHREAD) + return; + + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp || !grp->pcpu_sched) return; epoch = rq->cpu_epoch; /* avoid moving backwards */ - if (time_after_eq(mm->sc_stat.epoch, epoch)) + if (time_after_eq(grp->epoch, epoch)) return; - guard(raw_spinlock)(&mm->sc_stat.lock); + guard(raw_spinlock)(&grp->lock); if (work->next == work) { task_work_add(p, work, TWA_RESUME); - WRITE_ONCE(mm->sc_stat.epoch, epoch); + WRITE_ONCE(grp->epoch, epoch); } } -static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p, + struct sched_cache_group *grp) { #ifdef CONFIG_NUMA_BALANCING int cpu, curr_cpu, nid, pref_nid; @@ -1845,7 +1911,7 @@ static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p) if (!static_branch_likely(&sched_numa_balancing)) goto out; - cpu = READ_ONCE(p->mm->sc_stat.cpu); + cpu = READ_ONCE(grp->cpu); if (cpu != -1) nid = cpu_to_node(cpu); curr_cpu = task_cpu(p); @@ -1906,6 +1972,7 @@ static void task_cache_work(struct callback_head *work) unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; + struct sched_cache_group *grp; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; cpumask_var_t cpus; @@ -1917,12 +1984,16 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - next_scan = READ_ONCE(mm->sc_stat.next_scan); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return; + + next_scan = READ_ONCE(grp->next_scan); if (time_before(now, next_scan)) return; /* only 1 thread is allowed to scan */ - if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan, + if (!try_cmpxchg(&grp->next_scan, &next_scan, now + max_t(unsigned long, READ_ONCE(llc_epoch_period), 1))) return; @@ -1930,8 +2001,8 @@ static void task_cache_work(struct callback_head *work) curr_cpu = task_cpu(p); if (invalid_llc_nr(mm, p, curr_cpu) || exceed_llc_capacity(mm, curr_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return; } @@ -1942,7 +2013,7 @@ static void task_cache_work(struct callback_head *work) scoped_guard (cpus_read_lock) { guard(rcu)(); - get_scan_cpumasks(cpus, p); + get_scan_cpumasks(cpus, p, grp); for_each_cpu(cpu, cpus) { /* XXX sched_cluster_active */ @@ -1955,7 +2026,7 @@ static void task_cache_work(struct callback_head *work) for_each_cpu(i, sched_domain_span(sd)) { occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + per_cpu_ptr(grp->pcpu_sched, i)); a_occ += occ; if (occ > m_occ) { m_occ = occ; @@ -1988,7 +2059,7 @@ static void task_cache_work(struct callback_head *work) m_a_cpu = m_cpu; } - if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu))) + if (llc_id(cpu) == llc_id(READ_ONCE(grp->cpu))) curr_m_a_occ = a_occ; cpumask_andnot(cpus, cpus, sched_domain_span(sd)); @@ -1997,7 +2068,7 @@ static void task_cache_work(struct callback_head *work) if (m_a_occ > (2 * curr_m_a_occ)) { /* - * Avoid switching sc_stat.cpu too fast. + * Avoid switching sched_cache_grp->cpu too fast. * The reason to choose 2X is because: * 1. It is better to keep the preferred LLC stable, * rather than changing it frequently and cause migrations @@ -2006,10 +2077,10 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu); + WRITE_ONCE(grp->cpu, m_a_cpu); } - update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); + update_avg_scale(&grp->nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -3726,6 +3797,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) static void task_numa_placement(struct task_struct *p) __context_unsafe(/* conditional locking */) { + struct sched_cache_group __maybe_unused *grp; int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; unsigned long fault_types[2] = { 0, 0 }; @@ -3818,19 +3890,23 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sc_stat.footprint, otherwise - * the mm->sc_stat.footprint will not converge: - * the exiting thread's footprint remains unchanged/undecayed - * in mm->sc_stat.footprint. See exit_mm(). + * be subtracted from the mm->sched_cache_grp->footprint, + * otherwise the mm->sched_cache_grp->footprint will not + * converge: the exiting thread's footprint remains + * unchanged/undecayed in mm->sched_cache_grp->footprint. + * See exit_mm(). * * Lost updates and unsynchronized subtraction * in exit_mm() can cause footprint + diff to * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff; - WRITE_ONCE(p->mm->sc_stat.footprint, - max(new_fp, 0L)); + grp = READ_ONCE(p->mm->sched_cache_grp); + if (!grp) + continue; + + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); #endif } @@ -10778,6 +10854,7 @@ static inline bool task_misfits_asym_cpu(struct lb_env *env, struct task_struct static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { + struct sched_cache_group *grp; struct mm_struct *mm; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10791,15 +10868,19 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, if (!mm) return mig_unrestricted; - cpu = READ_ONCE(mm->sc_stat.cpu); + grp = READ_ONCE(mm->sched_cache_grp); + if (!grp) + return mig_unrestricted; + + cpu = READ_ONCE(grp->cpu); if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu)) return mig_unrestricted; /* skip cache aware load balance for too many threads */ if (invalid_llc_nr(mm, p, dst_cpu) || exceed_llc_capacity(mm, dst_cpu)) { - if (READ_ONCE(mm->sc_stat.cpu) != -1) - WRITE_ONCE(mm->sc_stat.cpu, -1); + if (READ_ONCE(grp->cpu) != -1) + WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; } -- cgit v1.2.3 From b636fef85bda7d1bab9c0a45067ab1508d79d946 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:25 -0700 Subject: sched/cache: Introduce task_struct->sched_cache_grp to fix UAF Add a sched_cache_grp pointer to task_struct so that scheduler code can access the cache group directly via the task, without going through mm->sched_cache_grp. This decouples the scheduler's hot-path accesses from the mm_struct. Each task holds its own refcount on the sched_cache_group, separate from the reference held by its mm_struct. The reference is acquired in copy_mm() (fork) and exec_mmap() (exec), and released in exit_mm(). This fixes the use-after-free when account_mm_sched() reaches the group through a task whose mm is being switched, as reported by Hyunwoo: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Convert all scheduler code in fair.c and exit.c to use p->sched_cache_grp instead of p->mm->sched_cache_grp. Keep the fork/exec/exit reference management out of the generic mm paths: add sched_cache_fork(), sched_cache_fork_cleanup(), sched_cache_exec_mmap() and sched_cache_exit_mm() in kernel/sched/cache_sched.c (with empty stubs for !CONFIG_SCHED_CACHE), so fs/exec.c, kernel/fork.c and kernel/exit.c each call one helper instead of open-coding the refcounting under #ifdef. Also add sched_cache_group_get() and task_cache_group_get(). Fixes: df0d98475954 ("sched/cache: Introduce infrastructure for cache-aware load balancing") Closes: https://lore.kernel.org/lkml/apPb-Dr4nPYuHQOK@v4bel/ Closes: https://lore.kernel.org/all/343a7e07-7fad-4979-9c9b-82ec038c293c@linux.dev/ Reported-by: Hyunwoo Kim Reported-by: Zenghui Yu (Huawei) Co-developed-by: Chen Yu Signed-off-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Cc: #7.2.x Link: https://patch.msgid.link/ae7081dc54736bf115215f9867abb2711a7403fb.1790035273.git.tim.c.chen@linux.intel.com --- fs/exec.c | 1 + include/linux/sched.h | 14 ++++ kernel/exit.c | 33 +-------- kernel/fork.c | 2 + kernel/sched/fair.c | 196 ++++++++++++++++++++++++++++++++++---------------- 5 files changed, 154 insertions(+), 92 deletions(-) (limited to 'include/linux') diff --git a/fs/exec.c b/fs/exec.c index 819643408e6d..a5269b5e00df 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -882,6 +882,7 @@ static int exec_mmap(struct linux_binprm *bprm) active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; + sched_cache_exec_mmap(tsk, mm); mm_init_cid(mm, tsk); exec_state = task_exec_state_replace(tsk, exec_state); /* diff --git a/include/linux/sched.h b/include/linux/sched.h index e14ad43522c8..d35ae49a991f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1433,6 +1433,7 @@ struct task_struct { #ifdef CONFIG_SCHED_CACHE struct callback_head cache_work; + struct sched_cache_group __rcu *sched_cache_grp; int preferred_llc; /* 1: task was enqueued to its preferred LLC, 0 otherwise */ int pref_llc_queued; @@ -2417,10 +2418,23 @@ struct sched_cache_group { struct rcu_head rcu; } ____cacheline_aligned_in_smp; +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp); +struct sched_cache_group *task_cache_group_get(struct task_struct *p); + +void sched_cache_fork(struct task_struct *p); +void sched_cache_fork_cleanup(struct task_struct *p); +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm); +void sched_cache_exit_mm(struct task_struct *p); + #else struct sched_cache_group { }; +static inline void sched_cache_fork(struct task_struct *p) { } +static inline void sched_cache_fork_cleanup(struct task_struct *p) { } +static inline void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) { } +static inline void sched_cache_exit_mm(struct task_struct *p) { } + #endif #ifndef MODULE diff --git a/kernel/exit.c b/kernel/exit.c index 024350e9b48c..282328d2b4cf 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -551,37 +551,6 @@ void mm_update_next_owner(struct mm_struct *mm) } #endif /* CONFIG_MEMCG */ -#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING) -/* - * Subtract the memory footprint of the current task from - * mm. - */ -static void exit_mm_sched_cache(struct mm_struct *mm) -{ - struct sched_cache_group *grp; - unsigned long fp, sub; - - if (!current->total_numa_faults) - return; - /* - * No lock protection due to performance considerations. - * Make sure the group footprint does not become - * negative. - */ - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp) - return; - - fp = READ_ONCE(grp->footprint); - sub = min(fp, current->total_numa_faults); - WRITE_ONCE(grp->footprint, fp - sub); -} -#else -static inline void exit_mm_sched_cache(struct mm_struct *mm) -{ -} -#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */ - /* * Turn us into a lazy TLB process if we * aren't already.. @@ -594,7 +563,7 @@ static void exit_mm(void) if (!mm) return; - exit_mm_sched_cache(mm); + sched_cache_exit_mm(current); mmap_read_lock(mm); mmgrab_lazy_tlb(mm); diff --git a/kernel/fork.c b/kernel/fork.c index 5ef413368912..10f2d05d816a 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1599,6 +1599,7 @@ static int copy_mm(u64 clone_flags, struct task_struct *tsk) tsk->mm = mm; tsk->active_mm = mm; + sched_cache_fork(tsk); return 0; } @@ -2602,6 +2603,7 @@ bad_fork_cleanup_io: bad_fork_cleanup_namespaces: exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: + sched_cache_fork_cleanup(p); if (p->mm) { mm_clear_owner(p->mm, p); mmput(p->mm); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f0a9586332fd..974a7dfe3215 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1478,7 +1478,7 @@ static inline int get_sched_cache_scale(int mul) return (1 + (tol - 1) * mul); } -static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) +static bool exceed_llc_capacity(struct sched_cache_group *grp, int cpu) { #ifdef CONFIG_NUMA_BALANCING unsigned long llc, footprint; @@ -1492,11 +1492,6 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return true; if (static_branch_likely(&sched_numa_balancing)) { - struct sched_cache_group *grp = READ_ONCE(mm->sched_cache_grp); - - if (!grp) - return true; - /* * TBD: RDT exclusive LLC ways reserved should be * excluded. @@ -1531,10 +1526,9 @@ static bool exceed_llc_capacity(struct mm_struct *mm, int cpu) return false; } -static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, +static bool invalid_llc_nr(struct sched_cache_group *grp, struct task_struct *p, int cpu) { - struct sched_cache_group *grp; int scale; if (get_nr_threads(p) <= 1) @@ -1548,10 +1542,6 @@ static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p, if (scale == INT_MAX) return false; - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp) - return true; - return !fits_capacity((READ_ONCE(grp->nr_running_avg) * cpu_smt_num_threads), (scale * per_cpu(sd_llc_size, cpu))); } @@ -1723,6 +1713,96 @@ static void sched_cache_group_put(struct sched_cache_group *grp) call_rcu(&grp->rcu, sched_cache_group_free_rcu); } +DEFINE_FREE(sched_cache_group_put, struct sched_cache_group *, + sched_cache_group_put(_T)); + +#define rcu_deref_sched_cache_grp(tsk) \ + rcu_dereference_check((tsk)->sched_cache_grp, (tsk) == current) + +static struct sched_cache_group *sched_cache_replace_grp(struct task_struct *p, + struct sched_cache_group *new) +{ + struct sched_cache_group *old; + + old = rcu_deref_sched_cache_grp(p); + rcu_assign_pointer(p->sched_cache_grp, new); + + return old; +} + +struct sched_cache_group *sched_cache_group_get(struct sched_cache_group *grp) +{ + /* + * refcount_inc_not_zero() is the acquire primitive for lockless + * (RCU) lookups; plain refcount_inc() would scribble the count if + * it already reached zero. Return NULL in that case. + */ + if (grp && !refcount_inc_not_zero(&grp->refcnt)) + grp = NULL; + + return grp; +} + +struct sched_cache_group *task_cache_group_get(struct task_struct *p) +{ + guard(rcu)(); + return sched_cache_group_get(rcu_dereference(p->sched_cache_grp)); +} + +void sched_cache_fork(struct task_struct *p) +{ + /* + * The child takes its own reference on the mm's cache group, separate + * from the reference held by the mm. @p is not yet visible to readers, + * so a plain initializing store is enough. + */ + RCU_INIT_POINTER(p->sched_cache_grp, + sched_cache_group_get(p->mm->sched_cache_grp)); +} + +void sched_cache_fork_cleanup(struct task_struct *p) +{ + /* + * A fork that fails after sched_cache_fork() never reaches exit_mm(), + * so drop the reference here. @p never became visible, so there are no + * concurrent readers and the reference we hold keeps the group alive. + */ + sched_cache_group_put(rcu_access_pointer(p->sched_cache_grp)); + RCU_INIT_POINTER(p->sched_cache_grp, NULL); +} + +void sched_cache_exec_mmap(struct task_struct *p, struct mm_struct *mm) +{ + struct sched_cache_group *old; + + /* + * Acquire the new reference before publishing the pointer, then drop + * the old one. @p is current and the only writer of its own pointer. + */ + old = sched_cache_replace_grp(p, sched_cache_group_get(mm->sched_cache_grp)); + sched_cache_group_put(old); +} + +void sched_cache_exit_mm(struct task_struct *p) +{ + struct sched_cache_group *grp = sched_cache_replace_grp(p, NULL); + +#ifdef CONFIG_NUMA_BALANCING + /* + * Subtract this task's footprint from the group before dropping the + * reference, so the group footprint converges as its threads exit. + * Unlocked for performance; clamp to avoid underflow. + */ + if (grp && p->total_numa_faults) { + unsigned long fp = READ_ONCE(grp->footprint); + unsigned long sub = min(fp, p->total_numa_faults); + + WRITE_ONCE(grp->footprint, fp - sub); + } +#endif + sched_cache_group_put(grp); +} + void mm_destroy_sched(struct mm_struct *mm) { sched_cache_group_put(mm->sched_cache_grp); @@ -1777,15 +1857,10 @@ static unsigned long fraction_mm_sched(struct rq *rq, return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1); } -static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) +static int get_pref_llc(struct task_struct *p, struct sched_cache_group *grp) { int mm_sched_llc = -1, mm_sched_cpu; - struct sched_cache_group *grp; - if (!mm) - return -1; - - grp = READ_ONCE(mm->sched_cache_grp); if (!grp) return -1; @@ -1819,9 +1894,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p); static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct sched_cache_time *pcpu_sched; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; int mm_sched_llc = -1; unsigned long epoch; @@ -1832,12 +1906,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) return; /* * init_task, kthreads and user thread created - * by user_mode_thread() don't have mm. + * by user_mode_thread() don't have a cache group. */ - if (!mm) - return; - - grp = READ_ONCE(mm->sched_cache_grp); if (!grp || !grp->pcpu_sched) return; @@ -1855,13 +1925,13 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) * its preferred state. */ if ((long)(epoch - READ_ONCE(grp->epoch)) > llc_epoch_affinity_timeout || - invalid_llc_nr(mm, p, cpu_of(rq)) || - exceed_llc_capacity(mm, cpu_of(rq))) { + invalid_llc_nr(grp, p, cpu_of(rq)) || + exceed_llc_capacity(grp, cpu_of(rq))) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); } - mm_sched_llc = get_pref_llc(p, mm); + mm_sched_llc = get_pref_llc(p, grp); /* task not on rq accounted later in account_entity_enqueue() */ if (task_running_on_cpu(rq->cpu, p) && @@ -1874,19 +1944,15 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) static void task_tick_cache(struct rq *rq, struct task_struct *p) { + struct sched_cache_group *grp = rcu_dereference_all(p->sched_cache_grp); struct callback_head *work = &p->cache_work; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; unsigned long epoch; if (!sched_cache_enabled()) return; - if (!mm || p->flags & PF_KTHREAD) - return; - - grp = READ_ONCE(mm->sched_cache_grp); - if (!grp || !grp->pcpu_sched) + if (!grp || p->flags & PF_KTHREAD || + !grp->pcpu_sched) return; epoch = rq->cpu_epoch; @@ -1968,14 +2034,13 @@ static inline void update_avg_scale(u64 *avg, u64 sample) static void task_cache_work(struct callback_head *work) { + struct sched_cache_group *grp __free(sched_cache_group_put) = NULL; + cpumask_var_t cpus __free(free_cpumask_var) = CPUMASK_VAR_NULL; int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; unsigned long next_scan, now = jiffies; struct task_struct *p = current, *cur; unsigned long curr_m_a_occ = 0; - struct sched_cache_group *grp; - struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1984,7 +2049,12 @@ static void task_cache_work(struct callback_head *work) if (p->flags & PF_EXITING) return; - grp = READ_ONCE(mm->sched_cache_grp); + /* + * A reference makes sure grp is not released by others. The rcu + * lock can not be held till after zalloc_cpumask_var() below, + * because the latter might sleep. + */ + grp = task_cache_group_get(p); if (!grp) return; @@ -1999,8 +2069,8 @@ static void task_cache_work(struct callback_head *work) return; curr_cpu = task_cpu(p); - if (invalid_llc_nr(mm, p, curr_cpu) || - exceed_llc_capacity(mm, curr_cpu)) { + if (invalid_llc_nr(grp, p, curr_cpu) || + exceed_llc_capacity(grp, curr_cpu)) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); @@ -2033,9 +2103,13 @@ static void task_cache_work(struct callback_head *work) m_cpu = i; } + /* + * rcu_access_pointer() is used because the + * pointer is only compared, never dereferenced. + */ cur = rcu_dereference_all(cpu_rq(i)->curr); if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) + rcu_access_pointer(cur->sched_cache_grp) == grp) nr_running++; } @@ -2081,7 +2155,6 @@ static void task_cache_work(struct callback_head *work) } update_avg_scale(&grp->nr_running_avg, nr_running); - free_cpumask_var(cpus); } void init_sched_mm(struct task_struct *p) @@ -2090,6 +2163,13 @@ void init_sched_mm(struct task_struct *p) init_task_work(work, task_cache_work); work->next = work; + /* + * dup_task_struct() copies the parent's task_struct, including its + * sched_cache_grp, for which the child holds no reference. Clear it + * here - before copy_mm() runs - so the child never carries a + * borrowed pointer that the fork error path would put. + */ + RCU_INIT_POINTER(p->sched_cache_grp, NULL); /* * Reset new task's preference to avoid * polluting account_llc_enqueue(). @@ -3890,10 +3970,9 @@ static void task_numa_placement(struct task_struct *p) * heuristic and occasional lost updates are tolerable. * * If a task exits, its corresponding footprint must - * be subtracted from the mm->sched_cache_grp->footprint, - * otherwise the mm->sched_cache_grp->footprint will not - * converge: the exiting thread's footprint remains - * unchanged/undecayed in mm->sched_cache_grp->footprint. + * be subtracted from p->sched_cache_grp->footprint, + * otherwise the footprint will not converge: the + * exiting thread's footprint remains unchanged/undecayed. * See exit_mm(). * * Lost updates and unsynchronized subtraction @@ -3901,12 +3980,14 @@ static void task_numa_placement(struct task_struct *p) * go negative. Clamp to zero to prevent the * unsigned footprint from wrapping. */ - grp = READ_ONCE(p->mm->sched_cache_grp); - if (!grp) - continue; + scoped_guard(rcu) { + grp = rcu_dereference(p->sched_cache_grp); - new_fp = (long)READ_ONCE(grp->footprint) + diff; - WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + if (grp) { + new_fp = (long)READ_ONCE(grp->footprint) + diff; + WRITE_ONCE(grp->footprint, max(new_fp, 0L)); + } + } #endif } @@ -10855,7 +10936,6 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, struct task_struct *p) { struct sched_cache_group *grp; - struct mm_struct *mm; bool to_pref; int cpu, src_cpu, dst_cpu; @@ -10864,11 +10944,7 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, src_cpu = env->src_cpu; dst_cpu = env->dst_cpu; - mm = p->mm; - if (!mm) - return mig_unrestricted; - - grp = READ_ONCE(mm->sched_cache_grp); + grp = rcu_dereference_all(p->sched_cache_grp); if (!grp) return mig_unrestricted; @@ -10877,8 +10953,8 @@ static enum llc_mig can_migrate_llc_task(struct lb_env *env, return mig_unrestricted; /* skip cache aware load balance for too many threads */ - if (invalid_llc_nr(mm, p, dst_cpu) || - exceed_llc_capacity(mm, dst_cpu)) { + if (invalid_llc_nr(grp, p, dst_cpu) || + exceed_llc_capacity(grp, dst_cpu)) { if (READ_ONCE(grp->cpu) != -1) WRITE_ONCE(grp->cpu, -1); return mig_unrestricted; -- cgit v1.2.3 From 3cb0243767fd033bdce95f4f1b5882172a2f8119 Mon Sep 17 00:00:00 2001 From: Davi Chaves Azevedo Date: Mon, 21 Sep 2026 17:37:27 -0700 Subject: sched/cache: Refresh LLC capacity across CPU hotplug, to fix capacity underestimation bug The scheduler scales LLC capacity by the fraction of cache-sharing CPUs covered by a domain: llc_bytes = cache_size * span_weight / shared_weight During CPU teardown, sched_cpu_deactivate() rebuilds scheduler domains before cacheinfo_cpu_pre_down() removes the CPU from shared_cpu_map. The new domains therefore use the old sharing weight. The later call to sched_update_llc_bytes() looks up the departing CPU's sd_llc, which has already been detached, and returns without correcting the surviving CPUs. On a Ryzen 5 7535U with twelve logical CPUs sharing a 16 MiB LLC, offlining one SMT sibling left the remaining CPUs with: llc_bytes = floor(16777216 * 11 / 12) = 15379114 bytes The correct capacity is still 16777216 bytes. On systems with active cache-aware scheduling, an underestimated capacity can cause exceed_llc_capacity() to reject aggregation for a process whose footprint would fit. Unchanged cpuset partitions sharing the physical cache can also retain stale capacity when a CPU comes online in another partition. Pass the cache-sharing mask already retained by cacheinfo to the scheduler update. Refresh every surviving CPU using its own LLC domain so that each partition receives the correct share. This also preserves the correction needed as cache-sharing maps grow during boot. Keep the existing CPU-hotplug and scheduler-domain synchronization. The update remains on the hotplug path; no steady-state scheduling operation or persistent allocation is added. Fixes: 7030513a0877 ("sched/cache: Calculate the LLC size and store it in sched_domain") Signed-off-by: Davi Chaves Azevedo Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Chen Yu Reviewed-by: Tim Chen Reviewed-by: K Prateek Nayak Tested-by: Chen Yu Tested-by: K Prateek Nayak Cc: # v7.2.x Link: https://patch.msgid.link/6751d93e15889e624796c74db0bfe66603d60b1b.1790035273.git.tim.c.chen@linux.intel.com --- drivers/base/cacheinfo.c | 11 ++++++----- include/linux/sched/topology.h | 4 ++-- kernel/sched/topology.c | 22 +++++++++++++--------- 3 files changed, 21 insertions(+), 16 deletions(-) (limited to 'include/linux') diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 9f9c72727a05..7a47a392568a 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -1040,9 +1040,10 @@ static int cacheinfo_cpu_online(unsigned int cpu) rc = cache_add_dev(cpu); if (rc) goto err; - if (cpu_map_shared_cache(true, cpu, &cpu_map)) + if (cpu_map_shared_cache(true, cpu, &cpu_map)) { update_per_cpu_data_slice_size(true, cpu, cpu_map); - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; err: free_cache_attributes(cpu); @@ -1059,10 +1060,10 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu) cpu_cache_sysfs_exit(cpu); free_cache_attributes(cpu); - if (nr_shared > 1) + if (nr_shared > 1) { update_per_cpu_data_slice_size(false, cpu, cpu_map); - - sched_update_llc_bytes(cpu); + sched_update_llc_bytes(cpu_map); + } return 0; } diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index b5d9d7c2b8ad..f96812d71c51 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -281,9 +281,9 @@ static inline int task_node(const struct task_struct *p) } #ifdef CONFIG_SCHED_CACHE -extern void sched_update_llc_bytes(unsigned int cpu); +extern void sched_update_llc_bytes(const struct cpumask *cpus); #else -static inline void sched_update_llc_bytes(unsigned int cpu) { } +static inline void sched_update_llc_bytes(const struct cpumask *cpus) { } #endif #endif /* _LINUX_SCHED_TOPOLOGY_H */ diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 0248227d983a..3dab0253976f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -985,8 +985,8 @@ void sched_cache_active_set(void) } /* - * Update the bottom sched_domain's llc_bytes for @cpu and all its - * LLC siblings. Called from cacheinfo_cpu_online() or + * Update the bottom sched_domain's llc_bytes for @cpus sharing a physical + * LLC. Called from cacheinfo_cpu_online() or * cacheinfo_cpu_pre_down() with cpu hotplug lock held. * * Note: get_effective_llc_bytes() returns 0 on PowerPC. @@ -996,17 +996,13 @@ void sched_cache_active_set(void) * and does not populates the per-CPU struct cpu_cacheinfo array * that get_cpu_cacheinfo_llc() reads. */ -void sched_update_llc_bytes(unsigned int cpu) +void sched_update_llc_bytes(const struct cpumask *cpus) { struct sched_domain *sd, *sdp; unsigned int i; sched_domains_mutex_lock(); - sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu)); - if (!sdp) - goto unlock; - /* * ci->shared_cpu_map is built incrementally as CPUs come * online, so the first CPU in an LLC initially sees @@ -1014,14 +1010,22 @@ void sched_update_llc_bytes(unsigned int cpu) * get_effective_llc_bytes(). Re-evaluating every LLC * sibling on each online event corrects this once the full * shared_cpu_map is known. + * + * The departing CPU's domains have already been detached when + * cacheinfo removes it. Use the surviving cache siblings instead. + * They may belong to different cpuset partitions, so use each CPU's + * own LLC domain to scale its share of the physical cache. */ - for_each_cpu(i, sched_domain_span(sdp)) { + for_each_cpu(i, cpus) { + sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, i)); + if (!sdp) + continue; + sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); if (sd) sd->llc_bytes = get_effective_llc_bytes(i, sdp); } -unlock: sched_domains_mutex_unlock(); } -- cgit v1.2.3 From ed6eec97b534979dcf28b40c389cee57bd6561d4 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Tue, 22 Sep 2026 17:20:18 +0000 Subject: bpf: Fix bounds check for skb-backed dynptrs The skb_pointer_if_linear() function checks whether a memory region of length len starting at offset off into the skb is in the linear area, and returns a pointer to the region if so. The check currently subtracts between skb_headlen and offset of the check, and since skb_headlen is unsigned the subtraction can underflow. This causes the bounds check to spuriously pass and generate an arbitrary pointer of the form *(skb->data + off). The only user of this helper is currently skb-backed BPF dynptr code. Returning the wrong pointer leads to the dynptr erroneously being backed with invalid memory. Ensure the subtraction cannot underflow, and fail the check if it would. Use u64 arithmetic to also prevent overflow when calculating (skb_headlen(skb) - off) since off is unsigned. Fixes: 6f5a630d7c57 ("bpf, net: Introduce skb_pointer_if_linear().") Reported-by: Nicholas Carlini Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Reviewed-by: Jiayuan Chen Link: https://patch.msgid.link/20260922172028.6269-2-emil@etsalapatis.com --- include/linux/skbuff.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 421f6fc45451..c8e21903074c 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -4372,7 +4372,10 @@ skb_header_pointer_careful(const struct sk_buff *skb, int offset, static inline void * __must_check skb_pointer_if_linear(const struct sk_buff *skb, int offset, int len) { - if (likely(skb_headlen(skb) - offset >= len)) + unsigned int uoffset = (unsigned int)offset; + + if (likely(uoffset <= skb_headlen(skb) && + (unsigned int)len <= skb_headlen(skb) - uoffset)) return skb->data + offset; return NULL; } -- cgit v1.2.3 From f85f5917aa2fbd861c72ea71ecb14a2fa915c3f6 Mon Sep 17 00:00:00 2001 From: Emil Tsalapatis Date: Tue, 22 Sep 2026 17:20:24 +0000 Subject: bpf: Prevent variable arena/non-arena register contents The verifier marks ALU instructions that include at least one arena operand with needs_zext: These instructions are fixed up after verification to be ALU32 instructions to ensure that the result is a valid offset into an arena. However, different code paths may provide two non-arena 64-bit arguments to the same instruction. The result of the operation in that code path is wrong, since it is now unexpectedly truncated to 32 bits and zero-extended. Add logic to the verifier to ensure every instruction either always has at least one PTR_TO_ARENA argument, or never does. Since needs_zext already tracks the first scenario, add a prevent_zext field in bpf_insn_aux to track the latter. Reject instructions that use arena arguments and have prevent_zext set, or do not have arena arguments and have needs_zext set. Fixes: 6082b6c328b5 ("bpf: Recognize addr_space_cast instruction in the verifier.") Reported-by: Nicholas Carlini Suggested-by: Nicholas Carlini Signed-off-by: Emil Tsalapatis Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260922172028.6269-8-emil@etsalapatis.com --- include/linux/bpf_verifier.h | 1 + kernel/bpf/verifier.c | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 6fe8e5dc57aa..b83ec99f1a13 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -680,6 +680,7 @@ struct bpf_insn_aux_data { bool nospec_result; /* result is unsafe under speculation, nospec must follow */ bool zext_dst; /* this insn zero extends dst reg */ bool needs_zext; /* alu op needs to clear upper bits */ + bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ bool is_iter_next; /* bpf_iter__next() kfunc call */ bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7ffbb804184f..41b49c56e123 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -15741,6 +15741,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64); + struct bpf_insn_aux_data *aux = cur_aux(env); u8 opcode = BPF_OP(insn->code); int err; @@ -15752,12 +15753,23 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, /* Case where at least one operand is an arena. */ if (dst_reg->type == PTR_TO_ARENA || (src_reg && src_reg->type == PTR_TO_ARENA)) { - struct bpf_insn_aux_data *aux = cur_aux(env); if (dst_reg->type != PTR_TO_ARENA) *dst_reg = *src_reg; if (BPF_CLASS(insn->code) == BPF_ALU64) { + /* + * Only arena pointers set needs_zext, but doing so + * modifies the instruction at fixup time to an ALU32 + * and makes it unsuitable for 64-bit scalar args. We + * prevent zext from being set if the instruction has + * been previously called with non-arena registers. + */ + if (aux->prevent_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + /* * 32-bit operations zero upper bits automatically. * 64-bit operations need to be converted to 32. @@ -15770,6 +15782,16 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, return 0; } + /* Prevent the instruction from being used with arena pointers (see above). */ + if (env->prog->aux->arena && BPF_CLASS(insn->code) == BPF_ALU64) { + if (aux->needs_zext) { + verbose(env, "same insn cannot be used with and without arena pointer\n"); + return -EINVAL; + } + + aux->prevent_zext = true; + } + if (dst_reg->type != SCALAR_VALUE) ptr_reg = dst_reg; -- cgit v1.2.3 From 9518405613863d0bf0700927a21367f5942cf058 Mon Sep 17 00:00:00 2001 From: Willem de Bruijn Date: Fri, 18 Sep 2026 20:47:29 -0400 Subject: packet: use ubuf_info completion for TX_RING packets tpacket_snd sends skbs with frags pointing into its ring slots. Slots are released when skb->destructor is called. A call to skb_orphan calls skb->destructor before the skb is freed. This can cause the slot to be reused while still linked into the skb. Switch to standard zerocopy completion (ubuf_info) so the slot is only released once all references to the payload are freed or copied. Restore skb->destructor to standard sock_wfree. The ubuf_info completion callback can be called with a NULL skb, but only from net_zcopy_put and related API, used by zerocopy implementations that hold their own reference on the uarg, such as MSG_ZEROCOPY. This uarg is only ever completed from skb_zcopy_clear, so skb is always set. To prevent userspace from aliasing in-flight state on shared ring slots, allocate tpacket_uarg per packet, rather than per slot. This adds a small allocation to the transmit path. Use standard kmalloc to allow backporting to stable kernels. The uarg holds an sk_wmem_alloc reference, rather than an sk_refcnt reference. packet_free_tx_ring waits on sk_wmem_alloc before freeing the ring pages. Always allocate vec->deferred for tx_ring so page-backed rings also wait on sk_wmem_alloc when skb_copy_ubufs drops page refs before calling tpacket_ubuf_complete. Drop the tx_ring.pg_vec test that tpacket_destruct_skb performed before accessing the slot. The sk_wmem_alloc reference now guarantees that the slot is valid. The test is also not sufficient by itself, as it reads pg_vec without pg_vec_lock, so it can race with packet_set_ring. As a result a slot is released when its payload is copied, which can be before transmission (e.g., in skb_orphan_frags_rx). If copied before skb_tx_timestamp() is called, no slot timestamp is recorded, similar to when skb_orphan() was called early in the datapath before this patch. Revert the now unused previous skb_zcopy_.._nouarg infra. Depends on commit 992cc9f94ca9 ("net/packet: defer vmalloc TX_RING free until skbs finish"). Reported-by: Katherine Leaver Reported-by: Bjoern Doebel Closes: https://lore.kernel.org/netdev/20260909085542.3370986-1-doebel@amazon.de/ Fixes: 5cd8d46ea156 ("packet: copy user buffers before orphan or clone") Cc: stable@vger.kernel.org Signed-off-by: Willem de Bruijn Link: https://patch.msgid.link/20260919004748.1463985-3-willemdebruijn.kernel@gmail.com Signed-off-by: Jakub Kicinski --- include/linux/skbuff.h | 19 +-------- net/packet/af_packet.c | 102 +++++++++++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 56 deletions(-) (limited to 'include/linux') diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 421f6fc45451..b14d6be7370b 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1834,22 +1834,6 @@ static inline void skb_zcopy_set(struct sk_buff *skb, struct ubuf_info *uarg, } } -static inline void skb_zcopy_set_nouarg(struct sk_buff *skb, void *val) -{ - skb_shinfo(skb)->destructor_arg = (void *)((uintptr_t) val | 0x1UL); - skb_shinfo(skb)->flags |= SKBFL_ZEROCOPY_FRAG; -} - -static inline bool skb_zcopy_is_nouarg(struct sk_buff *skb) -{ - return (uintptr_t) skb_shinfo(skb)->destructor_arg & 0x1UL; -} - -static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb) -{ - return (void *)((uintptr_t) skb_shinfo(skb)->destructor_arg & ~0x1UL); -} - static inline void net_zcopy_put(struct ubuf_info *uarg) { if (uarg) @@ -1872,8 +1856,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success) struct ubuf_info *uarg = skb_zcopy(skb); if (uarg) { - if (!skb_zcopy_is_nouarg(skb)) - uarg->ops->complete(skb, uarg, zerocopy_success); + uarg->ops->complete(skb, uarg, zerocopy_success); skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY; } diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 50cae32ae269..64b501db660a 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -2530,26 +2530,6 @@ drop_n_account: goto drop_n_restore; } -static void tpacket_destruct_skb(struct sk_buff *skb) -{ - struct packet_sock *po = pkt_sk(skb->sk); - - if (likely(po->tx_ring.pg_vec)) { - void *ph; - __u32 ts; - - ph = skb_zcopy_get_nouarg(skb); - - ts = __packet_set_timestamp(po, ph, skb); - __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); - - packet_dec_pending(&po->tx_ring); - complete(&po->skb_completion); - } - - sock_wfree(skb); -} - static int __packet_snd_vnet_parse(struct virtio_net_hdr *vnet_hdr, size_t len) { if ((vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && @@ -2589,27 +2569,56 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, return 0; } +struct tpacket_uarg { + struct ubuf_info ubuf; + struct packet_sock *po; + void *ph; +}; + +static void tpacket_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg, + bool success) +{ + struct tpacket_uarg *tu = container_of(uarg, struct tpacket_uarg, ubuf); + struct packet_sock *po = tu->po; + void *ph = tu->ph; + __u32 ts; + + DEBUG_NET_WARN_ON_ONCE(!skb); + + if (!refcount_dec_and_test(&uarg->refcnt)) + return; + + ts = __packet_set_timestamp(po, ph, skb); + __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); + + packet_dec_pending(&po->tx_ring); + complete(&po->skb_completion); + + kfree(tu); + sk_free(&po->sk); +} + +static const struct ubuf_info_ops tpacket_ubuf_ops = { + .complete = tpacket_ubuf_complete, +}; + static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, - void *frame, struct net_device *dev, void *data, int tp_len, + struct net_device *dev, void *data, int tp_len, __be16 proto, unsigned char *addr, int hlen, int copylen, int hard_header_len, const struct sockcm_cookie *sockc) { - union tpacket_uhdr ph; int to_write, offset, len, nr_frags, len_max; struct socket *sock = po->sk.sk_socket; struct page *page; int err; - ph.raw = frame; - skb->protocol = proto; skb->dev = dev; skb->priority = sockc->priority; skb->mark = sockc->mark; skb_set_delivery_type_by_clockid(skb, sockc->transmit_time, po->sk.sk_clockid); skb_setup_tx_timestamp(skb, sockc); - skb_zcopy_set_nouarg(skb, ph.raw); skb_reserve(skb, hlen); skb_reset_network_header(skb); @@ -2749,6 +2758,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) struct virtio_net_hdr vnet_hdr; bool has_vnet_hdr = false; struct sockcm_cookie sockc; + struct tpacket_uarg *uarg; __be16 proto; int err, reserve = 0; void *ph; @@ -2876,7 +2886,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) err = len_sum; goto out_status; } - tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, + tp_len = tpacket_fill_skb(po, skb, dev, data, tp_len, proto, addr, hlen, copylen, hard_header_len, &sockc); if (likely(tp_len >= 0) && @@ -2908,7 +2918,24 @@ tpacket_error: virtio_net_hdr_set_proto(skb, &vnet_hdr); } - skb->destructor = tpacket_destruct_skb; + uarg = kmalloc(sizeof(*uarg), GFP_KERNEL); + if (unlikely(!uarg)) { + if (likely(len_sum > 0)) + err = len_sum; + else + err = -ENOMEM; + goto out_status; + } + uarg->po = po; + uarg->ph = ph; + uarg->ubuf.ops = &tpacket_ubuf_ops; + uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG; + refcount_set(&uarg->ubuf.refcnt, 1); + + /* Hold a sk_wmem_alloc reference until completion */ + refcount_inc(&po->sk.sk_wmem_alloc); + skb_zcopy_init(skb, &uarg->ubuf); + __packet_set_status(po, ph, TP_STATUS_SENDING); packet_inc_pending(&po->tx_ring); @@ -4486,21 +4513,20 @@ static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring vec->len = block_nr; pg_vec = vec->pg_vec; + if (tx_ring) { + vec->deferred = kzalloc_obj(*vec->deferred, + GFP_KERNEL | __GFP_NOWARN); + if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; + INIT_DELAYED_WORK(&vec->deferred->work, + packet_free_pg_vec_work); + } + for (i = 0; i < block_nr; i++) { pg_vec[i].buffer = alloc_one_pg_vec_page(order); if (unlikely(!pg_vec[i].buffer)) goto out_free_pgvec; - - if (tx_ring && !vec->deferred && - is_vmalloc_addr(pg_vec[i].buffer)) { - vec->deferred = kzalloc_obj(*vec->deferred, - GFP_KERNEL | __GFP_NOWARN); - if (!vec->deferred) - goto out_free_pgvec; - vec->deferred->vec = vec; - INIT_DELAYED_WORK(&vec->deferred->work, - packet_free_pg_vec_work); - } } out: -- cgit v1.2.3 From 24b620729e53d978b3e425f55bc66efd3bab1f59 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 06:35:36 -0700 Subject: perf/core: Fill branch entries with a single assignment perf_clear_branch_entry_bitfields() clears the bitfields of struct perf_branch_entry one by one and leaves from/to alone, since callers overwrite those straight away. The list has to be kept in sync with the struct by hand and has already fallen behind: new_type and priv were added to perf_branch_entry and never added here. Only BRBE writes those two, and neither for every record. brbe_set_perf_entry_type() leaves new_type alone for a branch type it does not recognise, and priv is not set for source-only records. arm_pmuv3.c allocates the per-CPU branch stack with kmalloc(), so such a record reaches userspace with whatever the slot held: uninitialised kmalloc() data on the first pass over the buffer, the previous record's values after that. Nothing under arch/x86/events/ writes either field, so only arm64 is affected. Assign the whole entry at each site instead. Everything not named is then zero, and there is no list to keep in sync. The bitfields add up to exactly 64 bits, so the struct has no padding to leave undefined. perf_clear_branch_entry_bitfields() has no callers left, so remove it. perf_entry_from_brbe_regset() assigns an empty literal instead, since it fills from/to conditionally. PERF_BR_SPEC_NA is 0, so dropping the explicit spec assignment changes nothing. Fixes: b190bc4ac9e6 ("perf: Extend branch type classification") Fixes: 5402d25aa571 ("perf: Capture branch privilege information") Suggested-by: Peter Zijlstra Signed-off-by: Puranjay Mohan Signed-off-by: Peter Zijlstra (Intel) Tested-by: Yifan Wu Link: https://patch.msgid.link/20260810133540.1947118-4-puranjay@kernel.org --- arch/x86/events/amd/brs.c | 9 +++---- arch/x86/events/amd/lbr.c | 16 +++++------ arch/x86/events/intel/lbr.c | 65 ++++++++++++++++++++++++--------------------- drivers/perf/arm_brbe.c | 2 +- include/linux/perf_event.h | 17 ------------ 5 files changed, 48 insertions(+), 61 deletions(-) (limited to 'include/linux') diff --git a/arch/x86/events/amd/brs.c b/arch/x86/events/amd/brs.c index dc564688f3d7..54b13faba116 100644 --- a/arch/x86/events/amd/brs.c +++ b/arch/x86/events/amd/brs.c @@ -343,11 +343,10 @@ void amd_brs_drain(void) if (!amd_brs_match_plm(event, from, to)) continue; - perf_clear_branch_entry_bitfields(br+nr); - - br[nr].from = from; - br[nr].to = to; - + br[nr] = (struct perf_branch_entry){ + .from = from, + .to = to, + }; nr++; } empty: diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c index 9d9c961989d5..a55646fcb846 100644 --- a/arch/x86/events/amd/lbr.c +++ b/arch/x86/events/amd/lbr.c @@ -184,13 +184,6 @@ void amd_pmu_lbr_read(void) entry.to.split.reserved) continue; - perf_clear_branch_entry_bitfields(br + out); - - br[out].from = sign_ext_branch_ip(entry.from.split.ip); - br[out].to = sign_ext_branch_ip(entry.to.split.ip); - br[out].mispred = entry.from.split.mispredict; - br[out].predicted = !br[out].mispred; - /* * Set branch speculation information using the status of * the valid and spec bits. @@ -208,7 +201,14 @@ void amd_pmu_lbr_read(void) * speculative and took the correct path */ idx = (entry.to.split.valid << 1) | entry.to.split.spec; - br[out].spec = lbr_spec_map[idx]; + + br[out] = (struct perf_branch_entry){ + .from = sign_ext_branch_ip(entry.from.split.ip), + .to = sign_ext_branch_ip(entry.to.split.ip), + .mispred = entry.from.split.mispredict, + .predicted = !entry.from.split.mispredict, + .spec = lbr_spec_map[idx], + }; out++; } diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cbe5c762008d..22e2a06d5786 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -756,10 +756,10 @@ void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc) rdmsrq(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr); - perf_clear_branch_entry_bitfields(br); - - br->from = msr_lastbranch.from; - br->to = msr_lastbranch.to; + *br = (struct perf_branch_entry){ + .from = msr_lastbranch.from, + .to = msr_lastbranch.to, + }; br++; } cpuc->lbr_stack.nr = i; @@ -847,14 +847,15 @@ void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc) if (abort && x86_pmu.lbr_double_abort && out > 0) out--; - perf_clear_branch_entry_bitfields(br+out); - br[out].from = from; - br[out].to = to; - br[out].mispred = mis; - br[out].predicted = pred; - br[out].in_tx = in_tx; - br[out].abort = abort; - br[out].cycles = cycles; + br[out] = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mis, + .predicted = pred, + .in_tx = in_tx, + .abort = abort, + .cycles = cycles, + }; out++; } cpuc->lbr_stack.nr = out; @@ -905,6 +906,7 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, struct perf_branch_entry *e; struct lbr_entry *lbr; u64 from, to, info; + bool mispred; int i; for (i = 0; i < x86_pmu.lbr_nr; i++) { @@ -921,24 +923,27 @@ static void intel_pmu_store_lbr(struct cpu_hw_events *cpuc, to = rdlbr_to(i, lbr); info = rdlbr_info(i, lbr); - perf_clear_branch_entry_bitfields(e); - - e->from = from; - e->to = to; - e->mispred = get_lbr_mispred(info); - e->predicted = !e->mispred; - e->in_tx = !!(info & LBR_INFO_IN_TX); - e->abort = !!(info & LBR_INFO_ABORT); - e->cycles = get_lbr_cycles(info); - e->type = get_lbr_br_type(info); - - /* - * Leverage the reserved field of cpuc->lbr_entries[i] to - * temporarily store the branch counters information. - * The later code will decide what content can be disclosed - * to the perf tool. Pleae see intel_pmu_lbr_counters_reorder(). - */ - e->reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & LBR_INFO_BR_CNTR_FULL_MASK; + mispred = get_lbr_mispred(info); + + *e = (struct perf_branch_entry){ + .from = from, + .to = to, + .mispred = mispred, + .predicted = !mispred, + .in_tx = !!(info & LBR_INFO_IN_TX), + .abort = !!(info & LBR_INFO_ABORT), + .cycles = get_lbr_cycles(info), + .type = get_lbr_br_type(info), + /* + * Leverage the reserved field of + * cpuc->lbr_entries[i] to temporarily store the + * branch counters information. The later code will + * decide what content can be disclosed to the perf + * tool. Pleae see intel_pmu_lbr_counters_reorder(). + */ + .reserved = (info >> LBR_INFO_BR_CNTR_OFFSET) & + LBR_INFO_BR_CNTR_FULL_MASK, + }; } cpuc->lbr_stack.nr = i; diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c index ba554e0c846c..254be4da8ae2 100644 --- a/drivers/perf/arm_brbe.c +++ b/drivers/perf/arm_brbe.c @@ -604,7 +604,7 @@ static bool perf_entry_from_brbe_regset(int index, struct perf_branch_entry *ent return false; brbinf = bregs.brbinf; - perf_clear_branch_entry_bitfields(entry); + *entry = (struct perf_branch_entry){ }; if (brbe_record_is_complete(brbinf)) { entry->from = bregs.brbsrc; entry->to = bregs.brbtgt; diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 5842552294c1..915c6fd3f084 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1467,23 +1467,6 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } -/* - * Clear all bitfields in the perf_branch_entry. - * The to and from fields are not cleared because they are - * systematically modified by caller. - */ -static inline void perf_clear_branch_entry_bitfields(struct perf_branch_entry *br) -{ - br->mispred = 0; - br->predicted = 0; - br->in_tx = 0; - br->abort = 0; - br->cycles = 0; - br->type = 0; - br->spec = PERF_BR_SPEC_NA; - br->reserved = 0; -} - extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, -- cgit v1.2.3 From 845a2737e0f5ffe5ca8d38de556012b0c9472cdd Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Thu, 3 Sep 2026 18:37:36 +0200 Subject: tty: abort break signalling on hangup The break ioctls can race with hangup and end up calling into a tty driver for a device that is already gone or powered down. Drivers must handle this race, but TCSBRK and TCSBRKP should still be aborted to avoid calling back into the driver after a user-controlled timeout (possibly even after the tty has been reopened). Note that drivers should disable any break state on shutdown so just return on hangup (calling break_ctl() again is racy and will most likely fail for drivers handling the race). Fixes: f34d7a5b7010 ("tty: The big operations rework") Cc: stable Signed-off-by: Johan Hovold Link: https://patch.msgid.link/20260903163736.1499280-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_io.c | 36 +++++++++++++++++++++++++++--------- include/linux/tty.h | 1 + 2 files changed, 28 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 48569035da56..1c30faae9ec1 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -620,6 +620,8 @@ static void __tty_hangup(struct tty_struct *tty, int exit_session) tty_ldisc_hangup(tty, cons_filp != NULL); + wake_up_interruptible(&tty->break_wait); + spin_lock_irq(&tty->ctrl.lock); clear_bit(TTY_THROTTLED, &tty->flags); clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); @@ -2431,6 +2433,7 @@ static int tiocgetd(struct tty_struct *tty, int __user *p) * send_break - performed time break * @tty: device to break on * @duration: timeout in mS + * @file: file object * * Perform a timed break on hardware that lacks its own driver level timed * break functionality. @@ -2438,8 +2441,9 @@ static int tiocgetd(struct tty_struct *tty, int __user *p) * Locking: * @tty->atomic_write_lock serializes */ -static int send_break(struct tty_struct *tty, unsigned int duration) +static int send_break(struct file *file, struct tty_struct *tty, unsigned int duration) { + long timeout; int retval; if (tty->ops->break_ctl == NULL) @@ -2453,13 +2457,26 @@ static int send_break(struct tty_struct *tty, unsigned int duration) return -EINTR; retval = tty->ops->break_ctl(tty, -1); - if (!retval) { - msleep_interruptible(duration); - retval = tty->ops->break_ctl(tty, 0); - } else if (retval == -EOPNOTSUPP) { - /* some drivers can tell only dynamically */ - retval = 0; + if (retval) { + if (retval == -EOPNOTSUPP) { + /* some drivers can tell only dynamically */ + retval = 0; + } + goto out_unlock; + } + + timeout = msecs_to_jiffies(duration); + timeout = wait_event_interruptible_timeout(tty->break_wait, + tty_hung_up_p(file), + timeout); + /* return early on hangup only */ + if (timeout > 0) { + retval = -EIO; + goto out_unlock; } + + retval = tty->ops->break_ctl(tty, 0); +out_unlock: tty_write_unlock(tty); if (signal_pending(current)) @@ -2727,10 +2744,10 @@ long tty_ioctl(struct file *file, unsigned int cmd, unsigned long arg) * This is used by the tcdrain() termios function. */ if (!arg) - return send_break(tty, 250); + return send_break(file, tty, 250); return 0; case TCSBRKP: /* support for POSIX tcsendbreak() */ - return send_break(tty, arg ? arg*100 : 250); + return send_break(file, tty, arg ? arg * 100 : 250); case TIOCMGET: return tty_tiocmget(tty, p); @@ -3090,6 +3107,7 @@ struct tty_struct *alloc_tty_struct(struct tty_driver *driver, int idx) init_ldsem(&tty->ldisc_sem); init_waitqueue_head(&tty->write_wait); init_waitqueue_head(&tty->read_wait); + init_waitqueue_head(&tty->break_wait); INIT_WORK(&tty->hangup_work, do_tty_hangup); mutex_init(&tty->atomic_write_lock); spin_lock_init(&tty->ctrl.lock); diff --git a/include/linux/tty.h b/include/linux/tty.h index 0a46e4054dec..ad4e4a835f90 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -230,6 +230,7 @@ struct tty_struct { struct fasync_struct *fasync; wait_queue_head_t write_wait; wait_queue_head_t read_wait; + wait_queue_head_t break_wait; struct work_struct hangup_work; void *disc_data; void *driver_data; -- cgit v1.2.3 From b48820de8c1a596c5b99775524dfce73c6576c5d Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Thu, 10 Sep 2026 14:51:14 +0200 Subject: tty: port: fix tty_port_tty_vhangup() race Drivers rely on tty_port_tty_vhangup() to synchronously hang up their ports before tearing down resources as part of deregistration (e.g. on physical disconnect). If tty_port_tty_vhangup() races with another hangup it may however return before the port has been shut down. This is in turn can lead to use-after-free and NULL-pointer dereferences in racing ioctls (including a synchronous hangup) and in processing of racing asynchronous hangups (carrier loss). Add the missing serialisation to tty_port_tty_vhangup() and tty_port_hangup() so that the former does not return until the port has been shut down also when there is a racing hangup. Note that serial core does not use tty_port_hangup(), but it already holds the port mutex when clearing port->tty and shutting down the port in uart_hangup(). Fixes: 7ca0ff9ab321 ("tty: Add a full port_close function") Cc: stable@kernel.org # 2.6.32: 2b5eac0f8c6e: tty: introduce and use tty_port_tty_vhangup() helper Cc: stable@kernel.org # 2.6.32 Reported-by: syzbot+5fabc1ae99ff40690d84@syzkaller.appspotmail.com Link: https://lore.kernel.org/6a944641.4d659fcc.734b4.003c.GAE@google.com Signed-off-by: Johan Hovold Link: https://patch.msgid.link/20260910125114.640880-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/tty_port.c | 87 ++++++++++++++++++++++++++++++++---------------- include/linux/tty_port.h | 22 ++---------- 2 files changed, 60 insertions(+), 49 deletions(-) (limited to 'include/linux') diff --git a/drivers/tty/tty_port.c b/drivers/tty/tty_port.c index ae33987207b6..b22ff9eb7778 100644 --- a/drivers/tty/tty_port.c +++ b/drivers/tty/tty_port.c @@ -340,19 +340,9 @@ void tty_port_tty_set(struct tty_port *port, struct tty_struct *tty) } EXPORT_SYMBOL(tty_port_tty_set); -/** - * tty_port_shutdown - internal helper to shutdown the device - * @port: tty port to be shut down - * @tty: the associated tty - * - * It is used by tty_port_hangup() and tty_port_close(). Its task is to - * shutdown the device if it was initialized (note consoles remain - * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes - * @port->ops->shutdown(). - */ -static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) +static void tty_port_shutdown_locked(struct tty_port *port, struct tty_struct *tty) { - guard(mutex)(&port->mutex); + lockdep_assert_held(&port->mutex); if (port->console) return; @@ -372,6 +362,23 @@ static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) port->ops->shutdown(port); } +/** + * tty_port_shutdown - internal helper to shutdown the device + * @port: tty port to be shut down + * @tty: the associated tty + * + * It is used by tty_port_hangup() and tty_port_close(). Its task is to + * shutdown the device if it was initialized (note consoles remain + * functioning). It lowers DTR/RTS (if @tty has HUPCL set) and invokes + * @port->ops->shutdown(). + */ +static void tty_port_shutdown(struct tty_port *port, struct tty_struct *tty) +{ + guard(mutex)(&port->mutex); + + tty_port_shutdown_locked(port, tty); +} + /** * tty_port_hangup - hangup helper * @port: tty port @@ -385,36 +392,58 @@ void tty_port_hangup(struct tty_port *port) { struct tty_struct *tty; - scoped_guard(spinlock_irqsave, &port->lock) { - port->count = 0; - tty = port->tty; - if (tty) - set_bit(TTY_IO_ERROR, &tty->flags); - port->tty = NULL; - } + scoped_guard(mutex, &port->mutex) { + scoped_guard(spinlock_irqsave, &port->lock) { + port->count = 0; + tty = port->tty; + if (tty) + set_bit(TTY_IO_ERROR, &tty->flags); + port->tty = NULL; + } - tty_port_set_active(port, false); - tty_port_shutdown(port, tty); + tty_port_set_active(port, false); + tty_port_shutdown_locked(port, tty); + } tty_kref_put(tty); wake_up_interruptible(&port->open_wait); wake_up_interruptible(&port->delta_msr_wait); } EXPORT_SYMBOL(tty_port_hangup); -void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async) +/** + * tty_port_tty_hangup - helper to hang up a tty asynchronously + * @port: tty port + * @check_clocal: hang only ttys with %CLOCAL unset? + */ +void tty_port_tty_hangup(struct tty_port *port, bool check_clocal) { scoped_guard(tty_port_tty, port) { struct tty_struct *tty = scoped_tty(); - if (!check_clocal || !C_CLOCAL(tty)) { - if (async) - tty_hangup(tty); - else - tty_vhangup(tty); - } + if (!check_clocal || !C_CLOCAL(tty)) + tty_hangup(tty); + } +} +EXPORT_SYMBOL_GPL(tty_port_tty_hangup); + +/** + * tty_port_tty_vhangup - helper to hang up a tty synchronously + * @port: tty port + */ +void tty_port_tty_vhangup(struct tty_port *port) +{ + struct tty_struct *tty; + + mutex_lock(&port->mutex); + tty = tty_port_tty_get(port); + mutex_unlock(&port->mutex); + + if (tty) { + tty_vhangup(tty); + tty_kref_put(tty); } } -EXPORT_SYMBOL_GPL(__tty_port_tty_hangup); +EXPORT_SYMBOL_GPL(tty_port_tty_vhangup); /** * tty_port_tty_wakeup - helper to wake up a tty diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 23cad403bb8f..8d22c59c6f15 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -245,7 +245,8 @@ bool tty_port_carrier_raised(struct tty_port *port); void tty_port_raise_dtr_rts(struct tty_port *port); void tty_port_lower_dtr_rts(struct tty_port *port); void tty_port_hangup(struct tty_port *port); -void __tty_port_tty_hangup(struct tty_port *port, bool check_clocal, bool async); +void tty_port_tty_hangup(struct tty_port *port, bool check_clocal); +void tty_port_tty_vhangup(struct tty_port *port); void tty_port_tty_wakeup(struct tty_port *port); int tty_port_block_til_ready(struct tty_port *port, struct tty_struct *tty, struct file *filp); @@ -264,25 +265,6 @@ static inline int tty_port_users(struct tty_port *port) return port->count + port->blocked_open; } -/** - * tty_port_tty_hangup - helper to hang up a tty asynchronously - * @port: tty port - * @check_clocal: hang only ttys with %CLOCAL unset? - */ -static inline void tty_port_tty_hangup(struct tty_port *port, bool check_clocal) -{ - __tty_port_tty_hangup(port, check_clocal, true); -} - -/** - * tty_port_tty_vhangup - helper to hang up a tty synchronously - * @port: tty port - */ -static inline void tty_port_tty_vhangup(struct tty_port *port) -{ - __tty_port_tty_hangup(port, false, false); -} - #ifdef CONFIG_TTY void tty_kref_put(struct tty_struct *tty); __DEFINE_CLASS_IS_CONDITIONAL(tty_port_tty, true); -- cgit v1.2.3 From 1cd9b1be5a9c56fd69e3f555b200a1844f9b873e Mon Sep 17 00:00:00 2001 From: Viken Dadhaniya Date: Tue, 15 Sep 2026 18:56:27 +0530 Subject: soc: qcom: geni-se: Correct QUP Core ICC vote constants The GENI_TO_CORE ("qup-core") ICC vote selects the QUP Core 2X clock rate. The CORE_2X_*_MHZ constants are expressed in Bps, but their values are several orders of magnitude too small. For example, the 50 MHz threshold is represented by 2500 rather than 25000000 Bps. As a result, clients using these constants can severely under-vote the QUP Core clock. Correct the constants to their intended Bps thresholds so that the ICC provider selects the corresponding QUP Core 2X clock rate. Drop the unused definitions. Fixes: 58ffbba6a399 ("soc: qcom: geni: Support for ICC voting") Cc: stable@kernel.org Signed-off-by: Viken Dadhaniya Reviewed-by: Konrad Dybcio Link: https://patch.msgid.link/20260915-correct-icc-bandwidth-vote-constants-v2-1-f78eb611204e@oss.qualcomm.com Signed-off-by: Greg Kroah-Hartman --- include/linux/soc/qcom/geni-se.h | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'include/linux') diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 29a53bbc0dd4..5f18d281e6a4 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -347,17 +347,12 @@ struct geni_se { #define QUP_SE_VERSION_2_5 0x20050000 /* - * Define bandwidth thresholds that cause the underlying Core 2X interconnect - * clock to run at the named frequency. These baseline values are recommended - * by the hardware team, and are not dynamically scaled with GENI bandwidth - * beyond basic on/off. + * QUP Core 2X clock votes used by GENI clients through the "qup-core" ICC + * path. Values are in Bps and must be converted with Bps_to_icc() before + * setting avg_bw. */ -#define CORE_2X_19_2_MHZ 960 -#define CORE_2X_50_MHZ 2500 -#define CORE_2X_100_MHZ 5000 -#define CORE_2X_150_MHZ 7500 -#define CORE_2X_200_MHZ 10000 -#define CORE_2X_236_MHZ 16383 +#define CORE_2X_19_2_MHZ 9600000 +#define CORE_2X_50_MHZ 25000000 #define GENI_DEFAULT_BW Bps_to_icc(1000) -- cgit v1.2.3 From 5dad87615c9861cfa366ca984b52f581e861df20 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Fri, 25 Sep 2026 11:42:22 +0200 Subject: tty: add break_wait kernel-doc Add the missing kernel-doc description for the new break_wait field in struct tty_struct. Fixes: 845a2737e0f5 ("tty: abort break signalling on hangup") Reported-by: Randy Dunlap Link: https://lore.kernel.org/20d75a7a-2e7a-484f-a267-c21db82dd922@infradead.org Signed-off-by: Johan Hovold Link: https://patch.msgid.link/20260925094222.18303-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/tty.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/tty.h b/include/linux/tty.h index ad4e4a835f90..833bb7b97ceb 100644 --- a/include/linux/tty.h +++ b/include/linux/tty.h @@ -168,6 +168,7 @@ struct tty_operations; * @write_wait: concurrent writers are waiting in this queue until they are * allowed to write * @read_wait: readers wait for data in this queue + * @break_wait: wait queue for timed breaks * @hangup_work: normally a work to perform a hangup (do_tty_hangup()); while * freeing the tty, (re)used to release_one_tty() * @disc_data: pointer to @ldisc's private data (e.g. to &struct n_tty_data) -- cgit v1.2.3 From b78b728e21c32ec4c330b299f657fb1eb02dffc2 Mon Sep 17 00:00:00 2001 From: Hao Ge Date: Wed, 23 Sep 2026 14:37:59 +0800 Subject: netfs: Fix missing alloc tagging of direct mempool allocations Commit 1d78d56c43ef ("netfs: Fix folio_queue ENOMEM in writeback by adding a mempool") added a mempool for the folio_queues and made the request, subrequest and folio_queue allocations distinguish between writeback and everything else. Writeback is part of memory reclaim and must not fail due to ENOMEM, so it allocates under GFP_NOFS through mempool_alloc(), which may dip into the pool's reserve and, if that runs empty, wait for elements to be returned. The GFP_KERNEL paths, which can return -ENOMEM to their callers, invoke the pool's ->alloc() callback directly instead. The direct call, however, skips the alloc_hooks() wrapper that the mempool_alloc() macro provides. The pool callbacks, mempool_alloc_slab() and mempool_kmalloc(), call kmem_cache_alloc_noprof() and kmalloc_noprof() and rely on current->alloc_tag having been set by the caller. With CONFIG_MEM_ALLOC_PROFILING_DEBUG=y this leads to current->alloc_tag not set WARNING: ./include/linux/alloc_tag.h:161 at __alloc_tagging_slab_alloc_hook alloc_tag was not set WARNING: ./include/linux/alloc_tag.h:166 at __alloc_tagging_slab_free_hook at allocation and free time respectively, as reported when reading files on a CIFS mount. The allocations are also missing from /proc/allocinfo. Wrap the direct ->alloc() invocations in alloc_hooks() with a new mempool_alloc_noreserve() helper in include/linux/mempool.h, next to the other alloc_hooks()-wrapped macros such as mempool_alloc(). The GFP_KERNEL paths keep their failable allocation semantics, they just get tagged now. Fixes: 1d78d56c43ef ("netfs: Fix folio_queue ENOMEM in writeback by adding a mempool") Reported-by: Erhard Furtner Closes: https://lore.kernel.org/all/0b004319-9ef7-437c-a4dd-174d6a9a83db@mailbox.org/ Tested-by: Erhard Furtner Suggested-by: Suren Baghdasaryan Cc: stable@vger.kernel.org Signed-off-by: Hao Ge Link: https://patch.msgid.link/20260923063759.34667-1-hao.ge@linux.dev Acked-by: Vlastimil Babka (SUSE) Signed-off-by: Christian Brauner (Amutable) --- fs/netfs/objects.c | 4 ++-- fs/netfs/rolling_buffer.c | 2 +- include/linux/mempool.h | 7 +++++++ 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'include/linux') diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c index 7f6a3e912602..ad549daa9c79 100644 --- a/fs/netfs/objects.c +++ b/fs/netfs/objects.c @@ -34,7 +34,7 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, rreq = mempool_alloc(mempool, gfp); } else { - rreq = mempool->alloc(gfp, mempool->pool_data); + rreq = mempool_alloc_noreserve(mempool, gfp); if (!rreq) return ERR_PTR(-ENOMEM); } @@ -214,7 +214,7 @@ struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq struct kmem_cache *cache = mempool->pool_data; if (rreq->gfp == GFP_KERNEL) - subreq = mempool->alloc(rreq->gfp, mempool->pool_data); + subreq = mempool_alloc_noreserve(mempool, rreq->gfp); else subreq = mempool_alloc(mempool, rreq->gfp); if (!subreq) diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c index 424e77a9a109..d30d5ef6d86e 100644 --- a/fs/netfs/rolling_buffer.c +++ b/fs/netfs/rolling_buffer.c @@ -29,7 +29,7 @@ struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, struct folio_queue *fq; if (gfp == GFP_KERNEL) - fq = netfs_folioq_pool.alloc(gfp, netfs_folioq_pool.pool_data); + fq = mempool_alloc_noreserve(&netfs_folioq_pool, gfp); else fq = mempool_alloc(&netfs_folioq_pool, gfp); if (fq) { diff --git a/include/linux/mempool.h b/include/linux/mempool.h index a0fa6d43e0dc..6da502aef2f7 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -70,6 +70,13 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, #define mempool_alloc_bulk(...) \ alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) +/* + * Allocate a new element without dipping into the pool's reserves or + * waiting. Returns NULL on failure. + */ +#define mempool_alloc_noreserve(_pool, _gfp) \ + alloc_hooks((_pool)->alloc(_gfp, (_pool)->pool_data)) + void *mempool_alloc_preallocated(struct mempool *pool) __malloc; void mempool_free(void *element, struct mempool *pool); unsigned int mempool_free_bulk(struct mempool *pool, void **elem, -- cgit v1.2.3 From 700f8d24436c45cb3cc8956f35c90a1b6a6487c2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 21 Sep 2026 10:44:44 -0700 Subject: Revert "KVM: Check for duplicate vcpu_id as early as possible" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that KVM uses kvm_get_vcpu_by_id() to check for an existing vCPU ID before doing any meaningful work, which was made possible by holding kvm->lock for the entirety of vCPU creation, revert the now-redundant "early" vCPU ID tracking. The claims about the impact of kvm->vcpu_ids on the memory footprint were a wee bit wrong: the worst case scenario isn't 256 bytes per VM, it's 256 "unsigned longs" per VM, i.e. 2048 bytes per VM. Increasing the size of "struct kvm" by 2048 nearly doubled the total size on many architectures, and tripped x86's KVM_SANITY_CHECK_VM_STRUCT_SIZE, which was added to detect this *exact* scenario, where a single change significantly increased the size of "struct kvm". I.e. attempting to build KVM with CONFIG_DEBUG_KERNEL=n fails on x86 (the build failures got missed because all build bots apparently test only CONFIG_DEBUG_KERNEL=y kernels, and maintainers' test flows were similarly lacking). This reverts commit 97d65b544f48b2ee49f6aea32145e3e7969955dc. Fixes: 97d65b544f48 ("KVM: Check for duplicate vcpu_id as early as possible") Reported-by: Jean-Christophe Guillain Closes: https://lore.kernel.org/all/56a4bc35ee605588b7cc36c8e45c12b5f3b506cb.camel@guillain.net Reported-by: Paweł S Closes: https://lore.kernel.org/all/CABD%3DWFOS4j4hDv%2BpW-eEM9HAM2q2GY_iYdAG%2BqvYcUEinUrcQQ@mail.gmail.com Tested-by: Jean-Christophe Guillain Signed-off-by: Sean Christopherson Tested-by: Naveen N Rao (AMD) Message-ID: <20260921174445.911676-7-seanjc@google.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 1 - virt/kvm/kvm_main.c | 5 ----- 2 files changed, 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index cf7fe835c4ad..c844def6dd95 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -791,7 +791,6 @@ struct kvm { /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; - DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS); /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0e5b639559ba..02bab6923e03 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4208,15 +4208,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) if (kvm_get_vcpu_by_id(kvm, id)) return -EEXIST; - if (WARN_ON_ONCE(test_bit(id, kvm->vcpu_ids))) - return -EEXIST; - r = kvm_arch_vcpu_precreate(kvm, id); if (r) return r; kvm->created_vcpus++; - __set_bit(id, kvm->vcpu_ids); vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); if (!vcpu) { @@ -4299,7 +4295,6 @@ vcpu_free: kmem_cache_free(kvm_vcpu_cache, vcpu); vcpu_decrement: kvm->created_vcpus--; - __clear_bit(id, kvm->vcpu_ids); return r; } -- cgit v1.2.3 From 0fb5751a6e411b1b466c06e50021818d9391df98 Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:58 +0000 Subject: bpf: Wait for an RCU tasks grace period before freeing trampoline progs When a prog is detached from a trampoline, it is freed after an RCU grace period, or an RCU tasks trace one if it is sleepable. This covers the tasks that are running the prog, since the prog's enter helper takes the matching read lock before the prog is called. On a preemptible kernel, it doesn't cover a task that was preempted in the trampoline just before the enter helper. That task holds no lock yet, and it calls the prog after it was freed: BUG: KASAN: vmalloc-out-of-bounds in __bpf_prog_enter_recur+0x3a5/0x3f0 Read of size 8 at addr ffffc90000055040 by task candidate/110 CPU: 1 UID: 0 PID: 110 Comm: candidate Not tainted 7.3.0-rc2-00014-g15071f2a1263-dirty #2 PREEMPT(full) Call Trace: __bpf_prog_enter_recur+0x3a5/0x3f0 bpf_trampoline_6442509193+0x37/0xf1 __x64_sys_futex+0x9/0x410 do_syscall_64+0xb0/0x530 ... Wait for an RCU tasks grace period before the existing one when freeing a prog that was linked to a trampoline. An RCU tasks grace period only ends once the tasks that were preempted have run again, and bpf_tramp_image_put() already relies on it to free the image. It doesn't wait for tasks that sleep in the trampoline, the next commit takes care of those. Only progs that were linked to a trampoline can be called this way, so bpf_trampoline_add_prog() marks them and other progs are still freed as before. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Junseo Lim Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-2-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/aqdrwVpanH3WGurX@omen-arch/ --- include/linux/bpf.h | 1 + kernel/bpf/syscall.c | 19 ++++++++++++++++++- kernel/bpf/trampoline.c | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 1d2676782d70..a8ec85127571 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1770,6 +1770,7 @@ struct bpf_prog_aux { bool offload_requested; /* Program is bound and offloaded to the netdev. */ bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ + bool tramp_linked; /* true if it was ever linked to a trampoline */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 244a939b9d2d..96217b99399d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2448,6 +2448,21 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) bpf_prog_free(aux->prog); } +/* + * Progs called from a trampoline can also be reached by a task that was + * preempted in the trampoline before the prog's enter helper took its RCU + * read lock, wait for those first. + */ +static void __bpf_prog_put_rcu_tasks(struct rcu_head *rcu) +{ + struct bpf_prog *prog = container_of(rcu, struct bpf_prog_aux, rcu)->prog; + + if (prog->sleepable) + call_rcu_tasks_trace(rcu, __bpf_prog_put_rcu); + else + call_rcu(rcu, __bpf_prog_put_rcu); +} + static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { bpf_prog_kallsyms_del_all(prog); @@ -2461,7 +2476,9 @@ static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) btf_put(prog->aux->attach_btf); if (deferred) { - if (prog->sleepable) + if (IS_ENABLED(CONFIG_TASKS_RCU) && prog->aux->tramp_linked) + call_rcu_tasks(&prog->aux->rcu, __bpf_prog_put_rcu_tasks); + else if (prog->sleepable) call_rcu_tasks_trace(&prog->aux->rcu, __bpf_prog_put_rcu); else call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 90b70ea0d370..9d69c066a817 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -907,6 +907,7 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, } hlist_add_head(&node->tramp_hlist, prog_list); + node->link->prog->aux->tramp_linked = true; if (kind == BPF_TRAMP_FSESSION) { tr->progs_cnt[BPF_TRAMP_FENTRY]++; fexit = fsession_exit(node); -- cgit v1.2.3 From 17637e1a581a22466ac3620a91e099683ff9cc6f Mon Sep 17 00:00:00 2001 From: "Florent Revest (Anthropic)" Date: Sat, 26 Sep 2026 13:55:59 +0000 Subject: bpf: Skip detached progs in trampoline images that are still in use bpf_tramp_image_put() makes sure a trampoline image is not freed while a task may still be running in it, but nothing similar is done for the progs called by that image. Detach drops the last prog reference right away and the prog is freed after grace periods, on the basis that a task still in the traced function skips the fexit progs once the nop at ip_after_call is patched to a jump. That leaves out a task sleeping in a sleepable prog that runs before the detached one, which no grace period waits for: CPU 0 CPU 1 in image I, sleeping in prog S detach P from I's trampoline -> new image, bpf_tramp_image_put(I) bpf_prog_put(P), last ref grace periods, P freed back from S __bpf_prog_enter(P) call P->bpf_func If S and P are fexit progs the task is already past the patched jump, and fentry only images don't have one. On x86 this is an int3 in poisoned bpf_prog_pack memory: Oops: int3: 0000 [#1] SMP NOPTI CPU: 18 UID: 0 PID: 94573 Comm: x169 Not tainted 6.18.44 #1 PREEMPT(lazy) RIP: 0010:0xffffffffc0601d8d Call Trace: ? bpf_trampoline_6442515411+0x1a4/0x21b bpf_lsm_bprm_committed_creds+0x5/0x10 security_bprm_committed_creds+0x5f/0x70 begin_new_exec+0x2d6/0x410 ... We hit this in production when progs attached through trampolines got detached while their hooks were busy, and it was independently found with a fuzzer and KASAN. Have the JITs emit a patchable nop in front of each prog call sequence and record it in the image. When a prog is detached, patch its nop to a jump over the call sequence. Progs that stay attached keep running for the tasks that are in the image, and ip_after_call isn't needed anymore. The task can be in any image that isn't freed yet, not only in the current one. It sleeps in image I1 that calls S, P and Q, then P is detached and the trampoline moves to image I2, then Q is detached and its call is still in I1. So the trampoline keeps a list of its images until they are freed, and detaching a prog patches its nop in all of them. Images hold a reference on the trampoline for that long. On riscv and loongarch a jump of any range takes several instructions, and a task preempted in the middle of them could resume into half of the new sequence. The nop is a single instruction there, patched to a near branch through arch_bpf_trampoline_skip(). With the extra nops, BPF_MAX_TRAMP_LINKS progs no longer fit in a page on x86 and arm64 (and already didn't on powerpc), so lower the limit there like s390 does. Fixes: e21aa341785c ("bpf: Fix fexit trampoline.") Reported-by: Sechang Lim Suggested-by: Alexei Starovoitov Signed-off-by: Florent Revest (Anthropic) Signed-off-by: Alexei Starovoitov Link: https://patch.msgid.link/20260926135605.1217928-3-florent.revest@linux.dev Closes: https://lore.kernel.org/bpf/20260815071927.147049-1-zirajs7@gmail.com/ --- arch/arm64/net/bpf_jit_comp.c | 37 ++++++++++-------- arch/loongarch/net/bpf_jit.c | 60 +++++++++++++++++++---------- arch/powerpc/net/bpf_jit_comp.c | 45 +++++++++++----------- arch/riscv/net/bpf_jit_comp64.c | 56 ++++++++++++++++++--------- arch/s390/net/bpf_jit_comp.c | 46 ++++++++++++++-------- arch/x86/net/bpf_jit_comp.c | 26 ++++++++----- include/linux/bpf.h | 44 +++++++++++++++++++-- kernel/bpf/trampoline.c | 84 ++++++++++++++++++++++++++++++++--------- 8 files changed, 277 insertions(+), 121 deletions(-) (limited to 'include/linux') diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c5f55d6161fe..c1279fabce47 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2418,10 +2418,11 @@ bool bpf_jit_supports_subprog_tailcalls(void) return true; } -static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, - int bargs_off, int retval_off, int run_ctx_off, - bool save_ret) +static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *node, int bargs_off, + int retval_off, int run_ctx_off, bool save_ret) { + void *skip; __le32 *branch; u64 enter_prog; u64 exit_prog; @@ -2431,6 +2432,10 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, enter_prog = (u64)bpf_trampoline_enter(p); exit_prog = (u64)bpf_trampoline_exit(p); + /* nop, patched to skip this prog when it is detached */ + skip = ctx->ro_image + ctx->idx; + emit(A64_NOP, ctx); + if (node->cookie == 0) { /* if cookie is zero, one instruction is enough to store it */ emit(A64_STR64I(A64_ZR, A64_SP, run_ctx_off + cookie_off), ctx); @@ -2483,11 +2488,13 @@ static void invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *node, emit(A64_ADD_I(1, A64_R(2), A64_SP, run_ctx_off), ctx); emit_call(exit_prog, ctx); + + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); } -static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int bargs_off, int retval_off, int run_ctx_off, - __le32 **branches) +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int bargs_off, + int retval_off, int run_ctx_off, __le32 **branches) { int i; @@ -2496,7 +2503,7 @@ static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, */ emit(A64_STR64I(A64_ZR, A64_SP, retval_off), ctx); for (i = 0; i < tn->nr_nodes; i++) { - invoke_bpf_prog(ctx, tn->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, tn->nodes[i], bargs_off, retval_off, run_ctx_off, true); /* if (*(u64 *)(sp + retval_off) != 0) * goto do_fexit; @@ -2882,7 +2889,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fentry->nodes[i], bargs_off, + invoke_bpf_prog(ctx, im, fentry->nodes[i], bargs_off, retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); } @@ -2893,7 +2900,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (!branches) return -ENOMEM; - invoke_bpf_mod_ret(ctx, fmod_ret, bargs_off, retval_off, + invoke_bpf_mod_ret(ctx, im, fmod_ret, bargs_off, retval_off, run_ctx_off, branches); } @@ -2906,9 +2913,6 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_RET(A64_R(10)), ctx); /* store return value */ emit(A64_STR64I(A64_R(0), A64_SP, retval_off), ctx); - /* reserve a nop for bpf_tramp_image_put */ - im->ip_after_call = ctx->ro_image + ctx->idx; - emit(A64_NOP, ctx); } /* update the branches saved in invoke_bpf_mod_ret with cbnz */ @@ -2930,12 +2934,11 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, store_func_meta(ctx, meta, func_meta_off); cookie_bargs_off--; } - invoke_bpf_prog(ctx, fexit->nodes[i], bargs_off, retval_off, + invoke_bpf_prog(ctx, im, fexit->nodes[i], bargs_off, retval_off, run_ctx_off, false); } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; /* for the first pass, assume the worst case */ if (!ctx->image) ctx->idx += 4; @@ -2994,7 +2997,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, .image = NULL, .idx = 0, }; - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct arg_aux aaux; int ret; @@ -3281,6 +3284,10 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, * longer reachable, since bpf_tramp_image_put() function already * uses percpu_ref and task-based rcu to do the sync, no need to call * the sync version here, see bpf_tramp_image_put() for details. + * + * 3. when a detached prog is patched out of a trampoline, a CPU that + * still executes the old nop calls the prog before it went through + * a quiescent state, and the prog is freed after grace periods. */ ret = aarch64_insn_patch_text_nosync(ip, new_insn); out: diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 4da278900938..e78eb582c300 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -1578,6 +1578,24 @@ void *bpf_arch_text_copy(void *dst, void *src, size_t len) return ret ? ERR_PTR(-EINVAL) : dst; } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = INSN_NOP; + u32 new_insn = larch_insn_gen_b((unsigned long)nop, (unsigned long)target); + int ret; + + if (memcmp(nop, &old_insn, LOONGARCH_INSN_SIZE)) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = larch_insn_text_copy(nop, &new_insn, LOONGARCH_INSN_SIZE); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -1696,13 +1714,18 @@ static void restore_stk_args(struct jit_ctx *ctx, int nr_stk_args, int args_off, } } -static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, - int args_off, int retval_off, int run_ctx_off, bool save_ret) +static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_node *n, int args_off, int retval_off, + int run_ctx_off, bool save_ret) { int ret; u32 *branch; struct bpf_prog *p = n->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_image + ctx->idx; + + /* nop, patched to a b over this prog when it is detached */ + emit_insn(ctx, nop); if (n->cookie) emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, @@ -1755,13 +1778,17 @@ static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_node *n, /* arg3: &run_ctx */ emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_image + ctx->idx); + return 0; } -static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, - int args_off, int retval_off, int run_ctx_off, - int func_meta_off, bool save_ret, u64 func_meta, int cookie_off) +static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_image *im, + struct bpf_tramp_nodes *tn, int args_off, int retval_off, + int run_ctx_off, int func_meta_off, bool save_ret, + u64 func_meta, int cookie_off) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -1774,7 +1801,8 @@ static int invoke_bpf(struct jit_ctx *ctx, struct bpf_tramp_nodes *tn, emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, meta); cur_cookie--; } - err = invoke_bpf_prog(ctx, tn->nodes[i], args_off, retval_off, run_ctx_off, save_ret); + err = invoke_bpf_prog(ctx, im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret); if (err) return err; } @@ -2017,7 +2045,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i } if (fentry->nr_nodes) { - ret = invoke_bpf(ctx, fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(ctx, im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off); if (ret) return ret; @@ -2029,7 +2057,7 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(ctx, fmod_ret->nodes[i], + ret = invoke_bpf_prog(ctx, im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true); if (ret) goto out; @@ -2051,10 +2079,6 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i goto out; emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); - im->ip_after_call = ctx->ro_image + ctx->idx; - /* Reserve space for the move_imm + jirl instruction */ - for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) - emit_insn(ctx, nop); } for (i = 0; ctx->image && i < fmod_ret->nr_nodes; i++) { @@ -2068,14 +2092,13 @@ static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_i emit_store_stack_imm64(ctx, LOONGARCH_GPR_T1, -func_meta_off, func_meta); if (fexit->nr_nodes) { - ret = invoke_bpf(ctx, fexit, args_off, retval_off, run_ctx_off, + ret = invoke_bpf(ctx, im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_image + ctx->idx; move_addr(ctx, LOONGARCH_GPR_A0, (const u64)im); ret = emit_call(ctx, (const u64)__bpf_tramp_exit); if (ret) @@ -2177,11 +2200,8 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { int ret; - struct jit_ctx ctx; - struct bpf_tramp_image im; - - ctx.image = NULL; - ctx.idx = 0; + struct jit_ctx ctx = {}; + struct bpf_tramp_image im = {}; ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tnodes, func_addr, flags); diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7b07b43575f1..7a612688e7a2 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -602,14 +602,18 @@ int arch_protect_bpf_trampoline(void *image, unsigned int size) } static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_node *n, int regs_off, int retval_off, - int run_ctx_off, bool save_ret) + struct bpf_tramp_image *im, struct bpf_tramp_node *n, + int regs_off, int retval_off, int run_ctx_off, bool save_ret) { struct bpf_prog *p = n->link->prog; ppc_inst_t branch_insn; - u32 jmp_idx; + u32 jmp_idx, skip_idx; int ret = 0; + /* nop, patched to skip this prog when it is detached */ + skip_idx = ctx->idx; + EMIT(PPC_RAW_NOP()); + /* Save cookie */ if (IS_ENABLED(CONFIG_PPC64)) { PPC_LI64(_R3, n->cookie); @@ -679,13 +683,17 @@ static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ct EMIT(PPC_RAW_ADDI(_R5, _R1, run_ctx_off)); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)bpf_trampoline_exit(p)); + if (ret) + return ret; - return ret; + if (ro_image) /* image is NULL for dummy pass */ + bpf_tramp_image_add_skip(im, p, &ro_image[skip_idx], &ro_image[ctx->idx]); + return 0; } static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context *ctx, - struct bpf_tramp_nodes *tn, int regs_off, int retval_off, - int run_ctx_off, u32 *branches) + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int regs_off, int retval_off, int run_ctx_off, u32 *branches) { int i; @@ -696,8 +704,8 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context EMIT(PPC_RAW_LI(_R3, 0)); EMIT(PPC_RAW_STL(_R3, _R1, retval_off)); for (i = 0; i < tn->nr_nodes; i++) { - if (invoke_bpf_prog(image, ro_image, ctx, tn->nodes[i], regs_off, retval_off, - run_ctx_off, true)) + if (invoke_bpf_prog(image, ro_image, ctx, im, tn->nodes[i], regs_off, + retval_off, run_ctx_off, true)) return -EINVAL; /* @@ -1043,8 +1051,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fentry->nodes[i], regs_off, retval_off, - run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) + if (invoke_bpf_prog(image, ro_image, ctx, im, fentry->nodes[i], regs_off, + retval_off, run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET)) return -EINVAL; } @@ -1053,7 +1061,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(image, ro_image, ctx, fmod_ret, regs_off, retval_off, + if (invoke_bpf_mod_ret(image, ro_image, ctx, im, fmod_ret, regs_off, retval_off, run_ctx_off, branches)) { ret = -EINVAL; goto cleanup; @@ -1090,11 +1098,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Restore updated tail_call_cnt */ if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) bpf_trampoline_restore_tail_call_cnt(image, ctx, bpf_frame_size, r4_off); - - /* Reserve space to patch branch instruction to skip fexit progs */ - if (ro_image) /* image is NULL for dummy pass */ - im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; - EMIT(PPC_RAW_NOP()); } /* Update branches saved in invoke_bpf_mod_ret with address of do_fexit */ @@ -1123,16 +1126,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im cookie_ctx_off--; } - if (invoke_bpf_prog(image, ro_image, ctx, fexit->nodes[i], regs_off, retval_off, - run_ctx_off, false)) { + if (invoke_bpf_prog(image, ro_image, ctx, im, fexit->nodes[i], regs_off, + retval_off, run_ctx_off, false)) { ret = -EINVAL; goto cleanup; } } if (flags & BPF_TRAMP_F_CALL_ORIG) { - if (ro_image) /* image is NULL for dummy pass */ - im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; PPC_LI_ADDR(_R3, im); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)__bpf_tramp_exit); @@ -1192,7 +1193,7 @@ cleanup: int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; int ret; ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tnodes, func_addr); @@ -1320,7 +1321,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, /* * If we are not poking at bpf prog entry, then we are simply patching in/out - * an unconditional branch instruction at im->ip_after_call + * an unconditional branch instruction in a trampoline image */ if (offset) { if (old_t == BPF_MOD_CALL || new_t == BPF_MOD_CALL) { diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 151031e97a24..01fe66774f02 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -822,6 +822,24 @@ static int gen_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) return emit_jump_and_link(is_call ? RV_REG_T0 : RV_REG_ZERO, rvoff, false, &ctx); } +int arch_bpf_trampoline_skip(void *nop, void *target) +{ + u32 old_insn = rv_nop(); + u32 new_insn = rv_jal(RV_REG_ZERO, (target - nop) >> 1); + int ret; + + if (memcmp(nop, &old_insn, sizeof(old_insn))) + return -EFAULT; + + cpus_read_lock(); + mutex_lock(&text_mutex); + ret = patch_text(nop, &new_insn, sizeof(new_insn)); + mutex_unlock(&text_mutex); + cpus_read_unlock(); + + return ret; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type old_t, enum bpf_text_poke_type new_t, void *old_addr, void *new_addr) @@ -904,12 +922,17 @@ static void emit_store_stack_imm64(u8 reg, int stack_off, u64 imm64, emit_sd(RV_REG_FP, stack_off, reg, ctx); } -static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval_off, - int run_ctx_off, bool save_ret, struct rv_jit_context *ctx) +static int invoke_bpf_prog(struct bpf_tramp_image *im, struct bpf_tramp_node *node, + int args_off, int retval_off, int run_ctx_off, bool save_ret, + struct rv_jit_context *ctx) { int ret, branch_off; struct bpf_prog *p = node->link->prog; int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + void *skip = ctx->ro_insns + ctx->ninsns; + + /* nop, patched to a jal over this prog when it is detached */ + emit(rv_nop(), ctx); if (node->cookie) emit_store_stack_imm64(RV_REG_T1, -run_ctx_off + cookie_off, node->cookie, ctx); @@ -962,13 +985,17 @@ static int invoke_bpf_prog(struct bpf_tramp_node *node, int args_off, int retval /* arg3: &run_ctx */ emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); ret = emit_call((const u64)bpf_trampoline_exit(p), true, ctx); + if (ret) + return ret; - return ret; + bpf_tramp_image_add_skip(im, p, skip, ctx->ro_insns + ctx->ninsns); + return 0; } -static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, - int run_ctx_off, int func_meta_off, bool save_ret, u64 func_meta, - int cookie_off, struct rv_jit_context *ctx) +static int invoke_bpf(struct bpf_tramp_image *im, struct bpf_tramp_nodes *tn, + int args_off, int retval_off, int run_ctx_off, int func_meta_off, + bool save_ret, u64 func_meta, int cookie_off, + struct rv_jit_context *ctx) { int i, cur_cookie = (cookie_off - args_off) / 8; @@ -981,8 +1008,8 @@ static int invoke_bpf(struct bpf_tramp_nodes *tn, int args_off, int retval_off, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, meta, ctx); cur_cookie--; } - err = invoke_bpf_prog(tn->nodes[i], args_off, retval_off, run_ctx_off, - save_ret, ctx); + err = invoke_bpf_prog(im, tn->nodes[i], args_off, retval_off, + run_ctx_off, save_ret, ctx); if (err) return err; } @@ -1170,7 +1197,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, } if (fentry->nr_nodes) { - ret = invoke_bpf(fentry, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fentry, args_off, retval_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off, ctx); if (ret) return ret; @@ -1184,7 +1211,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* cleanup to avoid garbage return value confusion */ emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); for (i = 0; i < fmod_ret->nr_nodes; i++) { - ret = invoke_bpf_prog(fmod_ret->nodes[i], args_off, retval_off, + ret = invoke_bpf_prog(im, fmod_ret->nodes[i], args_off, retval_off, run_ctx_off, true, ctx); if (ret) goto out; @@ -1211,10 +1238,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_FP, -tcc_off, RV_REG_TCC, ctx); emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); - im->ip_after_call = ctx->ro_insns + ctx->ninsns; - /* 2 nops reserved for auipc+jalr pair */ - emit(rv_nop(), ctx); - emit(rv_nop(), ctx); } /* update branches saved in invoke_bpf_mod_ret with bnez */ @@ -1230,14 +1253,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(RV_REG_T1, -func_meta_off, func_meta, ctx); if (fexit->nr_nodes) { - ret = invoke_bpf(fexit, args_off, retval_off, run_ctx_off, func_meta_off, + ret = invoke_bpf(im, fexit, args_off, retval_off, run_ctx_off, func_meta_off, false, func_meta, cookie_off, ctx); if (ret) goto out; } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = ctx->ro_insns + ctx->ninsns; emit_imm(RV_REG_A0, ctx->insns ? (const s64)im : RV_MAX_COUNT_IMM, ctx); ret = emit_call((const u64)__bpf_tramp_exit, true, ctx); if (ret) @@ -1299,7 +1321,7 @@ out: int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct rv_jit_context ctx; int ret; diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index c4b47070bb59..1b2566012b63 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -2566,6 +2566,8 @@ struct bpf_tramp_jit { int r14_off; /* Offset of saved %r14, has to be at the * bottom */ int do_fexit; /* do_fexit: label */ + int skip[BPF_MAX_TRAMP_LINKS]; /* skip: labels after each prog */ + int nr_progs; }; static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val) @@ -2584,6 +2586,7 @@ static void emit_store_stack_imm64(struct bpf_jit *jit, int tmp_reg, int stack_o } static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_node *node, bool save_ret) { @@ -2591,8 +2594,20 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, int cookie_off = tjit->run_ctx_off + offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; + void *skip = jit->prg_buf + jit->prg; + int idx = tjit->nr_progs++; int patch; + if (idx >= ARRAY_SIZE(tjit->skip)) + return -E2BIG; + + /* + * nop, patched to skip this prog when it is detached + */ + + /* brcl 0,skip */ + EMIT6_PCREL_RILC(0xc0040000, 0, tjit->skip[idx]); + /* * run_ctx.cookie = node->cookie; */ @@ -2652,10 +2667,15 @@ static int invoke_bpf_prog(struct bpf_tramp_jit *tjit, /* brasl %r14,__bpf_prog_exit */ EMIT6_PCREL_RILB_PTR(0xc0050000, REG_14, bpf_trampoline_exit(p)); + /* skip: */ + tjit->skip[idx] = jit->prg; + bpf_tramp_image_add_skip(im, p, skip, jit->prg_buf + jit->prg); + return 0; } static int invoke_bpf(struct bpf_tramp_jit *tjit, + struct bpf_tramp_image *im, const struct btf_func_model *m, struct bpf_tramp_nodes *tn, bool save_ret, u64 func_meta, int cookie_off) @@ -2670,7 +2690,7 @@ static int invoke_bpf(struct bpf_tramp_jit *tjit, emit_store_stack_imm64(jit, REG_0, tjit->func_meta_off, meta); cur_cookie--; } - if (invoke_bpf_prog(tjit, m, tn->nodes[i], save_ret)) + if (invoke_bpf_prog(tjit, im, m, tn->nodes[i], save_ret)) return -EINVAL; } @@ -2712,6 +2732,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, u64 func_meta; int i, j; + /* The skip labels are taken from the previous pass. */ + tjit->nr_progs = 0; + if (im) + im->nr_skips = 0; + /* Support as many stack arguments as "mvc" instruction can handle. */ nr_reg_args = min_t(int, m->nr_args, MAX_NR_REG_ARGS); nr_stack_args = m->nr_args - nr_reg_args; @@ -2875,7 +2900,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_store_stack_imm64(jit, REG_0, tjit->retval_off, 0); } - if (invoke_bpf(tjit, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, + if (invoke_bpf(tjit, im, m, fentry, flags & BPF_TRAMP_F_RET_FENTRY_RET, func_meta, cookie_off)) return -EINVAL; @@ -2889,7 +2914,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, 0xf000 | tjit->retval_off); for (i = 0; i < fmod_ret->nr_nodes; i++) { - if (invoke_bpf_prog(tjit, m, fmod_ret->nodes[i], true)) + if (invoke_bpf_prog(tjit, im, m, fmod_ret->nodes[i], true)) return -EINVAL; /* @@ -2943,15 +2968,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* mvc tccnt_off(%r15),tail_call_cnt(4,%r15) */ _EMIT6(0xd203f000 | tjit->tccnt_off, 0xf000 | offsetof(struct prog_frame, tail_call_cnt)); - - im->ip_after_call = jit->prg_buf + jit->prg; - - /* - * The following nop will be patched by bpf_tramp_image_put(). - */ - - /* brcl 0,im->ip_epilogue */ - EMIT6_PCREL_RILC(0xc0040000, 0, (u64)im->ip_epilogue); } /* Set the "is_return" flag for fsession. */ @@ -2962,12 +2978,10 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, /* do_fexit: */ tjit->do_fexit = jit->prg; - if (invoke_bpf(tjit, m, fexit, false, func_meta, cookie_off)) + if (invoke_bpf(tjit, im, m, fexit, false, func_meta, cookie_off)) return -EINVAL; if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = jit->prg_buf + jit->prg; - /* * __bpf_tramp_exit(im); */ @@ -3016,7 +3030,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *orig_call) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; struct bpf_tramp_jit tjit; int ret; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2853e87797a7..6bca87457e87 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -3217,16 +3217,21 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, } static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_node *node, int stack_size, int run_ctx_off, bool save_ret, void *image, void *rw_image) { u8 *prog = *pprog; - u8 *jmp_insn; + u8 *jmp_insn, *skip; int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); struct bpf_prog *p = node->link->prog; u64 cookie = node->cookie; + /* nop, patched to skip this prog when it is detached */ + skip = image + (prog - (u8 *)rw_image); + emit_nops(&prog, X86_PATCH_SIZE); + /* mov rdi, cookie */ emit_mov_imm64(&prog, BPF_REG_1, (long) cookie >> 32, (u32) (long) cookie); @@ -3301,6 +3306,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, if (emit_rsb_call(&prog, bpf_trampoline_exit(p), image + (prog - (u8 *)rw_image))) return -EINVAL; + bpf_tramp_image_add_skip(im, p, skip, image + (prog - (u8 *)rw_image)); + *pprog = prog; return 0; } @@ -3332,6 +3339,7 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) } static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, int func_meta_off, bool save_ret, void *image, void *rw_image, u64 func_meta, @@ -3346,7 +3354,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, func_meta | (cur_cookie << BPF_TRAMP_COOKIE_INDEX_SHIFT)); cur_cookie--; } - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, save_ret, image, rw_image)) return -EINVAL; } @@ -3355,6 +3363,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog, } static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, + struct bpf_tramp_image *im, struct bpf_tramp_nodes *tl, int stack_size, int run_ctx_off, u8 **branches, void *image, void *rw_image) @@ -3368,7 +3377,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, emit_mov_imm32(&prog, false, BPF_REG_0, 0); emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); for (i = 0; i < tl->nr_nodes; i++) { - if (invoke_bpf_prog(m, &prog, tl->nodes[i], stack_size, run_ctx_off, true, + if (invoke_bpf_prog(m, &prog, im, tl->nodes[i], stack_size, run_ctx_off, true, image, rw_image)) return -EINVAL; @@ -3640,7 +3649,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (fentry->nr_nodes) { - if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fentry, regs_off, run_ctx_off, func_meta_off, flags & BPF_TRAMP_F_RET_FENTRY_RET, image, rw_image, func_meta, cookie_off)) return -EINVAL; @@ -3652,7 +3661,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im if (!branches) return -ENOMEM; - if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off, + if (invoke_bpf_mod_ret(m, &prog, im, fmod_ret, regs_off, run_ctx_off, branches, image, rw_image)) { ret = -EINVAL; goto cleanup; @@ -3682,8 +3691,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } /* remember return value in a stack for bpf prog to access */ emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8); - im->ip_after_call = image + (prog - (u8 *)rw_image); - emit_nops(&prog, X86_PATCH_SIZE); } if (fmod_ret->nr_nodes) { @@ -3708,7 +3715,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta); if (fexit->nr_nodes) { - if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, func_meta_off, + if (invoke_bpf(m, &prog, im, fexit, regs_off, run_ctx_off, func_meta_off, false, image, rw_image, func_meta, cookie_off)) { ret = -EINVAL; goto cleanup; @@ -3723,7 +3730,6 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * restored to R0. */ if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = image + (prog - (u8 *)rw_image); /* arg1: mov rdi, im */ emit_mov_imm64(&prog, BPF_REG_1, (long) im >> 32, (u32) (long) im); if (emit_rsb_call(&prog, __bpf_tramp_exit, image + (prog - (u8 *)rw_image))) { @@ -3811,7 +3817,7 @@ out: int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr) { - struct bpf_tramp_image im; + struct bpf_tramp_image im = {}; void *image; int ret; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index a8ec85127571..0ecb9418dfbd 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -1258,11 +1258,15 @@ struct btf_func_model { #define BPF_TRAMP_F_INDIRECT BIT(8) /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 - * bytes on x86. + * bytes on x86. The trampoline image has to fit in PAGE_SIZE. */ enum { -#if defined(__s390x__) +#if defined(__s390x__) || defined(__powerpc64__) BPF_MAX_TRAMP_LINKS = 27, +#elif defined(__x86_64__) + BPF_MAX_TRAMP_LINKS = 36, +#elif defined(__aarch64__) + BPF_MAX_TRAMP_LINKS = 37, #else BPF_MAX_TRAMP_LINKS = 38, #endif @@ -1315,6 +1319,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size); +int arch_bpf_trampoline_skip(void *nop, void *target); int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_nodes *tnodes, void *func_addr); @@ -1363,19 +1368,48 @@ enum bpf_tramp_prog_type { BPF_TRAMP_FSESSION, }; +/* + * Each prog call in a trampoline image is preceded by a nop. When the prog is + * detached, the nop is patched to a jump to target, right after the call, so + * that tasks still running in the image skip the prog. + */ +struct bpf_tramp_skip { + struct bpf_prog *prog; + void *nop; + void *target; +}; + struct bpf_tramp_image { void *image; int size; struct bpf_ksym ksym; struct percpu_ref pcref; - void *ip_after_call; - void *ip_epilogue; + bool call_orig; + /* entry in tr->images, the image holds a reference on tr */ + struct bpf_trampoline *tr; + struct list_head list; + int nr_skips; + struct bpf_tramp_skip *skips; union { struct rcu_head rcu; struct work_struct work; }; }; +static inline void bpf_tramp_image_add_skip(struct bpf_tramp_image *im, struct bpf_prog *prog, + void *nop, void *target) +{ + struct bpf_tramp_skip *skip; + + /* struct_ops trampolines and arch_bpf_trampoline_size() have no image */ + if (!im || !im->skips) + return; + skip = &im->skips[im->nr_skips++]; + skip->prog = prog; + skip->nop = nop; + skip->target = target; +} + struct bpf_trampoline { /* hlist for trampoline_key_table */ struct hlist_node hlist_key; @@ -1402,6 +1436,8 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; + /* Images not freed yet, cur_image and older ones still in use */ + struct list_head images; /* Used as temporary old image storage for multi_attach */ struct { struct bpf_tramp_image *old_image; diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c index 9d69c066a817..bf4cb3dd444d 100644 --- a/kernel/bpf/trampoline.c +++ b/kernel/bpf/trampoline.c @@ -401,6 +401,7 @@ static struct bpf_trampoline *bpf_trampoline_lookup(u64 key, unsigned long ip) head = &trampoline_ip_table[hash_64(tr->ip, TRAMPOLINE_HASH_BITS)]; hlist_add_head(&tr->hlist_ip, head); refcount_set(&tr->refcnt, 1); + INIT_LIST_HEAD(&tr->images); for (i = 0; i < BPF_TRAMP_MAX; i++) INIT_HLIST_HEAD(&tr->progs_hlist[i]); out: @@ -565,15 +566,22 @@ static void bpf_tramp_image_free(struct bpf_tramp_image *im) arch_free_bpf_trampoline(im->image, im->size); bpf_jit_uncharge_modmem(im->size); percpu_ref_exit(&im->pcref); + kfree(im->skips); kfree_rcu(im, rcu); } static void __bpf_tramp_image_put_deferred(struct work_struct *work) { struct bpf_tramp_image *im; + struct bpf_trampoline *tr; im = container_of(work, struct bpf_tramp_image, work); + tr = im->tr; + trampoline_lock(tr); + list_del(&im->list); + trampoline_unlock(tr); bpf_tramp_image_free(im); + bpf_trampoline_put(tr); } /* callback, fexit step 3 or fentry step 2 */ @@ -601,7 +609,7 @@ static void __bpf_tramp_image_put_rcu_tasks(struct rcu_head *rcu) struct bpf_tramp_image *im; im = container_of(rcu, struct bpf_tramp_image, rcu); - if (im->ip_after_call) + if (im->call_orig) /* the case of fmod_ret/fexit trampoline and CONFIG_PREEMPTION=y */ percpu_ref_kill(&im->pcref); else @@ -621,9 +629,9 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * * The trampoline is unreachable before bpf_tramp_image_put(). * - * First, patch the trampoline to avoid calling into fexit progs. - * The progs will be freed even if the original function is still - * executing or sleeping. + * Progs are patched out of the image when they are detached, see + * bpf_trampoline_skip_prog(), so they can be freed even if a task is + * still in the image. * In case of CONFIG_PREEMPT=y use call_rcu_tasks() to wait on * first few asm instructions to execute and call into * __bpf_tramp_enter->percpu_ref_get. @@ -637,11 +645,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) * percpu_ref_kill will be waiting for. Hence the first * call_rcu_tasks() is not necessary. */ - if (im->ip_after_call) { - int err = bpf_arch_text_poke(im->ip_after_call, BPF_MOD_NOP, - BPF_MOD_JUMP, NULL, - im->ip_epilogue); - WARN_ON(err); + if (im->call_orig) { if (IS_ENABLED(CONFIG_TASKS_RCU)) call_rcu_tasks(&im->rcu, __bpf_tramp_image_put_rcu_tasks); else @@ -658,7 +662,7 @@ static void bpf_tramp_image_put(struct bpf_tramp_image *im) call_rcu_tasks_trace(&im->rcu, __bpf_tramp_image_put_rcu_tasks); } -static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) +static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size, int nr_progs) { struct bpf_tramp_image *im; struct bpf_ksym *ksym; @@ -669,6 +673,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) if (!im) goto out; + im->skips = kzalloc_objs(*im->skips, nr_progs); + if (!im->skips) + goto out_free_im; + err = bpf_jit_charge_modmem(size); if (err) goto out_free_im; @@ -695,6 +703,7 @@ out_free_image: out_uncharge: bpf_jit_uncharge_modmem(size); out_free_im: + kfree(im->skips); kfree(im); out: return ERR_PTR(err); @@ -771,11 +780,12 @@ again: goto out; } - im = bpf_tramp_image_alloc(tr->key, size); + im = bpf_tramp_image_alloc(tr->key, size, total); if (IS_ERR(im)) { err = PTR_ERR(im); goto out; } + im->call_orig = tr->flags & BPF_TRAMP_F_CALL_ORIG; err = arch_prepare_bpf_trampoline(im, im->image, im->image + size, &tr->func.model, tr->flags, tnodes, @@ -806,8 +816,14 @@ again: #endif out_free: - if (err) + if (err) { bpf_tramp_image_free(im); + } else { + /* track the image until it is freed, for bpf_trampoline_skip_prog() */ + refcount_inc(&tr->refcnt); + im->tr = tr; + list_add(&im->list, &tr->images); + } out: /* If any error happens, restore previous flags */ if (err) @@ -921,6 +937,41 @@ static int bpf_trampoline_add_prog(struct bpf_trampoline *tr, return 0; } +/* + * Patch the nop in front of a prog call to a jump over it. A task can be + * preempted anywhere in the image, so archs that need several instructions for + * a jump of any range patch a single near branch here instead. + */ +int __weak arch_bpf_trampoline_skip(void *nop, void *target) +{ + return bpf_arch_text_poke(nop, BPF_MOD_NOP, BPF_MOD_JUMP, NULL, target); +} + +/* + * prog was detached and can be freed, but tasks may still be running in images + * that call it, sleeping in an earlier prog for example. They can be in any + * image that is not freed yet, not only in cur_image, so patch all of them to + * jump over prog. + */ +static void bpf_trampoline_skip_prog(struct bpf_trampoline *tr, struct bpf_prog *prog) +{ + struct bpf_tramp_image *im; + int i, err; + + list_for_each_entry(im, &tr->images, list) { + for (i = 0; i < im->nr_skips; i++) { + struct bpf_tramp_skip *skip = &im->skips[i]; + + if (skip->prog != prog) + continue; + err = arch_bpf_trampoline_skip(skip->nop, skip->target); + WARN_ON_ONCE(err); + /* not a nop anymore, and prog's address can be reused */ + skip->prog = NULL; + } + } +} + static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, struct bpf_tramp_node *node) { @@ -938,6 +989,7 @@ static void bpf_trampoline_remove_prog(struct bpf_trampoline *tr, } hlist_del_init(&node->tramp_hlist); tr->progs_cnt[kind]--; + bpf_trampoline_skip_prog(tr, node->link->prog); } static int __bpf_trampoline_link_prog(struct bpf_tramp_node *node, @@ -1246,11 +1298,9 @@ void bpf_trampoline_put(struct bpf_trampoline *tr) if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[i]))) goto out; - /* This code will be executed even when the last bpf_tramp_image - * is alive. All progs are detached from the trampoline and the - * trampoline image is patched with jmp into epilogue to skip - * fexit progs. The fentry-only trampoline will be freed via - * multiple rcu callbacks. + /* + * All progs are detached and the last image has been freed, images + * hold a reference on the trampoline until then. */ hlist_del(&tr->hlist_key); hlist_del(&tr->hlist_ip); -- cgit v1.2.3 From f341dc4d934a69ffbe6c21e3609ee725790188f1 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 21 Sep 2026 21:12:00 +0200 Subject: cpufreq: intel_pstate: Fix max_freq fallback in cpufreq_update_pressure() After commit d2d5c129d07e ("cpufreq: Make cpufreq_update_pressure() fall back to cpuinfo.max_freq"), cpufreq pressure appears in the CPU load balancer unexpectedly in some cases in which it was not present before, leading to confusion and uncertainty. Clearly, the scheduler assumes that cpufreq pressure will not be set unless the capacity reference frequency of the CPU is known, and the commit mentioned above violates that assumption. However, in some cases the capacity reference frequency of the CPU is in fact known even though arch_scale_freq_ref() returns 0 and in those cases it should be possible to set cpufreq pressure as appropriate. For this purpose, introduce a new cpufreq driver callback returning the CPU capacity reference frequency, .scale_freq_ref(), and make cpufreq_update_pressure() invoke it, if present, instead of falling back to cpuinfo.max_freq unconditionally. Add that callback to the intel_pstate driver and make it return 0 unless the scale-invariant capacity of the given CPU has been explicitly set, in which cases its reference frequency is always cpuinfo.max_freq. Fixes: d2d5c129d07e ("cpufreq: Make cpufreq_update_pressure() fall back to cpuinfo.max_freq") Reported-by: Jianyong Wu Closes: https://lore.kernel.org/linux-pm/20260915065747.1671965-1-wujianyong@hygon.cn/ Tested-by: Jianyong Wu Signed-off-by: Rafael J. Wysocki Tested-by: Ricardo Neri # Intel hybrid parts Tested-by: Chen Yu [ rjw: Add READ_ONCE() around a capacity_perf read ] Link: https://patch.msgid.link/12975163.O9o76ZdvQC@rafael.j.wysocki Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/cpufreq.c | 4 ++-- drivers/cpufreq/intel_pstate.c | 10 ++++++++++ include/linux/cpufreq.h | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 96515880b4ac..54dde8419bdc 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -2590,8 +2590,8 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy) cpu = cpumask_first(policy->related_cpus); max_freq = arch_scale_freq_ref(cpu); - if (!max_freq) - max_freq = policy->cpuinfo.max_freq; + if (!max_freq && cpufreq_driver->scale_freq_ref) + max_freq = cpufreq_driver->scale_freq_ref(policy); capped_freq = policy->max; diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index ceb340f7a110..17158cb25dd8 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -1135,6 +1135,14 @@ static bool hybrid_clear_max_perf_cpu(void) return ret; } +static unsigned int intel_pstate_scale_freq_ref(struct cpufreq_policy *policy) +{ + if (READ_ONCE(all_cpu_data[policy->cpu]->capacity_perf)) + return policy->cpuinfo.max_freq; + + return 0; +} + static void intel_pstate_update_freq_limits(struct cpudata *cpu) { int scaling = cpu->pstate.scaling; @@ -3088,6 +3096,7 @@ static struct cpufreq_driver intel_pstate = { .offline = intel_pstate_cpu_offline, .online = intel_pstate_cpu_online, .update_limits = intel_pstate_update_limits, + .scale_freq_ref = intel_pstate_scale_freq_ref, .name = "intel_pstate", }; @@ -3411,6 +3420,7 @@ static struct cpufreq_driver intel_cpufreq = { .suspend = intel_cpufreq_suspend, .resume = intel_pstate_resume, .update_limits = intel_pstate_update_limits, + .scale_freq_ref = intel_pstate_scale_freq_ref, .name = "intel_cpufreq", }; diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 35ce665edfd8..d3d0d9d02aa4 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -420,6 +420,9 @@ struct cpufreq_driver { /* Will be called after the driver is fully initialized */ void (*ready)(struct cpufreq_policy *policy); + /* Return the capacity reference frequency for policy. */ + unsigned int (*scale_freq_ref)(struct cpufreq_policy *policy); + struct freq_attr **attr; /* platform specific boost support code */ -- cgit v1.2.3 From 28fa9af353bed2bb738c46e31f9b7d0347b759d1 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 19 Sep 2026 16:40:27 +0200 Subject: hrtimer: Use the mask to clear TIF_HRTIMER_REARM from the exit work hrtimer_rearm_deferred_user_irq() removes the rearm bit from the local copy of the TIF work with: *tif_work &= ~TIF_HRTIMER_REARM; TIF_HRTIMER_REARM is the bit number (12), not the mask. That clears TIF_NOTIFY_SIGNAL (bit 2) and TIF_MEMDIE (bit 3) from the copy and leaves bit 12 set. So the function never returns true, and the copy handed to exit_to_user_mode_loop() lacks TIF_NOTIFY_SIGNAL. If that was the only work for the loop, the task returns to user space with the task work still pending. hrtimer_interrupt() sets TIF_HRTIMER_REARM every time, so the next tick repeats that. Task work queued with TWA_SIGNAL from a hrtimer callback stays pending until the task does a syscall, takes an interrupt without deferred rearm or has to reschedule. A task which polls the io_uring completion ring in user space sees a timeout completion after 30ms (median) instead of 70us. 2 CPU QEMU guest, HZ=1000. Use the mask. Fixes: 15dd3a948855 ("hrtimer: Push reprogramming timers into the interrupt return path") Signed-off-by: Karl Mehltretter Signed-off-by: Thomas Gleixner Assisted-by: LLM Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260919144027.70250-1-kmehltretter@gmail.com --- include/linux/hrtimer_rearm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h index 17a81826bd9a..12e42721c8e0 100644 --- a/include/linux/hrtimer_rearm.h +++ b/include/linux/hrtimer_rearm.h @@ -45,7 +45,7 @@ hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif clear_thread_flag(TIF_HRTIMER_REARM); __hrtimer_rearm_deferred(); /* Don't go into the loop if HRTIMER_REARM was the only flag */ - *tif_work &= ~TIF_HRTIMER_REARM; + *tif_work &= ~_TIF_HRTIMER_REARM; return !*tif_work; } return false; -- cgit v1.2.3 From ea4d4b5dddb5121e64f56fd8e0720bd8b447b63d Mon Sep 17 00:00:00 2001 From: Jamal Hadi Salim Date: Mon, 28 Sep 2026 08:46:16 -0400 Subject: net: cap skb->queue_mapping when the tx queue is picked skbedit can set skb->queue_mapping and raise the per-CPU skip_txqueue flag so __dev_queue_xmit() honours the mapping. __dev_queue_xmit() cleared the flag before sch_handle_egress() and only read it afterwards, so the flag was not confined to the xmit that set it: a nested xmit (mirred redirect or mirror, or a drop after skbedit) could set the flag and the outer xmit would consume it for an skb that never went through skbedit. A forwarded packet still carries the ingress NIC's rx_queue + 1 in skb->queue_mapping, so the outer device then indexes its tx queue state with that stale value. Taprio's child array q->qdiscs[] is sized to the device's queue count, so taprio_enqueue() indexes past its allocation and dereferences the result as a struct Qdisc *. We (ab)use the skb->nf_skip_egress which means "skip netfilter egress for this packet" to tag to "am I in tc egress?". Despite the overload I dont see it as a conflict since the marker is set only around the single sch_handle_egress() call and ingress path is guarded by tc_at_ingress. I will send a followup(net-next) patch once this hits net-next to rename the skb->nf_skip_egress bit/flag to skb->skip_egress Arm the flag only from the egress classifier that can use it: raise skip_txqueue from tcf_skbedit_act() only when it runs inside sch_handle_egress(), thanks to skb->nf_skip_egress. An egress qdisc classifier runs in q->enqueue(), after the tx queue has been picked, so a mapping it sets cannot affect the current packet; arming the flag there only pollutes it for a later xmit. Then own the flag for the xmit frame the egress hook runs in: save the incoming value and clear it just before sch_handle_egress(), and restore it after the hook - on the consumed (drop) path, or, in the same call that reads it, on the surviving path. The save and the restores stay inside the egress_needed_key static branch, so a packet pays for them only when egress hooks are active (2f1e85b1aee4). Store the value netdev_cap_txqueue() selected back into skb->queue_mapping in netdev_tx_queue_mapping(), as netdev_core_pick_tx() already does, so the skip_txqueue path never hands a later reader on the xmit path a mapping the device cannot serve. A store made still later in the same frame, by a tc BPF program attached to a transmit qdisc, is outside this path and is not re-capped; a separate followup will resolve that path. netdev_xmit_skip_txqueue() returns the previous flag value so the save-and-clear is one call, and a no-op stub is provided when CONFIG_NET_EGRESS is disabled. skb->nf_skip_egress is compiled under CONFIG_NET_EGRESS rather than CONFIG_NETFILTER_SKIP_EGRESS, so skb_at_tc_egress() is valid whenever the egress path is built. A local user in a network namespace can redirect a packet from a device with more TX queues to one with fewer after setting a mapping valid only on the larger device. That reaches these reads and, under KASAN, faults with "slab-out-of-bounds in taprio_enqueue". Conditions to recreate the bug: the report's own trigger is a local user with CAP_NET_ADMIN in a network namespace, so no eBPF program is needed. With CONFIG_NET_SCH_TAPRIO=y, CONFIG_NET_ACT_SKBEDIT=y, CONFIG_NET_ACT_MIRRED=y, CONFIG_NET_CLS_MATCHALL=y, CONFIG_NET_SCH_PRIO=y and KASAN enabled, create qa (3 queues), qb (2 queues) and qc (1 queue) as dummy devices; put a taprio root on qb (num_tc 1, queues 2@0) and clsact on all three; then add an egress matchall filter on every device. On qa: "action skbedit queue_mapping 2 pipe action mirred egress redirect dev qb". On qb: "action mirred egress mirror dev qc". On qc: "action skbedit queue_mapping 0 pipe". Send one packet out qa. qc's skbedit sets the flag while qb's outer xmit is in flight; without the fix qb consumes it and reads its two-entry taprio child array with the forwarded packet's stale mapping. A qc whose skbedit is instead installed in a transmit-qdisc classifier (a matchall filter on the qc root qdisc) reaches the same code path the same way without the fix. Testing: on a KASAN build with panic_on_warn=1 the unfixed kernel panics with "BUG: KASAN: slab-out-of-bounds in taprio_enqueue", a read 0 bytes past a 16-byte taprio_init() allocation, for the clsact-setter and the transmit-qdisc-classifier reproducers and for a clsact skbedit-then-tc-BPF store; the fixed kernel runs all three with no report, and the BPF store variant additionally shows the expected "selects TX queue" clamp notice from the write-back. Fixes: 2f1e85b1aee4 ("net: sched: use queue_mapping to pick tx queue") Reported-by: Zero Day Initiative Link: https://lore.kernel.org/netdev/CANn89iLwYx8nCVf0pCEk_MmEiyC6kQaMwCQT9WkQVeeNzNQHqQ@mail.gmail.com/ Link: https://lore.kernel.org/netdev/179008581937.2160803.7117814290574262942@kernel.org/ Link: https://lore.kernel.org/netdev/179033713973.2160803.4914570693994398206@kernel.org/ Link: https://lore.kernel.org/netdev/20260925180407.63647514@kernel.org/ Link: https://lore.kernel.org/netdev/CANn89i+k-mZKDQVtvws_MEXeuMTAdaCcOXFZE-RfhcGTu90sjA@mail.gmail.com/ Suggested-by: Eric Dumazet Suggested-by: Jakub Kicinski Tested-by: hybris Signed-off-by: Jamal Hadi Salim Reviewed-by: Eric Dumazet Link: https://patch.msgid.link/QDISC-9R8V.v4.20260928081529@mojatatu.com Signed-off-by: Jakub Kicinski --- include/linux/netfilter_netdev.h | 2 +- include/linux/rtnetlink.h | 7 ++++++- include/linux/skbuff.h | 2 +- include/net/sch_generic.h | 9 +++++++++ net/core/dev.c | 31 ++++++++++++++++++++++++------- net/sched/act_skbedit.c | 5 ++--- 6 files changed, 43 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/netfilter_netdev.h b/include/linux/netfilter_netdev.h index 3175073a66ba..2a854a0bbd4b 100644 --- a/include/linux/netfilter_netdev.h +++ b/include/linux/netfilter_netdev.h @@ -133,7 +133,7 @@ static inline struct sk_buff *nf_hook_egress(struct sk_buff *skb, int *rc, static inline void nf_skip_egress(struct sk_buff *skb, bool skip) { -#ifdef CONFIG_NETFILTER_SKIP_EGRESS +#ifdef CONFIG_NET_EGRESS skb->nf_skip_egress = skip; #endif } diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index 95729339e7a5..a54ec40d095c 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -186,7 +186,12 @@ void net_dec_ingress_queue(void); #ifdef CONFIG_NET_EGRESS void net_inc_egress_queue(void); void net_dec_egress_queue(void); -void netdev_xmit_skip_txqueue(bool skip); +bool netdev_xmit_skip_txqueue(bool skip); +#else +static inline bool netdev_xmit_skip_txqueue(bool skip) +{ + return false; +} #endif void rtnetlink_init(void); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 84308498a3a8..a3ff380d43f0 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1020,7 +1020,7 @@ struct sk_buff { #ifdef CONFIG_NET_REDIRECT __u8 from_ingress:1; #endif -#ifdef CONFIG_NETFILTER_SKIP_EGRESS +#ifdef CONFIG_NET_EGRESS __u8 nf_skip_egress:1; #endif #ifdef CONFIG_SKB_DECRYPTED diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index f35bd06a6bad..1acaadb3e2cf 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -810,6 +810,15 @@ static inline bool skb_at_tc_ingress(const struct sk_buff *skb) #endif } +static inline bool skb_at_tc_egress(const struct sk_buff *skb) +{ +#ifdef CONFIG_NET_EGRESS + return skb->nf_skip_egress && !skb_at_tc_ingress(skb); +#else + return false; +#endif +} + static inline bool skb_skip_tc_classify(struct sk_buff *skb) { #ifdef CONFIG_NET_CLS_ACT diff --git a/net/core/dev.c b/net/core/dev.c index f660fccfc0db..8638c994bbcb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4404,9 +4404,14 @@ EXPORT_SYMBOL(dev_loopback_xmit); static struct netdev_queue * netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) { - int qm = skb_get_queue_mapping(skb); + int queue = skb_get_queue_mapping(skb); + int capped; - return netdev_get_tx_queue(dev, netdev_cap_txqueue(dev, qm)); + capped = netdev_cap_txqueue(dev, queue); + if (unlikely(capped != queue)) + skb_set_queue_mapping(skb, capped); + + return netdev_get_tx_queue(dev, capped); } #ifndef CONFIG_PREEMPT_RT @@ -4415,9 +4420,13 @@ static bool netdev_xmit_txqueue_skipped(void) return __this_cpu_read(softnet_data.xmit.skip_txqueue); } -void netdev_xmit_skip_txqueue(bool skip) +bool netdev_xmit_skip_txqueue(bool skip) { + bool prev = netdev_xmit_txqueue_skipped(); + __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); + + return prev; } EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); @@ -4427,9 +4436,13 @@ static bool netdev_xmit_txqueue_skipped(void) return current->net_xmit.skip_txqueue; } -void netdev_xmit_skip_txqueue(bool skip) +bool netdev_xmit_skip_txqueue(bool skip) { + bool prev = netdev_xmit_txqueue_skipped(); + current->net_xmit.skip_txqueue = skip; + + return prev; } EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); #endif @@ -4848,21 +4861,25 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) tcx_set_ingress(skb, false); #ifdef CONFIG_NET_EGRESS if (static_branch_unlikely(&egress_needed_key)) { + bool skip_txq; + if (nf_hook_egress_active()) { skb = nf_hook_egress(skb, &rc, dev); if (!skb) goto out; } - netdev_xmit_skip_txqueue(false); + skip_txq = netdev_xmit_skip_txqueue(false); nf_skip_egress(skb, true); skb = sch_handle_egress(skb, &rc, dev); - if (!skb) + if (!skb) { + netdev_xmit_skip_txqueue(skip_txq); goto out; + } nf_skip_egress(skb, false); - if (netdev_xmit_txqueue_skipped()) + if (netdev_xmit_skip_txqueue(skip_txq)) txq = netdev_tx_queue_mapping(dev, skb); } #endif diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index bfec6b668410..26258b353cde 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -72,9 +72,8 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, } if (params->flags & SKBEDIT_F_QUEUE_MAPPING && skb->dev->real_num_tx_queues > params->queue_mapping) { -#ifdef CONFIG_NET_EGRESS - netdev_xmit_skip_txqueue(true); -#endif + if (skb_at_tc_egress(skb)) + netdev_xmit_skip_txqueue(true); skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb)); } if (params->flags & SKBEDIT_F_MARK) { -- cgit v1.2.3 From 3ae37eafd36694bb2d3227f60ac98fbf602470a2 Mon Sep 17 00:00:00 2001 From: Pablo Neira Ayuso Date: Wed, 23 Sep 2026 20:46:18 +0200 Subject: netfilter: flowtable: restore ieee80211 forward path Before commit 871df5007eda ("netfilter: flowtable: bail out if forward path cannot be discovered"), there was a fallback to set up a forward path in case .ndo_fill_forward_path fails or DEV_PATH_MTK_WDMA was used. Such fallback was used by commit d787a3e38f01 ("mac80211: add support for .ndo_fill_forward_path"). One possibility is to handle DEV_PATH_MTK_WDMA from the flowtable forward path discovery. However, this is only used internally by drivers to retrieve mtk_wdma information to set up hardware offload. Felix decided to use the .fill_forward_path interface for this purpose due to the lack of a better interface at that time. Add a new DEV_PATH_IEEE80211 path which is offered if the new ieee80211 flag is set on in the struct net_device_path_ctx to restore the flowtable with a ieee80211 netdevice. Handle this new DEV_PATH_IEEE80211 path just like DEV_PATH_ETHERNET and DEV_PATH_DSA, ie. this is the last netdevice in the stack. This new ieee80211 flag is implicitly unset for mtk_ppe and airoha which call dev_fill_forward_path() to retrieve a DEV_PATH_MTK_WDMA path. Fixes: 871df5007eda ("netfilter: flowtable: bail out if forward path cannot be discovered") Signed-off-by: Pablo Neira Ayuso --- include/linux/netdevice.h | 3 +++ net/mac80211/iface.c | 7 +++++++ net/netfilter/nf_flow_table_path.c | 3 +++ 3 files changed, 13 insertions(+) (limited to 'include/linux') diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 87cafc932e9e..3cff2174dc03 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -887,6 +887,7 @@ enum net_device_path_type { DEV_PATH_DSA, DEV_PATH_MTK_WDMA, DEV_PATH_TUN, + DEV_PATH_IEEE80211, }; struct net_device_path { @@ -953,6 +954,8 @@ struct net_device_path_ctx { u16 id; __be16 proto; } vlan[NET_DEVICE_PATH_VLAN_MAX]; + + bool ieee80211; }; enum tc_setup_type { diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 889c32fd8de1..c5584435fde9 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -1023,6 +1023,13 @@ static int ieee80211_netdev_fill_forward_path(struct net_device_path_ctx *ctx, struct sta_info *sta; int ret = -ENOENT; + if (ctx->ieee80211) { + path->type = DEV_PATH_IEEE80211; + path->dev = ctx->dev; + ctx->dev = NULL; + return 0; + } + sdata = IEEE80211_DEV_TO_SUB_IF(ctx->dev); local = sdata->local; diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 1e55644f2edb..d90013685bf1 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -53,6 +53,7 @@ static int nft_dev_fill_forward_path(const struct dst_entry *dst_cache, struct net_device_path_ctx ctx = { .dev = dev, .ether_type = ether_type, + .ieee80211 = true, }; struct neighbour *n; u8 nud_state; @@ -114,6 +115,7 @@ static int nft_dev_path_info(struct net_device_path_stack *stack, path = &stack->path[i]; switch (path->type) { case DEV_PATH_ETHERNET: + case DEV_PATH_IEEE80211: case DEV_PATH_DSA: case DEV_PATH_VLAN: case DEV_PATH_PPPOE: @@ -123,6 +125,7 @@ static int nft_dev_path_info(struct net_device_path_stack *stack, memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN); if (path->type == DEV_PATH_ETHERNET || + path->type == DEV_PATH_IEEE80211 || path->type == DEV_PATH_DSA) break; -- cgit v1.2.3 From 69bf14300b50631e7c2271289d347bbd1b19a06c Mon Sep 17 00:00:00 2001 From: Paulo Alcantara Date: Mon, 21 Sep 2026 11:17:26 -0300 Subject: netfs: clear post-EOF pagecache when extending a file via write Fix netfs to erase the contents of a hole created after the EOF by an ordinary write if dirty data has been previously left there by writes through an mmapped region. Neither the buffered nor the unbuffered/DIO write path clears that stale pagecache. Zero the tail of the folio straddling the EOF before an extending write. That is the only folio that can hold data written past the EOF through an mmap, as pages wholly beyond the EOF can't be faulted in. The folio is zeroed rather than dropped so a concurrent extending write can't lose data. Both write paths downgrade the i_rwsem to shared, so extending writes can run concurrently and the i_size read by the caller may be stale by the time the folio is locked. Re-read i_size under the folio lock and clamp the zeroed range up to it, so a racing write that already put data into the folio isn't clobbered. Wait for any writeback on the folio to finish before zeroing it so that the pagecache isn't modified while it may still be read by the transport during transmission. Honour IOCB_NOWAIT by returning -EAGAIN rather than blocking on the folio lock, on writeback, or in folio_mkclean()'s rmap walk when the folio is mapped. truncate_pagecache() can't be used here: it must be called with the i_rwsem held exclusively, but these write paths only hold it shared, and it would block unconditionally, breaking IOCB_NOWAIT. Callers that hold i_rwsem exclusively for the whole resize (truncate, setattr, fallocate, clone) exclude any genuine concurrent buffered writer, so staleness can instead be decided from the folio's dirty state, as pagecache_isize_extended() already does for filesystems that serialise writes against truncate/setattr via a single i_rwsem. Export netfs_clear_stale_post_isize() helper to handle such case. The helper is required by the CIFS client to fix generic/363. Closes: https://sashiko.dev/#/patchset/20260921230755.1133425-1-pc%40manguebit.org Fixes: 938e13a73b24 ("netfs: Implement buffered write API") Fixes: 153a9961b551 ("netfs: Implement unbuffered/DIO write support") Reviewed-by: David Howells Reviewed-by: Namjae Jeon Signed-off-by: Paulo Alcantara Cc: Christian Brauner Cc: Matthew Wilcox Cc: Ronnie Sahlberg Cc: Shyam Prasad N Cc: Tom Talpey Cc: Bharath SM Cc: stable@vger.kernel.org --- Documentation/filesystems/netfs_library.rst | 26 ++++++++ fs/netfs/buffered_write.c | 9 +++ fs/netfs/direct_write.c | 9 +++ fs/netfs/internal.h | 2 + fs/netfs/misc.c | 99 +++++++++++++++++++++++++++++ include/linux/netfs.h | 2 + 6 files changed, 147 insertions(+) (limited to 'include/linux') diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst index ddd799df6ce3..a9de281db8ee 100644 --- a/Documentation/filesystems/netfs_library.rst +++ b/Documentation/filesystems/netfs_library.rst @@ -451,6 +451,32 @@ one. The inode should be marked ``NETFS_ICTX_SINGLE_NO_UPLOAD`` if this API is to be used. The writeback function requires the buffer to be of ITER_FOLIOQ type. +Clearing Stale Post-EOF Pagecache +--------------------------------- + +When a file is extended, data left in the pagecache past the old EOF by a write +through an mmap must not be exposed as file content. Netfslib clears this on +its own write paths, and exports a helper so a filesystem can do the same from +a resize path (truncate, setattr, fallocate and the like) that holds the +inode's ``i_rwsem`` exclusively across the whole resize:: + + void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to); + +This zeroes any such data within the ``[from, to)`` hole to be made, where +@from is the old EOF and @to is the new one, and the caller must have already +updated ``i_size`` to @to before calling it. Only the folio straddling @from +can hold data written past the EOF through an mmap, as pages wholly beyond the +EOF can't be faulted in, so the zeroing is limited to that folio. The folio is +zeroed rather than dropped so that a concurrent extending write can't lose +data. + +This overlaps with ``pagecache_isize_extended()`` but can't reuse it: that +helper is keyed on a sub-page block size and is a no-op when the block size is +``>= PAGE_SIZE`` (as on network filesystems), and it doesn't wait for +writeback. As both address the same problem, a change to one should probably +be reflected in the other to keep them in sync. + High-Level VM API ================== diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c index 2cdb68e6b16f..ecf119b4f916 100644 --- a/fs/netfs/buffered_write.c +++ b/fs/netfs/buffered_write.c @@ -469,6 +469,7 @@ ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *fr struct netfs_group *netfs_group) { struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); ssize_t ret; trace_netfs_write_iter(iocb, from); @@ -481,6 +482,14 @@ ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *fr if (ret) return ret; + if (iocb->ki_pos > i_size_read(inode)) { + ret = netfs_clear_stale_pre_isize(inode, i_size_read(inode), + iocb->ki_pos, + iocb->ki_flags & IOCB_NOWAIT); + if (ret) + return ret; + } + return netfs_perform_write(iocb, from, netfs_group); } EXPORT_SYMBOL(netfs_buffered_write_iter_locked); diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index 2361277416c7..8be74706d85e 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -359,6 +359,15 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from) ret = file_update_time(file); if (ret < 0) goto out; + + if (iocb->ki_pos > i_size_read(inode)) { + ret = netfs_clear_stale_pre_isize(inode, i_size_read(inode), + iocb->ki_pos, + iocb->ki_flags & IOCB_NOWAIT); + if (ret < 0) + goto out; + } + if (iocb->ki_flags & IOCB_NOWAIT) { /* We could block if there are any pages in the range. */ ret = -EAGAIN; diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index c79c8e69d60c..786af76da98c 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -80,6 +80,8 @@ ssize_t netfs_wait_for_write(struct netfs_io_request *rreq); void netfs_wait_for_paused_read(struct netfs_io_request *rreq); void netfs_wait_for_paused_write(struct netfs_io_request *rreq); void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq); +int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait); /* * objects.c diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index f5c1c463f4ff..523057390d6f 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -6,6 +6,7 @@ */ #include +#include #include "internal.h" /** @@ -582,3 +583,101 @@ void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq) trace_netfs_rreq(rreq, netfs_rreq_trace_waited_put_ra_refs); finish_wait(&rreq->waitq, &myself); } + +/** + * netfs_clear_stale_isize - Clear stale pagecache in a to-be-created hole + * @inode: The inode to act upon. + * @from: The base of the hole to be made. + * @to: The top of the hole to be made. + * @nowait: True to return -EAGAIN rather than block. + * @exclusive: True if the caller holds i_rwsem exclusively for the resize. + * + * Zero any data left in the pagecache within the [@from, @to) hole by a + * write through an mmap so that it isn't exposed as file content once the + * file is extended. Only the uptodate folio straddling @from can hold such + * data as pages wholly beyond the EOF can't be faulted in, so the zeroing + * is limited to that folio. The folio is zeroed rather than dropped so + * that a concurrent extending write can't lose data. + * + * If @exclusive is false, @from is re-read from i_size and used to clamp + * the zeroed range, for callers that may race with another writer also + * extending the file (eg. multiple buffered writes extending the same file + * under a shared i_rwsem). If @exclusive is true, for callers that hold + * i_rwsem exclusively across the whole resize and have already updated + * i_size to @to, staleness is decided from the folio's dirty state instead: + * since no genuine concurrent buffered writer can be racing, a lockless + * stat() adopting a server-confirmed size mid-resize has no data behind it + * and never dirties the folio, so it can't fool this check into skipping + * the zeroing the way it could fool the @exclusive false clamp. + * + * pagecache_isize_extended() can't be reused here: it is keyed on a + * sub-page block size (a no-op when the block size is >= PAGE_SIZE, as on + * network filesystems), runs after i_size is updated, can't honour + * @nowait, and doesn't wait for writeback. Keep the two in sync if either + * is changed. + * + * Return: 0 on success, or -EAGAIN if @nowait is set and the folio is + * mapped or under writeback and so can't be cleaned without blocking. + */ +static int netfs_clear_stale_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait, bool exclusive) +{ + struct address_space *mapping = inode->i_mapping; + fgf_t fgp = FGP_LOCK; + struct folio *folio; + int ret; + + if (from >= to) + return 0; + + if (nowait) + fgp |= FGP_NOWAIT; + + folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, fgp, 0); + if (IS_ERR(folio)) + return PTR_ERR(folio) == -EAGAIN ? -EAGAIN : 0; + + ret = 0; + if (nowait && (folio_mapped(folio) || folio_test_writeback(folio))) { + ret = -EAGAIN; + goto out; + } + + folio_wait_writeback(folio); + + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + + if (folio_test_uptodate(folio) && + (!exclusive || folio_test_dirty(folio))) { + uoff_t fpos = folio_pos(folio); + + if (!exclusive) + from = umax(from, i_size_read(inode)); + if (from < to && from < fpos + folio_size(folio)) { + size_t end = umin(to - fpos, folio_size(folio)); + size_t offset = from - fpos; + + folio_zero_segment(folio, offset, end); + } + } +out: + folio_unlock(folio); + folio_put(folio); + return ret; +} + +/* Clear stale pagecache before an extending buffered/DIO write. */ +int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from, + uoff_t to, bool nowait) +{ + return netfs_clear_stale_isize(inode, from, to, nowait, false); +} + +/* Clear stale pagecache when extending a file under an exclusive resize. */ +void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to) +{ + netfs_clear_stale_isize(inode, from, to, false, true); +} +EXPORT_SYMBOL(netfs_clear_stale_post_isize); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index b4dd32863dd4..fe6275e55fc2 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -397,6 +397,8 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from); ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *iter, struct netfs_group *netfs_group); ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from); +void netfs_clear_stale_post_isize(struct inode *inode, uoff_t from, + uoff_t to); /* Single, monolithic object read/write API. */ void netfs_single_mark_inode_dirty(struct inode *inode); -- cgit v1.2.3 From 8df07fe93573e9e548d6645273e9bcb6eb74059e Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Wed, 30 Sep 2026 15:18:45 +0200 Subject: tty: fix saved termios reset race Resetting saved termios state on device registration is needed where a minor number can be reused for an entirely different device and where the old settings may prevent the port from even being opened (e.g. when CLOCAL is not set). Not all TTY drivers guarantee that the minor number is no longer in use when registering devices however, something which can lead to a use-after-free when closing a TTY (and saving its termios) races with re-registration. Add a new TTY_DRIVER_RESET_SAVED_TERMIOS flag to request that any saved termios state is reset on registration and only set it for drivers that make sure that the minor number is no longer in use. Fixes: 93857edd9829 ("tty: reset termios state on device registration") Reported-by: Chengfeng Ye Link: https://lore.kernel.org/20260926184154.3017929-1-nicoyip.dev@gmail.com Cc: stable@kernel.org # 4.12 Signed-off-by: Johan Hovold Link: https://patch.msgid.link/20260930131845.1809256-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/staging/greybus/uart.c | 3 ++- drivers/tty/tty_io.c | 18 ++++++++++-------- drivers/usb/class/cdc-acm.c | 2 +- drivers/usb/serial/usb-serial.c | 3 ++- include/linux/tty_driver.h | 6 ++++++ 5 files changed, 21 insertions(+), 11 deletions(-) (limited to 'include/linux') diff --git a/drivers/staging/greybus/uart.c b/drivers/staging/greybus/uart.c index 24b4dab069c3..172cf900eaf0 100644 --- a/drivers/staging/greybus/uart.c +++ b/drivers/staging/greybus/uart.c @@ -948,7 +948,8 @@ static int gb_tty_init(void) int retval = 0; gb_tty_driver = tty_alloc_driver(GB_NUM_MINORS, TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_DYNAMIC_DEV | + TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(gb_tty_driver)) { pr_err("Can not allocate tty driver\n"); retval = PTR_ERR(gb_tty_driver); diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index 1c30faae9ec1..1a6c8a1bc764 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3256,14 +3256,16 @@ struct device *tty_register_device_attr(struct tty_driver *driver, goto err_put; if (!(driver->flags & TTY_DRIVER_DYNAMIC_ALLOC)) { - /* - * Free any saved termios data so that the termios state is - * reset when reusing a minor number. - */ - tp = driver->termios[index]; - if (tp) { - driver->termios[index] = NULL; - kfree(tp); + if (driver->flags & TTY_DRIVER_RESET_SAVED_TERMIOS) { + /* + * Free any saved termios data so that the termios + * state is reset when reusing a minor number. + */ + tp = driver->termios[index]; + if (tp) { + driver->termios[index] = NULL; + kfree(tp); + } } retval = tty_cdev_add(driver, devt, index, 1); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 7bc5329fa3ed..eeb401c0e5e1 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -2140,7 +2140,7 @@ static int __init acm_init(void) { int retval; acm_tty_driver = tty_alloc_driver(ACM_TTY_MINORS, TTY_DRIVER_REAL_RAW | - TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_DYNAMIC_DEV | TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(acm_tty_driver)) return PTR_ERR(acm_tty_driver); acm_tty_driver->driver_name = "acm", diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c index 17edc057a311..06e246426361 100644 --- a/drivers/usb/serial/usb-serial.c +++ b/drivers/usb/serial/usb-serial.c @@ -1313,7 +1313,8 @@ static int __init usb_serial_init(void) int result; usb_serial_tty_driver = tty_alloc_driver(USB_SERIAL_TTY_MINORS, - TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); + TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV | + TTY_DRIVER_RESET_SAVED_TERMIOS); if (IS_ERR(usb_serial_tty_driver)) return PTR_ERR(usb_serial_tty_driver); diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 1f2896e56e77..62e459c5d3ae 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -73,6 +73,11 @@ struct serial_struct; * @TTY_DRIVER_NO_WORKQUEUE: * Do not create workqueue when tty_register_driver(). Whenever set, flip * buffer workqueue can be set by tty_port_link_wq() for every port. + * + * @TTY_DRIVER_RESET_SAVED_TERMIOS + * Reset any saved termios settings on device registration when reusing a + * minor number. Must only be set by drivers that guarantee that the minor + * number is no longer in use. */ enum tty_driver_flag { TTY_DRIVER_INSTALLED = BIT(0), @@ -84,6 +89,7 @@ enum tty_driver_flag { TTY_DRIVER_DYNAMIC_ALLOC = BIT(6), TTY_DRIVER_UNNUMBERED_NODE = BIT(7), TTY_DRIVER_NO_WORKQUEUE = BIT(8), + TTY_DRIVER_RESET_SAVED_TERMIOS = BIT(9), }; enum tty_driver_type { -- cgit v1.2.3 From b9d1fdc6f4ac1b6e49f9deafaf137da1407d9af1 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Tue, 29 Sep 2026 15:23:32 -0700 Subject: perf: Replace perf_event_header__init_id with full header init perf_iterate_sb() invokes its callback for each matching perf_event on the CPU and task context, passing a shared caller-allocated event structure. perf_event_header__init_id() mutated header->size in place by adding event->id_header_size, requiring sideband output callbacks to save and restore header fields across iterations. Three sideband callbacks failed to save and restore header.size around perf_event_header__init_id(): - perf_event_ksymbol_output() - perf_event_bpf_output() - perf_event_text_poke_output() When multiple events with attr.ksymbol, attr.bpf_event, or attr.text_poke and sample_id_all are active on the same CPU, each subsequent event receives a record whose header.size is inflated by all preceding events' id_header_size values while only a single id_sample is written, leaving uninitialized ring-buffer bytes at the end of the record and causing userspace perf to fail with -EFAULT ("Bad address") when parsing the sample_id trailer. Similarly, perf_event_mmap_output() set PERF_RECORD_MISC_MMAP_BUILD_ID in mmap_event->event_id.header.misc when event->attr.build_id was enabled, but only saved and restored header.size and header.type. If an event with attr.build_id was followed by an event with attr.mmap2 and !attr.build_id, the second event received PERF_RECORD_MISC_MMAP_BUILD_ID in header.misc while its payload contained maj/min/ino/ino_generation instead of a build ID. Rather than splitting header initialization between callers and output callbacks and saving/restoring mutated header fields, replace perf_event_header__init_id() with perf_event_header__init(), which initializes header->type, header->misc, and header->size alongside the sample_id fields on each invocation. Fixes: 76193a94522f ("perf, bpf: Introduce PERF_RECORD_KSYMBOL") Fixes: 6ee52e2a3fe4 ("perf, bpf: Introduce PERF_RECORD_BPF_EVENT") Fixes: e17d43b93e54 ("perf: Add perf text poke event") Fixes: 88a16a130933 ("perf: Add build id data in mmap2 event") Assisted-by: Antigravity:gemini-3.1-pro Signed-off-by: Ian Rogers Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260929222332.973435-1-irogers@google.com Cc: stable@vger.kernel.org --- arch/powerpc/perf/imc-pmu.c | 23 ++-- include/linux/perf_event.h | 7 +- kernel/events/core.c | 307 +++++++++++++++++++------------------------- kernel/events/ring_buffer.c | 7 +- 4 files changed, 148 insertions(+), 196 deletions(-) (limited to 'include/linux') diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index f401181d8669..a9f8464997a2 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -1275,6 +1275,8 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, struct perf_event_header *header, struct perf_event *event) { + u16 misc = 0; + /* Sanity checks for a valid record */ if (be64_to_cpu(READ_ONCE(mem->tb1)) > *prev_tb) *prev_tb = be64_to_cpu(READ_ONCE(mem->tb1)); @@ -1289,23 +1291,19 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, data->ip = be64_to_cpu(READ_ONCE(mem->ip)); data->period = event->hw.last_period; - header->type = PERF_RECORD_SAMPLE; - header->size = sizeof(*header) + event->header_size; - header->misc = 0; - if (cpu_has_feature(CPU_FTR_ARCH_31)) { switch (IMC_TRACE_RECORD_VAL_HVPR(be64_to_cpu(READ_ONCE(mem->val)))) { case 0:/* when MSR HV and PR not set in the trace-record */ - header->misc |= PERF_RECORD_MISC_GUEST_KERNEL; + misc |= PERF_RECORD_MISC_GUEST_KERNEL; break; case 1: /* MSR HV is 0 and PR is 1 */ - header->misc |= PERF_RECORD_MISC_GUEST_USER; + misc |= PERF_RECORD_MISC_GUEST_USER; break; case 2: /* MSR HV is 1 and PR is 0 */ - header->misc |= PERF_RECORD_MISC_KERNEL; + misc |= PERF_RECORD_MISC_KERNEL; break; case 3: /* MSR HV is 1 and PR is 1 */ - header->misc |= PERF_RECORD_MISC_USER; + misc |= PERF_RECORD_MISC_USER; break; default: pr_info("IMC: Unable to set the flag based on MSR bits\n"); @@ -1313,11 +1311,14 @@ static int trace_imc_prepare_sample(struct trace_imc_data *mem, } } else { if (is_kernel_addr(data->ip)) - header->misc |= PERF_RECORD_MISC_KERNEL; + misc |= PERF_RECORD_MISC_KERNEL; else - header->misc |= PERF_RECORD_MISC_USER; + misc |= PERF_RECORD_MISC_USER; } - perf_event_header__init_id(header, data, event); + perf_event_header__init(header, data, + PERF_RECORD_SAMPLE, misc, + sizeof(*header) + event->header_size, + event); return 0; } diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 915c6fd3f084..2d5e7687875b 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1506,9 +1506,10 @@ is_default_overflow_handler(struct perf_event *event) } extern void -perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event); +perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event); extern void perf_event__output_id_sample(struct perf_event *event, struct perf_output_handle *handle, diff --git a/kernel/events/core.c b/kernel/events/core.c index c2882b5be7c3..3aa223595e7d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8131,10 +8131,15 @@ static void __perf_event_header__init_id(struct perf_sample_data *data, } } -void perf_event_header__init_id(struct perf_event_header *header, - struct perf_sample_data *data, - struct perf_event *event) +void perf_event_header__init(struct perf_event_header *header, + struct perf_sample_data *data, + u32 type, u16 misc, u16 size, + struct perf_event *event) { + header->type = type; + header->misc = misc; + header->size = size; + if (event->attr.sample_id_all) { header->size += event->id_header_size; __perf_event_header__init_id(data, event, event->attr.sample_type); @@ -8971,17 +8976,16 @@ perf_event_read_event(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct perf_read_event read_event = { - .header = { - .type = PERF_RECORD_READ, - .misc = 0, - .size = sizeof(read_event) + event->read_size, - }, .pid = perf_event_pid(event, task), .tid = perf_event_tid(event, task), }; int ret; - perf_event_header__init_id(&read_event.header, &sample, event); + perf_event_header__init(&read_event.header, &sample, + PERF_RECORD_READ, + /* misc= */ 0, + sizeof(read_event) + event->read_size, + event); ret = perf_output_begin(&handle, &sample, event, read_event.header.size); if (ret) return; @@ -9222,6 +9226,7 @@ struct perf_task_event { u32 ptid; u64 time; } event_id; + int new; }; static int perf_event_task_match(struct perf_event *event) @@ -9238,17 +9243,21 @@ static void perf_event_task_output(struct perf_event *event, struct perf_output_handle handle; struct perf_sample_data sample; struct task_struct *task = task_event->task; - int ret, size = task_event->event_id.header.size; + int ret; if (!perf_event_task_match(event)) return; - perf_event_header__init_id(&task_event->event_id.header, &sample, event); + perf_event_header__init(&task_event->event_id.header, &sample, + task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, + /* misc= */ 0, + sizeof(task_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, task_event->event_id.header.size); if (ret) - goto out; + return; task_event->event_id.pid = perf_event_pid(event, task); task_event->event_id.tid = perf_event_tid(event, task); @@ -9270,8 +9279,6 @@ static void perf_event_task_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - task_event->event_id.header.size = size; } static void perf_event_task(struct task_struct *task, @@ -9288,18 +9295,7 @@ static void perf_event_task(struct task_struct *task, task_event = (struct perf_task_event){ .task = task, .task_ctx = task_ctx, - .event_id = { - .header = { - .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, - .misc = 0, - .size = sizeof(task_event.event_id), - }, - /* .pid */ - /* .ppid */ - /* .tid */ - /* .ptid */ - /* .time */ - }, + .new = new, }; perf_iterate_sb(perf_event_task_output, @@ -9376,6 +9372,7 @@ struct perf_comm_event { u32 pid; u32 tid; } event_id; + bool exec; }; static int perf_event_comm_match(struct perf_event *event) @@ -9389,18 +9386,21 @@ static void perf_event_comm_output(struct perf_event *event, struct perf_comm_event *comm_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = comm_event->event_id.header.size; int ret; if (!perf_event_comm_match(event)) return; - perf_event_header__init_id(&comm_event->event_id.header, &sample, event); + perf_event_header__init(&comm_event->event_id.header, &sample, + PERF_RECORD_COMM, + comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0, + sizeof(comm_event->event_id) + comm_event->comm_size, + event); ret = perf_output_begin(&handle, &sample, event, comm_event->event_id.header.size); if (ret) - goto out; + return; comm_event->event_id.pid = perf_event_pid(event, comm_event->task); comm_event->event_id.tid = perf_event_tid(event, comm_event->task); @@ -9412,8 +9412,6 @@ static void perf_event_comm_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - comm_event->event_id.header.size = size; } static void perf_event_comm_event(struct perf_comm_event *comm_event) @@ -9428,8 +9426,6 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) comm_event->comm = comm; comm_event->comm_size = size; - comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; - perf_iterate_sb(perf_event_comm_output, comm_event, NULL); @@ -9446,15 +9442,8 @@ void perf_event_comm(struct task_struct *task, bool exec) .task = task, /* .comm */ /* .comm_size */ - .event_id = { - .header = { - .type = PERF_RECORD_COMM, - .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, - /* .size */ - }, - /* .pid */ - /* .tid */ - }, + /* .event_id */ + .exec = exec, }; perf_event_comm_event(&comm_event); @@ -9488,18 +9477,20 @@ static void perf_event_namespaces_output(struct perf_event *event, struct perf_namespaces_event *namespaces_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = namespaces_event->event_id.header.size; int ret; if (!perf_event_namespaces_match(event)) return; - perf_event_header__init_id(&namespaces_event->event_id.header, - &sample, event); + perf_event_header__init(&namespaces_event->event_id.header, &sample, + PERF_RECORD_NAMESPACES, + /* misc= */ 0, + sizeof(namespaces_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, namespaces_event->event_id.header.size); if (ret) - goto out; + return; namespaces_event->event_id.pid = perf_event_pid(event, namespaces_event->task); @@ -9511,8 +9502,6 @@ static void perf_event_namespaces_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - namespaces_event->event_id.header.size = header_size; } static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, @@ -9543,11 +9532,7 @@ void perf_event_namespaces(struct task_struct *task) namespaces_event = (struct perf_namespaces_event){ .task = task, .event_id = { - .header = { - .type = PERF_RECORD_NAMESPACES, - .misc = 0, - .size = sizeof(namespaces_event.event_id), - }, + /* .header */ /* .pid */ /* .tid */ .nr_namespaces = NR_NAMESPACES, @@ -9615,18 +9600,19 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) struct perf_cgroup_event *cgroup_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - u16 header_size = cgroup_event->event_id.header.size; int ret; + u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size; if (!perf_event_cgroup_match(event)) return; - perf_event_header__init_id(&cgroup_event->event_id.header, - &sample, event); + perf_event_header__init(&cgroup_event->event_id.header, &sample, + PERF_RECORD_CGROUP, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, cgroup_event->event_id.header.size); if (ret) - goto out; + return; perf_output_put(&handle, cgroup_event->event_id); __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); @@ -9634,8 +9620,6 @@ static void perf_event_cgroup_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - cgroup_event->event_id.header.size = header_size; } static void perf_event_cgroup(struct cgroup *cgrp) @@ -9650,11 +9634,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) cgroup_event = (struct perf_cgroup_event){ .event_id = { - .header = { - .type = PERF_RECORD_CGROUP, - .misc = 0, - .size = sizeof(cgroup_event.event_id), - }, .id = cgroup_id(cgrp), }, }; @@ -9677,7 +9656,6 @@ static void perf_event_cgroup(struct cgroup *cgrp) while (!IS_ALIGNED(size, sizeof(u64))) cgroup_event.path[size++] = '\0'; - cgroup_event.event_id.header.size += size; cgroup_event.path_size = size; perf_iterate_sb(perf_event_cgroup_output, @@ -9733,38 +9711,40 @@ static void perf_event_mmap_output(struct perf_event *event, struct perf_mmap_event *mmap_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int size = mmap_event->event_id.header.size; - u32 type = mmap_event->event_id.header.type; - bool use_build_id; + int size = sizeof(mmap_event->event_id) + mmap_event->file_size; + u32 type = PERF_RECORD_MMAP; + u16 misc = PERF_RECORD_MISC_USER; + bool use_build_id = false; int ret; if (!perf_event_mmap_match(event, data)) return; if (event->attr.mmap2) { - mmap_event->event_id.header.type = PERF_RECORD_MMAP2; - mmap_event->event_id.header.size += sizeof(mmap_event->maj); - mmap_event->event_id.header.size += sizeof(mmap_event->min); - mmap_event->event_id.header.size += sizeof(mmap_event->ino); - mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); - mmap_event->event_id.header.size += sizeof(mmap_event->prot); - mmap_event->event_id.header.size += sizeof(mmap_event->flags); - } - - perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); + type = PERF_RECORD_MMAP2; + size += sizeof(mmap_event->maj); + size += sizeof(mmap_event->min); + size += sizeof(mmap_event->ino); + size += sizeof(mmap_event->ino_generation); + size += sizeof(mmap_event->prot); + size += sizeof(mmap_event->flags); + use_build_id = event->attr.build_id && mmap_event->build_id_size; + if (use_build_id) + misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; + } + if (!(mmap_event->vma->vm_flags & VM_EXEC)) + misc |= PERF_RECORD_MISC_MMAP_DATA; + + perf_event_header__init(&mmap_event->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, mmap_event->event_id.header.size); if (ret) - goto out; + return; mmap_event->event_id.pid = perf_event_pid(event, current); mmap_event->event_id.tid = perf_event_tid(event, current); - use_build_id = event->attr.build_id && mmap_event->build_id_size; - - if (event->attr.mmap2 && use_build_id) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; - perf_output_put(&handle, mmap_event->event_id); if (event->attr.mmap2) { @@ -9789,9 +9769,6 @@ static void perf_event_mmap_output(struct perf_event *event, perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - mmap_event->event_id.header.size = size; - mmap_event->event_id.header.type = type; } static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) @@ -9887,11 +9864,6 @@ got_name: mmap_event->prot = prot; mmap_event->flags = flags; - if (!(vma->vm_flags & VM_EXEC)) - mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; - - mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; - if (atomic_read(&nr_build_id_events)) build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); @@ -10011,11 +9983,7 @@ void perf_event_mmap(struct vm_area_struct *vma) /* .file_name */ /* .file_size */ .event_id = { - .header = { - .type = PERF_RECORD_MMAP, - .misc = PERF_RECORD_MISC_USER, - /* .size */ - }, + /* .header */ /* .pid */ /* .tid */ .start = vma->vm_start, @@ -10045,18 +10013,15 @@ void perf_event_aux_event(struct perf_event *event, unsigned long head, u64 size; u64 flags; } rec = { - .header = { - .type = PERF_RECORD_AUX, - .misc = 0, - .size = sizeof(rec), - }, .offset = head, .size = size, .flags = flags, }; int ret; - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10081,15 +10046,14 @@ void perf_log_lost_samples(struct perf_event *event, u64 lost) struct perf_event_header header; u64 lost; } lost_samples_event = { - .header = { - .type = PERF_RECORD_LOST_SAMPLES, - .misc = 0, - .size = sizeof(lost_samples_event), - }, .lost = lost, }; - perf_event_header__init_id(&lost_samples_event.header, &sample, event); + perf_event_header__init(&lost_samples_event.header, &sample, + PERF_RECORD_LOST_SAMPLES, + /* misc= */ 0, + sizeof(lost_samples_event), + event); ret = perf_output_begin(&handle, &sample, event, lost_samples_event.header.size); @@ -10114,6 +10078,8 @@ struct perf_switch_event { u32 next_prev_pid; u32 next_prev_tid; } event_id; + bool sched_in; + bool preempt; }; static int perf_event_switch_match(struct perf_event *event) @@ -10126,6 +10092,9 @@ static void perf_event_switch_output(struct perf_event *event, void *data) struct perf_switch_event *se = data; struct perf_output_handle handle; struct perf_sample_data sample; + u32 type; + u16 misc; + u16 size; int ret; if (!perf_event_switch_match(event)) @@ -10133,18 +10102,22 @@ static void perf_event_switch_output(struct perf_event *event, void *data) /* Only CPU-wide events are allowed to see next/prev pid/tid */ if (event->ctx->task) { - se->event_id.header.type = PERF_RECORD_SWITCH; - se->event_id.header.size = sizeof(se->event_id.header); + type = PERF_RECORD_SWITCH; + size = sizeof(se->event_id.header); } else { - se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; - se->event_id.header.size = sizeof(se->event_id); + type = PERF_RECORD_SWITCH_CPU_WIDE; + size = sizeof(se->event_id); se->event_id.next_prev_pid = perf_event_pid(event, se->next_prev); se->event_id.next_prev_tid = perf_event_tid(event, se->next_prev); } + misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT; + if (se->preempt) + misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - perf_event_header__init_id(&se->event_id.header, &sample, event); + perf_event_header__init(&se->event_id.header, &sample, + type, misc, size, event); ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); if (ret) @@ -10170,22 +10143,11 @@ static void perf_event_switch(struct task_struct *task, switch_event = (struct perf_switch_event){ .task = task, .next_prev = next_prev, - .event_id = { - .header = { - /* .type */ - .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, - /* .size */ - }, - /* .next_prev_pid */ - /* .next_prev_tid */ - }, + /* .event_id */ + .sched_in = sched_in, + .preempt = !sched_in && task_is_runnable(task), }; - if (!sched_in && task_is_runnable(task)) { - switch_event.event_id.header.misc |= - PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; - } - perf_iterate_sb(perf_event_switch_output, &switch_event, NULL); } @@ -10205,20 +10167,17 @@ static void perf_log_throttle(struct perf_event *event, int enable) u64 id; u64 stream_id; } throttle_event = { - .header = { - .type = PERF_RECORD_THROTTLE, - .misc = 0, - .size = sizeof(throttle_event), - }, .time = perf_event_clock(event), .id = primary_event_id(event), .stream_id = event->id, }; - if (enable) - throttle_event.header.type = PERF_RECORD_UNTHROTTLE; - - perf_event_header__init_id(&throttle_event.header, &sample, event); + perf_event_header__init(&throttle_event.header, &sample, + enable ? PERF_RECORD_UNTHROTTLE + : PERF_RECORD_THROTTLE, + /* misc= */ 0, + sizeof(throttle_event), + event); ret = perf_output_begin(&handle, &sample, event, throttle_event.header.size); @@ -10257,12 +10216,14 @@ static void perf_event_ksymbol_output(struct perf_event *event, void *data) struct perf_output_handle handle; struct perf_sample_data sample; int ret; + u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len; if (!perf_event_ksymbol_match(event)) return; - perf_event_header__init_id(&ksymbol_event->event_id.header, - &sample, event); + perf_event_header__init(&ksymbol_event->event_id.header, &sample, + PERF_RECORD_KSYMBOL, /* misc= */ 0, size, + event); ret = perf_output_begin(&handle, &sample, event, ksymbol_event->event_id.header.size); if (ret) @@ -10303,11 +10264,6 @@ void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, .name = name, .name_len = name_len, .event_id = { - .header = { - .type = PERF_RECORD_KSYMBOL, - .size = sizeof(ksymbol_event.event_id) + - name_len, - }, .addr = addr, .len = len, .ksym_type = ksym_type, @@ -10351,8 +10307,11 @@ static void perf_event_bpf_output(struct perf_event *event, void *data) if (!perf_event_bpf_match(event)) return; - perf_event_header__init_id(&bpf_event->event_id.header, - &sample, event); + perf_event_header__init(&bpf_event->event_id.header, &sample, + PERF_RECORD_BPF_EVENT, + /* misc= */ 0, + sizeof(bpf_event->event_id), + event); ret = perf_output_begin(&handle, &sample, event, bpf_event->event_id.header.size); if (ret) @@ -10408,10 +10367,6 @@ void perf_event_bpf_event(struct bpf_prog *prog, bpf_event = (struct perf_bpf_event){ .prog = prog, .event_id = { - .header = { - .type = PERF_RECORD_BPF_EVENT, - .size = sizeof(bpf_event.event_id), - }, .type = type, .flags = flags, .id = prog->aux->id, @@ -10439,18 +10394,23 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) struct perf_callchain_deferred_event *deferred_event = data; struct perf_output_handle handle; struct perf_sample_data sample; - int ret, size = deferred_event->event.header.size; + int ret; + u16 size = sizeof(deferred_event->event) + (deferred_event->trace->nr * sizeof(u64)); if (!event->attr.defer_output) return; /* XXX do we really need sample_id_all for this ??? */ - perf_event_header__init_id(&deferred_event->event.header, &sample, event); + perf_event_header__init(&deferred_event->event.header, &sample, + PERF_RECORD_CALLCHAIN_DEFERRED, + PERF_RECORD_MISC_USER, + size, + event); ret = perf_output_begin(&handle, &sample, event, deferred_event->event.header.size); if (ret) - goto out; + return; perf_output_put(&handle, deferred_event->event); for (int i = 0; i < deferred_event->trace->nr; i++) { @@ -10460,8 +10420,6 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data) perf_event__output_id_sample(event, &handle, &sample); perf_output_end(&handle); -out: - deferred_event->event.header.size = size; } static void perf_unwind_deferred_callback(struct unwind_work *work, @@ -10470,12 +10428,6 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_callchain_deferred_event deferred_event = { .trace = trace, .event = { - .header = { - .type = PERF_RECORD_CALLCHAIN_DEFERRED, - .misc = PERF_RECORD_MISC_USER, - .size = sizeof(deferred_event.event) + - (trace->nr * sizeof(u64)), - }, .cookie = cookie, .nr = trace->nr, }, @@ -10487,7 +10439,8 @@ static void perf_unwind_deferred_callback(struct unwind_work *work, struct perf_text_poke_event { const void *old_bytes; const void *new_bytes; - size_t pad; + u16 tot; + u16 pad; u16 old_len; u16 new_len; @@ -10508,13 +10461,18 @@ static void perf_event_text_poke_output(struct perf_event *event, void *data) struct perf_text_poke_event *text_poke_event = data; struct perf_output_handle handle; struct perf_sample_data sample; + u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad; u64 padding = 0; int ret; if (!perf_event_text_poke_match(event)) return; - perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); + perf_event_header__init(&text_poke_event->event_id.header, &sample, + PERF_RECORD_TEXT_POKE, + PERF_RECORD_MISC_KERNEL, + size, + event); ret = perf_output_begin(&handle, &sample, event, text_poke_event->event_id.header.size); @@ -10552,15 +10510,11 @@ void perf_event_text_poke(const void *addr, const void *old_bytes, text_poke_event = (struct perf_text_poke_event){ .old_bytes = old_bytes, .new_bytes = new_bytes, + .tot = tot, .pad = pad, .old_len = old_len, .new_len = new_len, .event_id = { - .header = { - .type = PERF_RECORD_TEXT_POKE, - .misc = PERF_RECORD_MISC_KERNEL, - .size = sizeof(text_poke_event.event_id) + tot + pad, - }, .addr = (unsigned long)addr, }, }; @@ -10591,13 +10545,12 @@ static void perf_log_itrace_start(struct perf_event *event) event->attach_state & PERF_ATTACH_ITRACE) return; - rec.header.type = PERF_RECORD_ITRACE_START; - rec.header.misc = 0; - rec.header.size = sizeof(rec); rec.pid = perf_event_pid(event, current); rec.tid = perf_event_tid(event, current); - perf_event_header__init_id(&rec.header, &sample, event); + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec), + event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) @@ -10622,12 +10575,10 @@ void perf_report_aux_output_id(struct perf_event *event, u64 hw_id) if (event->parent) event = event->parent; - rec.header.type = PERF_RECORD_AUX_OUTPUT_HW_ID; - rec.header.misc = 0; - rec.header.size = sizeof(rec); - rec.hw_id = hw_id; - - perf_event_header__init_id(&rec.header, &sample, event); + rec.hw_id = hw_id; + perf_event_header__init(&rec.header, &sample, + PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0, + sizeof(rec), event); ret = perf_output_begin(&handle, &sample, event, rec.header.size); if (ret) diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 1b1ffe0533e5..13bd42e5493d 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -246,14 +246,13 @@ __perf_output_begin(struct perf_output_handle *handle, handle->size = (1UL << page_shift) - offset; if (unlikely(have_lost)) { - lost_event.header.size = sizeof(lost_event); - lost_event.header.type = PERF_RECORD_LOST; - lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); /* XXX mostly redundant; @data is already fully initializes */ - perf_event_header__init_id(&lost_event.header, data, event); + perf_event_header__init(&lost_event.header, data, + PERF_RECORD_LOST, /* misc= */ 0, + sizeof(lost_event), event); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, data); } -- cgit v1.2.3 From 6c95ca52f27855dd2fb74131c8d4e8325d2af7de Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Fri, 2 Oct 2026 09:27:36 +0200 Subject: tty: add missing driver flag kernel-doc colon A recent change adding a new TTY driver flag left out a colon required for well-formed kernel-doc. Fixes: 8df07fe93573 ("tty: fix saved termios reset race") Reported-by: Randy Dunlap Link: https://lore.kernel.org/ec3a09d6-4ba4-41b8-abb1-b59e265f4532@infradead.org Signed-off-by: Johan Hovold Link: https://patch.msgid.link/20261002072736.2063004-1-johan@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/tty_driver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 62e459c5d3ae..40ae3bb13e11 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -74,7 +74,7 @@ struct serial_struct; * Do not create workqueue when tty_register_driver(). Whenever set, flip * buffer workqueue can be set by tty_port_link_wq() for every port. * - * @TTY_DRIVER_RESET_SAVED_TERMIOS + * @TTY_DRIVER_RESET_SAVED_TERMIOS: * Reset any saved termios settings on device registration when reusing a * minor number. Must only be set by drivers that guarantee that the minor * number is no longer in use. -- cgit v1.2.3