From 9d30872a4c09c2e8a97084ef1fe6913f6bf9e34d Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 23 Feb 2026 15:44:17 -0500 Subject: hazptr: Implement Hazard Pointers This API provides existence guarantees of objects through Hazard Pointers [1] (hazptr). Its main benefit over RCU is that it allows fast reclaim of HP-protected pointers without needing to wait for a grace period. This implementation has 4 statically allocated hazard pointer slots per cpu for the fast path, and relies on a on-stack backup slot allocated by the hazard pointer user as fallback in case no per-cpu slot is available. It integrates with the scheduler to migrate per-CPU slots to the backup slot on context switch. This ensures that the per-CPU slots won't be used by blocked or preempted tasks holding on hazard pointers for a long time. References: [1]: M. M. Michael, "Hazard pointers: safe memory reclamation for lock-free objects," in IEEE Transactions on Parallel and Distributed Systems, vol. 15, no. 6, pp. 491-504, June 2004 Link: https://lpc.events/event/19/contributions/2082/ Link: https://lore.kernel.org/lkml/j3scdl5iymjlxavomgc6u5ndg3svhab6ga23dr36o4f5mt333w@7xslvq6b6hmv/ Link: https://lpc.events/event/18/contributions/1731/ Signed-off-by: Mathieu Desnoyers Cc: Nicholas Piggin Cc: Michael Ellerman Cc: Greg Kroah-Hartman Cc: Sebastian Andrzej Siewior Cc: "Paul E. McKenney" Cc: Will Deacon Cc: Peter Zijlstra Cc: Boqun Feng Cc: Alan Stern Cc: John Stultz Cc: Linus Torvalds Cc: Andrew Morton Cc: Frederic Weisbecker Cc: Joel Fernandes Cc: Josh Triplett Cc: Uladzislau Rezki Cc: Steven Rostedt Cc: Lai Jiangshan Cc: Zqiang Cc: Ingo Molnar Cc: Waiman Long Cc: Mark Rutland Cc: Thomas Gleixner Cc: Vlastimil Babka Cc: maged.michael@gmail.com Cc: Mateusz Guzik Cc: Jonas Oberhauser Cc: Cc: Cc: Signed-off-by: Paul E. McKenney --- include/linux/hazptr.h | 197 ++++++++++++++++++++++++++++++++++++++++ init/main.c | 2 + kernel/Makefile | 2 +- kernel/hazptr.c | 242 +++++++++++++++++++++++++++++++++++++++++++++++++ kernel/sched/core.c | 2 + 5 files changed, 444 insertions(+), 1 deletion(-) create mode 100644 include/linux/hazptr.h create mode 100644 kernel/hazptr.c diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h new file mode 100644 index 000000000000..b121f7779cda --- /dev/null +++ b/include/linux/hazptr.h @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// +// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers + +#ifndef _LINUX_HAZPTR_H +#define _LINUX_HAZPTR_H + +/* + * hazptr: Hazard Pointers + * + * This API provides existence guarantees of objects through hazard + * pointers. + * + * Its main benefit over RCU is that it allows fast reclaim of + * HP-protected pointers without needing to wait for a grace period. + * + * References: + * + * [1]: M. M. Michael, "Hazard pointers: safe memory reclamation for + * lock-free objects," in IEEE Transactions on Parallel and + * Distributed Systems, vol. 15, no. 6, pp. 491-504, June 2004 + */ + +#include +#include +#include +#include + +/* 4 slots (each sizeof(hazptr_slot_item)) fit in a single 64-byte cache line. */ +#define NR_HAZPTR_PERCPU_SLOTS 4 +#define HAZPTR_WILDCARD ((void *) 0x1UL) + +/* + * Hazard pointer slot. + */ +struct hazptr_slot { + void *addr; +}; + +struct hazptr_overflow_list; + +struct hazptr_backup_slot { + struct hlist_node overflow_node; + struct hazptr_slot slot; + /* Overflow list where the backup slot is added. */ + struct hazptr_overflow_list *overflow_list; +}; + +struct hazptr_ctx { + struct hazptr_slot *slot; + /* Backup slot in case all per-CPU slots are used. */ + struct hazptr_backup_slot backup_slot; + struct hlist_node preempt_node; +}; + +struct hazptr_slot_ctx { + struct hazptr_ctx *ctx; +}; + +struct hazptr_slot_item { + struct hazptr_slot slot; + struct hazptr_slot_ctx ctx; +}; + +struct hazptr_percpu_slots { + struct hazptr_slot_item items[NR_HAZPTR_PERCPU_SLOTS]; +} ____cacheline_aligned; + +DECLARE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots); + +void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p); + +/* + * hazptr_synchronize: Wait until @addr is released from all slots. + * + * Wait to observe that each slot contains a value that differs from + * @addr before returning. + * Should be called from preemptible context. + */ +void hazptr_synchronize(void *addr); + +/* + * hazptr_chain_backup_slot: Chain backup slot into overflow list. + * + * Set backup slot address to @addr, and chain it into the overflow + * list. + */ +struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx); + +/* + * hazptr_unchain_backup_slot: Unchain backup slot from overflow list. + */ +void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx); + +static inline +bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot) +{ + return slot == &ctx->backup_slot.slot; +} + +static inline +void hazptr_note_context_switch(void) +{ + struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots); + unsigned int idx; + + for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) { + struct hazptr_slot_item *item = &percpu_slots->items[idx]; + struct hazptr_slot *slot = &item->slot, *backup_slot; + struct hazptr_ctx *ctx; + + if (!slot->addr) + continue; + ctx = item->ctx.ctx; + backup_slot = hazptr_chain_backup_slot(ctx); + /* + * Move hazard pointer from the per-CPU slot to the + * backup slot. This requires hazard pointer + * synchronize to iterate on per-CPU slots with + * load-acquire before iterating on the overflow list. + */ + WRITE_ONCE(backup_slot->addr, slot->addr); + /* + * store-release orders store to backup slot addr before + * store to per-CPU slot addr. + */ + smp_store_release(&slot->addr, NULL); + /* Use the backup slot for context. */ + ctx->slot = backup_slot; + } +} + +/* + * hazptr_acquire: Load pointer at address and protect with hazard pointer. + * + * Load @addr_p, and protect the loaded pointer with hazard pointer. + * When using hazptr_acquire from interrupt handlers, the acquired slots + * need to be released before returning from the interrupt handler. + * + * Returns a non-NULL protected address if the loaded pointer is non-NULL. + * Returns NULL if the loaded pointer is NULL. + * + * On success the protected hazptr slot is stored in @ctx->slot. + */ +static inline +void *hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p) +{ + struct hazptr_percpu_slots *percpu_slots; + struct hazptr_slot_item *slot_item; + struct hazptr_slot *slot; + void *addr; + + guard(preempt)(); + percpu_slots = this_cpu_ptr(&hazptr_percpu_slots); + slot_item = &percpu_slots->items[0]; + slot = &slot_item->slot; + if (unlikely(slot->addr)) + return __hazptr_acquire(ctx, addr_p); + WRITE_ONCE(slot->addr, HAZPTR_WILDCARD); /* Store B */ + + /* Memory ordering: Store B before Load A. */ + smp_mb(); + + /* + * Load @addr_p after storing wildcard to the hazard pointer slot. + */ + addr = READ_ONCE(*addr_p); /* Load A */ + + /* + * We don't care about ordering of Store C. It will simply + * replace the wildcard by a more specific address. If addr is + * NULL, we simply store NULL into the slot. + */ + WRITE_ONCE(slot->addr, addr); /* Store C */ + slot_item->ctx.ctx = ctx; + ctx->slot = slot; + return addr; +} + +/* Release the protected hazard pointer from @slot. */ +static inline +void hazptr_release(struct hazptr_ctx *ctx, void *addr) +{ + struct hazptr_slot *slot; + + if (!addr) + return; + guard(preempt)(); + slot = ctx->slot; + smp_store_release(&slot->addr, NULL); + if (unlikely(hazptr_slot_is_backup(ctx, slot))) + hazptr_unchain_backup_slot(ctx); +} + +void hazptr_init(void); + +#endif /* _LINUX_HAZPTR_H */ diff --git a/init/main.c b/init/main.c index e363232b428b..e41f3ae436d4 100644 --- a/init/main.c +++ b/init/main.c @@ -107,6 +107,7 @@ #include #include #include +#include #include #include @@ -1065,6 +1066,7 @@ void start_kernel(void) workqueue_init_early(); rcu_init(); + hazptr_init(); kvfree_rcu_init(); /* Trace events are available after this */ diff --git a/kernel/Makefile b/kernel/Makefile index 1e1a31673577..8961c8660d0d 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -7,7 +7,7 @@ obj-y = fork.o exec_domain.o exec_state.o panic.o \ cpu.o exit.o softirq.o resource.o \ sysctl.o capability.o ptrace.o user.o \ signal.o sys.o umh.o workqueue.o pid.o task_work.o \ - extable.o params.o \ + extable.o params.o hazptr.o \ kthread.o sys_ni.o nsproxy.o nstree.o nscommon.o \ notifier.o ksysfs.o cred.o reboot.o \ async.o range.o smpboot.o ucount.o regset.o ksyms_common.o diff --git a/kernel/hazptr.c b/kernel/hazptr.c new file mode 100644 index 000000000000..a9d3d68a1525 --- /dev/null +++ b/kernel/hazptr.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// +// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers + +/* + * hazptr: Hazard Pointers + */ + +#include +#include +#include +#include +#include +#include + +struct hazptr_overflow_list { + raw_spinlock_t lock; /* Lock protecting overflow list and list generation. */ + struct hlist_head head; /* Overflow list head. */ + uint64_t gen; /* Overflow list generation. */ +}; + +/* + * Flip between two lists to guarantee list scan forward progress even + * with frequent generation counter increments. The list additions are + * always done on a different list than the one used for scan. The scan + * successively iterates on both lists. Therefore, only list removals + * can cause the iteration to retry, and the number of removals is + * limited to the number of list elements. + */ +struct hazptr_overflow_list_flip { + struct mutex lock; /* Mutex protecting add_idx from concurrent updates. */ + unsigned int add_idx; /* Index of current flip-list to add to. */ + struct hazptr_overflow_list array[2]; +}; + +static DEFINE_PER_CPU(struct hazptr_overflow_list_flip, percpu_overflow_list_flip); + +DEFINE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots); +EXPORT_PER_CPU_SYMBOL_GPL(hazptr_percpu_slots); + +static +struct hazptr_slot *hazptr_get_free_percpu_slot(struct hazptr_ctx *ctx) +{ + struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots); + unsigned int idx; + + for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) { + struct hazptr_slot_item *item = &percpu_slots->items[idx]; + struct hazptr_slot *slot = &item->slot; + + if (!slot->addr) { + item->ctx.ctx = ctx; + return slot; + } + } + /* All slots are in use. */ + return NULL; +} + +/* + * Hazard pointer acquire slow path. + * Called with preemption disabled. + */ +void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p) +{ + struct hazptr_slot *slot = hazptr_get_free_percpu_slot(ctx); + void *addr; + + /* + * If all the per-CPU slots are already in use, fallback + * to the backup slot. + */ + if (unlikely(!slot)) + slot = hazptr_chain_backup_slot(ctx); + WRITE_ONCE(slot->addr, HAZPTR_WILDCARD); /* Store B */ + + /* Memory ordering: Store B before Load A. */ + smp_mb(); + + /* + * Load @addr_p after storing wildcard to the hazard pointer slot. + */ + addr = READ_ONCE(*addr_p); /* Load A */ + + /* + * We don't care about ordering of Store C. It will simply + * replace the wildcard by a more specific address. If addr is + * NULL, we simply store NULL into the slot. + */ + WRITE_ONCE(slot->addr, addr); /* Store C */ + ctx->slot = slot; + if (!addr && hazptr_slot_is_backup(ctx, slot)) + hazptr_unchain_backup_slot(ctx); + return addr; +} +EXPORT_SYMBOL_GPL(__hazptr_acquire); + +/* + * Perform piecewise iteration on overflow list waiting until "addr" is + * not present. Raw spinlock is released and taken between each list + * item and busy loop iteration. The overflow list generation is checked + * each time the lock is taken to validate that the list has not changed + * before resuming iteration or busy wait. If the generation has + * changed, retry the entire list traversal. + */ +static +void hazptr_synchronize_overflow_list(struct hazptr_overflow_list *overflow_list, void *addr) +{ + struct hazptr_backup_slot *backup_slot; + uint64_t snapshot_gen; + unsigned long flags; + + raw_spin_lock_irqsave(&overflow_list->lock, flags); +retry: + snapshot_gen = overflow_list->gen; + hlist_for_each_entry(backup_slot, &overflow_list->head, overflow_node) { + /* Busy-wait if node is found. */ + for (;;) { + void *load_addr = smp_load_acquire(&backup_slot->slot.addr); /* Load B */ + + if (load_addr != addr && load_addr != HAZPTR_WILDCARD) + break; + raw_spin_unlock_irqrestore(&overflow_list->lock, flags); + cpu_relax(); + raw_spin_lock_irqsave(&overflow_list->lock, flags); + if (overflow_list->gen != snapshot_gen) + goto retry; + } + raw_spin_unlock_irqrestore(&overflow_list->lock, flags); + /* + * Release raw spinlock, validate generation after + * re-acquiring the lock. + */ + raw_spin_lock_irqsave(&overflow_list->lock, flags); + if (overflow_list->gen != snapshot_gen) + goto retry; + } + raw_spin_unlock_irqrestore(&overflow_list->lock, flags); +} + +static +void hazptr_synchronize_cpu_slots(int cpu, void *addr) +{ + struct hazptr_percpu_slots *percpu_slots = per_cpu_ptr(&hazptr_percpu_slots, cpu); + unsigned int idx; + + for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) { + struct hazptr_slot_item *item = &percpu_slots->items[idx]; + + /* Busy-wait if node is found. */ + smp_cond_load_acquire(&item->slot.addr, VAL != addr && VAL != HAZPTR_WILDCARD); /* Load B */ + } +} + +/* + * hazptr_synchronize: Wait until @addr is released from all slots. + * + * Wait to observe that each slot contains a value that differs from + * @addr before returning. + * Should be called from preemptible context. + */ +void hazptr_synchronize(void *addr) +{ + int cpu; + + /* + * Busy-wait should only be done from preemptible context. + */ + lockdep_assert_preemption_enabled(); + + /* + * Store A precedes hazptr_scan(): it unpublishes addr (sets it to + * NULL or to a different value), and thus hides it from hazard + * pointer readers. + */ + if (!addr) + return; + /* Memory ordering: Store A before Load B. */ + smp_mb(); + /* Scan all CPUs slots. */ + for_each_possible_cpu(cpu) { + struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu); + unsigned int scan_idx; + + /* Scan CPU slots. */ + hazptr_synchronize_cpu_slots(cpu, addr); + + /* + * Scan backup slots in percpu overflow lists. + * Forward progress is guaranteed by scanning one list + * while new elements are added into the other list. + */ + guard(mutex)(&overflow_list_flip->lock); + scan_idx = overflow_list_flip->add_idx ^ 1; + hazptr_synchronize_overflow_list(&overflow_list_flip->array[scan_idx], addr); + /* Flip current list. */ + WRITE_ONCE(overflow_list_flip->add_idx, scan_idx); + hazptr_synchronize_overflow_list(&overflow_list_flip->array[scan_idx ^ 1], addr); + } +} +EXPORT_SYMBOL_GPL(hazptr_synchronize); + +struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx) +{ + struct hazptr_overflow_list_flip *overflow_list_flip = this_cpu_ptr(&percpu_overflow_list_flip); + unsigned int list_idx = READ_ONCE(overflow_list_flip->add_idx); + struct hazptr_overflow_list *overflow_list = &overflow_list_flip->array[list_idx]; + struct hazptr_slot *slot = &ctx->backup_slot.slot; + + slot->addr = NULL; + guard(raw_spinlock_irqsave)(&overflow_list->lock); + overflow_list->gen++; + hlist_add_head(&ctx->backup_slot.overflow_node, &overflow_list->head); + ctx->backup_slot.overflow_list = overflow_list; + return slot; +} +EXPORT_SYMBOL_GPL(hazptr_chain_backup_slot); + +void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx) +{ + struct hazptr_overflow_list *overflow_list = ctx->backup_slot.overflow_list; + + guard(raw_spinlock_irqsave)(&overflow_list->lock); + overflow_list->gen++; + hlist_del(&ctx->backup_slot.overflow_node); +} +EXPORT_SYMBOL_GPL(hazptr_unchain_backup_slot); + +void __init hazptr_init(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu); + + mutex_init(&overflow_list_flip->lock); + for (int i = 0; i < 2; i++) { + raw_spin_lock_init(&overflow_list_flip->array[i].lock); + INIT_HLIST_HEAD(&overflow_list_flip->array[i].head); + } + } +} diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..e3fb70c38436 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -7087,6 +7088,7 @@ static void __sched notrace __schedule(int sched_mode) local_irq_disable(); rcu_note_context_switch(preempt); migrate_disable_switch(rq, prev); + hazptr_note_context_switch(); /* * Make sure that signal_pending_state()->signal_pending() below -- cgit v1.2.3 From 2aaef4a4ff3387a2603d8f7ca9fee573d1d9ec73 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 23 Feb 2026 15:44:18 -0500 Subject: hazptr: Add refscale test Add the refscale test for hazptr to measure the reader side performance. Co-developed-by: Boqun Feng Signed-off-by: Boqun Feng Signed-off-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Cc: Cc: --- kernel/rcu/refscale.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c index a2d9d75d88a1..2320b3fedec7 100644 --- a/kernel/rcu/refscale.c +++ b/kernel/rcu/refscale.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1200,6 +1201,47 @@ static const struct ref_scale_ops typesafe_seqlock_ops = { .name = "typesafe_seqlock" }; +static void ref_hazptr_read_section(const int nloops) +{ + static void *ref_hazptr_read_section_ptr = ref_hazptr_read_section; + int i; + + for (i = nloops; i >= 0; i--) { + struct hazptr_ctx ctx; + void *addr; + + addr = hazptr_acquire(&ctx, &ref_hazptr_read_section_ptr); + hazptr_release(&ctx, addr); + } +} + +static void ref_hazptr_delay_section(const int nloops, const int udl, const int ndl) +{ + static void *ref_hazptr_delay_section_ptr = ref_hazptr_delay_section; + int i; + + for (i = nloops; i >= 0; i--) { + struct hazptr_ctx ctx; + void *addr; + + addr = hazptr_acquire(&ctx, &ref_hazptr_delay_section_ptr); + un_delay(udl, ndl); + hazptr_release(&ctx, addr); + } +} + +static bool ref_hazptr_init(void) +{ + return true; +} + +static const struct ref_scale_ops hazptr_ops = { + .init = ref_hazptr_init, + .readsection = ref_hazptr_read_section, + .delaysection = ref_hazptr_delay_section, + .name = "hazptr" +}; + static void rcu_scale_one_reader(void) { if (readdelay <= 0) @@ -1504,6 +1546,7 @@ ref_scale_init(void) &sched_clock_ops, &clock_ops, &jiffies_ops, &preempt_ops, &bh_ops, &irq_ops, &irqsave_ops, &typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops, + &hazptr_ops, }; if (!torture_init_begin(scale_type, verbose)) -- cgit v1.2.3 From ee7331b9cbbe412a63ad8e26a182dc6b94cce89c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 24 Mar 2026 14:35:45 -0700 Subject: torture: Add a hazptrtorture.c torture test This commit adds a torture test for hazard pointers. The initial version simply acquires and releases the hazard pointers without nesting, each from within the context of a single task. [ paulmck: Apply kernel test robot feedback. ] Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- include/linux/torture.h | 2 +- kernel/rcu/Kconfig.debug | 12 + kernel/rcu/Makefile | 1 + kernel/rcu/hazptrtorture.c | 681 +++++++++++++++++++++ kernel/rcu/update.c | 3 +- tools/testing/selftests/rcutorture/bin/kvm.sh | 6 +- .../selftests/rcutorture/configs/hazptr/CFLIST | 2 + .../selftests/rcutorture/configs/hazptr/CFcommon | 2 + .../selftests/rcutorture/configs/hazptr/NOPREEMPT | 17 + .../selftests/rcutorture/configs/hazptr/PREEMPT | 14 + .../rcutorture/configs/hazptr/ver_functions.sh | 40 ++ 11 files changed, 775 insertions(+), 5 deletions(-) create mode 100644 kernel/rcu/hazptrtorture.c create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/CFLIST create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/CFcommon create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh diff --git a/include/linux/torture.h b/include/linux/torture.h index c9b47d138302..66d2d444428a 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -131,7 +131,7 @@ void _torture_stop_kthread(char *m, struct task_struct **tp); #endif void torture_sched_set_normal(struct task_struct *t, int nice); -#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST) +#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST) long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn); #endif diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug index e078e988773d..fe64356f0088 100644 --- a/kernel/rcu/Kconfig.debug +++ b/kernel/rcu/Kconfig.debug @@ -113,6 +113,18 @@ config RCU_REF_SCALE_TEST Say M if you want to build it as a module instead. Say N if you are unsure. +config HAZPTR_TORTURE_TEST + tristate "Torture tests for hazard pointers" + depends on DEBUG_KERNEL + select TORTURE_TEST + default n + help + This option provides in-kernel hazard-pointer stress tests. + + Say Y here if you want hazard-pointer testing built into the kernel. + Say M if you want to build them as a module instead. + Say N if you are unsure. + config RCU_CPU_STALL_TIMEOUT int "RCU CPU stall timeout in seconds" depends on RCU_STALL_COMMON diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile index 0cfb009a99b9..478ef2237fee 100644 --- a/kernel/rcu/Makefile +++ b/kernel/rcu/Makefile @@ -10,6 +10,7 @@ endif obj-y += update.o sync.o obj-$(CONFIG_TREE_SRCU) += srcutree.o obj-$(CONFIG_TINY_SRCU) += srcutiny.o +obj-$(CONFIG_HAZPTR_TORTURE_TEST) += hazptrtorture.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o obj-$(CONFIG_RCU_SCALE_TEST) += rcuscale.o obj-$(CONFIG_RCU_REF_SCALE_TEST) += refscale.o diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c new file mode 100644 index 000000000000..c656ba980100 --- /dev/null +++ b/kernel/rcu/hazptrtorture.c @@ -0,0 +1,681 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Hazard-pointer module-based torture test facility + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * + * Author: Paul E. McKenney + */ + +#define pr_fmt(fmt) fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rcu.h" + +MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Paul E. McKenney "); + +torture_param(int, irqreader, 1, "Allow hazard-pointer readers from irq handlers"); +torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); +torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); +torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); +torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); +torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)"); +torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); +torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); +torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); +torture_param(int, stutter, 5, "Number of seconds to run/halt test"); +torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); + +static char *torture_type = "hazptr"; +module_param(torture_type, charp, 0444); +MODULE_PARM_DESC(torture_type, "Type of hazard pointers to torture (hazptr, ...)"); + +static int nrealreaders; +static struct task_struct *writer_task; +static struct task_struct *preempt_task; +static struct task_struct **reader_tasks; +static struct task_struct *stats_task; + +#define HAZPTR_TORTURE_PIPE_LEN 10 + +// Update-side data structure used to check RCU readers. +struct hazptr_torture { + void *obj_hazptr; + int htort_pipe_count; + struct list_head htort_free; +}; + +static LIST_HEAD(hazptr_torture_freelist); +static struct hazptr_torture *hazptr_torture_current; +static unsigned long hazptr_torture_current_version; +static struct hazptr_torture hazptr_tortures[10 * HAZPTR_TORTURE_PIPE_LEN]; +static DEFINE_SPINLOCK(hazptr_torture_lock); +static DEFINE_PER_CPU(long [HAZPTR_TORTURE_PIPE_LEN + 1], hazptr_torture_count); +static atomic_t hazptr_torture_wcount[HAZPTR_TORTURE_PIPE_LEN + 1]; +static atomic_t n_hazptr_torture_alloc; +static atomic_t n_hazptr_torture_alloc_fail; +static atomic_t n_hazptr_torture_free; +static atomic_t n_hazptr_torture_error; +static struct list_head hazptr_torture_removed; + +static int hazptr_torture_writer_state; +#define HTWS_FIXED_DELAY 0 +#define HTWS_DELAY 1 +#define HTWS_REPLACE 2 +#define HTWS_SYNC 3 +#define HTWS_STUTTER 4 +#define HTWS_STOPPING 5 +static const char * const hazptr_torture_writer_state_names[] = { + "HTWS_FIXED_DELAY", + "HTWS_DELAY", + "HTWS_REPLACE", + "HTWS_SYNC", + "HTWS_STUTTER", + "HTWS_STOPPING", +}; + +static const char *hazptr_torture_writer_state_getname(void) +{ + unsigned int i = READ_ONCE(hazptr_torture_writer_state); + + if (i >= ARRAY_SIZE(hazptr_torture_writer_state_names)) + return "???"; + return hazptr_torture_writer_state_names[i]; +} + +/* + * Allocate an element from the hazptr_tortures pool. + */ +static struct hazptr_torture *hazptr_torture_alloc(void) +{ + struct list_head *p; + + spin_lock_bh(&hazptr_torture_lock); + if (list_empty(&hazptr_torture_freelist)) { + atomic_inc(&n_hazptr_torture_alloc_fail); + spin_unlock_bh(&hazptr_torture_lock); + return NULL; + } + atomic_inc(&n_hazptr_torture_alloc); + p = hazptr_torture_freelist.next; + list_del_init(p); + spin_unlock_bh(&hazptr_torture_lock); + return container_of(p, struct hazptr_torture, htort_free); +} + +/* + * Free an element to the hazptr_tortures pool. + */ +static void +hazptr_torture_free(struct hazptr_torture *p) +{ + atomic_inc(&n_hazptr_torture_free); + spin_lock_bh(&hazptr_torture_lock); + list_add_tail(&p->htort_free, &hazptr_torture_freelist); + spin_unlock_bh(&hazptr_torture_lock); +} + +/* + * Update object in the pipe. This should be invoked after a suitable time. + */ +static bool +hazptr_torture_pipe_update_one(struct hazptr_torture *rp) +{ + int i; + + i = rp->htort_pipe_count; + if (i > HAZPTR_TORTURE_PIPE_LEN) + i = HAZPTR_TORTURE_PIPE_LEN; + atomic_inc(&hazptr_torture_wcount[i]); + WRITE_ONCE(rp->htort_pipe_count, i + 1); + ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count); + if (i + 1 >= HAZPTR_TORTURE_PIPE_LEN) + return true; + return false; +} + +/* + * Update all callbacks in the pipe each time period. + */ +static void +hazptr_torture_pipe_update(struct hazptr_torture *old_rp) +{ + struct hazptr_torture *rp; + struct hazptr_torture *rp1; + + if (old_rp) + list_add(&old_rp->htort_free, &hazptr_torture_removed); + list_for_each_entry_safe(rp, rp1, &hazptr_torture_removed, htort_free) { + if (hazptr_torture_pipe_update_one(rp)) { + list_del(&rp->htort_free); + hazptr_torture_free(rp); + } + } +} + +/* + * Operations vector for selecting different types of tests. + */ + +struct hazptr_torture_ops { + void (*init)(void); + void (*cleanup)(void); + struct hazptr_torture *((*readlock)(struct hazptr_ctx **hcpp)); + void (*read_delay)(struct torture_random_state *rrsp); + void (*readunlock)(struct hazptr_ctx *hcp, struct hazptr_torture *htp); + void (*sync)(void *htp); + int irq_capable; + int onstack_ctx; + const char *name; +}; + +static struct hazptr_torture_ops *cur_ops; + +/* + * Definitions for hazard-pointer torture testing. + */ + +static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx **hcpp) +{ + struct hazptr_ctx *hcp = kmalloc(sizeof(*hcp), GFP_KERNEL); + + *hcpp = hcp; + if (!hcp) + return NULL; + return (struct hazptr_torture *)hazptr_acquire(hcp, (void *)&hazptr_torture_current); +} + +static void hazptr_read_delay(struct torture_random_state *rrsp) +{ + const unsigned long shortdelay_us = 200; + unsigned long longdelay_ms = 300; + + // We want a short delay sometimes to make a reader delay the grace + // period, and we want a long delay occasionally to trigger + // force_quiescent_state. + + if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { + if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) + longdelay_ms = 5; /* Avoid triggering BH limits. */ + mdelay(longdelay_ms); + } + if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) + udelay(shortdelay_us); + if (!preempt_count() && !(torture_random(rrsp) % (nrealreaders * 500))) + torture_preempt_schedule(); /* QS only if preemptible. */ +} + +static void hazptr_torture_read_unlock(struct hazptr_ctx *hcp, struct hazptr_torture *htp) +{ + if (hcp) { + hazptr_release(hcp, htp); + if (cur_ops->onstack_ctx) + kfree(hcp); + } +} + +static void hazptr_sync_torture_init(void) +{ + INIT_LIST_HEAD(&hazptr_torture_removed); +} + +static struct hazptr_torture_ops hazptr_ops = { + .init = hazptr_sync_torture_init, + .readlock = hazptr_torture_read_lock, + .read_delay = hazptr_read_delay, + .readunlock = hazptr_torture_read_unlock, + .sync = hazptr_synchronize, + .irq_capable = 1, + .onstack_ctx = 1, + .name = "hazptr" +}; + +/* + * Hazard-pointer torture writer kthread. Repeatedly substitutes a new + * structure for that pointed to by hazptr_torture_current, freeing the + * old structure after a series of timeouts (the "pipeline"). + */ +static int +hazptr_torture_writer(void *arg) +{ + bool booting_still = false; + int i; + unsigned long j; + int oldnice = task_nice(current); + struct hazptr_torture *rp; + struct hazptr_torture *old_rp; + static DEFINE_TORTURE_RANDOM(rand); + bool stutter_waited; + + VERBOSE_TOROUT_STRING("hazptr_torture_writer task started"); + // If the system is still booting, let it finish. + j = jiffies; + while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) { + booting_still = true; + schedule_timeout_interruptible(HZ); + } + if (booting_still) + pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", + torture_type, jiffies - j); + + do { + hazptr_torture_writer_state = HTWS_FIXED_DELAY; + torture_hrtimeout_us(500, 1000, &rand); + rp = hazptr_torture_alloc(); + if (rp == NULL) + continue; + rp->htort_pipe_count = 0; + ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count); + hazptr_torture_writer_state = HTWS_DELAY; + udelay(torture_random(&rand) & 0x3ff); + hazptr_torture_writer_state = HTWS_REPLACE; + old_rp = READ_ONCE(hazptr_torture_current); + smp_store_release(&hazptr_torture_current, rp); // Publish new structure. + smp_wmb(); /* Mods to old_rp must follow smp_store_release() */ + if (old_rp) { + i = old_rp->htort_pipe_count; + if (i > HAZPTR_TORTURE_PIPE_LEN) + i = HAZPTR_TORTURE_PIPE_LEN; + atomic_inc(&hazptr_torture_wcount[i]); + WRITE_ONCE(old_rp->htort_pipe_count, + old_rp->htort_pipe_count + 1); + ASSERT_EXCLUSIVE_WRITER(old_rp->htort_pipe_count); + + hazptr_torture_writer_state = HTWS_SYNC; + cur_ops->sync((void *)old_rp); + hazptr_torture_pipe_update(old_rp); + } + + WRITE_ONCE(hazptr_torture_current_version, hazptr_torture_current_version + 1); + hazptr_torture_writer_state = HTWS_STUTTER; + stutter_waited = stutter_wait("hazptr_torture_writer"); + if (stutter_waited && !torture_must_stop()) + for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++) + if (list_empty(&hazptr_tortures[i].htort_free) && + READ_ONCE(hazptr_torture_current) != &hazptr_tortures[i]) { + tracing_off(); + WARN(1, "%s: htort_pipe_count: %d\n", __func__, hazptr_tortures[i].htort_pipe_count); + rcu_ftrace_dump(DUMP_ALL); + break; + } + if (stutter_waited) + sched_set_normal(current, oldnice); + } while (!torture_must_stop()); + hazptr_torture_current = NULL; // Let stats task know that we are done. + hazptr_torture_writer_state = HTWS_STOPPING; + torture_kthread_stopping("hazptr_torture_writer"); + return 0; +} + +/* + * Hazard-pointer torture reader kthread. Repeatedly dereferences + * hazptr_torture_current, incrementing the corresponding element of the + * pipeline array. The counter in the element should never be greater + * than 1, otherwise, the hazard-pointer implementation is broken. + */ +static int hazptr_torture_reader(void *arg) +{ + struct hazptr_ctx *hcp; + struct hazptr_torture *htp; + unsigned long lastsleep = jiffies; + long myid = (long)arg; + int mynumonline = myid; + int pipe_count; + DEFINE_TORTURE_RANDOM(rand); + + VERBOSE_TOROUT_STRING("hazptr_torture_reader task started"); + set_user_nice(current, MAX_NICE); + do { + htp = cur_ops->readlock(&hcp); + if (!htp) { + schedule_timeout_interruptible(HZ / 10); + continue; + } + if (time_after(jiffies, lastsleep) && !torture_must_stop()) { + torture_hrtimeout_us(500, 1000, &rand); + lastsleep = jiffies + 10; + } + cur_ops->read_delay(&rand); + preempt_disable(); + pipe_count = READ_ONCE(htp->htort_pipe_count); + if (pipe_count > HAZPTR_TORTURE_PIPE_LEN) { + // Should not happen in a correct RCU implementation, + // happens quite often for torture_type=busted. + pipe_count = HAZPTR_TORTURE_PIPE_LEN; + } + if (pipe_count > 1) + rcu_ftrace_dump(DUMP_ALL); + __this_cpu_inc(hazptr_torture_count[pipe_count]); + preempt_enable(); + cur_ops->readunlock(hcp, htp); + while (!torture_must_stop() && + (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) + schedule_timeout_interruptible(HZ / 5); + stutter_wait("hazptr_torture_reader"); + } while (!torture_must_stop()); + torture_kthread_stopping("hazptr_torture_reader"); + return 0; +} + +/* + * Print torture statistics. Caller must ensure that there is only one + * call to this function at a given time!!! This is normally accomplished + * by relying on the module system to only have one copy of the module + * loaded, and then by giving the hazptr_torture_stats kthread full control + * (or the init/cleanup functions when hazptr_torture_stats thread is + * not running). + */ +static void +hazptr_torture_stats_print(void) +{ + const char *cp = hazptr_torture_writer_state_getname(); + int cpu; + int i; + long pipesummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 }; + long batchsummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 }; + struct hazptr_torture *rtcp; + static unsigned long rtcv_snap = ULONG_MAX; + static bool splatted; + struct task_struct *wtp; + + for_each_possible_cpu(cpu) + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + pipesummary[i] += READ_ONCE(per_cpu(hazptr_torture_count, cpu)[i]); + for (i = HAZPTR_TORTURE_PIPE_LEN; i >= 0; i--) { + if (pipesummary[i] != 0) + break; + } // The value of variable "i" is used later, so don't clobber it! + + pr_alert("%s%s ", torture_type, TORTURE_FLAG); + rtcp = READ_ONCE(hazptr_torture_current); + pr_cont("rtc: %p %s: %lu %s tfle: %d rta: %d rtaf: %d rtf: %d ", + rtcp, + rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER", + hazptr_torture_current_version, + cp, + list_empty(&hazptr_torture_freelist), + atomic_read(&n_hazptr_torture_alloc), + atomic_read(&n_hazptr_torture_alloc_fail), + atomic_read(&n_hazptr_torture_free)); + torture_onoff_stats(); + + pr_alert("%s%s ", torture_type, TORTURE_FLAG); + if (i > 1) { + pr_cont("%s", "!!! "); + atomic_inc(&n_hazptr_torture_error); + WARN_ON_ONCE(i > 1); // Too-short grace period + } + pr_cont("Reader Pipe: "); + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + pr_cont(" %ld", pipesummary[i]); + pr_cont("\n"); + + pr_alert("%s%s ", torture_type, TORTURE_FLAG); + pr_cont("Reader Batch: "); + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + pr_cont(" %ld", batchsummary[i]); + pr_cont("\n"); + + pr_alert("%s%s ", torture_type, TORTURE_FLAG); + pr_cont("Free-Block Circulation: "); + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + pr_cont(" %d", atomic_read(&hazptr_torture_wcount[i])); + pr_cont("\n"); + + if (rtcv_snap == hazptr_torture_current_version && + READ_ONCE(hazptr_torture_current) && + rcu_inkernel_boot_has_ended()) { + int __maybe_unused flags = 0; + unsigned long __maybe_unused gp_seq = 0; + + wtp = READ_ONCE(writer_task); + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + hazptr_torture_writer_state_getname(), + hazptr_torture_writer_state, gp_seq, flags, + wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? -1 : (int)task_cpu(wtp)); + if (!splatted && wtp) { + sched_show_task(wtp); + splatted = true; + } + rcu_ftrace_dump(DUMP_ALL); + } + rtcv_snap = hazptr_torture_current_version; +} + +/* + * Periodically prints torture statistics, if periodic statistics printing + * was specified via the stat_interval module parameter. + */ +static int +hazptr_torture_stats(void *arg) +{ + VERBOSE_TOROUT_STRING("hazptr_torture_stats task started"); + do { + schedule_timeout_interruptible(stat_interval * HZ); + hazptr_torture_stats_print(); + torture_shutdown_absorb("hazptr_torture_stats"); + } while (!torture_must_stop()); + torture_kthread_stopping("hazptr_torture_stats"); + return 0; +} + +static void +hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char *tag) +{ + pr_alert("%s" TORTURE_FLAG + "--- %s: nreaders=%d " + "stat_interval=%d verbose=%d " + "shuffle_interval=%d stutter=%d irqreader=%d " + "onoff_interval=%d onoff_holdoff=%d\n", + torture_type, tag, nrealreaders, + stat_interval, verbose, + shuffle_interval, stutter, irqreader, + onoff_interval, onoff_holdoff); +} + +// Randomly preempt online CPUs. +static int hazptr_torture_preempt(void *unused) +{ + int cpu = -1; + DEFINE_TORTURE_RANDOM(rand); + + schedule_timeout_idle(onoff_holdoff * HZ); + do { + // Wait for preempt_interval ms with up to 100us fuzz. + torture_hrtimeout_ms(preempt_interval, 100, &rand); + // Select online CPU. + cpu = cpumask_next(cpu, cpu_online_mask); + if (cpu >= nr_cpu_ids) + cpu = cpumask_next(-1, cpu_online_mask); + WARN_ON_ONCE(cpu >= nr_cpu_ids); + // Move to that CPU, if can't do so, retry later. + if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) + continue; + // Preempt at high-ish priority, then reset to normal. + sched_set_fifo(current); + torture_sched_setaffinity(current->pid, cpu_present_mask, true); + mdelay(preempt_duration); + sched_set_normal(current, 0); + stutter_wait("hazptr_torture_preempt"); + } while (!torture_must_stop()); + torture_kthread_stopping("hazptr_torture_preempt"); + return 0; +} + +static void +hazptr_torture_cleanup(void) +{ + int i; + + if (torture_cleanup_begin()) + return; + if (!cur_ops) { + torture_cleanup_end(); + return; + } + + torture_stop_kthread(hazptr_torture_preempt, preempt_task); + torture_stop_kthread(hazptr_torture_writer, writer_task); + + if (reader_tasks) { + for (i = 0; i < nrealreaders; i++) + torture_stop_kthread(hazptr_torture_reader, + reader_tasks[i]); + kfree(reader_tasks); + reader_tasks = NULL; + } + + torture_stop_kthread(hazptr_torture_stats, stats_task); + + /* Do torture-type-specific cleanup operations. */ + if (cur_ops->cleanup != NULL) + cur_ops->cleanup(); + + hazptr_torture_stats_print(); /* -After- the stats thread is stopped! */ + if (atomic_read(&n_hazptr_torture_error)) + hazptr_torture_print_module_parms(cur_ops, "End of test: FAILURE"); + else if (torture_onoff_failures()) + hazptr_torture_print_module_parms(cur_ops, "End of test: HAZPTR_HOTPLUG"); + else + hazptr_torture_print_module_parms(cur_ops, "End of test: SUCCESS"); + torture_cleanup_end(); +} + +static int __init hazptr_torture_init(void) +{ + long i; + int cpu; + int firsterr = 0; + static struct hazptr_torture_ops *torture_ops[] = { &hazptr_ops, }; + + if (!torture_init_begin(torture_type, verbose)) + return -EBUSY; + + /* Process args and tell the world that the torturer is on the job. */ + for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { + cur_ops = torture_ops[i]; + if (strcmp(torture_type, cur_ops->name) == 0) + break; + } + if (i == ARRAY_SIZE(torture_ops)) { + pr_alert("hazptr-torture: invalid torture type: \"%s\"\n", torture_type); + pr_alert("hazptr-torture types:"); + for (i = 0; i < ARRAY_SIZE(torture_ops); i++) + pr_cont(" %s", torture_ops[i]->name); + pr_cont("\n"); + firsterr = -EINVAL; + cur_ops = NULL; + goto unwind; + } + + if (cur_ops->init) + cur_ops->init(); + + if (nreaders >= 0) { + nrealreaders = nreaders; + } else { + nrealreaders = num_online_cpus() - 2 - nreaders; + if (nrealreaders <= 0) + nrealreaders = 1; + } + hazptr_torture_print_module_parms(cur_ops, "Start of test"); + + /* Set up the freelist. */ + INIT_LIST_HEAD(&hazptr_torture_freelist); + for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++) + list_add_tail(&hazptr_tortures[i].htort_free, &hazptr_torture_freelist); + + /* Initialize the statistics so that each run gets its own numbers. */ + + hazptr_torture_current = NULL; + hazptr_torture_current_version = 0; + atomic_set(&n_hazptr_torture_alloc, 0); + atomic_set(&n_hazptr_torture_alloc_fail, 0); + atomic_set(&n_hazptr_torture_free, 0); + atomic_set(&n_hazptr_torture_error, 0); + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + atomic_set(&hazptr_torture_wcount[i], 0); + for_each_possible_cpu(cpu) { + for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++) + per_cpu(hazptr_torture_count, cpu)[i] = 0; + } + + /* Start up the kthreads. */ + + reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); + for (i = 0; i < nrealreaders; i++) { + firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i, + reader_tasks[i]); + if (torture_init_error(firsterr)) + goto unwind; + } + + firsterr = torture_create_kthread(hazptr_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + + firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); + if (torture_init_error(firsterr)) + goto unwind; + + if (stat_interval > 0) { + firsterr = torture_create_kthread(hazptr_torture_stats, NULL, stats_task); + if (torture_init_error(firsterr)) + goto unwind; + } + if (shuffle_interval > 0) { + firsterr = torture_shuffle_init(shuffle_interval * HZ); + if (torture_init_error(firsterr)) + goto unwind; + } + if (stutter < 0) + stutter = 0; + if (stutter) { + int t; + + t = stutter * HZ; + firsterr = torture_stutter_init(stutter * HZ, t); + if (torture_init_error(firsterr)) + goto unwind; + } + firsterr = torture_shutdown_init(shutdown_secs, hazptr_torture_cleanup); + if (torture_init_error(firsterr)) + goto unwind; + if (preempt_duration > 0) { + firsterr = torture_create_kthread(hazptr_torture_preempt, NULL, preempt_task); + if (torture_init_error(firsterr)) + goto unwind; + } + + torture_init_end(); + return 0; + +unwind: + torture_init_end(); + hazptr_torture_cleanup(); + if (shutdown_secs) { + WARN_ON(!IS_MODULE(CONFIG_HAZPTR_TORTURE_TEST)); + kernel_power_off(); + } + return firsterr; +} + +module_init(hazptr_torture_init); +module_exit(hazptr_torture_cleanup); diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index b62735a67884..2a778b8ab4ad 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -44,6 +44,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS @@ -525,7 +526,7 @@ EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read); do { } while (0) #endif -#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST) +#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST) /* Get rcutorture access to sched_setaffinity(). */ long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn) { diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh index 65b04b832733..32c319997638 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm.sh @@ -91,7 +91,7 @@ usage () { echo " --remote" echo " --results absolute-pathname" echo " --shutdown-grace seconds" - echo " --torture lock|rcu|rcuscale|refscale|scf|X*" + echo " --torture hazptr|lock|rcu|rcuscale|refscale|scf|X*" echo " --trust-make" exit 1 } @@ -256,9 +256,9 @@ do shift ;; --torture) - checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\|rcuscale\|refscale\|scf\|X.*\)$' '^--' + checkarg --torture "(suite name)" "$#" "$2" '^\(hazptr\|lock\|rcu\|rcuscale\|refscale\|scf\|X.*\)$' '^--' TORTURE_SUITE=$2 - TORTURE_MOD="`echo $TORTURE_SUITE | sed -e 's/^\(lock\|rcu\|scf\)$/\1torture/'`" + TORTURE_MOD="`echo $TORTURE_SUITE | sed -e 's/^\(hazptr\|lock\|rcu\|scf\)$/\1torture/'`" shift if test "$TORTURE_SUITE" = rcuscale || test "$TORTURE_SUITE" = refscale then diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST b/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST new file mode 100644 index 000000000000..4d62eb4a39f9 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST @@ -0,0 +1,2 @@ +NOPREEMPT +PREEMPT diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon b/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon new file mode 100644 index 000000000000..c440d227007d --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon @@ -0,0 +1,2 @@ +CONFIG_HAZPTR_TORTURE_TEST=y +CONFIG_PRINTK_TIME=y diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT new file mode 100644 index 000000000000..e2da430abe4d --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT @@ -0,0 +1,17 @@ +CONFIG_SMP=y +CONFIG_NR_CPUS=16 +CONFIG_PREEMPT_LAZY=y +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT=n +CONFIG_PREEMPT_DYNAMIC=n +CONFIG_HZ_PERIODIC=n +CONFIG_NO_HZ_IDLE=y +CONFIG_NO_HZ_FULL=n +CONFIG_HOTPLUG_CPU=y +CONFIG_SUSPEND=n +CONFIG_HIBERNATION=n +CONFIG_DEBUG_LOCK_ALLOC=n +CONFIG_PROVE_LOCKING=n +CONFIG_KPROBES=n +CONFIG_FTRACE=n diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT new file mode 100644 index 000000000000..b8ea4364b20b --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT @@ -0,0 +1,14 @@ +CONFIG_SMP=y +CONFIG_NR_CPUS=16 +CONFIG_PREEMPT_NONE=n +CONFIG_PREEMPT_VOLUNTARY=n +CONFIG_PREEMPT=y +CONFIG_HZ_PERIODIC=n +CONFIG_NO_HZ_IDLE=y +CONFIG_NO_HZ_FULL=n +CONFIG_HOTPLUG_CPU=y +CONFIG_SUSPEND=n +CONFIG_HIBERNATION=n +CONFIG_DEBUG_LOCK_ALLOC=n +CONFIG_PROVE_LOCKING=n +CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh new file mode 100644 index 000000000000..a28ea2f292e4 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0+ +# +# Kernel-version-dependent shell functions for the rest of the scripts. +# +# Claude created this file, and I quote: +# +# "I created [this file] modeled on the lock torture +# version. It defines per_version_boot_params to pass +# hazptrtorture.shutdown_secs=$3, hazptrtorture.stat_interval=15, +# hazptrtorture.verbose=1, and optional CPU-hotplug parameters to +# the kernel command line." +# +# I therefore kept locktorture's ver_functions.sh copyright notice: +# +# Copyright (C) Meta Platforms, Inc. and affiliates. +# +# Authors: Paul E. McKenney + +# hazptrtorture_param_onoff bootparam-string config-file +# +# Adds onoff hazptrtorture module parameters to kernels having it. +hazptrtorture_param_onoff () { + if ! bootparam_hotplug_cpu "$1" && configfrag_hotplug_cpu "$2" + then + echo CPU-hotplug kernel, adding hazptrtorture onoff. 1>&2 + echo hazptrtorture.onoff_interval=3 hazptrtorture.onoff_holdoff=30 + fi +} + +# per_version_boot_params bootparam-string config-file seconds +# +# Adds per-version torture-module parameters to kernels supporting them. +per_version_boot_params () { + echo `hazptrtorture_param_onoff "$1" "$2"` \ + hazptrtorture.stat_interval=15 \ + hazptrtorture.shutdown_secs=$3 \ + hazptrtorture.verbose=1 \ + $1 +} -- cgit v1.2.3 From f0b95b94649dd3848daf03fbd44796f278e8b802 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 27 Mar 2026 03:04:59 -0700 Subject: hazptrtorture: Add testing of on-stack hazptr_ctx structures This commit adds a test using on-stack hazptr_ctx structures, in contrast with the per-CPU structures used by the initial test. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 48 +++++++++++++++++----- .../rcutorture/configs/hazptr/NOPREEMPT.boot | 1 + 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index c656ba980100..22a0610d0dbd 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -30,7 +30,6 @@ MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul E. McKenney "); -torture_param(int, irqreader, 1, "Allow hazard-pointer readers from irq handlers"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); @@ -188,7 +187,8 @@ struct hazptr_torture_ops { static struct hazptr_torture_ops *cur_ops; /* - * Definitions for hazard-pointer torture testing. + * Definitions for hazard-pointer torture testing using per-CPU hazptr_ctx + * structures. */ static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx **hcpp) @@ -242,10 +242,33 @@ static struct hazptr_torture_ops hazptr_ops = { .readunlock = hazptr_torture_read_unlock, .sync = hazptr_synchronize, .irq_capable = 1, - .onstack_ctx = 1, .name = "hazptr" }; +/* + * Definitions for hazard-pointer torture testing using on-stack + * hazptr_ctx structures. + */ + +static struct hazptr_torture *hazptr_torture_read_lock_stack(struct hazptr_ctx **hcpp) +{ + struct hazptr_torture *htp; + + htp = (struct hazptr_torture *)hazptr_acquire(*hcpp, (void *)&hazptr_torture_current); + return htp; +} + +static struct hazptr_torture_ops hazptr_stack_ops = { + .init = hazptr_sync_torture_init, + .readlock = hazptr_torture_read_lock_stack, + .read_delay = hazptr_read_delay, + .readunlock = hazptr_torture_read_unlock, + .sync = hazptr_synchronize, + .irq_capable = 1, + .onstack_ctx = 1, + .name = "hazptr-stack" +}; + /* * Hazard-pointer torture writer kthread. Repeatedly substitutes a new * structure for that pointed to by hazptr_torture_current, freeing the @@ -331,7 +354,8 @@ hazptr_torture_writer(void *arg) */ static int hazptr_torture_reader(void *arg) { - struct hazptr_ctx *hcp; + struct hazptr_ctx hc; + struct hazptr_ctx *hcp = &hc; struct hazptr_torture *htp; unsigned long lastsleep = jiffies; long myid = (long)arg; @@ -481,13 +505,15 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char { pr_alert("%s" TORTURE_FLAG "--- %s: nreaders=%d " - "stat_interval=%d verbose=%d " - "shuffle_interval=%d stutter=%d irqreader=%d " - "onoff_interval=%d onoff_holdoff=%d\n", + "onoff_interval=%d onoff_holdoff=%d " + "preempt_duration=%d preempt_interval=%d " + "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " + "verbose=%d\n", torture_type, tag, nrealreaders, - stat_interval, verbose, - shuffle_interval, stutter, irqreader, - onoff_interval, onoff_holdoff); + onoff_interval, onoff_holdoff, + preempt_duration, preempt_interval, + shuffle_interval, shutdown_secs, stat_interval, stutter, + verbose); } // Randomly preempt online CPUs. @@ -563,7 +589,7 @@ static int __init hazptr_torture_init(void) long i; int cpu; int firsterr = 0; - static struct hazptr_torture_ops *torture_ops[] = { &hazptr_ops, }; + static struct hazptr_torture_ops *torture_ops[] = { &hazptr_ops, &hazptr_stack_ops, }; if (!torture_init_begin(torture_type, verbose)) return -EBUSY; diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot new file mode 100644 index 000000000000..1d09a6446080 --- /dev/null +++ b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot @@ -0,0 +1 @@ +hazptrtorture.torture_type=hazptr-stack -- cgit v1.2.3 From 8967123e64836489861830a3ad79de99ca8a8d54 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Apr 2026 13:25:29 -0700 Subject: hazptrtorture: Add microsecond-scale sleep in readers This commit adds a default-disabled reader_sleep_us module parameter that causes the hazard-pointer reader to unconditionally sleep for the specified number of microseconds. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 22a0610d0dbd..9ecb86520551 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -35,6 +35,7 @@ torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)"); +torture_param(int, reader_sleep_us, 0, "Reader sleep duration (us)"); torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles"); torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); @@ -219,6 +220,8 @@ static void hazptr_read_delay(struct torture_random_state *rrsp) udelay(shortdelay_us); if (!preempt_count() && !(torture_random(rrsp) % (nrealreaders * 500))) torture_preempt_schedule(); /* QS only if preemptible. */ + if (reader_sleep_us > 0) + torture_hrtimeout_us(reader_sleep_us, 0, NULL); } static void hazptr_torture_read_unlock(struct hazptr_ctx *hcp, struct hazptr_torture *htp) @@ -507,11 +510,13 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char "--- %s: nreaders=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " + "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", torture_type, tag, nrealreaders, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, + reader_sleep_us, shuffle_interval, shutdown_secs, stat_interval, stutter, verbose); } -- cgit v1.2.3 From a328871fc90ffd138fc892c3bff8bc44f15c0877 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Apr 2026 17:33:53 -0700 Subject: hazptrtorture: Enable system-independent CPU overcommit This commit interprets negative values of the nreaders module parameter as a number of readers per CPU, so that hazptrtorture.nreaders=-5 would spawn five hazard-pointer reader kthreads per CPU. As CPUs go offline, a number of readers determined by the overcommit are idled, so that again when hazptrtorture.nreaders=-5, five readers would be idled for each offline CPU. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 9ecb86520551..e9ba25f156ac 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -362,7 +362,7 @@ static int hazptr_torture_reader(void *arg) struct hazptr_torture *htp; unsigned long lastsleep = jiffies; long myid = (long)arg; - int mynumonline = myid; + int mynumonline = myid % nr_cpu_ids; int pipe_count; DEFINE_TORTURE_RANDOM(rand); @@ -622,7 +622,7 @@ static int __init hazptr_torture_init(void) if (nreaders >= 0) { nrealreaders = nreaders; } else { - nrealreaders = num_online_cpus() - 2 - nreaders; + nrealreaders = num_online_cpus() * -nreaders; if (nrealreaders <= 0) nrealreaders = 1; } -- cgit v1.2.3 From 19774ca9c3ca15b2843a5c05cc055b2087a93e0c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 1 Jun 2026 17:33:14 -0700 Subject: torture: Add a stutter_will_wait() function This commit adds a stutter_will_wait() function that returns true if a call to stutter_wait() at that same time would have waited. Of course, the passage of time means that the return value might become immediately stale, so this should be periodically polled on the one hand, or used only for heuristic purposes on the other. The initial use case for this function is to clean up references to objects that might otherwise be held across the stutter interval, which could result in false-positive failures. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- include/linux/torture.h | 1 + kernel/torture.c | 23 +++++++++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/include/linux/torture.h b/include/linux/torture.h index 66d2d444428a..b8e5d0f3c2d8 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -98,6 +98,7 @@ void torture_shutdown_absorb(const char *title); int torture_shutdown_init(int ssecs, void (*cleanup)(void)); /* Task stuttering, which forces load/no-load transitions. */ +bool stutter_will_wait(void); bool stutter_wait(const char *title); int torture_stutter_init(int s, int sgap); diff --git a/kernel/torture.c b/kernel/torture.c index 77cb3589b19f..bcd2e9c8a263 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -727,6 +727,26 @@ static ktime_t stutter_till_abs_time; static int stutter; static int stutter_gap; +static bool _stutter_will_wait(ktime_t *till_ns) +{ + *till_ns = READ_ONCE(stutter_till_abs_time); + if (*till_ns && ktime_before(ktime_get(), *till_ns)) + return true; + return false; +} + +/* + * Will stutter_wait() actually stutter? The returned result is of + * course immediately stale due to the passage of time. + */ +bool stutter_will_wait(void) +{ + ktime_t till_ns; + + return _stutter_will_wait(&till_ns); +} +EXPORT_SYMBOL_GPL(stutter_will_wait); + /* * Block until the stutter interval ends. This must be called periodically * by all running kthreads that need to be subject to stuttering. @@ -737,8 +757,7 @@ bool stutter_wait(const char *title) ktime_t till_ns; cond_resched_tasks_rcu_qs(); - till_ns = READ_ONCE(stutter_till_abs_time); - if (till_ns && ktime_before(ktime_get(), till_ns)) { + if (_stutter_will_wait(&till_ns)) { torture_hrtimeout_ns(till_ns, 0, HRTIMER_MODE_ABS, NULL); ret = true; } -- cgit v1.2.3 From 488445203f1688ed14d5bc621398137c49bea414 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 2 Jun 2026 11:29:01 -0700 Subject: hazptrtorture: Use mnemonic local variables for context information This commit introduces can_sleep and short_spin local variables to hazptr_read_delay(). The can_sleep variable records whether unconstrained sleeping is feasible, and the short_spin variable checks for the NMI handlers, IRQ handlers, or disabled interrupts/softirqs suggesting short spin times. While in the area, it disables the reader_sleep_us module parameter when invoked where sleeping is not permitted. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index e9ba25f156ac..c0ebabdcc72e 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -204,23 +204,25 @@ static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx **hcpp) static void hazptr_read_delay(struct torture_random_state *rrsp) { + const bool can_sleep = !preempt_count() && !irqs_disabled(); const unsigned long shortdelay_us = 200; unsigned long longdelay_ms = 300; + const bool short_spin = irqs_disabled() || irq_count(); // We want a short delay sometimes to make a reader delay the grace // period, and we want a long delay occasionally to trigger // force_quiescent_state. if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { - if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) + if (short_spin) longdelay_ms = 5; /* Avoid triggering BH limits. */ mdelay(longdelay_ms); } if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us))) udelay(shortdelay_us); - if (!preempt_count() && !(torture_random(rrsp) % (nrealreaders * 500))) + if (can_sleep && !(torture_random(rrsp) % (nrealreaders * 500))) torture_preempt_schedule(); /* QS only if preemptible. */ - if (reader_sleep_us > 0) + if (can_sleep && reader_sleep_us > 0) torture_hrtimeout_us(reader_sleep_us, 0, NULL); } -- cgit v1.2.3 From 8f30a8a9044cd08e01e13be7b5d2b8a97d1832f6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 2 Jun 2026 13:15:37 -0700 Subject: hazptrtorture: Split hazptr_torture_reader_tail() from hazptr_torture_reader() This commit splits a new hazptr_torture_reader_tail() function out of hazptr_torture_reader(). This will allow hazptr_torture_reader() to pass hazard pointers off to other tasks and to various types of handlers, and those hazard pointers can in turn be passed to this new hazptr_torture_reader_tail() function to complete processing. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index c0ebabdcc72e..8672169f7868 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -351,6 +351,31 @@ hazptr_torture_writer(void *arg) return 0; } +/* + * Do the delay, the accounting, and the release. This in intended to + * be invoked from hazptr_torture_reader, but also for hazard pointers + * sent off to interrupt handlers and the like. + */ +static void hazptr_torture_reader_tail(struct hazptr_ctx *hcp, struct hazptr_torture *htp, + struct torture_random_state *trsp) +{ + int pipe_count; + + cur_ops->read_delay(trsp); + preempt_disable(); + pipe_count = READ_ONCE(htp->htort_pipe_count); + if (pipe_count > HAZPTR_TORTURE_PIPE_LEN) { + // Should not happen in a correct hazptr implementation, + // happens quite often for TBD torture_type=busted. + pipe_count = HAZPTR_TORTURE_PIPE_LEN; + } + if (pipe_count > 1) + rcu_ftrace_dump(DUMP_ALL); + __this_cpu_inc(hazptr_torture_count[pipe_count]); + preempt_enable(); + cur_ops->readunlock(hcp, htp); +} + /* * Hazard-pointer torture reader kthread. Repeatedly dereferences * hazptr_torture_current, incrementing the corresponding element of the @@ -365,7 +390,6 @@ static int hazptr_torture_reader(void *arg) unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid % nr_cpu_ids; - int pipe_count; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("hazptr_torture_reader task started"); @@ -373,6 +397,8 @@ static int hazptr_torture_reader(void *arg) do { htp = cur_ops->readlock(&hcp); if (!htp) { + // Still starting up or allocation failure, + // so get out of the way. schedule_timeout_interruptible(HZ / 10); continue; } @@ -380,19 +406,7 @@ static int hazptr_torture_reader(void *arg) torture_hrtimeout_us(500, 1000, &rand); lastsleep = jiffies + 10; } - cur_ops->read_delay(&rand); - preempt_disable(); - pipe_count = READ_ONCE(htp->htort_pipe_count); - if (pipe_count > HAZPTR_TORTURE_PIPE_LEN) { - // Should not happen in a correct RCU implementation, - // happens quite often for torture_type=busted. - pipe_count = HAZPTR_TORTURE_PIPE_LEN; - } - if (pipe_count > 1) - rcu_ftrace_dump(DUMP_ALL); - __this_cpu_inc(hazptr_torture_count[pipe_count]); - preempt_enable(); - cur_ops->readunlock(hcp, htp); + hazptr_torture_reader_tail(hcp, htp, &rand); while (!torture_must_stop() && (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) schedule_timeout_interruptible(HZ / 5); -- cgit v1.2.3 From 94d2e93c222ee251db4c88facffd238ba3adaa28 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 12 Jun 2026 10:26:45 -0700 Subject: hazptrtorture: Add kthread to release deferred hazard pointers This commit adds a kthread to release deferred hazard pointers, which will be used to test acquiring a hazard pointer in one task and releasing it in another. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 8672169f7868..97663f22b36f 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -30,6 +30,8 @@ MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul E. McKenney "); +torture_param(int, kthread_do_pending_ms, -1, + "Delay between cleanups for deferred hazard pointers (ms), zero to disable"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); @@ -50,6 +52,7 @@ static int nrealreaders; static struct task_struct *writer_task; static struct task_struct *preempt_task; static struct task_struct **reader_tasks; +static struct task_struct *do_pending_task; static struct task_struct *stats_task; #define HAZPTR_TORTURE_PIPE_LEN 10 @@ -74,6 +77,14 @@ static atomic_t n_hazptr_torture_free; static atomic_t n_hazptr_torture_error; static struct list_head hazptr_torture_removed; +// State for a deferred (AKA pending) hazard pointer +struct hazptr_pending { + struct llist_node hpp_node; + struct hazptr_ctx hpp_hc; + struct hazptr_torture *hpp_htp; +}; +static DEFINE_PER_CPU(struct llist_head, hazptr_pending); + static int hazptr_torture_writer_state; #define HTWS_FIXED_DELAY 0 #define HTWS_DELAY 1 @@ -416,6 +427,76 @@ static int hazptr_torture_reader(void *arg) return 0; } +/* + * Release the specified CPU's set of deferred/pending hazard pointers. + */ +static void hazptr_torture_do_one_pending(int cpu, struct torture_random_state *trsp) +{ + struct hazptr_pending *hppp; + struct hazptr_pending *hppp1; + struct llist_head *llhp; + struct llist_node *llnp; + + llhp = per_cpu_ptr(&hazptr_pending, cpu); + llnp = llist_del_all(llhp); + if (!llnp) + return; + llist_for_each_entry_safe(hppp, hppp1, llnp, hpp_node) { + hazptr_torture_reader_tail(&hppp->hpp_hc, hppp->hpp_htp, trsp); + kfree(hppp); + } +} + +/* + * Hazard-pointer release of deferred/pending hazard pointers. + */ +static int hazptr_torture_do_pending(void *arg) +{ + int cpu = 0; + DEFINE_TORTURE_RANDOM(rand); + + VERBOSE_TOROUT_STRING("hazptr_torture_do_pending task started"); + do { + if (stutter_will_wait()) { + for_each_possible_cpu(cpu) + hazptr_torture_do_one_pending(cpu, &rand); + } else { + cpu = cpumask_next_wrap(cpu, cpu_possible_mask); + hazptr_torture_do_one_pending(cpu, &rand); + } + if (torture_must_stop()) + torture_hrtimeout_ms(kthread_do_pending_ms, USEC_PER_MSEC, &rand); + // Omit stutter_wait() because this function needs to do cleanup. + } while (!torture_must_stop()); + torture_kthread_stopping("hazptr_torture_do_pending"); + return 0; +} + +/* + * Spawn hazptr_torture_do_pending() if there is something for it to do. + */ +static int hazptr_torture_do_pending_init(void) +{ + if (kthread_do_pending_ms == -1) + kthread_do_pending_ms = cur_ops->onstack_ctx ? 0 : 3; + if (kthread_do_pending_ms < 0) { + pr_alert("Cannot have negative kthread_do_pending_ms, disabling deferral.\n"); + goto err_out; + } + if (cur_ops->onstack_ctx && kthread_do_pending_ms) { + pr_alert("Cannot defer onstack hazptr_ctx, disabling deferral.\n"); + goto err_out; + } + if (!kthread_do_pending_ms) + return 0; + return torture_create_kthread(hazptr_torture_do_pending, NULL, do_pending_task); + +err_out: + WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST)); + kthread_do_pending_ms = 0; + return 0; +} + /* * Print torture statistics. Caller must ensure that there is only one * call to this function at a given time!!! This is normally accomplished @@ -524,12 +605,14 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char { pr_alert("%s" TORTURE_FLAG "--- %s: nreaders=%d " + "kthread_do_pending_ms=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", torture_type, tag, nrealreaders, + kthread_do_pending_ms, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, reader_sleep_us, @@ -578,6 +661,7 @@ hazptr_torture_cleanup(void) return; } + torture_stop_kthread(hazptr_torture_do_pending, do_pending_task); torture_stop_kthread(hazptr_torture_preempt, preempt_task); torture_stop_kthread(hazptr_torture_writer, writer_task); @@ -666,6 +750,12 @@ static int __init hazptr_torture_init(void) /* Start up the kthreads. */ + // This must be before the readers in order to set up the module + // parameters used by the readers. + firsterr = hazptr_torture_do_pending_init(); + if (torture_init_error(firsterr)) + goto unwind; + reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); for (i = 0; i < nrealreaders; i++) { firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i, -- cgit v1.2.3 From 9b08a09f6d073bebd38e7be08781b0a4758c924e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 12 Jun 2026 12:50:11 -0700 Subject: hazptrtorture: Defer release of hazard pointers This commit creates the defer_modulus module parameter, so that the releases of one out of defer_modulus hazard-pointer acquisitions will be deferred. This parameter defaults to -1, which results in a value of 1000*nr_cpu_ids to be used. This parameter must be zero for hazptr_torture runs that set cur_ops->onstack_ctx, because otherwise we would get on-stack data races. It must also be zero when the kthread_do_pending_ms module parameter is zero. Setting the defer_modulus module parameter to a value less than -1 will result in disabling hazard-pointer deferral. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 87 ++++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 97663f22b36f..ff253fe35ca2 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -30,6 +30,7 @@ MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul E. McKenney "); +torture_param(int, defer_modulus, -1, "Defer once per specified # of hazptr ops, zero to disable"); torture_param(int, kthread_do_pending_ms, -1, "Delay between cleanups for deferred hazard pointers (ms), zero to disable"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); @@ -187,7 +188,7 @@ hazptr_torture_pipe_update(struct hazptr_torture *old_rp) struct hazptr_torture_ops { void (*init)(void); void (*cleanup)(void); - struct hazptr_torture *((*readlock)(struct hazptr_ctx **hcpp)); + struct hazptr_torture *((*readlock)(struct hazptr_ctx *hcpp)); void (*read_delay)(struct torture_random_state *rrsp); void (*readunlock)(struct hazptr_ctx *hcp, struct hazptr_torture *htp); void (*sync)(void *htp); @@ -203,14 +204,12 @@ static struct hazptr_torture_ops *cur_ops; * structures. */ -static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx **hcpp) +static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx *hcpp) { - struct hazptr_ctx *hcp = kmalloc(sizeof(*hcp), GFP_KERNEL); + struct hazptr_torture *htp; - *hcpp = hcp; - if (!hcp) - return NULL; - return (struct hazptr_torture *)hazptr_acquire(hcp, (void *)&hazptr_torture_current); + htp = (struct hazptr_torture *)hazptr_acquire(hcpp, (void *)&hazptr_torture_current); + return htp; } static void hazptr_read_delay(struct torture_random_state *rrsp) @@ -241,8 +240,6 @@ static void hazptr_torture_read_unlock(struct hazptr_ctx *hcp, struct hazptr_tor { if (hcp) { hazptr_release(hcp, htp); - if (cur_ops->onstack_ctx) - kfree(hcp); } } @@ -266,17 +263,9 @@ static struct hazptr_torture_ops hazptr_ops = { * hazptr_ctx structures. */ -static struct hazptr_torture *hazptr_torture_read_lock_stack(struct hazptr_ctx **hcpp) -{ - struct hazptr_torture *htp; - - htp = (struct hazptr_torture *)hazptr_acquire(*hcpp, (void *)&hazptr_torture_current); - return htp; -} - static struct hazptr_torture_ops hazptr_stack_ops = { .init = hazptr_sync_torture_init, - .readlock = hazptr_torture_read_lock_stack, + .readlock = hazptr_torture_read_lock, .read_delay = hazptr_read_delay, .readunlock = hazptr_torture_read_unlock, .sync = hazptr_synchronize, @@ -387,6 +376,20 @@ static void hazptr_torture_reader_tail(struct hazptr_ctx *hcp, struct hazptr_tor cur_ops->readunlock(hcp, htp); } +/* + * Defer the specified hazard pointer to some other context. + */ +static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_random_state *trsp) +{ + int cpu = torture_random(trsp) % nr_cpu_ids; + struct llist_head *llhp; + + guard(preempt)(); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); + llhp = per_cpu_ptr(&hazptr_pending, cpu); + llist_add(&hppp->hpp_node, llhp); +} + /* * Hazard-pointer torture reader kthread. Repeatedly dereferences * hazptr_torture_current, incrementing the corresponding element of the @@ -395,9 +398,9 @@ static void hazptr_torture_reader_tail(struct hazptr_ctx *hcp, struct hazptr_tor */ static int hazptr_torture_reader(void *arg) { - struct hazptr_ctx hc; - struct hazptr_ctx *hcp = &hc; - struct hazptr_torture *htp; + bool can_defer = !cur_ops->onstack_ctx && kthread_do_pending_ms && defer_modulus; + struct hazptr_pending hpp; + struct hazptr_pending *hppp = cur_ops->onstack_ctx ? &hpp : NULL; unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid % nr_cpu_ids; @@ -406,10 +409,17 @@ static int hazptr_torture_reader(void *arg) VERBOSE_TOROUT_STRING("hazptr_torture_reader task started"); set_user_nice(current, MAX_NICE); do { - htp = cur_ops->readlock(&hcp); - if (!htp) { - // Still starting up or allocation failure, - // so get out of the way. + if (!hppp) { + hppp = kmalloc_obj(*hppp, GFP_KERNEL); + if (!hppp) { + // Allocation failure, so get out of the way. + schedule_timeout_interruptible(HZ / 10); + continue; + } + } + hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + if (!hppp->hpp_htp) { + // Still starting up, so get out of the way. schedule_timeout_interruptible(HZ / 10); continue; } @@ -417,7 +427,12 @@ static int hazptr_torture_reader(void *arg) torture_hrtimeout_us(500, 1000, &rand); lastsleep = jiffies + 10; } - hazptr_torture_reader_tail(hcp, htp, &rand); + if (can_defer && !(torture_random(&rand) % defer_modulus)) { + hazptr_torture_defer(hppp, &rand); + hppp = NULL; + } else { + hazptr_torture_reader_tail(&hppp->hpp_hc, hppp->hpp_htp, &rand); + } while (!torture_must_stop() && (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) schedule_timeout_interruptible(HZ / 5); @@ -478,7 +493,10 @@ static int hazptr_torture_do_pending(void *arg) static int hazptr_torture_do_pending_init(void) { if (kthread_do_pending_ms == -1) - kthread_do_pending_ms = cur_ops->onstack_ctx ? 0 : 3; + kthread_do_pending_ms = (cur_ops->onstack_ctx || defer_modulus == 0) ? 0 : 3; + if (defer_modulus == -1) + defer_modulus = (cur_ops->onstack_ctx || + kthread_do_pending_ms == 0) ? 0 : 1000 * nr_cpu_ids; if (kthread_do_pending_ms < 0) { pr_alert("Cannot have negative kthread_do_pending_ms, disabling deferral.\n"); goto err_out; @@ -487,6 +505,16 @@ static int hazptr_torture_do_pending_init(void) pr_alert("Cannot defer onstack hazptr_ctx, disabling deferral.\n"); goto err_out; } + if (defer_modulus < 0) { + pr_alert("Cannot have negative defer_modulus (%d), disabling deferral.\n", + defer_modulus); + goto err_out; + } + if (!kthread_do_pending_ms != !defer_modulus) { + pr_alert("Pending kthread (%d) & deferral (%d) don't match, disabling deferral.\n", + kthread_do_pending_ms, defer_modulus); + goto err_out; + } if (!kthread_do_pending_ms) return 0; return torture_create_kthread(hazptr_torture_do_pending, NULL, do_pending_task); @@ -494,6 +522,7 @@ static int hazptr_torture_do_pending_init(void) err_out: WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST)); kthread_do_pending_ms = 0; + defer_modulus = 0; return 0; } @@ -605,14 +634,14 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char { pr_alert("%s" TORTURE_FLAG "--- %s: nreaders=%d " - "kthread_do_pending_ms=%d " + "defer_modulus=%d kthread_do_pending_ms=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", torture_type, tag, nrealreaders, - kthread_do_pending_ms, + defer_modulus, kthread_do_pending_ms, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, reader_sleep_us, -- cgit v1.2.3 From 08953d3d485f4f1d6553b99f72733a9dbe006124 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 14 Jun 2026 16:02:06 -0700 Subject: hazptrtorture: Add irq_acquire to acquire hazptr from irq This commit adds the irq_acquire module parameter, which specifies how often hazard pointers will be acquired from an smp_call_function_single() handler. For example, a value of 10 would result in one of ten hazard-pointer acquisitions taking place in a handler, and probably also death by excessive numbers of handlers. A value of zero disables, and a value of -1 makes the frequency decrease as a function of the number of CPUs. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index ff253fe35ca2..dc148894786b 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -31,6 +31,8 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul E. McKenney "); torture_param(int, defer_modulus, -1, "Defer once per specified # of hazptr ops, zero to disable"); +torture_param(int, irq_acquire, -1, + "Acquire hazard pointers from irq handlers once per specified #, zero to disable"); torture_param(int, kthread_do_pending_ms, -1, "Delay between cleanups for deferred hazard pointers (ms), zero to disable"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); @@ -351,6 +353,16 @@ hazptr_torture_writer(void *arg) return 0; } +/* + * Acquire a hazard pointer from an smp_call_function handler. + */ +static void hazptr_torture_acquire(void *hppp_in) +{ + struct hazptr_pending *hppp = hppp_in; + + hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); +} + /* * Do the delay, the accounting, and the release. This in intended to * be invoked from hazptr_torture_reader, but also for hazard pointers @@ -399,6 +411,7 @@ static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_ran static int hazptr_torture_reader(void *arg) { bool can_defer = !cur_ops->onstack_ctx && kthread_do_pending_ms && defer_modulus; + int cpu = 0; struct hazptr_pending hpp; struct hazptr_pending *hppp = cur_ops->onstack_ctx ? &hpp : NULL; unsigned long lastsleep = jiffies; @@ -417,7 +430,16 @@ static int hazptr_torture_reader(void *arg) continue; } } - hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + if (irq_acquire && !(torture_random(&rand) % irq_acquire)) { + guard(preempt)(); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); + if (cpu != smp_processor_id()) + smp_call_function_single(cpu, hazptr_torture_acquire, hppp, 1); + else + hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + } else { + hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + } if (!hppp->hpp_htp) { // Still starting up, so get out of the way. schedule_timeout_interruptible(HZ / 10); @@ -634,14 +656,14 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char { pr_alert("%s" TORTURE_FLAG "--- %s: nreaders=%d " - "defer_modulus=%d kthread_do_pending_ms=%d " + "defer_modulus=%d irq_acquire=%d kthread_do_pending_ms=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", torture_type, tag, nrealreaders, - defer_modulus, kthread_do_pending_ms, + defer_modulus, irq_acquire, kthread_do_pending_ms, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, reader_sleep_us, @@ -785,6 +807,13 @@ static int __init hazptr_torture_init(void) if (torture_init_error(firsterr)) goto unwind; + if (irq_acquire == -1) { + irq_acquire = 1000 * nr_cpu_ids; + } else if (irq_acquire < 0) { + pr_alert("Cannot have irq_acquire (%d) < -1, disabling.\n", irq_acquire); + WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST)); + irq_acquire = 0; + } reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); for (i = 0; i < nrealreaders; i++) { firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i, -- cgit v1.2.3 From ceccd5ee5a60cef642d214263d5a86646a85cf98 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 12 Jun 2026 10:34:14 +0800 Subject: hazptrtorture: Use task_state_to_char() for task-state reporting Use the kernel's standard symbolic task-state representation instead of printing raw hexadecimal task-state values. Suggested-by: Zqiang Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index dc148894786b..51fd0562f9fa 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -620,10 +620,10 @@ hazptr_torture_stats_print(void) unsigned long __maybe_unused gp_seq = 0; wtp = READ_ONCE(writer_task); - pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", hazptr_torture_writer_state_getname(), hazptr_torture_writer_state, gp_seq, flags, - wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? '?' : task_state_to_char(wtp), wtp == NULL ? -1 : (int)task_cpu(wtp)); if (!splatted && wtp) { sched_show_task(wtp); -- cgit v1.2.3 From e7dd3939f46fb122c640b759c954fcf84130fe28 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 15 Jun 2026 12:04:11 -0700 Subject: hazptrtorture: Pass hazptr_pending to hazptr_torture_reader_tail() This commit passes a hazptr_pending structure instead of a pair of pointers to the hazptr_torture_reader_tail() function in order to make it easier to test the releasing of hazard pointers from interrupt handlers. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 51fd0562f9fa..ece38f444be6 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -368,9 +368,11 @@ static void hazptr_torture_acquire(void *hppp_in) * be invoked from hazptr_torture_reader, but also for hazard pointers * sent off to interrupt handlers and the like. */ -static void hazptr_torture_reader_tail(struct hazptr_ctx *hcp, struct hazptr_torture *htp, - struct torture_random_state *trsp) +static void +hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_state *trsp) { + struct hazptr_ctx *hcp = &hppp->hpp_hc; + struct hazptr_torture *htp = hppp->hpp_htp; int pipe_count; cur_ops->read_delay(trsp); @@ -453,7 +455,7 @@ static int hazptr_torture_reader(void *arg) hazptr_torture_defer(hppp, &rand); hppp = NULL; } else { - hazptr_torture_reader_tail(&hppp->hpp_hc, hppp->hpp_htp, &rand); + hazptr_torture_reader_tail(hppp, &rand); } while (!torture_must_stop() && (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended())) @@ -479,7 +481,7 @@ static void hazptr_torture_do_one_pending(int cpu, struct torture_random_state * if (!llnp) return; llist_for_each_entry_safe(hppp, hppp1, llnp, hpp_node) { - hazptr_torture_reader_tail(&hppp->hpp_hc, hppp->hpp_htp, trsp); + hazptr_torture_reader_tail(hppp, trsp); kfree(hppp); } } -- cgit v1.2.3 From 9b5e5791aba7836471ee364b96b2f688c892f7f0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 15 Jun 2026 21:37:47 -0700 Subject: hazptrtorture: Add the ability to disable the writer kthread This commit adds a hazptrtorture.nwriters module parameter that, when set to zero, disables hazard-pointer update-side activity. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index ece38f444be6..138c7e695821 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -36,6 +36,7 @@ torture_param(int, irq_acquire, -1, torture_param(int, kthread_do_pending_ms, -1, "Delay between cleanups for deferred hazard pointers (ms), zero to disable"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); +torture_param(int, nwriters, 1, "Number of hazard-pointer writer threads, 0 or 1"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable"); @@ -657,14 +658,14 @@ static void hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s: nreaders=%d " + "--- %s: nreaders=%d nwriters=%d " "defer_modulus=%d irq_acquire=%d kthread_do_pending_ms=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", - torture_type, tag, nrealreaders, + torture_type, tag, nrealreaders, nwriters, defer_modulus, irq_acquire, kthread_do_pending_ms, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, @@ -824,9 +825,12 @@ static int __init hazptr_torture_init(void) goto unwind; } - firsterr = torture_create_kthread(hazptr_torture_writer, NULL, writer_task); - if (torture_init_error(firsterr)) - goto unwind; + if (nwriters) { + WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST) && nwriters != 1); + firsterr = torture_create_kthread(hazptr_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + } firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); if (torture_init_error(firsterr)) -- cgit v1.2.3 From 11bea7ad52760c1be45c4184866032582df4febd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 Jun 2026 14:42:03 -0700 Subject: hazptrtorture: Add irq_release to release hazptr from irq This commit adds the irq_release module parameter, which specifies how often hazard pointers will be released from an smp_call_function_single() handler. For example, a value of 10 would result in one of ten hazard-pointer acquisitions taking place in a handler, and probably also death by excessive numbers of handlers. A value of zero disables, and a value of -1 makes the frequency decrease as a function of the number of CPUs. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- kernel/rcu/hazptrtorture.c | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 138c7e695821..895c4551dabe 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -33,6 +33,8 @@ MODULE_AUTHOR("Paul E. McKenney "); torture_param(int, defer_modulus, -1, "Defer once per specified # of hazptr ops, zero to disable"); torture_param(int, irq_acquire, -1, "Acquire hazard pointers from irq handlers once per specified #, zero to disable"); +torture_param(int, irq_release, -1, + "Release hazard pointers from irq handlers once per specified #, zero to disable"); torture_param(int, kthread_do_pending_ms, -1, "Delay between cleanups for deferred hazard pointers (ms), zero to disable"); torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads"); @@ -364,6 +366,16 @@ static void hazptr_torture_acquire(void *hppp_in) hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); } +/* + * Release a hazard pointer from an smp_call_function handler. + */ +static void hazptr_torture_release(void *hppp_in) +{ + struct hazptr_pending *hppp = hppp_in; + + cur_ops->readunlock(&hppp->hpp_hc, hppp->hpp_htp); +} + /* * Do the delay, the accounting, and the release. This in intended to * be invoked from hazptr_torture_reader, but also for hazard pointers @@ -372,6 +384,7 @@ static void hazptr_torture_acquire(void *hppp_in) static void hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_state *trsp) { + int cpu; struct hazptr_ctx *hcp = &hppp->hpp_hc; struct hazptr_torture *htp = hppp->hpp_htp; int pipe_count; @@ -388,7 +401,13 @@ hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_st rcu_ftrace_dump(DUMP_ALL); __this_cpu_inc(hazptr_torture_count[pipe_count]); preempt_enable(); - cur_ops->readunlock(hcp, htp); + if (irq_release && !(torture_random(trsp) % irq_release)) { + guard(preempt)(); + cpu = cpumask_next_wrap(smp_processor_id(), cpu_online_mask); + smp_call_function_single(cpu, hazptr_torture_release, hppp, 1); + } else { + cur_ops->readunlock(hcp, htp); + } } /* @@ -659,14 +678,14 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char { pr_alert("%s" TORTURE_FLAG "--- %s: nreaders=%d nwriters=%d " - "defer_modulus=%d irq_acquire=%d kthread_do_pending_ms=%d " + "defer_modulus=%d irq_acquire=%d irq_release=%d kthread_do_pending_ms=%d " "onoff_interval=%d onoff_holdoff=%d " "preempt_duration=%d preempt_interval=%d " "reader_sleep_us=%d " "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d " "verbose=%d\n", torture_type, tag, nrealreaders, nwriters, - defer_modulus, irq_acquire, kthread_do_pending_ms, + defer_modulus, irq_acquire, irq_release, kthread_do_pending_ms, onoff_interval, onoff_holdoff, preempt_duration, preempt_interval, reader_sleep_us, @@ -817,6 +836,13 @@ static int __init hazptr_torture_init(void) WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST)); irq_acquire = 0; } + if (irq_release == -1) { + irq_release = 1000 * nr_cpu_ids; + } else if (irq_release < 0) { + pr_alert("Cannot have irq_release (%d) < -1, disabling.\n", irq_release); + WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST)); + irq_release = 0; + } reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders); for (i = 0; i < nrealreaders; i++) { firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i, -- cgit v1.2.3 From 833bfc05b756f48cbc2dfcdbd884ef995c646a33 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 24 Jun 2026 15:16:31 -0700 Subject: hazptrtorture: Accumulate operation statistics This commit accumulates and prints counts of the number and types of hazard-pointer operations that the test performed. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- include/linux/torture.h | 3 +++ kernel/rcu/hazptrtorture.c | 28 ++++++++++++++++++++++++++-- kernel/torture.c | 15 +++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/include/linux/torture.h b/include/linux/torture.h index b8e5d0f3c2d8..1b6d5be641f8 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -136,4 +136,7 @@ void torture_sched_set_normal(struct task_struct *t, int nice); long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn); #endif +/* Atomic per-CPU counters. */ +s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp); + #endif /* __LINUX_TORTURE_H */ diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 895c4551dabe..4c9b1814e7ca 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -81,6 +81,12 @@ static atomic_t n_hazptr_torture_alloc; static atomic_t n_hazptr_torture_alloc_fail; static atomic_t n_hazptr_torture_free; static atomic_t n_hazptr_torture_error; +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires); +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases); +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires_irq); +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_irq); +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_defer); +static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_undefer); static struct list_head hazptr_torture_removed; // State for a deferred (AKA pending) hazard pointer @@ -364,6 +370,7 @@ static void hazptr_torture_acquire(void *hppp_in) struct hazptr_pending *hppp = hppp_in; hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires_irq, raw_smp_processor_id())); } /* @@ -374,6 +381,7 @@ static void hazptr_torture_release(void *hppp_in) struct hazptr_pending *hppp = hppp_in; cur_ops->readunlock(&hppp->hpp_hc, hppp->hpp_htp); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_irq, raw_smp_processor_id())); } /* @@ -407,6 +415,7 @@ hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_st smp_call_function_single(cpu, hazptr_torture_release, hppp, 1); } else { cur_ops->readunlock(hcp, htp); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases, raw_smp_processor_id())); } } @@ -422,6 +431,7 @@ static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_ran cpu = cpumask_next_wrap(cpu, cpu_online_mask); llhp = per_cpu_ptr(&hazptr_pending, cpu); llist_add(&hppp->hpp_node, llhp); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_defer, raw_smp_processor_id())); } /* @@ -455,12 +465,17 @@ static int hazptr_torture_reader(void *arg) if (irq_acquire && !(torture_random(&rand) % irq_acquire)) { guard(preempt)(); cpu = cpumask_next_wrap(cpu, cpu_online_mask); - if (cpu != smp_processor_id()) + if (cpu != smp_processor_id()) { smp_call_function_single(cpu, hazptr_torture_acquire, hppp, 1); - else + } else { hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires, + raw_smp_processor_id())); + } } else { hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires, + raw_smp_processor_id())); } if (!hppp->hpp_htp) { // Still starting up, so get out of the way. @@ -502,6 +517,8 @@ static void hazptr_torture_do_one_pending(int cpu, struct torture_random_state * return; llist_for_each_entry_safe(hppp, hppp1, llnp, hpp_node) { hazptr_torture_reader_tail(hppp, trsp); + atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_undefer, + raw_smp_processor_id())); kfree(hppp); } } @@ -611,6 +628,13 @@ hazptr_torture_stats_print(void) atomic_read(&n_hazptr_torture_alloc_fail), atomic_read(&n_hazptr_torture_free)); torture_onoff_stats(); + pr_cont("acq: %lld rel: %lld acqirq: %lld relirq: %lld reldefer: %lld relundefer %lld\n", + torture_sum_pcpu_atomic_long(&hazptr_torture_acquires), + torture_sum_pcpu_atomic_long(&hazptr_torture_releases), + torture_sum_pcpu_atomic_long(&hazptr_torture_acquires_irq), + torture_sum_pcpu_atomic_long(&hazptr_torture_releases_irq), + torture_sum_pcpu_atomic_long(&hazptr_torture_releases_defer), + torture_sum_pcpu_atomic_long(&hazptr_torture_releases_undefer)); pr_alert("%s%s ", torture_type, TORTURE_FLAG); if (i > 1) { diff --git a/kernel/torture.c b/kernel/torture.c index bcd2e9c8a263..bb9e1349418a 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -1007,3 +1007,18 @@ void torture_sched_set_normal(struct task_struct *t, int nice) sched_set_normal(t, realnice); } EXPORT_SYMBOL_GPL(torture_sched_set_normal); + +/* + * Sum the specified per-CPU atomic_long_t variable. + */ +s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp) +{ + int cpu; + s64 sum = 0; + + for_each_possible_cpu(cpu) { + sum += atomic_long_read(per_cpu_ptr(pcp, cpu)); + } + return sum; +} +EXPORT_SYMBOL_GPL(torture_sum_pcpu_atomic_long); -- cgit v1.2.3 From 4fe14b9fc1f3bd79d816e7a3195fe292717bf734 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 24 Jun 2026 20:21:48 -0700 Subject: doc: Add hazptrtorture module parameters Add the hazard-pointer-torture module parameters to kernel-parameters.txt. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- Documentation/admin-guide/kernel-parameters.txt | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..7977ce806a26 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1951,6 +1951,102 @@ Kernel parameters for 64-bit NUMA, off otherwise. Format: 0 | 1 (for off | on) + hazptrtorture.defer_modulus= [KNL] + Defer release of the hazard pointer on average + one out of the specified number of times. Zero + disables. Negative one defaults to 1,000 times + nr_cpu_ids, unless hazptr.kthread_do_pending_ms is + equal to zero, in which case it instead defaults + to zero (disabled). + + hazptrtorture.irq_acquire= [KNL] + Acquire hazard pointers from an irq handler on average + one out of the specified number of times. Zero + disables. Negative one defaults to 1,000 times + nr_cpu_ids. + + hazptrtorture.irq_release= [KNL] + Release hazard pointers from an irq handler on average + one out of the specified number of times. Zero + disables. Negative one defaults to 1,000 times + nr_cpu_ids. + + hazptrtorture.kthread_do_pending_ms= [KNL] + Interval between cleanup of pending (deferred) + hazard-pointer releases in milliseconds. + Zero disables. Negative one defaults to three + milliseconds, unless hazptr.kthread_do_pending_ms + is equal to zero, in which case it instead + defaults to zero (disabled). + + hazptrtorture.nreaders= [KNL] + Number of hazard-pointer reader kthreads, each + of which repeatedly acquires and releases hazard + pointers. If the value is zero, one reader + is spawned. If the value is less than zero, + the absolute value is multiplied by the number + of online CPUs at initialization time. + + hazptrtorture.nwriters= [KNL] + Controls whether or not there is a writer. + There is at most one writer, so this value must + be zero or one. + + hazptrtorture.onoff_holdoff= [KNL] + Set time (s) after boot for CPU-hotplug testing. + + hazptrtorture.onoff_interval= [KNL] + Set time (jiffies) between CPU-hotplug operations, + or zero to disable CPU-hotplug testing. + + hazptrtorture.preempt_duration= [KNL] + Set duration (in milliseconds) of preemptions + by a high-priority FIFO real-time task. Set to + zero (the default) to disable. The CPUs to + preempt are selected randomly from the set that + are online at a given point in time. Races with + CPUs going offline are ignored, with that attempt + at preemption skipped. + + hazptrtorture.preempt_interval= [KNL] + Set interval (in milliseconds, defaulting to one + second) between preemptions by a high-priority + FIFO real-time task. This delay is mediated + by an hrtimer and is further fuzzed to avoid + inadvertent synchronizations. + + hazptrtorture.reader_sleep_us= [KNL] + Set occassional read-side sleep time in + microseconds, defaulting to zero for disabled. + + hazptrtorture.shuffle_interval= [KNL] + Set task-shuffle interval (seconds). Shuffling + tasks allows some CPUs to go into dyntick-idle + mode during the hazptrtorture test. + + hazptrtorture.shutdown_secs= [KNL] + Set time (s) after boot system shutdown. This + is useful for hands-off automated testing. + + hazptrtorture.stat_interval= [KNL] + Time (s) between statistics printk()s. + + hazptrtorture.stutter= [KNL] + Time (s) to stutter testing, for example, + specifying three seconds (the default) causes + the test to run for three seconds, wait + for five seconds, and so on. This tests the + hazard-pointers primitives' ability to transition + abruptly to and from idle. Setting this to zero + disables stuttering. + + hazptrtorture.torture_type= [KNL] + Specify the hazard-pointers implementation + to test. + + hazptrtorture.verbose= [KNL] + Enable additional printk() statements. + hd= [EIDE] (E)IDE hard drive subsystem geometry Format: ,, -- cgit v1.2.3 From c77c79c6837bb62f5bc3a17e7fb7f7dca21f6e22 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 7 Jul 2026 13:23:06 -0700 Subject: hazptr: Permit detaching hazard pointers from contexts Provide a new hazptr_detach() function that detaches a given hazard pointer from its acquisition context. This context might be a task or an interrupt handler. [ paulmck: s/hazptr_detach_from_task/hazptr_detach/ per Mathieu. ] Signed-off-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Cc: --- include/linux/hazptr.h | 55 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h index b121f7779cda..8197a51b9f7a 100644 --- a/include/linux/hazptr.h +++ b/include/linux/hazptr.h @@ -98,6 +98,41 @@ bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot) return slot == &ctx->backup_slot.slot; } +/* Internal helper. */ +static inline +void hazptr_promote_to_backup_slot(struct hazptr_ctx *ctx, struct hazptr_slot *slot) +{ + struct hazptr_slot *backup_slot; + + backup_slot = hazptr_chain_backup_slot(ctx); + /* + * Move hazard pointer from the per-CPU slot to the + * backup slot. This requires hazard pointer + * synchronize to iterate on per-CPU slots with + * load-acquire before iterating on the overflow list. + */ + WRITE_ONCE(backup_slot->addr, slot->addr); + /* + * store-release orders store to backup slot addr before + * store to per-CPU slot addr. + */ + smp_store_release(&slot->addr, NULL); + /* Use the backup slot for context. */ + ctx->slot = backup_slot; +} + +static inline +void hazptr_detach(struct hazptr_ctx *ctx) +{ + struct hazptr_slot *slot; + + guard(preempt)(); + slot = ctx->slot; + if (unlikely(hazptr_slot_is_backup(ctx, slot))) + return; + hazptr_promote_to_backup_slot(ctx, slot); +} + static inline void hazptr_note_context_switch(void) { @@ -106,27 +141,11 @@ void hazptr_note_context_switch(void) for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) { struct hazptr_slot_item *item = &percpu_slots->items[idx]; - struct hazptr_slot *slot = &item->slot, *backup_slot; - struct hazptr_ctx *ctx; + struct hazptr_slot *slot = &item->slot; if (!slot->addr) continue; - ctx = item->ctx.ctx; - backup_slot = hazptr_chain_backup_slot(ctx); - /* - * Move hazard pointer from the per-CPU slot to the - * backup slot. This requires hazard pointer - * synchronize to iterate on per-CPU slots with - * load-acquire before iterating on the overflow list. - */ - WRITE_ONCE(backup_slot->addr, slot->addr); - /* - * store-release orders store to backup slot addr before - * store to per-CPU slot addr. - */ - smp_store_release(&slot->addr, NULL); - /* Use the backup slot for context. */ - ctx->slot = backup_slot; + hazptr_promote_to_backup_slot(item->ctx.ctx, slot); } } -- cgit v1.2.3 From 09d4bb5b23c0d8d3e0847b1926f2d2ec351acd9a Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Tue, 7 Jul 2026 14:34:42 -0700 Subject: hazptrtorture: Detach deferred and IPIed hazard pointers Pass to hazptr_detach() those hazard pointers that are to be released in some other task or within the context of an IPI handler. [ paulmck: s/hazptr_detach_from_task/hazptr_detach/ per Mathieu. ] Signed-off-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Cc: --- kernel/rcu/hazptrtorture.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 4c9b1814e7ca..3cbfe7c15a8c 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -411,6 +411,7 @@ hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_st preempt_enable(); if (irq_release && !(torture_random(trsp) % irq_release)) { guard(preempt)(); + hazptr_detach(&hppp->hpp_hc); cpu = cpumask_next_wrap(smp_processor_id(), cpu_online_mask); smp_call_function_single(cpu, hazptr_torture_release, hppp, 1); } else { @@ -428,6 +429,7 @@ static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_ran struct llist_head *llhp; guard(preempt)(); + hazptr_detach(&hppp->hpp_hc); cpu = cpumask_next_wrap(cpu, cpu_online_mask); llhp = per_cpu_ptr(&hazptr_pending, cpu); llist_add(&hppp->hpp_node, llhp); -- cgit v1.2.3 From 1658c11c1992780820276ad77d1e65bbba00fd16 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 9 Jul 2026 14:29:33 -0400 Subject: hazptr: Introduce CONFIG_HAZPTR_DEBUG misuse detection Introduce Hazard Pointers debug assert, which detects misuse of hazard pointers, namely failure to detach the hazard pointer from its owner thread before releasing it from a different thread. Prints the following to the console when a failure is detected: Hazard Pointer (addr=000000006885a05f) released on remote task without being detached from task. Acquire: caller=hazptr_torture_read_lock+0x43/0xa0 [hazptrtorture], pid=3727, cpu=1. Release: pid=3725, cpu=139. WARNING: ./include/linux/hazptr.h:225 at hazptr_torture_read_unlock+0x68/0xf0 [hazptrtorture], CPU#139: hazptr_torture_/3725 Modules linked in: hazptrtorture torture nft_masq nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 nf_tables nfnetlink CPU: 139 UID: 0 PID: 3725 Comm: hazptr_torture_ Not tainted 7.1.0-rc4+ #9 PREEMPT(full) Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 RIP: 0010:hazptr_torture_read_unlock+0x74/0xf0 [hazptrtorture] Code: 74 31 8b 4f 40 4c 8b 43 48 49 c7 c2 22 c9 56 c0 48 c7 c7 3b c9 56 c0 4c 8d 1d b8 32 f1 ff 52 48 89 fa 4c 89 df 50 51 4c 89 d1 <67> 48 0f b9 3a 48 83 c4 18 48 8b 03 48 8d 53 18 48 c7 00 00 00 00 RSP: 0018:ff621a2a479b3de0 EFLAGS: 00010293 RAX: 0000000000000e8d RBX: ff12ea30d3913488 RCX: ffffffffc056c922 RDX: ffffffffc056c93b RSI: ffffffffc0562280 RDI: ffffffffc0562030 RBP: ff621a2a479b3e50 R08: ffffffffc064ee53 R09: 0000000000000e8f R10: ffffffffc056c922 R11: ffffffffc0562030 R12: 0000000000000000 R13: ffffffffc0562280 R14: ff12ea30d3913488 R15: ff621a2a479b3e50 FS: 0000000000000000(0000) GS:ff12ea5011a84000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007f39b6e4b010 CR3: 0000000114c6e005 CR4: 0000000000771ef0 PKRU: 55555554 Call Trace: hazptr_torture_reader_tail+0x8e/0x210 [hazptrtorture] hazptr_torture_reader+0x145/0xb30 [hazptrtorture] ? srso_alias_return_thunk+0x5/0xfbef5 ? set_cpus_allowed_ptr+0x36/0x60 ? srso_alias_return_thunk+0x5/0xfbef5 ? srso_alias_return_thunk+0x5/0xfbef5 ? __pfx_hazptr_torture_reader+0x10/0x10 [hazptrtorture] kthread+0xdf/0x120 ? __pfx_kthread+0x10/0x10 ret_from_fork+0x216/0x2d0 ? __pfx_kthread+0x10/0x10 ret_from_fork_asm+0x1a/0x30 Signed-off-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Cc: --- include/linux/hazptr.h | 38 ++++++++++++++++++++++++++++++++++++++ kernel/rcu/Kconfig.debug | 9 +++++++++ 2 files changed, 47 insertions(+) diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h index 8197a51b9f7a..415316282142 100644 --- a/include/linux/hazptr.h +++ b/include/linux/hazptr.h @@ -51,6 +51,11 @@ struct hazptr_ctx { /* Backup slot in case all per-CPU slots are used. */ struct hazptr_backup_slot backup_slot; struct hlist_node preempt_node; +#ifdef CONFIG_HAZPTR_DEBUG + bool detach_task, detach_cpu; /* Whether the ctx has been detached from task/cpu. */ + int acquire_pid, acquire_cpu; /* Note the task and cpu number at acquire. */ + unsigned long acquire_caller; /* Acquire instruction pointer. */ +#endif }; struct hazptr_slot_ctx { @@ -127,6 +132,9 @@ void hazptr_detach(struct hazptr_ctx *ctx) struct hazptr_slot *slot; guard(preempt)(); +#ifdef CONFIG_HAZPTR_DEBUG + ctx->detach_task = ctx->detach_cpu = true; +#endif slot = ctx->slot; if (unlikely(hazptr_slot_is_backup(ctx, slot))) return; @@ -145,6 +153,9 @@ void hazptr_note_context_switch(void) if (!slot->addr) continue; +#ifdef CONFIG_HAZPTR_DEBUG + item->ctx.ctx->detach_cpu = true; +#endif hazptr_promote_to_backup_slot(item->ctx.ctx, slot); } } @@ -173,6 +184,12 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p) percpu_slots = this_cpu_ptr(&hazptr_percpu_slots); slot_item = &percpu_slots->items[0]; slot = &slot_item->slot; +#ifdef CONFIG_HAZPTR_DEBUG + ctx->detach_cpu = ctx->detach_task = false; + ctx->acquire_pid = current->pid; + ctx->acquire_cpu = smp_processor_id(); + ctx->acquire_caller = _THIS_IP_; +#endif if (unlikely(slot->addr)) return __hazptr_acquire(ctx, addr_p); WRITE_ONCE(slot->addr, HAZPTR_WILDCARD); /* Store B */ @@ -196,6 +213,26 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p) return addr; } +#ifdef CONFIG_HAZPTR_DEBUG +/* Called with preemption disabled. */ +static inline +void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) +{ + int pid = current->pid, cpu = smp_processor_id(); + bool warn_remote_cpu = !ctx->detach_cpu && ctx->acquire_cpu != cpu, + warn_remote_task = !ctx->detach_task && ctx->acquire_pid != pid; + + WARN_ONCE(warn_remote_cpu || warn_remote_task, + "Hazard Pointer (addr=%p) released on remote %s without %s. Acquire: caller=%pS, pid=%d, cpu=%d. Release: pid=%d, cpu=%d.", + addr, + warn_remote_task ? "task" : "cpu", + warn_remote_task ? "being detached from task" : "context switch", + (void *) ctx->acquire_caller, ctx->acquire_pid, ctx->acquire_cpu, pid, cpu); +} +#else +static inline void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) { } +#endif + /* Release the protected hazard pointer from @slot. */ static inline void hazptr_release(struct hazptr_ctx *ctx, void *addr) @@ -205,6 +242,7 @@ void hazptr_release(struct hazptr_ctx *ctx, void *addr) if (!addr) return; guard(preempt)(); + hazptr_release_debug(ctx, addr); slot = ctx->slot; smp_store_release(&slot->addr, NULL); if (unlikely(hazptr_slot_is_backup(ctx, slot))) diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug index fe64356f0088..9c3f72474cb5 100644 --- a/kernel/rcu/Kconfig.debug +++ b/kernel/rcu/Kconfig.debug @@ -251,4 +251,13 @@ config TRIVIAL_PREEMPT_RCU This has no value for production and is only for testing. +config HAZPTR_DEBUG + bool "Provide debugging asserts for Hazard Pointers" + depends on DEBUG_KERNEL + default n + help + This option provides consistency checks for Hazard pointers. + + Say Y here if you want to enable those assert, N otherwise. + endmenu # "RCU Debugging" -- cgit v1.2.3 From b288d381b900a13fda9470e8e2fa16a37d795df7 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 9 Jul 2026 14:29:32 -0400 Subject: hazptrtorture: Fix hazptr ownership issue hazptr_torture_acquire is invoked from IPI on remote CPUs. It needs to immediately detach the hazptr from its local task so it can be released from other tasks. Without this fix, the hazptrtorture test fails loudly with a OOPS. It passes with the fix applied. [ This applies on top of linux-rcu dev branch at commit e5ee2ea2d00ae5a0ca53ccf9c9100f2357e1bd08 ] [ paulmck: s/hazptr_detach_from_task/hazptr_detach/ per Mathieu. ] Signed-off-by: Mathieu Desnoyers Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Cc: --- kernel/rcu/hazptrtorture.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c index 3cbfe7c15a8c..7526ea65ff7c 100644 --- a/kernel/rcu/hazptrtorture.c +++ b/kernel/rcu/hazptrtorture.c @@ -370,6 +370,11 @@ static void hazptr_torture_acquire(void *hppp_in) struct hazptr_pending *hppp = hppp_in; hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc); + /* + * Acquiring a hazard pointer from a remote CPU. + * Detach hazptr from its task so it can be released by another task. + */ + hazptr_detach(&hppp->hpp_hc); atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires_irq, raw_smp_processor_id())); } @@ -411,6 +416,10 @@ hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_st preempt_enable(); if (irq_release && !(torture_random(trsp) % irq_release)) { guard(preempt)(); + /* + * Handling release from a remote CPU. + * Detach hazptr from its task so it can be released by another task. + */ hazptr_detach(&hppp->hpp_hc); cpu = cpumask_next_wrap(smp_processor_id(), cpu_online_mask); smp_call_function_single(cpu, hazptr_torture_release, hppp, 1); @@ -429,6 +438,10 @@ static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_ran struct llist_head *llhp; guard(preempt)(); + /* + * Handling release from a remote CPU. + * Detach hazptr from its task so it can be released by another task. + */ hazptr_detach(&hppp->hpp_hc); cpu = cpumask_next_wrap(cpu, cpu_online_mask); llhp = per_cpu_ptr(&hazptr_pending, cpu); -- cgit v1.2.3 From 88f35d0588688935c0d8d88683758b5a299ae2fc Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Jul 2026 13:12:56 -0700 Subject: hazptrtorture: Enable CONFIG_HAZPTR_DEBUG This commit enables the CONFIG_HAZPTR_DEBUG Kconfig option in order to more easily debug hazard-pointer handoff issues. Signed-off-by: Paul E. McKenney Cc: Boqun Feng Cc: Mathieu Desnoyers Cc: Cc: --- tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT | 2 ++ tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT index e2da430abe4d..405941f3b50f 100644 --- a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT +++ b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT @@ -15,3 +15,5 @@ CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_PROVE_LOCKING=n CONFIG_KPROBES=n CONFIG_FTRACE=n +CONFIG_DEBUG_KERNEL=y +CONFIG_HAZPTR_DEBUG=y diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT index b8ea4364b20b..c5b12f3bef74 100644 --- a/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT +++ b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT @@ -12,3 +12,5 @@ CONFIG_HIBERNATION=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_PROVE_LOCKING=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n +CONFIG_DEBUG_KERNEL=y +CONFIG_HAZPTR_DEBUG=y -- cgit v1.2.3 From 2d457348e3a07c59953880bbf4dae85b1e91355f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Jul 2026 13:39:09 -0700 Subject: hazptr: Upgrade kernel-doc headers Upgrade the kernel-doc headers for hazptr_acquire(), hazptr_release(), and hazptr_detach_from_task(). [ paulmck: s/hazptr_detach_from_task/hazptr_detach/ per Mathieu. ] Signed-off-by: Paul E. McKenney Cc: Mathieu Desnoyers Cc: Boqun Feng Cc: Cc: --- include/linux/hazptr.h | 75 +++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h index 415316282142..082fde224508 100644 --- a/include/linux/hazptr.h +++ b/include/linux/hazptr.h @@ -75,12 +75,16 @@ DECLARE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots); void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p); -/* - * hazptr_synchronize: Wait until @addr is released from all slots. +/** + * hazptr_synchronize: Wait for release from hazard-pointer protection + * + * @addr: The address to be released from hazard-pointer protection * - * Wait to observe that each slot contains a value that differs from - * @addr before returning. - * Should be called from preemptible context. + * Wait for the specified @addr to be released from protection from all + * hazard pointers. The caller should make @addr inaccessible to all + * hazard-pointer readers before invoking this function. + * + * Must be called from preemptible context. */ void hazptr_synchronize(void *addr); @@ -126,6 +130,28 @@ void hazptr_promote_to_backup_slot(struct hazptr_ctx *ctx, struct hazptr_slot *s ctx->slot = backup_slot; } +/** + * hazptr_detach - Allow a hazard pointer to be released in some other context + * + * @ctx: The hazard-pointer context to be detached. + * + * By default, a given hazptr_acquire() and the corresponding + * hazptr_release() must run in a single execution context, for example, + * the context of a single task or a single interrupt handler. When you + * have acquired a hazard pointer in one context and need to release it + * in another, you must invoke hazptr_detach() on that hazard pointer's + * context. It is permissible to invoke hazptr_detach() multiple times + * on the same @ctx while it is protecting the same pointer, however, + * the first invocation absolutely must be in the same context that did + * the hazptr_acquire(), and must take place after the return from that + * hazptr_acquire(). + * + * For example, if a hazard pointer is acquired by a task and released + * by a timer handler, that task would need to pass the hazard pointer's + * context to hazptr_detach() after return from the hazptr_acquire() and + * before arming the timer (or at least before the handler had a chance + * to access that hazard-pointer context). + */ static inline void hazptr_detach(struct hazptr_ctx *ctx) { @@ -160,12 +186,25 @@ void hazptr_note_context_switch(void) } } -/* - * hazptr_acquire: Load pointer at address and protect with hazard pointer. +/** + * hazptr_acquire - Load pointer at address and protect with hazard pointer. + * + * @ctx: The hazard-pointer context to be passed to hazptr_release(). + * @addr_p: Pointer to the pointer that is to be hazard-pointer protected. * * Load @addr_p, and protect the loaded pointer with hazard pointer. - * When using hazptr_acquire from interrupt handlers, the acquired slots - * need to be released before returning from the interrupt handler. + * This protection is roughly similar to (but way faster than) that of a + * reference counter, and ends with a later call to hazptr_release(). + * + * By default, the call to hazptr_release() must be running in the same + * execution context as the corresponding hazptr_acquire(), for example, + * within the same task or interrupt handler. When it is necessary to + * instead call hazptr_release() from some other context, pass @ctx to + * hazptr_detach() in the original context after invoking hazptr_acquire() + * but before making the hazard pointer available to that other context. + * + * It is not permissible to invoke hazptr_acquire() twice on the same @ctx + * without an intervening hazptr_release(). * * Returns a non-NULL protected address if the loaded pointer is non-NULL. * Returns NULL if the loaded pointer is NULL. @@ -233,7 +272,23 @@ void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) static inline void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) { } #endif -/* Release the protected hazard pointer from @slot. */ +/** + * hazptr_release - Release the specified hazard pointer + * + * @ctx: The hazard-pointer context that was passed to hazptr_acquire(). + * @addr_p: The pointer that is to be hazard-pointer unprotected. + * + * Release the protected hazard pointer recorded in @ctx. + * + * By default, hazptr_release() must execute in the same execution context + * that invoked the corresponding hazptr_acquire(), for example, within the + * same task or the same interrupt handler. However, if this restriction + * is problematic for your use case, please see hazptr_detach(). + * + * It is permissible (though unwise from a maintainability viewpoint) + * to invoke hazptr_release() twice on the same @ctx without an intervening + * hazptr_acquire(). + */ static inline void hazptr_release(struct hazptr_ctx *ctx, void *addr) { -- cgit v1.2.3 From 12bec72d6f3caceaf27fe18197e892379ae88817 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 09:30:09 -0700 Subject: arm64: Mark set_preempt_need_resched() access to .need_resched The .need_resched field can be accessed from both task level and from interrrupt handlers, so apply WRITE_ONCE() to the update in set_preempt_need_resched(). This also brings arm64 in line with s390 (which uses atomic operations) and x86 (which uses inline assembly). Other architectures avoid this issue via the empty definition in include/asm-generic/preempt.h. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Catalin Marinas Cc: Will Deacon Cc: Jinjie Ruan Cc: Ada Couprie Diaz Cc: --- arch/arm64/include/asm/preempt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h index 932ea4b62042..610853da140a 100644 --- a/arch/arm64/include/asm/preempt.h +++ b/arch/arm64/include/asm/preempt.h @@ -28,7 +28,7 @@ static inline void preempt_count_set(u64 pc) static inline void set_preempt_need_resched(void) { - current_thread_info()->preempt.need_resched = 0; + WRITE_ONCE(current_thread_info()->preempt.need_resched, 0); } static inline void clear_preempt_need_resched(void) -- cgit v1.2.3 From 98d3f664fa12834d968af2c24e63130da0d34259 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:07:20 -0700 Subject: hrtimer: Mark data-racy accesses to hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion. Therefore, mark data-racy accesses to the hrtimer_sleeper ->task field using READ_ONCE() (using a new hrtimer_sleeper_task_get() access function) and WRITE_ONCE() (using a new hrtimer_sleeper_task_set() access function). KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner --- include/linux/hrtimer.h | 8 ++++++++ kernel/time/hrtimer.c | 14 +++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 6862dea0acc5..838ea7bce99c 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -350,6 +350,14 @@ extern int schedule_hrtimeout_range_clock(ktime_t *expires, const enum hrtimer_mode mode, clockid_t clock_id); extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); +static inline struct task_struct *hrtimer_sleeper_task_get(struct hrtimer_sleeper *sl) +{ + return READ_ONCE(sl->task); +} +static inline void hrtimer_sleeper_task_set(struct hrtimer_sleeper *sl, struct task_struct *t) +{ + WRITE_ONCE(sl->task, t); +} /* Soft interrupt function to run the hrtimer queues: */ extern void hrtimer_run_queues(void); diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 313dcea127fe..84ea341a6efc 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -2286,9 +2286,9 @@ void hrtimer_run_queues(void) static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) { struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); - struct task_struct *task = t->task; + struct task_struct *task = hrtimer_sleeper_task_get(t); - t->task = NULL; + hrtimer_sleeper_task_set(t, NULL); if (task) wake_up_process(task); @@ -2317,7 +2317,7 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode /* If already expired, clear the task pointer and set current state to running */ if (!hrtimer_start_expires_user(&sl->timer, mode)) { - sl->task = NULL; + hrtimer_sleeper_task_set(sl, NULL); __set_current_state(TASK_RUNNING); } } @@ -2351,7 +2351,7 @@ static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_ } __hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode); - sl->task = current; + hrtimer_sleeper_task_set(sl, current); } /** @@ -2395,17 +2395,17 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); hrtimer_sleeper_start_expires(t, mode); - if (likely(t->task)) + if (likely(hrtimer_sleeper_task_get(t))) schedule(); hrtimer_cancel(&t->timer); mode = HRTIMER_MODE_ABS; - } while (t->task && !signal_pending(current)); + } while (hrtimer_sleeper_task_get(t) && !signal_pending(current)); __set_current_state(TASK_RUNNING); - if (!t->task) + if (!hrtimer_sleeper_task_get(t)) return 0; restart = ¤t->restart_block; -- cgit v1.2.3 From e2961ab96411fb756f0a31865eb0cccb5da7e422 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: aio: Use accessor for hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Benjamin LaHaise Cc: Alexander Viro Cc: Christian Brauner Cc: Jan Kara Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner Cc: Cc: --- fs/aio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/aio.c b/fs/aio.c index f57fa21a2503..e8cd65fce41e 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1402,7 +1402,7 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr, w.min_nr = min_nr - ret; ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE); - if (!ret2 && !t.task) + if (!ret2 && !hrtimer_sleeper_task_get(&t)) ret2 = -ETIME; if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2) -- cgit v1.2.3 From f31608bca0e94136da43e784b4d516c373899a30 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: wait: Use accessor for hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to the __wait_event_hrtimeout() macro. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Juri Lelli Cc: Vincent Guittot Cc: Dietmar Eggemann Cc: Steven Rostedt Cc: Ben Segall Cc: Mel Gorman Cc: Valentin Schneider Cc: K Prateek Nayak Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner --- include/linux/wait.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/wait.h b/include/linux/wait.h index dce055e6add3..00cdce75837e 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -556,7 +556,7 @@ do { \ } \ \ __ret = ___wait_event(wq_head, condition, state, 0, 0, \ - if (!__t.task) { \ + if (!hrtimer_sleeper_task_get(&__t)) { \ __ret = -ETIME; \ break; \ } \ -- cgit v1.2.3 From 838ac24f031e28a775babb67ba40e5a9afa047a8 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: io-uring/rw: Use accessor for hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to the io_hybrid_iopoll_delay() function. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Jens Axboe Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner Cc: --- io_uring/rw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io_uring/rw.c b/io_uring/rw.c index 63b6519e498c..4afbc3ceea0b 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -1291,7 +1291,7 @@ static u64 io_hybrid_iopoll_delay(struct io_ring_ctx *ctx, struct io_kiocb *req) set_current_state(TASK_INTERRUPTIBLE); hrtimer_sleeper_start_expires(&timer, mode); - if (timer.task) + if (hrtimer_sleeper_task_get(&timer)) io_schedule(); hrtimer_cancel(&timer.timer); -- cgit v1.2.3 From 2a707bc4bc5176fa0505490b2718506f6e8a6954 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: futex: Use accessor for hrtimer_sleeper ->task field in waitwake.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to kernel/futex/waitwake.c. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Darren Hart Cc: Davidlohr Bueso Cc: "AndrĂ© Almeida" Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner --- kernel/futex/waitwake.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index d4483d15d30a..cf18309e5770 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -383,7 +383,7 @@ void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout) * flagged for rescheduling. Only call schedule if there * is no timeout, or if it has yet to expire. */ - if (!timeout || timeout->task) + if (!timeout || hrtimer_sleeper_task_get(timeout)) schedule(); } __set_current_state(TASK_RUNNING); @@ -539,7 +539,7 @@ retry: static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to) { - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return; for (; count; count--, vs++) { @@ -590,7 +590,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, if (ret >= 0) return ret; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; else if (signal_pending(current)) return -ERESTARTSYS; @@ -725,7 +725,7 @@ retry: if (!futex_unqueue(&q)) return 0; - if (to && !to->task) + if (to && !hrtimer_sleeper_task_get(to)) return -ETIMEDOUT; /* -- cgit v1.2.3 From 65f23137b638c9319d054f2cc21924c8808966b0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: timers: Use accessor for hrtimer_sleeper ->task field in sleep_timeout.c The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to schedule_hrtimeout_range_clock(). KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner Signed-off-by: Paul E. McKenney --- kernel/time/sleep_timeout.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c index 3c90574bd904..ad8c415851ae 100644 --- a/kernel/time/sleep_timeout.c +++ b/kernel/time/sleep_timeout.c @@ -212,7 +212,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, hrtimer_set_expires_range_ns(&t.timer, *expires, delta); hrtimer_sleeper_start_expires(&t, mode); - if (likely(t.task)) + if (likely(hrtimer_sleeper_task_get(&t))) schedule(); hrtimer_cancel(&t.timer); @@ -220,7 +220,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta, __set_current_state(TASK_RUNNING); - return !t.task ? 0 : -EINTR; + return !hrtimer_sleeper_task_get(&t) ? 0 : -EINTR; } EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock); -- cgit v1.2.3 From 5fd353768124d29f51322f069a369492ce7dc49e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: net: pktgen: Use accessor for hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to the pktgen spin() function. KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Cc: Simon Horman Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner Cc: --- net/core/pktgen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/core/pktgen.c b/net/core/pktgen.c index 8e185b318288..f4a6be18267d 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -2345,11 +2345,11 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until) set_current_state(TASK_INTERRUPTIBLE); hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS); - if (likely(t.task)) + if (likely(hrtimer_sleeper_task_get(&t))) schedule(); hrtimer_cancel(&t.timer); - } while (t.task && pkt_dev->running && !signal_pending(current)); + } while (hrtimer_sleeper_task_get(&t) && pkt_dev->running && !signal_pending(current)); __set_current_state(TASK_RUNNING); end_time = ktime_get(); } -- cgit v1.2.3 From 6eec6da5c7bb1806a0af047571886e11a45d269a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: rtmutex: Use accessor for hrtimer_sleeper ->task field The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to rt_mutex_slowlock_block(). KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Peter Zijlstra Cc: Ingo Molnar Cc: Will Deacon Cc: Boqun Feng Cc: Waiman Long Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner --- kernel/locking/rtmutex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c index 4728631ae719..5a9534c715b8 100644 --- a/kernel/locking/rtmutex.c +++ b/kernel/locking/rtmutex.c @@ -1644,7 +1644,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock, break; } - if (timeout && !timeout->task) { + if (timeout && !hrtimer_sleeper_task_get(timeout)) { ret = -ETIMEDOUT; break; } -- cgit v1.2.3 From 3159bc28e37577ca3514a6d79c67a7332f649cb7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Jul 2026 15:11:28 -0700 Subject: futex: Use accessor for hrtimer_sleeper ->task field in requeue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hrtimer_sleeper structure's ->task field is used as a flag to indicate that the associated hrtimer has expired. This means that the hrtimer handler can be storing to this field while other code is loading from it to check for expiry. Note that additional races appear for hrtimers that can be restarted, which could be argued to be a user error. However, that is no reason to let the compiler introduce additional confusion, and to this end, the hrtimer_sleeper_task_get() was introduced, use of which also has the benefit of avoiding open-code access to hrtimer_sleeper innards. Therefore, apply this accessor to handle_early_requeue_pi_wakeup(). KCSAN located this issue. Signed-off-by: Paul E. McKenney Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Darren Hart Cc: Davidlohr Bueso Cc: "AndrĂ© Almeida" Cc: Anna-Maria Behnsen Cc: Frederic Weisbecker Cc: Thomas Gleixner Signed-off-by: Paul E. McKenney --- kernel/futex/requeue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index 79823ad13683..196d6ff148a4 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -736,7 +736,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, /* Handle spurious wakeups gracefully */ ret = -EWOULDBLOCK; - if (timeout && !timeout->task) + if (timeout && !hrtimer_sleeper_task_get(timeout)) ret = -ETIMEDOUT; else if (signal_pending(current)) ret = -ERESTARTNOINTR; -- cgit v1.2.3 From 34ff636f322c6d5e4f0a52338fa63a7dff09a495 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:36 -0400 Subject: scftorture: Count single_rpc offline failures in statistics output scf_torture_stats_print() aggregates each invoker thread's counters into a local scf_statistics structure before printing, but the n_single_rpc_ofl field is missing from the aggregation loop. As a result, the "single_rpc_ofl" value printed in the statistics line is always zero, even when smp_call_function_single() invocations for the RPC test have failed due to offline CPUs and been counted by the invoker threads. Add the missing accumulation so that the printed value reflects the actual counts. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 327c315f411c..ffd6c83732ee 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -193,6 +193,7 @@ static void scf_torture_stats_print(void) scfs.n_single += scf_stats_p[i].n_single; scfs.n_single_ofl += scf_stats_p[i].n_single_ofl; scfs.n_single_rpc += scf_stats_p[i].n_single_rpc; + scfs.n_single_rpc_ofl += scf_stats_p[i].n_single_rpc_ofl; scfs.n_single_wait += scf_stats_p[i].n_single_wait; scfs.n_single_wait_ofl += scf_stats_p[i].n_single_wait_ofl; scfs.n_many += scf_stats_p[i].n_many; -- cgit v1.2.3 From 8b5b048277e2c43b9012f0196f4097c4e92025a1 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:37 -0400 Subject: scftorture: Make invoker threads actually wait for all threads to start Each scftorture_invoker() thread decrements n_started, which is initialized to the number of threads, and is then supposed to wait until all of its siblings have also checked in before starting the test proper. However, the wait loop is guarded by !atomic_dec_return(&n_started), which is true only for the final thread to arrive, and by then n_started is already zero, so the final thread does not wait either. The side-effect (possibly positive) is that no thread ever waits and the start-synchronization barrier is dead code, with early threads beginning to hammer smp_call_function*() while later threads are still being spawned. Invert the test so that every thread other than the last spins until n_started reaches zero, making the threads start testing together as intended. The existing torture_must_stop() check in the wait loop continues to bound the wait during shutdown. We can also drop the spinning entirely if the intent is to leave it as dead code, however for the current intent, this patches fixes the code. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/scftorture.c b/kernel/scftorture.c index ffd6c83732ee..392df1b4d4ed 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -497,7 +497,7 @@ static int scftorture_invoker(void *arg) "%s: Wanted CPU %d, running on %d, nr_cpu_ids = %d\n", __func__, scfp->cpu, curcpu, nr_cpu_ids); - if (!atomic_dec_return(&n_started)) + if (atomic_dec_return(&n_started)) while (atomic_read_acquire(&n_started)) { if (torture_must_stop()) { VERBOSE_SCFTORTOUT("scftorture_invoker %d ended before starting", scfp->cpu); -- cgit v1.2.3