Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git

32 files changed, 1821 insertions, 30 deletions

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index d0cbafb08e01..ab23a97fe0c7 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1958,6 +1958,102 @@ Kernel parameters
 			for 64-bit NUMA, off otherwise.
 			Format: 0 | 1 (for off | on)
 
+	hazptrtorture.defer_modulus= [KNL]
+			Defer release of the hazard pointer on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids, unless hazptr.kthread_do_pending_ms is
+			equal to zero, in which case it instead defaults
+			to zero (disabled).
+
+	hazptrtorture.irq_acquire= [KNL]
+			Acquire hazard pointers from an irq handler on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids.
+
+	hazptrtorture.irq_release= [KNL]
+			Release hazard pointers from an irq handler on average
+			one out of the specified number of times.  Zero
+			disables.  Negative one defaults to 1,000 times
+			nr_cpu_ids.
+
+	hazptrtorture.kthread_do_pending_ms= [KNL]
+			Interval between cleanup of pending (deferred)
+			hazard-pointer releases in milliseconds.
+			Zero disables.	Negative one defaults to three
+			milliseconds, unless hazptr.kthread_do_pending_ms
+			is equal to zero, in which case it instead
+			defaults to zero (disabled).
+
+	hazptrtorture.nreaders= [KNL]
+			Number of hazard-pointer reader kthreads, each
+			of which repeatedly acquires and releases hazard
+			pointers.  If the value is zero, one reader
+			is spawned.  If the value is less than zero,
+			the absolute value is multiplied by the number
+			of online CPUs at initialization time.
+
+	hazptrtorture.nwriters= [KNL]
+			Controls whether or not there is a writer.
+			There is at most one writer, so this value must
+			be zero or one.
+
+	hazptrtorture.onoff_holdoff= [KNL]
+			Set time (s) after boot for CPU-hotplug testing.
+
+	hazptrtorture.onoff_interval= [KNL]
+			Set time (jiffies) between CPU-hotplug operations,
+			or zero to disable CPU-hotplug testing.
+
+	hazptrtorture.preempt_duration= [KNL]
+			Set duration (in milliseconds) of preemptions
+			by a high-priority FIFO real-time task.  Set to
+			zero (the default) to disable.	The CPUs to
+			preempt are selected randomly from the set that
+			are online at a given point in time.  Races with
+			CPUs going offline are ignored, with that attempt
+			at preemption skipped.
+
+	hazptrtorture.preempt_interval= [KNL]
+			Set interval (in milliseconds, defaulting to one
+			second) between preemptions by a high-priority
+			FIFO real-time task.  This delay is mediated
+			by an hrtimer and is further fuzzed to avoid
+			inadvertent synchronizations.
+
+	hazptrtorture.reader_sleep_us= [KNL]
+			Set occassional read-side sleep time in
+			microseconds, defaulting to zero for disabled.
+
+	hazptrtorture.shuffle_interval= [KNL]
+			Set task-shuffle interval (seconds).  Shuffling
+			tasks allows some CPUs to go into dyntick-idle
+			mode during the hazptrtorture test.
+
+	hazptrtorture.shutdown_secs= [KNL]
+			Set time (s) after boot system shutdown.  This
+			is useful for hands-off automated testing.
+
+	hazptrtorture.stat_interval= [KNL]
+			Time (s) between statistics printk()s.
+
+	hazptrtorture.stutter= [KNL]
+			Time (s) to stutter testing, for example,
+			specifying three seconds (the default) causes
+			the test to run for three seconds, wait
+			for five seconds, and so on.  This tests the
+			hazard-pointers primitives' ability to transition
+			abruptly to and from idle.  Setting this to zero
+			disables stuttering.
+
+	hazptrtorture.torture_type= [KNL]
+			Specify the hazard-pointers implementation
+			to test.
+
+	hazptrtorture.verbose= [KNL]
+			Enable additional printk() statements.
+
 	hd=		[EIDE] (E)IDE hard drive subsystem geometry
 			Format: <cyl>,<head>,<sect>
 
diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h
index 932ea4b62042..610853da140a 100644
--- a/arch/arm64/include/asm/preempt.h
+++ b/arch/arm64/include/asm/preempt.h
@@ -28,7 +28,7 @@ static inline void preempt_count_set(u64 pc)
 
 static inline void set_preempt_need_resched(void)
 {
-	current_thread_info()->preempt.need_resched = 0;
+	WRITE_ONCE(current_thread_info()->preempt.need_resched, 0);
 }
 
 static inline void clear_preempt_need_resched(void)
diff --git a/fs/aio.c b/fs/aio.c
index 3de586da197b..9844e9725896 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -1402,7 +1402,7 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
 		w.min_nr = min_nr - ret;
 
 		ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE);
-		if (!ret2 && !t.task)
+		if (!ret2 && !hrtimer_sleeper_task_get(&t))
 			ret2 = -ETIME;
 
 		if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2)
diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h
new file mode 100644
index 000000000000..082fde224508
--- /dev/null
+++ b/include/linux/hazptr.h
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+#ifndef _LINUX_HAZPTR_H
+#define _LINUX_HAZPTR_H
+
+/*
+ * hazptr: Hazard Pointers
+ *
+ * This API provides existence guarantees of objects through hazard
+ * pointers.
+ *
+ * Its main benefit over RCU is that it allows fast reclaim of
+ * HP-protected pointers without needing to wait for a grace period.
+ *
+ * References:
+ *
+ * [1]: M. M. Michael, "Hazard pointers: safe memory reclamation for
+ *      lock-free objects," in IEEE Transactions on Parallel and
+ *      Distributed Systems, vol. 15, no. 6, pp. 491-504, June 2004
+ */
+
+#include <linux/percpu.h>
+#include <linux/types.h>
+#include <linux/cleanup.h>
+#include <linux/sched.h>
+
+/* 4 slots (each sizeof(hazptr_slot_item)) fit in a single 64-byte cache line. */
+#define NR_HAZPTR_PERCPU_SLOTS	4
+#define HAZPTR_WILDCARD		((void *) 0x1UL)
+
+/*
+ * Hazard pointer slot.
+ */
+struct hazptr_slot {
+	void *addr;
+};
+
+struct hazptr_overflow_list;
+
+struct hazptr_backup_slot {
+	struct hlist_node overflow_node;
+	struct hazptr_slot slot;
+	/* Overflow list where the backup slot is added. */
+	struct hazptr_overflow_list *overflow_list;
+};
+
+struct hazptr_ctx {
+	struct hazptr_slot *slot;
+	/* Backup slot in case all per-CPU slots are used. */
+	struct hazptr_backup_slot backup_slot;
+	struct hlist_node preempt_node;
+#ifdef CONFIG_HAZPTR_DEBUG
+	bool detach_task, detach_cpu;	/* Whether the ctx has been detached from task/cpu. */
+	int acquire_pid, acquire_cpu;	/* Note the task and cpu number at acquire. */
+	unsigned long acquire_caller;	/* Acquire instruction pointer. */
+#endif
+};
+
+struct hazptr_slot_ctx {
+	struct hazptr_ctx *ctx;
+};
+
+struct hazptr_slot_item {
+	struct hazptr_slot slot;
+	struct hazptr_slot_ctx ctx;
+};
+
+struct hazptr_percpu_slots {
+	struct hazptr_slot_item items[NR_HAZPTR_PERCPU_SLOTS];
+} ____cacheline_aligned;
+
+DECLARE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots);
+
+void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p);
+
+/**
+ * hazptr_synchronize: Wait for release from hazard-pointer protection
+ *
+ * @addr: The address to be released from hazard-pointer protection
+ *
+ * Wait for the specified @addr to be released from protection from all
+ * hazard pointers.  The caller should make @addr inaccessible to all
+ * hazard-pointer readers before invoking this function.
+ *
+ * Must be called from preemptible context.
+ */
+void hazptr_synchronize(void *addr);
+
+/*
+ * hazptr_chain_backup_slot: Chain backup slot into overflow list.
+ *
+ * Set backup slot address to @addr, and chain it into the overflow
+ * list.
+ */
+struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx);
+
+/*
+ * hazptr_unchain_backup_slot: Unchain backup slot from overflow list.
+ */
+void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx);
+
+static inline
+bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
+{
+	return slot == &ctx->backup_slot.slot;
+}
+
+/* Internal helper. */
+static inline
+void hazptr_promote_to_backup_slot(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
+{
+	struct hazptr_slot *backup_slot;
+
+	backup_slot = hazptr_chain_backup_slot(ctx);
+	/*
+	 * Move hazard pointer from the per-CPU slot to the
+	 * backup slot. This requires hazard pointer
+	 * synchronize to iterate on per-CPU slots with
+	 * load-acquire before iterating on the overflow list.
+	 */
+	WRITE_ONCE(backup_slot->addr, slot->addr);
+	/*
+	 * store-release orders store to backup slot addr before
+	 * store to per-CPU slot addr.
+	 */
+	smp_store_release(&slot->addr, NULL);
+	/* Use the backup slot for context. */
+	ctx->slot = backup_slot;
+}
+
+/**
+ * hazptr_detach - Allow a hazard pointer to be released in some other context
+ *
+ * @ctx: The hazard-pointer context to be detached.
+ *
+ * By default, a given hazptr_acquire() and the corresponding
+ * hazptr_release() must run in a single execution context, for example,
+ * the context of a single task or a single interrupt handler.  When you
+ * have acquired a hazard pointer in one context and need to release it
+ * in another, you must invoke hazptr_detach() on that hazard pointer's
+ * context.  It is permissible to invoke hazptr_detach() multiple times
+ * on the same @ctx while it is protecting the same pointer, however,
+ * the first invocation absolutely must be in the same context that did
+ * the hazptr_acquire(), and must take place after the return from that
+ * hazptr_acquire().
+ *
+ * For example, if a hazard pointer is acquired by a task and released
+ * by a timer handler, that task would need to pass the hazard pointer's
+ * context to hazptr_detach() after return from the hazptr_acquire() and
+ * before arming the timer (or at least before the handler had a chance
+ * to access that hazard-pointer context).
+ */
+static inline
+void hazptr_detach(struct hazptr_ctx *ctx)
+{
+	struct hazptr_slot *slot;
+
+	guard(preempt)();
+#ifdef CONFIG_HAZPTR_DEBUG
+	ctx->detach_task = ctx->detach_cpu = true;
+#endif
+	slot = ctx->slot;
+	if (unlikely(hazptr_slot_is_backup(ctx, slot)))
+		return;
+	hazptr_promote_to_backup_slot(ctx, slot);
+}
+
+static inline
+void hazptr_note_context_switch(void)
+{
+	struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+		struct hazptr_slot *slot = &item->slot;
+
+		if (!slot->addr)
+			continue;
+#ifdef CONFIG_HAZPTR_DEBUG
+		item->ctx.ctx->detach_cpu = true;
+#endif
+		hazptr_promote_to_backup_slot(item->ctx.ctx, slot);
+	}
+}
+
+/**
+ * hazptr_acquire - Load pointer at address and protect with hazard pointer.
+ *
+ * @ctx: The hazard-pointer context to be passed to hazptr_release().
+ * @addr_p: Pointer to the pointer that is to be hazard-pointer protected.
+ *
+ * Load @addr_p, and protect the loaded pointer with hazard pointer.
+ * This protection is roughly similar to (but way faster than) that of a
+ * reference counter, and ends with a later call to hazptr_release().
+ *
+ * By default, the call to hazptr_release() must be running in the same
+ * execution context as the corresponding hazptr_acquire(), for example,
+ * within the same task or interrupt handler.  When it is necessary to
+ * instead call hazptr_release() from some other context, pass @ctx to
+ * hazptr_detach() in the original context after invoking hazptr_acquire()
+ * but before making the hazard pointer available to that other context.
+ *
+ * It is not permissible to invoke hazptr_acquire() twice on the same @ctx
+ * without an intervening hazptr_release().
+ *
+ * Returns a non-NULL protected address if the loaded pointer is non-NULL.
+ * Returns NULL if the loaded pointer is NULL.
+ *
+ * On success the protected hazptr slot is stored in @ctx->slot.
+ */
+static inline
+void *hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p)
+{
+	struct hazptr_percpu_slots *percpu_slots;
+	struct hazptr_slot_item *slot_item;
+	struct hazptr_slot *slot;
+	void *addr;
+
+	guard(preempt)();
+	percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	slot_item = &percpu_slots->items[0];
+	slot = &slot_item->slot;
+#ifdef CONFIG_HAZPTR_DEBUG
+	ctx->detach_cpu = ctx->detach_task = false;
+	ctx->acquire_pid = current->pid;
+	ctx->acquire_cpu = smp_processor_id();
+	ctx->acquire_caller = _THIS_IP_;
+#endif
+	if (unlikely(slot->addr))
+		return __hazptr_acquire(ctx, addr_p);
+	WRITE_ONCE(slot->addr, HAZPTR_WILDCARD);	/* Store B */
+
+	/* Memory ordering: Store B before Load A. */
+	smp_mb();
+
+	/*
+	 * Load @addr_p after storing wildcard to the hazard pointer slot.
+	 */
+	addr = READ_ONCE(*addr_p);	/* Load A */
+
+	/*
+	 * We don't care about ordering of Store C. It will simply
+	 * replace the wildcard by a more specific address. If addr is
+	 * NULL, we simply store NULL into the slot.
+	 */
+	WRITE_ONCE(slot->addr, addr);	/* Store C */
+	slot_item->ctx.ctx = ctx;
+	ctx->slot = slot;
+	return addr;
+}
+
+#ifdef CONFIG_HAZPTR_DEBUG
+/* Called with preemption disabled. */
+static inline
+void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr)
+{
+	int pid = current->pid, cpu = smp_processor_id();
+	bool warn_remote_cpu = !ctx->detach_cpu && ctx->acquire_cpu != cpu,
+	     warn_remote_task = !ctx->detach_task && ctx->acquire_pid != pid;
+
+	WARN_ONCE(warn_remote_cpu || warn_remote_task,
+		"Hazard Pointer (addr=%p) released on remote %s without %s. Acquire: caller=%pS, pid=%d, cpu=%d. Release: pid=%d, cpu=%d.",
+		addr,
+		warn_remote_task ? "task" : "cpu",
+		warn_remote_task ? "being detached from task" : "context switch",
+		(void *) ctx->acquire_caller, ctx->acquire_pid, ctx->acquire_cpu, pid, cpu);
+}
+#else
+static inline void hazptr_release_debug(struct hazptr_ctx *ctx, void *addr) { }
+#endif
+
+/**
+ * hazptr_release - Release the specified hazard pointer
+ *
+ * @ctx: The hazard-pointer context that was passed to hazptr_acquire().
+ * @addr_p: The pointer that is to be hazard-pointer unprotected.
+ *
+ * Release the protected hazard pointer recorded in @ctx.
+ *
+ * By default, hazptr_release() must execute in the same execution context
+ * that invoked the corresponding hazptr_acquire(), for example, within the
+ * same task or the same interrupt handler.  However, if this restriction
+ * is problematic for your use case, please see hazptr_detach().
+ *
+ * It is permissible (though unwise from a maintainability viewpoint)
+ * to invoke hazptr_release() twice on the same @ctx without an intervening
+ * hazptr_acquire().
+ */
+static inline
+void hazptr_release(struct hazptr_ctx *ctx, void *addr)
+{
+	struct hazptr_slot *slot;
+
+	if (!addr)
+		return;
+	guard(preempt)();
+	hazptr_release_debug(ctx, addr);
+	slot = ctx->slot;
+	smp_store_release(&slot->addr, NULL);
+	if (unlikely(hazptr_slot_is_backup(ctx, slot)))
+		hazptr_unchain_backup_slot(ctx);
+}
+
+void hazptr_init(void);
+
+#endif /* _LINUX_HAZPTR_H */
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 29072d89e5cb..d44549c0a2c1 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -320,6 +320,14 @@ extern int schedule_hrtimeout_range_clock(ktime_t *expires,
 					  const enum hrtimer_mode mode,
 					  clockid_t clock_id);
 extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
+static inline struct task_struct *hrtimer_sleeper_task_get(struct hrtimer_sleeper *sl)
+{
+	return READ_ONCE(sl->task);
+}
+static inline void hrtimer_sleeper_task_set(struct hrtimer_sleeper *sl, struct task_struct *t)
+{
+	WRITE_ONCE(sl->task, t);
+}
 
 /* Soft interrupt function to run the hrtimer queues: */
 extern void hrtimer_run_queues(void);
diff --git a/include/linux/torture.h b/include/linux/torture.h
index c9b47d138302..1b6d5be641f8 100644
--- a/include/linux/torture.h
+++ b/include/linux/torture.h
@@ -98,6 +98,7 @@ void torture_shutdown_absorb(const char *title);
 int torture_shutdown_init(int ssecs, void (*cleanup)(void));
 
 /* Task stuttering, which forces load/no-load transitions. */
+bool stutter_will_wait(void);
 bool stutter_wait(const char *title);
 int torture_stutter_init(int s, int sgap);
 
@@ -131,8 +132,11 @@ void _torture_stop_kthread(char *m, struct task_struct **tp);
 #endif
 void torture_sched_set_normal(struct task_struct *t, int nice);
 
-#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST)
+#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST)
 long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn);
 #endif
 
+/* Atomic per-CPU counters. */
+s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp);
+
 #endif /* __LINUX_TORTURE_H */
diff --git a/include/linux/wait.h b/include/linux/wait.h
index dce055e6add3..00cdce75837e 100644
--- a/include/linux/wait.h
+++ b/include/linux/wait.h
@@ -556,7 +556,7 @@ do {										\
 	}									\
 										\
 	__ret = ___wait_event(wq_head, condition, state, 0, 0,			\
-		if (!__t.task) {						\
+		if (!hrtimer_sleeper_task_get(&__t)) {				\
 			__ret = -ETIME;						\
 			break;							\
 		}								\
diff --git a/init/main.c b/init/main.c
index 2613d3f9b3ce..d9d936707c19 100644
--- a/init/main.c
+++ b/init/main.c
@@ -108,6 +108,7 @@
 #include <linux/time_namespace.h>
 #include <linux/unaligned.h>
 #include <linux/vdso_datastore.h>
+#include <linux/hazptr.h>
 #include <net/net_namespace.h>
 
 #include <asm/io.h>
@@ -1075,6 +1076,7 @@ void start_kernel(void)
 	workqueue_init_early();
 
 	rcu_init();
+	hazptr_init();
 	kvfree_rcu_init();
 
 	/* Trace events are available after this */
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 95038cfda615..0c0827623cb6 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -1299,7 +1299,7 @@ static u64 io_hybrid_iopoll_delay(struct io_ring_ctx *ctx, struct io_kiocb *req)
 	set_current_state(TASK_INTERRUPTIBLE);
 	hrtimer_sleeper_start_expires(&timer, mode);
 
-	if (timer.task)
+	if (hrtimer_sleeper_task_get(&timer))
 		io_schedule();
 
 	hrtimer_cancel(&timer.timer);
diff --git a/kernel/Makefile b/kernel/Makefile
index 1e1a31673577..8961c8660d0d 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -7,7 +7,7 @@ obj-y     = fork.o exec_domain.o exec_state.o panic.o \
 	    cpu.o exit.o softirq.o resource.o \
 	    sysctl.o capability.o ptrace.o user.o \
 	    signal.o sys.o umh.o workqueue.o pid.o task_work.o \
-	    extable.o params.o \
+	    extable.o params.o hazptr.o \
 	    kthread.o sys_ni.o nsproxy.o nstree.o nscommon.o \
 	    notifier.o ksysfs.o cred.o reboot.o \
 	    async.o range.o smpboot.o ucount.o regset.o ksyms_common.o
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index 79823ad13683..196d6ff148a4 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -736,7 +736,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
 
 	/* Handle spurious wakeups gracefully */
 	ret = -EWOULDBLOCK;
-	if (timeout && !timeout->task)
+	if (timeout && !hrtimer_sleeper_task_get(timeout))
 		ret = -ETIMEDOUT;
 	else if (signal_pending(current))
 		ret = -ERESTARTNOINTR;
diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c
index d4483d15d30a..cf18309e5770 100644
--- a/kernel/futex/waitwake.c
+++ b/kernel/futex/waitwake.c
@@ -383,7 +383,7 @@ void futex_do_wait(struct futex_q *q, struct hrtimer_sleeper *timeout)
 		 * flagged for rescheduling. Only call schedule if there
 		 * is no timeout, or if it has yet to expire.
 		 */
-		if (!timeout || timeout->task)
+		if (!timeout || hrtimer_sleeper_task_get(timeout))
 			schedule();
 	}
 	__set_current_state(TASK_RUNNING);
@@ -539,7 +539,7 @@ retry:
 static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count,
 				 struct hrtimer_sleeper *to)
 {
-	if (to && !to->task)
+	if (to && !hrtimer_sleeper_task_get(to))
 		return;
 
 	for (; count; count--, vs++) {
@@ -590,7 +590,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
 		if (ret >= 0)
 			return ret;
 
-		if (to && !to->task)
+		if (to && !hrtimer_sleeper_task_get(to))
 			return -ETIMEDOUT;
 		else if (signal_pending(current))
 			return -ERESTARTSYS;
@@ -725,7 +725,7 @@ retry:
 	if (!futex_unqueue(&q))
 		return 0;
 
-	if (to && !to->task)
+	if (to && !hrtimer_sleeper_task_get(to))
 		return -ETIMEDOUT;
 
 	/*
diff --git a/kernel/hazptr.c b/kernel/hazptr.c
new file mode 100644
index 000000000000..a9d3d68a1525
--- /dev/null
+++ b/kernel/hazptr.c
@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+//
+// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+
+/*
+ * hazptr: Hazard Pointers
+ */
+
+#include <linux/hazptr.h>
+#include <linux/percpu.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/export.h>
+
+struct hazptr_overflow_list {
+	raw_spinlock_t lock;		/* Lock protecting overflow list and list generation. */
+	struct hlist_head head;		/* Overflow list head. */
+	uint64_t gen;			/* Overflow list generation. */
+};
+
+/*
+ * Flip between two lists to guarantee list scan forward progress even
+ * with frequent generation counter increments. The list additions are
+ * always done on a different list than the one used for scan. The scan
+ * successively iterates on both lists. Therefore, only list removals
+ * can cause the iteration to retry, and the number of removals is
+ * limited to the number of list elements.
+ */
+struct hazptr_overflow_list_flip {
+	struct mutex lock;		/* Mutex protecting add_idx from concurrent updates. */
+	unsigned int add_idx;		/* Index of current flip-list to add to. */
+	struct hazptr_overflow_list array[2];
+};
+
+static DEFINE_PER_CPU(struct hazptr_overflow_list_flip, percpu_overflow_list_flip);
+
+DEFINE_PER_CPU(struct hazptr_percpu_slots, hazptr_percpu_slots);
+EXPORT_PER_CPU_SYMBOL_GPL(hazptr_percpu_slots);
+
+static
+struct hazptr_slot *hazptr_get_free_percpu_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_percpu_slots *percpu_slots = this_cpu_ptr(&hazptr_percpu_slots);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+		struct hazptr_slot *slot = &item->slot;
+
+		if (!slot->addr) {
+			item->ctx.ctx = ctx;
+			return slot;
+		}
+	}
+	/* All slots are in use. */
+	return NULL;
+}
+
+/*
+ * Hazard pointer acquire slow path.
+ * Called with preemption disabled.
+ */
+void *__hazptr_acquire(struct hazptr_ctx *ctx, void * const *addr_p)
+{
+	struct hazptr_slot *slot = hazptr_get_free_percpu_slot(ctx);
+	void *addr;
+
+	/*
+	 * If all the per-CPU slots are already in use, fallback
+	 * to the backup slot.
+	 */
+	if (unlikely(!slot))
+		slot = hazptr_chain_backup_slot(ctx);
+	WRITE_ONCE(slot->addr, HAZPTR_WILDCARD);	/* Store B */
+
+	/* Memory ordering: Store B before Load A. */
+	smp_mb();
+
+	/*
+	 * Load @addr_p after storing wildcard to the hazard pointer slot.
+	 */
+	addr = READ_ONCE(*addr_p);	/* Load A */
+
+	/*
+	 * We don't care about ordering of Store C. It will simply
+	 * replace the wildcard by a more specific address. If addr is
+	 * NULL, we simply store NULL into the slot.
+	 */
+	WRITE_ONCE(slot->addr, addr);	/* Store C */
+	ctx->slot = slot;
+	if (!addr && hazptr_slot_is_backup(ctx, slot))
+		hazptr_unchain_backup_slot(ctx);
+	return addr;
+}
+EXPORT_SYMBOL_GPL(__hazptr_acquire);
+
+/*
+ * Perform piecewise iteration on overflow list waiting until "addr" is
+ * not present. Raw spinlock is released and taken between each list
+ * item and busy loop iteration. The overflow list generation is checked
+ * each time the lock is taken to validate that the list has not changed
+ * before resuming iteration or busy wait. If the generation has
+ * changed, retry the entire list traversal.
+ */
+static
+void hazptr_synchronize_overflow_list(struct hazptr_overflow_list *overflow_list, void *addr)
+{
+	struct hazptr_backup_slot *backup_slot;
+	uint64_t snapshot_gen;
+	unsigned long flags;
+
+	raw_spin_lock_irqsave(&overflow_list->lock, flags);
+retry:
+	snapshot_gen = overflow_list->gen;
+	hlist_for_each_entry(backup_slot, &overflow_list->head, overflow_node) {
+		/* Busy-wait if node is found. */
+		for (;;) {
+			void *load_addr = smp_load_acquire(&backup_slot->slot.addr);	/* Load B */
+
+			if (load_addr != addr && load_addr != HAZPTR_WILDCARD)
+				break;
+			raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+			cpu_relax();
+			raw_spin_lock_irqsave(&overflow_list->lock, flags);
+			if (overflow_list->gen != snapshot_gen)
+				goto retry;
+		}
+		raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+		/*
+		 * Release raw spinlock, validate generation after
+		 * re-acquiring the lock.
+		 */
+		raw_spin_lock_irqsave(&overflow_list->lock, flags);
+		if (overflow_list->gen != snapshot_gen)
+			goto retry;
+	}
+	raw_spin_unlock_irqrestore(&overflow_list->lock, flags);
+}
+
+static
+void hazptr_synchronize_cpu_slots(int cpu, void *addr)
+{
+	struct hazptr_percpu_slots *percpu_slots = per_cpu_ptr(&hazptr_percpu_slots, cpu);
+	unsigned int idx;
+
+	for (idx = 0; idx < NR_HAZPTR_PERCPU_SLOTS; idx++) {
+		struct hazptr_slot_item *item = &percpu_slots->items[idx];
+
+		/* Busy-wait if node is found. */
+		smp_cond_load_acquire(&item->slot.addr, VAL != addr && VAL != HAZPTR_WILDCARD); /* Load B */
+	}
+}
+
+/*
+ * hazptr_synchronize: Wait until @addr is released from all slots.
+ *
+ * Wait to observe that each slot contains a value that differs from
+ * @addr before returning.
+ * Should be called from preemptible context.
+ */
+void hazptr_synchronize(void *addr)
+{
+	int cpu;
+
+	/*
+	 * Busy-wait should only be done from preemptible context.
+	 */
+	lockdep_assert_preemption_enabled();
+
+	/*
+	 * Store A precedes hazptr_scan(): it unpublishes addr (sets it to
+	 * NULL or to a different value), and thus hides it from hazard
+	 * pointer readers.
+	 */
+	if (!addr)
+		return;
+	/* Memory ordering: Store A before Load B. */
+	smp_mb();
+	/* Scan all CPUs slots. */
+	for_each_possible_cpu(cpu) {
+		struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu);
+		unsigned int scan_idx;
+
+		/* Scan CPU slots. */
+		hazptr_synchronize_cpu_slots(cpu, addr);
+
+		/*
+		 * Scan backup slots in percpu overflow lists.
+		 * Forward progress is guaranteed by scanning one list
+		 * while new elements are added into the other list.
+		 */
+		guard(mutex)(&overflow_list_flip->lock);
+		scan_idx = overflow_list_flip->add_idx ^ 1;
+		hazptr_synchronize_overflow_list(&overflow_list_flip->array[scan_idx], addr);
+		/* Flip current list. */
+		WRITE_ONCE(overflow_list_flip->add_idx, scan_idx);
+		hazptr_synchronize_overflow_list(&overflow_list_flip->array[scan_idx ^ 1], addr);
+	}
+}
+EXPORT_SYMBOL_GPL(hazptr_synchronize);
+
+struct hazptr_slot *hazptr_chain_backup_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_overflow_list_flip *overflow_list_flip = this_cpu_ptr(&percpu_overflow_list_flip);
+	unsigned int list_idx = READ_ONCE(overflow_list_flip->add_idx);
+	struct hazptr_overflow_list *overflow_list = &overflow_list_flip->array[list_idx];
+	struct hazptr_slot *slot = &ctx->backup_slot.slot;
+
+	slot->addr = NULL;
+	guard(raw_spinlock_irqsave)(&overflow_list->lock);
+	overflow_list->gen++;
+	hlist_add_head(&ctx->backup_slot.overflow_node, &overflow_list->head);
+	ctx->backup_slot.overflow_list = overflow_list;
+	return slot;
+}
+EXPORT_SYMBOL_GPL(hazptr_chain_backup_slot);
+
+void hazptr_unchain_backup_slot(struct hazptr_ctx *ctx)
+{
+	struct hazptr_overflow_list *overflow_list = ctx->backup_slot.overflow_list;
+
+	guard(raw_spinlock_irqsave)(&overflow_list->lock);
+	overflow_list->gen++;
+	hlist_del(&ctx->backup_slot.overflow_node);
+}
+EXPORT_SYMBOL_GPL(hazptr_unchain_backup_slot);
+
+void __init hazptr_init(void)
+{
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct hazptr_overflow_list_flip *overflow_list_flip = per_cpu_ptr(&percpu_overflow_list_flip, cpu);
+
+		mutex_init(&overflow_list_flip->lock);
+		for (int i = 0; i < 2; i++) {
+			raw_spin_lock_init(&overflow_list_flip->array[i].lock);
+			INIT_HLIST_HEAD(&overflow_list_flip->array[i].head);
+		}
+	}
+}
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4728631ae719..5a9534c715b8 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -1644,7 +1644,7 @@ static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock,
 			break;
 		}
 
-		if (timeout && !timeout->task) {
+		if (timeout && !hrtimer_sleeper_task_get(timeout)) {
 			ret = -ETIMEDOUT;
 			break;
 		}
diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
index e078e988773d..9c3f72474cb5 100644
--- a/kernel/rcu/Kconfig.debug
+++ b/kernel/rcu/Kconfig.debug
@@ -113,6 +113,18 @@ config RCU_REF_SCALE_TEST
 	  Say M if you want to build it as a module instead.
 	  Say N if you are unsure.
 
+config HAZPTR_TORTURE_TEST
+	tristate "Torture tests for hazard pointers"
+	depends on DEBUG_KERNEL
+	select TORTURE_TEST
+	default n
+	help
+	  This option provides in-kernel hazard-pointer stress tests.
+
+	  Say Y here if you want hazard-pointer testing built into the kernel.
+	  Say M if you want to build them as a module instead.
+	  Say N if you are unsure.
+
 config RCU_CPU_STALL_TIMEOUT
 	int "RCU CPU stall timeout in seconds"
 	depends on RCU_STALL_COMMON
@@ -239,4 +251,13 @@ config TRIVIAL_PREEMPT_RCU
 
 	  This has no value for production and is only for testing.
 
+config HAZPTR_DEBUG
+	bool "Provide debugging asserts for Hazard Pointers"
+	depends on DEBUG_KERNEL
+	default n
+	help
+	  This option provides consistency checks for Hazard pointers.
+
+	  Say Y here if you want to enable those assert, N otherwise.
+
 endmenu # "RCU Debugging"
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index 0cfb009a99b9..478ef2237fee 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -10,6 +10,7 @@ endif
 obj-y += update.o sync.o
 obj-$(CONFIG_TREE_SRCU) += srcutree.o
 obj-$(CONFIG_TINY_SRCU) += srcutiny.o
+obj-$(CONFIG_HAZPTR_TORTURE_TEST) += hazptrtorture.o
 obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
 obj-$(CONFIG_RCU_SCALE_TEST) += rcuscale.o
 obj-$(CONFIG_RCU_REF_SCALE_TEST) += refscale.o
diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c
new file mode 100644
index 000000000000..7526ea65ff7c
--- /dev/null
+++ b/kernel/rcu/hazptrtorture.c
@@ -0,0 +1,947 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hazard-pointer module-based torture test facility
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ *
+ * Author: Paul E. McKenney <paulmck@kernel.org>
+ */
+
+#define pr_fmt(fmt) fmt
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/sched/debug.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/reboot.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/torture.h>
+#include <linux/hazptr.h>
+#include <linux/rcupdate.h>
+
+#include "rcu.h"
+
+MODULE_DESCRIPTION("Hazard-pointer module-based torture test facility");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Paul E. McKenney <paulmckrcu@meta.com>");
+
+torture_param(int, defer_modulus, -1, "Defer once per specified # of hazptr ops, zero to disable");
+torture_param(int, irq_acquire, -1,
+	      "Acquire hazard pointers from irq handlers once per specified #, zero to disable");
+torture_param(int, irq_release, -1,
+	      "Release hazard pointers from irq handlers once per specified #, zero to disable");
+torture_param(int, kthread_do_pending_ms, -1,
+	      "Delay between cleanups for deferred hazard pointers (ms), zero to disable");
+torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads");
+torture_param(int, nwriters, 1, "Number of hazard-pointer writer threads, 0 or 1");
+torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
+torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable");
+torture_param(int, preempt_duration, 0, "Preemption duration (ms), zero to disable");
+torture_param(int, preempt_interval, MSEC_PER_SEC, "Interval between preemptions (ms)");
+torture_param(int, reader_sleep_us, 0, "Reader sleep duration (us)");
+torture_param(int, shuffle_interval, 3, "Number of seconds between shuffles");
+torture_param(int, shutdown_secs, 0, "Shutdown time (s), <= zero to disable.");
+torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s");
+torture_param(int, stutter, 5, "Number of seconds to run/halt test");
+torture_param(int, verbose, 1, "Enable verbose debugging printk()s");
+
+static char *torture_type = "hazptr";
+module_param(torture_type, charp, 0444);
+MODULE_PARM_DESC(torture_type, "Type of hazard pointers to torture (hazptr, ...)");
+
+static int nrealreaders;
+static struct task_struct *writer_task;
+static struct task_struct *preempt_task;
+static struct task_struct **reader_tasks;
+static struct task_struct *do_pending_task;
+static struct task_struct *stats_task;
+
+#define HAZPTR_TORTURE_PIPE_LEN 10
+
+// Update-side data structure used to check RCU readers.
+struct hazptr_torture {
+	void *obj_hazptr;
+	int htort_pipe_count;
+	struct list_head htort_free;
+};
+
+static LIST_HEAD(hazptr_torture_freelist);
+static struct hazptr_torture *hazptr_torture_current;
+static unsigned long hazptr_torture_current_version;
+static struct hazptr_torture hazptr_tortures[10 * HAZPTR_TORTURE_PIPE_LEN];
+static DEFINE_SPINLOCK(hazptr_torture_lock);
+static DEFINE_PER_CPU(long [HAZPTR_TORTURE_PIPE_LEN + 1], hazptr_torture_count);
+static atomic_t hazptr_torture_wcount[HAZPTR_TORTURE_PIPE_LEN + 1];
+static atomic_t n_hazptr_torture_alloc;
+static atomic_t n_hazptr_torture_alloc_fail;
+static atomic_t n_hazptr_torture_free;
+static atomic_t n_hazptr_torture_error;
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_acquires_irq);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_irq);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_defer);
+static DEFINE_PER_CPU(atomic_long_t, hazptr_torture_releases_undefer);
+static struct list_head hazptr_torture_removed;
+
+// State for a deferred (AKA pending) hazard pointer
+struct hazptr_pending {
+	struct llist_node hpp_node;
+	struct hazptr_ctx hpp_hc;
+	struct hazptr_torture *hpp_htp;
+};
+static DEFINE_PER_CPU(struct llist_head, hazptr_pending);
+
+static int hazptr_torture_writer_state;
+#define HTWS_FIXED_DELAY	0
+#define HTWS_DELAY		1
+#define HTWS_REPLACE		2
+#define HTWS_SYNC		3
+#define HTWS_STUTTER		4
+#define HTWS_STOPPING		5
+static const char * const hazptr_torture_writer_state_names[] = {
+	"HTWS_FIXED_DELAY",
+	"HTWS_DELAY",
+	"HTWS_REPLACE",
+	"HTWS_SYNC",
+	"HTWS_STUTTER",
+	"HTWS_STOPPING",
+};
+
+static const char *hazptr_torture_writer_state_getname(void)
+{
+	unsigned int i = READ_ONCE(hazptr_torture_writer_state);
+
+	if (i >= ARRAY_SIZE(hazptr_torture_writer_state_names))
+		return "???";
+	return hazptr_torture_writer_state_names[i];
+}
+
+/*
+ * Allocate an element from the hazptr_tortures pool.
+ */
+static struct hazptr_torture *hazptr_torture_alloc(void)
+{
+	struct list_head *p;
+
+	spin_lock_bh(&hazptr_torture_lock);
+	if (list_empty(&hazptr_torture_freelist)) {
+		atomic_inc(&n_hazptr_torture_alloc_fail);
+		spin_unlock_bh(&hazptr_torture_lock);
+		return NULL;
+	}
+	atomic_inc(&n_hazptr_torture_alloc);
+	p = hazptr_torture_freelist.next;
+	list_del_init(p);
+	spin_unlock_bh(&hazptr_torture_lock);
+	return container_of(p, struct hazptr_torture, htort_free);
+}
+
+/*
+ * Free an element to the hazptr_tortures pool.
+ */
+static void
+hazptr_torture_free(struct hazptr_torture *p)
+{
+	atomic_inc(&n_hazptr_torture_free);
+	spin_lock_bh(&hazptr_torture_lock);
+	list_add_tail(&p->htort_free, &hazptr_torture_freelist);
+	spin_unlock_bh(&hazptr_torture_lock);
+}
+
+/*
+ * Update object in the pipe.  This should be invoked after a suitable time.
+ */
+static bool
+hazptr_torture_pipe_update_one(struct hazptr_torture *rp)
+{
+	int i;
+
+	i = rp->htort_pipe_count;
+	if (i > HAZPTR_TORTURE_PIPE_LEN)
+		i = HAZPTR_TORTURE_PIPE_LEN;
+	atomic_inc(&hazptr_torture_wcount[i]);
+	WRITE_ONCE(rp->htort_pipe_count, i + 1);
+	ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count);
+	if (i + 1 >= HAZPTR_TORTURE_PIPE_LEN)
+		return true;
+	return false;
+}
+
+/*
+ * Update all callbacks in the pipe each time period.
+ */
+static void
+hazptr_torture_pipe_update(struct hazptr_torture *old_rp)
+{
+	struct hazptr_torture *rp;
+	struct hazptr_torture *rp1;
+
+	if (old_rp)
+		list_add(&old_rp->htort_free, &hazptr_torture_removed);
+	list_for_each_entry_safe(rp, rp1, &hazptr_torture_removed, htort_free) {
+		if (hazptr_torture_pipe_update_one(rp)) {
+			list_del(&rp->htort_free);
+			hazptr_torture_free(rp);
+		}
+	}
+}
+
+/*
+ * Operations vector for selecting different types of tests.
+ */
+
+struct hazptr_torture_ops {
+	void (*init)(void);
+	void (*cleanup)(void);
+	struct hazptr_torture *((*readlock)(struct hazptr_ctx *hcpp));
+	void (*read_delay)(struct torture_random_state *rrsp);
+	void (*readunlock)(struct hazptr_ctx *hcp, struct hazptr_torture *htp);
+	void (*sync)(void *htp);
+	int irq_capable;
+	int onstack_ctx;
+	const char *name;
+};
+
+static struct hazptr_torture_ops *cur_ops;
+
+/*
+ * Definitions for hazard-pointer torture testing using per-CPU hazptr_ctx
+ * structures.
+ */
+
+static struct hazptr_torture *hazptr_torture_read_lock(struct hazptr_ctx *hcpp)
+{
+	struct hazptr_torture *htp;
+
+	htp = (struct hazptr_torture *)hazptr_acquire(hcpp, (void *)&hazptr_torture_current);
+	return htp;
+}
+
+static void hazptr_read_delay(struct torture_random_state *rrsp)
+{
+	const bool can_sleep = !preempt_count() && !irqs_disabled();
+	const unsigned long shortdelay_us = 200;
+	unsigned long longdelay_ms = 300;
+	const bool short_spin = irqs_disabled() || irq_count();
+
+	 // We want a short delay sometimes to make a reader delay the grace
+	 // period, and we want a long delay occasionally to trigger
+	 // force_quiescent_state.
+
+	if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
+		if (short_spin)
+			longdelay_ms = 5; /* Avoid triggering BH limits. */
+		mdelay(longdelay_ms);
+	}
+	if (!(torture_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
+		udelay(shortdelay_us);
+	if (can_sleep && !(torture_random(rrsp) % (nrealreaders * 500)))
+		torture_preempt_schedule();  /* QS only if preemptible. */
+	if (can_sleep && reader_sleep_us > 0)
+		torture_hrtimeout_us(reader_sleep_us, 0, NULL);
+}
+
+static void hazptr_torture_read_unlock(struct hazptr_ctx *hcp, struct hazptr_torture *htp)
+{
+	if (hcp) {
+		hazptr_release(hcp, htp);
+	}
+}
+
+static void hazptr_sync_torture_init(void)
+{
+	INIT_LIST_HEAD(&hazptr_torture_removed);
+}
+
+static struct hazptr_torture_ops hazptr_ops = {
+	.init			= hazptr_sync_torture_init,
+	.readlock		= hazptr_torture_read_lock,
+	.read_delay		= hazptr_read_delay,
+	.readunlock		= hazptr_torture_read_unlock,
+	.sync			= hazptr_synchronize,
+	.irq_capable		= 1,
+	.name			= "hazptr"
+};
+
+/*
+ * Definitions for hazard-pointer torture testing using on-stack
+ * hazptr_ctx structures.
+ */
+
+static struct hazptr_torture_ops hazptr_stack_ops = {
+	.init			= hazptr_sync_torture_init,
+	.readlock		= hazptr_torture_read_lock,
+	.read_delay		= hazptr_read_delay,
+	.readunlock		= hazptr_torture_read_unlock,
+	.sync			= hazptr_synchronize,
+	.irq_capable		= 1,
+	.onstack_ctx		= 1,
+	.name			= "hazptr-stack"
+};
+
+/*
+ * Hazard-pointer torture writer kthread.  Repeatedly substitutes a new
+ * structure for that pointed to by hazptr_torture_current, freeing the
+ * old structure after a series of timeouts (the "pipeline").
+ */
+static int
+hazptr_torture_writer(void *arg)
+{
+	bool booting_still = false;
+	int i;
+	unsigned long j;
+	int oldnice = task_nice(current);
+	struct hazptr_torture *rp;
+	struct hazptr_torture *old_rp;
+	static DEFINE_TORTURE_RANDOM(rand);
+	bool stutter_waited;
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_writer task started");
+	// If the system is still booting, let it finish.
+	j = jiffies;
+	while (!torture_must_stop() && !rcu_inkernel_boot_has_ended()) {
+		booting_still = true;
+		schedule_timeout_interruptible(HZ);
+	}
+	if (booting_still)
+		pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n",
+			 torture_type, jiffies - j);
+
+	do {
+		hazptr_torture_writer_state = HTWS_FIXED_DELAY;
+		torture_hrtimeout_us(500, 1000, &rand);
+		rp = hazptr_torture_alloc();
+		if (rp == NULL)
+			continue;
+		rp->htort_pipe_count = 0;
+		ASSERT_EXCLUSIVE_WRITER(rp->htort_pipe_count);
+		hazptr_torture_writer_state = HTWS_DELAY;
+		udelay(torture_random(&rand) & 0x3ff);
+		hazptr_torture_writer_state = HTWS_REPLACE;
+		old_rp = READ_ONCE(hazptr_torture_current);
+		smp_store_release(&hazptr_torture_current, rp); // Publish new structure.
+		smp_wmb(); /* Mods to old_rp must follow smp_store_release() */
+		if (old_rp) {
+			i = old_rp->htort_pipe_count;
+			if (i > HAZPTR_TORTURE_PIPE_LEN)
+				i = HAZPTR_TORTURE_PIPE_LEN;
+			atomic_inc(&hazptr_torture_wcount[i]);
+			WRITE_ONCE(old_rp->htort_pipe_count,
+				   old_rp->htort_pipe_count + 1);
+			ASSERT_EXCLUSIVE_WRITER(old_rp->htort_pipe_count);
+
+			hazptr_torture_writer_state = HTWS_SYNC;
+			cur_ops->sync((void *)old_rp);
+			hazptr_torture_pipe_update(old_rp);
+		}
+
+		WRITE_ONCE(hazptr_torture_current_version, hazptr_torture_current_version + 1);
+		hazptr_torture_writer_state = HTWS_STUTTER;
+		stutter_waited = stutter_wait("hazptr_torture_writer");
+		if (stutter_waited && !torture_must_stop())
+			for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++)
+				if (list_empty(&hazptr_tortures[i].htort_free) &&
+				    READ_ONCE(hazptr_torture_current) != &hazptr_tortures[i]) {
+					tracing_off();
+					WARN(1, "%s: htort_pipe_count: %d\n", __func__, hazptr_tortures[i].htort_pipe_count);
+					rcu_ftrace_dump(DUMP_ALL);
+					break;
+				}
+		if (stutter_waited)
+			sched_set_normal(current, oldnice);
+	} while (!torture_must_stop());
+	hazptr_torture_current = NULL;  // Let stats task know that we are done.
+	hazptr_torture_writer_state = HTWS_STOPPING;
+	torture_kthread_stopping("hazptr_torture_writer");
+	return 0;
+}
+
+/*
+ * Acquire a hazard pointer from an smp_call_function handler.
+ */
+static void hazptr_torture_acquire(void *hppp_in)
+{
+	struct hazptr_pending *hppp = hppp_in;
+
+	hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+	/*
+	 * Acquiring a hazard pointer from a remote CPU.
+	 * Detach hazptr from its task so it can be released by another task.
+	 */
+	hazptr_detach(&hppp->hpp_hc);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires_irq, raw_smp_processor_id()));
+}
+
+/*
+ * Release a hazard pointer from an smp_call_function handler.
+ */
+static void hazptr_torture_release(void *hppp_in)
+{
+	struct hazptr_pending *hppp = hppp_in;
+
+	cur_ops->readunlock(&hppp->hpp_hc, hppp->hpp_htp);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_irq, raw_smp_processor_id()));
+}
+
+/*
+ * Do the delay, the accounting, and the release.  This in intended to
+ * be invoked from hazptr_torture_reader, but also for hazard pointers
+ * sent off to interrupt handlers and the like.
+ */
+static void
+hazptr_torture_reader_tail(struct hazptr_pending *hppp, struct torture_random_state *trsp)
+{
+	int cpu;
+	struct hazptr_ctx *hcp = &hppp->hpp_hc;
+	struct hazptr_torture *htp = hppp->hpp_htp;
+	int pipe_count;
+
+	cur_ops->read_delay(trsp);
+	preempt_disable();
+	pipe_count = READ_ONCE(htp->htort_pipe_count);
+	if (pipe_count > HAZPTR_TORTURE_PIPE_LEN) {
+		// Should not happen in a correct hazptr implementation,
+		// happens quite often for TBD torture_type=busted.
+		pipe_count = HAZPTR_TORTURE_PIPE_LEN;
+	}
+	if (pipe_count > 1)
+		rcu_ftrace_dump(DUMP_ALL);
+	__this_cpu_inc(hazptr_torture_count[pipe_count]);
+	preempt_enable();
+	if (irq_release && !(torture_random(trsp) % irq_release)) {
+		guard(preempt)();
+		/*
+		 * Handling release from a remote CPU.
+		 * Detach hazptr from its task so it can be released by another task.
+		 */
+		hazptr_detach(&hppp->hpp_hc);
+		cpu = cpumask_next_wrap(smp_processor_id(), cpu_online_mask);
+		smp_call_function_single(cpu, hazptr_torture_release, hppp, 1);
+	} else {
+		cur_ops->readunlock(hcp, htp);
+		atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases, raw_smp_processor_id()));
+	}
+}
+
+/*
+ * Defer the specified hazard pointer to some other context.
+ */
+static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_random_state *trsp)
+{
+	int cpu = torture_random(trsp) % nr_cpu_ids;
+	struct llist_head *llhp;
+
+	guard(preempt)();
+	/*
+	 * Handling release from a remote CPU.
+	 * Detach hazptr from its task so it can be released by another task.
+	 */
+	hazptr_detach(&hppp->hpp_hc);
+	cpu = cpumask_next_wrap(cpu, cpu_online_mask);
+	llhp = per_cpu_ptr(&hazptr_pending, cpu);
+	llist_add(&hppp->hpp_node, llhp);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_defer, raw_smp_processor_id()));
+}
+
+/*
+ * Hazard-pointer torture reader kthread.  Repeatedly dereferences
+ * hazptr_torture_current, incrementing the corresponding element of the
+ * pipeline array.  The counter in the element should never be greater
+ * than 1, otherwise, the hazard-pointer implementation is broken.
+ */
+static int hazptr_torture_reader(void *arg)
+{
+	bool can_defer = !cur_ops->onstack_ctx && kthread_do_pending_ms && defer_modulus;
+	int cpu = 0;
+	struct hazptr_pending hpp;
+	struct hazptr_pending *hppp = cur_ops->onstack_ctx ? &hpp : NULL;
+	unsigned long lastsleep = jiffies;
+	long myid = (long)arg;
+	int mynumonline = myid % nr_cpu_ids;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_reader task started");
+	set_user_nice(current, MAX_NICE);
+	do {
+		if (!hppp) {
+			hppp = kmalloc_obj(*hppp, GFP_KERNEL);
+			if (!hppp) {
+				// Allocation failure, so get out of the way.
+				schedule_timeout_interruptible(HZ / 10);
+				continue;
+			}
+		}
+		if (irq_acquire && !(torture_random(&rand) % irq_acquire)) {
+			guard(preempt)();
+			cpu = cpumask_next_wrap(cpu, cpu_online_mask);
+			if (cpu != smp_processor_id()) {
+				smp_call_function_single(cpu, hazptr_torture_acquire, hppp, 1);
+			} else {
+				hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+				atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires,
+							    raw_smp_processor_id()));
+			}
+		} else {
+			hppp->hpp_htp = cur_ops->readlock(&hppp->hpp_hc);
+			atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires,
+						    raw_smp_processor_id()));
+		}
+		if (!hppp->hpp_htp) {
+			// Still starting up, so get out of the way.
+			schedule_timeout_interruptible(HZ / 10);
+			continue;
+		}
+		if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
+			torture_hrtimeout_us(500, 1000, &rand);
+			lastsleep = jiffies + 10;
+		}
+		if (can_defer && !(torture_random(&rand) % defer_modulus)) {
+			hazptr_torture_defer(hppp, &rand);
+			hppp = NULL;
+		} else {
+			hazptr_torture_reader_tail(hppp, &rand);
+		}
+		while (!torture_must_stop() &&
+		       (torture_num_online_cpus() < mynumonline || !rcu_inkernel_boot_has_ended()))
+			schedule_timeout_interruptible(HZ / 5);
+		stutter_wait("hazptr_torture_reader");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_reader");
+	return 0;
+}
+
+/*
+ * Release the specified CPU's set of deferred/pending hazard pointers.
+ */
+static void hazptr_torture_do_one_pending(int cpu, struct torture_random_state *trsp)
+{
+	struct hazptr_pending *hppp;
+	struct hazptr_pending *hppp1;
+	struct llist_head *llhp;
+	struct llist_node *llnp;
+
+	llhp = per_cpu_ptr(&hazptr_pending, cpu);
+	llnp = llist_del_all(llhp);
+	if (!llnp)
+		return;
+	llist_for_each_entry_safe(hppp, hppp1, llnp, hpp_node) {
+		hazptr_torture_reader_tail(hppp, trsp);
+		atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_undefer,
+					    raw_smp_processor_id()));
+		kfree(hppp);
+	}
+}
+
+/*
+ * Hazard-pointer release of deferred/pending hazard pointers.
+ */
+static int hazptr_torture_do_pending(void *arg)
+{
+	int cpu = 0;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	VERBOSE_TOROUT_STRING("hazptr_torture_do_pending task started");
+	do {
+		if (stutter_will_wait()) {
+			for_each_possible_cpu(cpu)
+				hazptr_torture_do_one_pending(cpu, &rand);
+		} else {
+			cpu = cpumask_next_wrap(cpu, cpu_possible_mask);
+			hazptr_torture_do_one_pending(cpu, &rand);
+		}
+		if (torture_must_stop())
+			torture_hrtimeout_ms(kthread_do_pending_ms, USEC_PER_MSEC, &rand);
+		// Omit stutter_wait() because this function needs to do cleanup.
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_do_pending");
+	return 0;
+}
+
+/*
+ * Spawn hazptr_torture_do_pending() if there is something for it to do.
+ */
+static int hazptr_torture_do_pending_init(void)
+{
+	if (kthread_do_pending_ms == -1)
+		kthread_do_pending_ms = (cur_ops->onstack_ctx || defer_modulus == 0) ? 0 : 3;
+	if (defer_modulus == -1)
+		defer_modulus = (cur_ops->onstack_ctx ||
+				 kthread_do_pending_ms == 0) ? 0 : 1000 * nr_cpu_ids;
+	if (kthread_do_pending_ms < 0) {
+		pr_alert("Cannot have negative kthread_do_pending_ms, disabling deferral.\n");
+		goto err_out;
+	}
+	if (cur_ops->onstack_ctx && kthread_do_pending_ms) {
+		pr_alert("Cannot defer onstack hazptr_ctx, disabling deferral.\n");
+		goto err_out;
+	}
+	if (defer_modulus < 0) {
+		pr_alert("Cannot have negative defer_modulus (%d), disabling deferral.\n",
+			 defer_modulus);
+		goto err_out;
+	}
+	if (!kthread_do_pending_ms != !defer_modulus) {
+		pr_alert("Pending kthread (%d) & deferral (%d) don't match, disabling deferral.\n",
+			 kthread_do_pending_ms, defer_modulus);
+		goto err_out;
+	}
+	if (!kthread_do_pending_ms)
+		return 0;
+	return torture_create_kthread(hazptr_torture_do_pending, NULL, do_pending_task);
+
+err_out:
+	WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+	kthread_do_pending_ms = 0;
+	defer_modulus = 0;
+	return 0;
+}
+
+/*
+ * Print torture statistics.  Caller must ensure that there is only one
+ * call to this function at a given time!!!  This is normally accomplished
+ * by relying on the module system to only have one copy of the module
+ * loaded, and then by giving the hazptr_torture_stats kthread full control
+ * (or the init/cleanup functions when hazptr_torture_stats thread is
+ * not running).
+ */
+static void
+hazptr_torture_stats_print(void)
+{
+	const char *cp = hazptr_torture_writer_state_getname();
+	int cpu;
+	int i;
+	long pipesummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 };
+	long batchsummary[HAZPTR_TORTURE_PIPE_LEN + 1] = { 0 };
+	struct hazptr_torture *rtcp;
+	static unsigned long rtcv_snap = ULONG_MAX;
+	static bool splatted;
+	struct task_struct *wtp;
+
+	for_each_possible_cpu(cpu)
+		for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+			pipesummary[i] += READ_ONCE(per_cpu(hazptr_torture_count, cpu)[i]);
+	for (i = HAZPTR_TORTURE_PIPE_LEN; i >= 0; i--) {
+		if (pipesummary[i] != 0)
+			break;
+	} // The value of variable "i" is used later, so don't clobber it!
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	rtcp = READ_ONCE(hazptr_torture_current);
+	pr_cont("rtc: %p %s: %lu %s tfle: %d rta: %d rtaf: %d rtf: %d ",
+		rtcp,
+		rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
+		hazptr_torture_current_version,
+		cp,
+		list_empty(&hazptr_torture_freelist),
+		atomic_read(&n_hazptr_torture_alloc),
+		atomic_read(&n_hazptr_torture_alloc_fail),
+		atomic_read(&n_hazptr_torture_free));
+	torture_onoff_stats();
+	pr_cont("acq: %lld rel: %lld acqirq: %lld relirq: %lld reldefer: %lld relundefer %lld\n",
+		torture_sum_pcpu_atomic_long(&hazptr_torture_acquires),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_acquires_irq),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_irq),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_defer),
+		torture_sum_pcpu_atomic_long(&hazptr_torture_releases_undefer));
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	if (i > 1) {
+		pr_cont("%s", "!!! ");
+		atomic_inc(&n_hazptr_torture_error);
+		WARN_ON_ONCE(i > 1); // Too-short grace period
+	}
+	pr_cont("Reader Pipe: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %ld", pipesummary[i]);
+	pr_cont("\n");
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	pr_cont("Reader Batch: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %ld", batchsummary[i]);
+	pr_cont("\n");
+
+	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	pr_cont("Free-Block Circulation: ");
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		pr_cont(" %d", atomic_read(&hazptr_torture_wcount[i]));
+	pr_cont("\n");
+
+	if (rtcv_snap == hazptr_torture_current_version &&
+	    READ_ONCE(hazptr_torture_current) &&
+	    rcu_inkernel_boot_has_ended()) {
+		int __maybe_unused flags = 0;
+		unsigned long __maybe_unused gp_seq = 0;
+
+		wtp = READ_ONCE(writer_task);
+		pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n",
+			 hazptr_torture_writer_state_getname(),
+			 hazptr_torture_writer_state, gp_seq, flags,
+			 wtp == NULL ? '?' : task_state_to_char(wtp),
+			 wtp == NULL ? -1 : (int)task_cpu(wtp));
+		if (!splatted && wtp) {
+			sched_show_task(wtp);
+			splatted = true;
+		}
+		rcu_ftrace_dump(DUMP_ALL);
+	}
+	rtcv_snap = hazptr_torture_current_version;
+}
+
+/*
+ * Periodically prints torture statistics, if periodic statistics printing
+ * was specified via the stat_interval module parameter.
+ */
+static int
+hazptr_torture_stats(void *arg)
+{
+	VERBOSE_TOROUT_STRING("hazptr_torture_stats task started");
+	do {
+		schedule_timeout_interruptible(stat_interval * HZ);
+		hazptr_torture_stats_print();
+		torture_shutdown_absorb("hazptr_torture_stats");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_stats");
+	return 0;
+}
+
+static void
+hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char *tag)
+{
+	pr_alert("%s" TORTURE_FLAG
+		 "--- %s: nreaders=%d nwriters=%d "
+		 "defer_modulus=%d irq_acquire=%d irq_release=%d kthread_do_pending_ms=%d "
+		 "onoff_interval=%d onoff_holdoff=%d "
+		 "preempt_duration=%d preempt_interval=%d "
+		 "reader_sleep_us=%d "
+		 "shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d "
+		 "verbose=%d\n",
+		 torture_type, tag, nrealreaders, nwriters,
+		 defer_modulus, irq_acquire, irq_release, kthread_do_pending_ms,
+		 onoff_interval, onoff_holdoff,
+		 preempt_duration, preempt_interval,
+		 reader_sleep_us,
+		 shuffle_interval, shutdown_secs, stat_interval, stutter,
+		 verbose);
+}
+
+// Randomly preempt online CPUs.
+static int hazptr_torture_preempt(void *unused)
+{
+	int cpu = -1;
+	DEFINE_TORTURE_RANDOM(rand);
+
+	schedule_timeout_idle(onoff_holdoff * HZ);
+	do {
+		// Wait for preempt_interval ms with up to 100us fuzz.
+		torture_hrtimeout_ms(preempt_interval, 100, &rand);
+		// Select online CPU.
+		cpu = cpumask_next(cpu, cpu_online_mask);
+		if (cpu >= nr_cpu_ids)
+			cpu = cpumask_next(-1, cpu_online_mask);
+		WARN_ON_ONCE(cpu >= nr_cpu_ids);
+		// Move to that CPU, if can't do so, retry later.
+		if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false))
+			continue;
+		// Preempt at high-ish priority, then reset to normal.
+		sched_set_fifo(current);
+		torture_sched_setaffinity(current->pid, cpu_present_mask, true);
+		mdelay(preempt_duration);
+		sched_set_normal(current, 0);
+		stutter_wait("hazptr_torture_preempt");
+	} while (!torture_must_stop());
+	torture_kthread_stopping("hazptr_torture_preempt");
+	return 0;
+}
+
+static void
+hazptr_torture_cleanup(void)
+{
+	int i;
+
+	if (torture_cleanup_begin())
+		return;
+	if (!cur_ops) {
+		torture_cleanup_end();
+		return;
+	}
+
+	torture_stop_kthread(hazptr_torture_do_pending, do_pending_task);
+	torture_stop_kthread(hazptr_torture_preempt, preempt_task);
+	torture_stop_kthread(hazptr_torture_writer, writer_task);
+
+	if (reader_tasks) {
+		for (i = 0; i < nrealreaders; i++)
+			torture_stop_kthread(hazptr_torture_reader,
+					     reader_tasks[i]);
+		kfree(reader_tasks);
+		reader_tasks = NULL;
+	}
+
+	torture_stop_kthread(hazptr_torture_stats, stats_task);
+
+	/* Do torture-type-specific cleanup operations.  */
+	if (cur_ops->cleanup != NULL)
+		cur_ops->cleanup();
+
+	hazptr_torture_stats_print();  /* -After- the stats thread is stopped! */
+	if (atomic_read(&n_hazptr_torture_error))
+		hazptr_torture_print_module_parms(cur_ops, "End of test: FAILURE");
+	else if (torture_onoff_failures())
+		hazptr_torture_print_module_parms(cur_ops, "End of test: HAZPTR_HOTPLUG");
+	else
+		hazptr_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
+	torture_cleanup_end();
+}
+
+static int __init hazptr_torture_init(void)
+{
+	long i;
+	int cpu;
+	int firsterr = 0;
+	static struct hazptr_torture_ops *torture_ops[] = { &hazptr_ops, &hazptr_stack_ops, };
+
+	if (!torture_init_begin(torture_type, verbose))
+		return -EBUSY;
+
+	/* Process args and tell the world that the torturer is on the job. */
+	for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
+		cur_ops = torture_ops[i];
+		if (strcmp(torture_type, cur_ops->name) == 0)
+			break;
+	}
+	if (i == ARRAY_SIZE(torture_ops)) {
+		pr_alert("hazptr-torture: invalid torture type: \"%s\"\n", torture_type);
+		pr_alert("hazptr-torture types:");
+		for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
+			pr_cont(" %s", torture_ops[i]->name);
+		pr_cont("\n");
+		firsterr = -EINVAL;
+		cur_ops = NULL;
+		goto unwind;
+	}
+
+	if (cur_ops->init)
+		cur_ops->init();
+
+	if (nreaders >= 0) {
+		nrealreaders = nreaders;
+	} else {
+		nrealreaders = num_online_cpus() * -nreaders;
+		if (nrealreaders <= 0)
+			nrealreaders = 1;
+	}
+	hazptr_torture_print_module_parms(cur_ops, "Start of test");
+
+	/* Set up the freelist. */
+	INIT_LIST_HEAD(&hazptr_torture_freelist);
+	for (i = 0; i < ARRAY_SIZE(hazptr_tortures); i++)
+		list_add_tail(&hazptr_tortures[i].htort_free, &hazptr_torture_freelist);
+
+	/* Initialize the statistics so that each run gets its own numbers. */
+
+	hazptr_torture_current = NULL;
+	hazptr_torture_current_version = 0;
+	atomic_set(&n_hazptr_torture_alloc, 0);
+	atomic_set(&n_hazptr_torture_alloc_fail, 0);
+	atomic_set(&n_hazptr_torture_free, 0);
+	atomic_set(&n_hazptr_torture_error, 0);
+	for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+		atomic_set(&hazptr_torture_wcount[i], 0);
+	for_each_possible_cpu(cpu) {
+		for (i = 0; i < HAZPTR_TORTURE_PIPE_LEN + 1; i++)
+			per_cpu(hazptr_torture_count, cpu)[i] = 0;
+	}
+
+	/* Start up the kthreads. */
+
+	// This must be before the readers in order to set up the module
+	// parameters used by the readers.
+	firsterr = hazptr_torture_do_pending_init();
+	if (torture_init_error(firsterr))
+		goto unwind;
+
+	if (irq_acquire == -1) {
+		irq_acquire = 1000 * nr_cpu_ids;
+	} else if (irq_acquire < 0) {
+		pr_alert("Cannot have irq_acquire (%d) < -1, disabling.\n", irq_acquire);
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+		irq_acquire = 0;
+	}
+	if (irq_release == -1) {
+		irq_release = 1000 * nr_cpu_ids;
+	} else if (irq_release < 0) {
+		pr_alert("Cannot have irq_release (%d) < -1, disabling.\n", irq_release);
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+		irq_release = 0;
+	}
+	reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders);
+	for (i = 0; i < nrealreaders; i++) {
+		firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i,
+						  reader_tasks[i]);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	if (nwriters) {
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST) && nwriters != 1);
+		firsterr = torture_create_kthread(hazptr_torture_writer, NULL, writer_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL);
+	if (torture_init_error(firsterr))
+		goto unwind;
+
+	if (stat_interval > 0) {
+		firsterr = torture_create_kthread(hazptr_torture_stats, NULL, stats_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	if (shuffle_interval > 0) {
+		firsterr = torture_shuffle_init(shuffle_interval * HZ);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	if (stutter < 0)
+		stutter = 0;
+	if (stutter) {
+		int t;
+
+		t = stutter * HZ;
+		firsterr = torture_stutter_init(stutter * HZ, t);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+	firsterr = torture_shutdown_init(shutdown_secs, hazptr_torture_cleanup);
+	if (torture_init_error(firsterr))
+		goto unwind;
+	if (preempt_duration > 0) {
+		firsterr = torture_create_kthread(hazptr_torture_preempt, NULL, preempt_task);
+		if (torture_init_error(firsterr))
+			goto unwind;
+	}
+
+	torture_init_end();
+	return 0;
+
+unwind:
+	torture_init_end();
+	hazptr_torture_cleanup();
+	if (shutdown_secs) {
+		WARN_ON(!IS_MODULE(CONFIG_HAZPTR_TORTURE_TEST));
+		kernel_power_off();
+	}
+	return firsterr;
+}
+
+module_init(hazptr_torture_init);
+module_exit(hazptr_torture_cleanup);
diff --git a/kernel/rcu/refscale.c b/kernel/rcu/refscale.c
index a2d9d75d88a1..2320b3fedec7 100644
--- a/kernel/rcu/refscale.c
+++ b/kernel/rcu/refscale.c
@@ -29,6 +29,7 @@
 #include <linux/reboot.h>
 #include <linux/sched.h>
 #include <linux/seq_buf.h>
+#include <linux/hazptr.h>
 #include <linux/spinlock.h>
 #include <linux/smp.h>
 #include <linux/stat.h>
@@ -1200,6 +1201,47 @@ static const struct ref_scale_ops typesafe_seqlock_ops = {
 	.name		= "typesafe_seqlock"
 };
 
+static void ref_hazptr_read_section(const int nloops)
+{
+	static void *ref_hazptr_read_section_ptr = ref_hazptr_read_section;
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		struct hazptr_ctx ctx;
+		void *addr;
+
+		addr = hazptr_acquire(&ctx, &ref_hazptr_read_section_ptr);
+		hazptr_release(&ctx, addr);
+	}
+}
+
+static void ref_hazptr_delay_section(const int nloops, const int udl, const int ndl)
+{
+	static void *ref_hazptr_delay_section_ptr = ref_hazptr_delay_section;
+	int i;
+
+	for (i = nloops; i >= 0; i--) {
+		struct hazptr_ctx ctx;
+		void *addr;
+
+		addr = hazptr_acquire(&ctx, &ref_hazptr_delay_section_ptr);
+		un_delay(udl, ndl);
+		hazptr_release(&ctx, addr);
+	}
+}
+
+static bool ref_hazptr_init(void)
+{
+	return true;
+}
+
+static const struct ref_scale_ops hazptr_ops = {
+	.init		= ref_hazptr_init,
+	.readsection	= ref_hazptr_read_section,
+	.delaysection	= ref_hazptr_delay_section,
+	.name		= "hazptr"
+};
+
 static void rcu_scale_one_reader(void)
 {
 	if (readdelay <= 0)
@@ -1504,6 +1546,7 @@ ref_scale_init(void)
 		&sched_clock_ops, &clock_ops, &jiffies_ops,
 		&preempt_ops, &bh_ops, &irq_ops, &irqsave_ops,
 		&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,
+		&hazptr_ops,
 	};
 
 	if (!torture_init_begin(scale_type, verbose))
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index b62735a67884..2a778b8ab4ad 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -44,6 +44,7 @@
 #include <linux/slab.h>
 #include <linux/irq_work.h>
 #include <linux/rcupdate_trace.h>
+#include <linux/torture.h>
 
 #define CREATE_TRACE_POINTS
 
@@ -525,7 +526,7 @@ EXPORT_SYMBOL_GPL(do_trace_rcu_torture_read);
 	do { } while (0)
 #endif
 
-#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST)
+#if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_ENABLED(CONFIG_HAZPTR_TORTURE_TEST)
 /* Get rcutorture access to sched_setaffinity(). */
 long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn)
 {
diff --git a/kernel/scftorture.c b/kernel/scftorture.c
index 2082f9b44370..603b2dba1536 100644
--- a/kernel/scftorture.c
+++ b/kernel/scftorture.c
@@ -193,6 +193,7 @@ static void scf_torture_stats_print(void)
 		scfs.n_single += scf_stats_p[i].n_single;
 		scfs.n_single_ofl += scf_stats_p[i].n_single_ofl;
 		scfs.n_single_rpc += scf_stats_p[i].n_single_rpc;
+		scfs.n_single_rpc_ofl += scf_stats_p[i].n_single_rpc_ofl;
 		scfs.n_single_wait += scf_stats_p[i].n_single_wait;
 		scfs.n_single_wait_ofl += scf_stats_p[i].n_single_wait_ofl;
 		scfs.n_many += scf_stats_p[i].n_many;
@@ -491,7 +492,7 @@ static int scftorture_invoker(void *arg)
 		  "%s: Wanted CPU %d, running on %d, nr_cpu_ids = %d\n",
 		  __func__, scfp->cpu, curcpu, nr_cpu_ids);
 
-	if (!atomic_dec_return(&n_started))
+	if (atomic_dec_return(&n_started))
 		while (atomic_read_acquire(&n_started)) {
 			if (torture_must_stop()) {
 				VERBOSE_SCFTORTOUT("scftorture_invoker %d ended before starting", scfp->cpu);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2e7cde033a31..71a886283cd9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -60,6 +60,7 @@
 #include <linux/profile.h>
 #include <linux/psi.h>
 #include <linux/rcuwait_api.h>
+#include <linux/hazptr.h>
 #include <linux/rseq.h>
 #include <linux/sched/wake_q.h>
 #include <linux/scs.h>
@@ -7084,6 +7085,7 @@ static void __sched notrace __schedule(int sched_mode)
 	local_irq_disable();
 	rcu_note_context_switch(preempt);
 	migrate_disable_switch(rq, prev);
+	hazptr_note_context_switch();
 
 	/*
 	 * Make sure that signal_pending_state()->signal_pending() below
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 01890b7a5f98..81d293d3a6c3 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -2312,9 +2312,9 @@ void hrtimer_run_queues(void)
 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
 {
 	struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer);
-	struct task_struct *task = t->task;
+	struct task_struct *task = hrtimer_sleeper_task_get(t);
 
-	t->task = NULL;
+	hrtimer_sleeper_task_set(t, NULL);
 	if (task)
 		wake_up_process(task);
 
@@ -2343,7 +2343,7 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode
 
 	/* If already expired, clear the task pointer and set current state to running */
 	if (!hrtimer_start_expires_user(&sl->timer, mode)) {
-		sl->task = NULL;
+		hrtimer_sleeper_task_set(sl, NULL);
 		__set_current_state(TASK_RUNNING);
 	}
 }
@@ -2377,7 +2377,7 @@ static void __hrtimer_setup_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_
 	}
 
 	__hrtimer_setup(&sl->timer, hrtimer_wakeup, clock_id, mode);
-	sl->task = current;
+	hrtimer_sleeper_task_set(sl, current);
 }
 
 /**
@@ -2421,17 +2421,17 @@ static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mod
 		set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
 		hrtimer_sleeper_start_expires(t, mode);
 
-		if (likely(t->task))
+		if (likely(hrtimer_sleeper_task_get(t)))
 			schedule();
 
 		hrtimer_cancel(&t->timer);
 		mode = HRTIMER_MODE_ABS;
 
-	} while (t->task && !signal_pending(current));
+	} while (hrtimer_sleeper_task_get(t) && !signal_pending(current));
 
 	__set_current_state(TASK_RUNNING);
 
-	if (!t->task)
+	if (!hrtimer_sleeper_task_get(t))
 		return 0;
 
 	restart = &current->restart_block;
diff --git a/kernel/time/sleep_timeout.c b/kernel/time/sleep_timeout.c
index 3c90574bd904..ad8c415851ae 100644
--- a/kernel/time/sleep_timeout.c
+++ b/kernel/time/sleep_timeout.c
@@ -212,7 +212,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
 	hrtimer_sleeper_start_expires(&t, mode);
 
-	if (likely(t.task))
+	if (likely(hrtimer_sleeper_task_get(&t)))
 		schedule();
 
 	hrtimer_cancel(&t.timer);
@@ -220,7 +220,7 @@ int __sched schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
 
 	__set_current_state(TASK_RUNNING);
 
-	return !t.task ? 0 : -EINTR;
+	return !hrtimer_sleeper_task_get(&t) ? 0 : -EINTR;
 }
 EXPORT_SYMBOL_GPL(schedule_hrtimeout_range_clock);
 
diff --git a/kernel/torture.c b/kernel/torture.c
index 77cb3589b19f..bb9e1349418a 100644
--- a/kernel/torture.c
+++ b/kernel/torture.c
@@ -727,6 +727,26 @@ static ktime_t stutter_till_abs_time;
 static int stutter;
 static int stutter_gap;
 
+static bool _stutter_will_wait(ktime_t *till_ns)
+{
+	*till_ns = READ_ONCE(stutter_till_abs_time);
+	if (*till_ns && ktime_before(ktime_get(), *till_ns))
+		return true;
+	return false;
+}
+
+/*
+ * Will stutter_wait() actually stutter?  The returned result is of
+ * course immediately stale due to the passage of time.
+ */
+bool stutter_will_wait(void)
+{
+	ktime_t till_ns;
+
+	return _stutter_will_wait(&till_ns);
+}
+EXPORT_SYMBOL_GPL(stutter_will_wait);
+
 /*
  * Block until the stutter interval ends.  This must be called periodically
  * by all running kthreads that need to be subject to stuttering.
@@ -737,8 +757,7 @@ bool stutter_wait(const char *title)
 	ktime_t till_ns;
 
 	cond_resched_tasks_rcu_qs();
-	till_ns = READ_ONCE(stutter_till_abs_time);
-	if (till_ns && ktime_before(ktime_get(), till_ns)) {
+	if (_stutter_will_wait(&till_ns)) {
 		torture_hrtimeout_ns(till_ns, 0, HRTIMER_MODE_ABS, NULL);
 		ret = true;
 	}
@@ -988,3 +1007,18 @@ void torture_sched_set_normal(struct task_struct *t, int nice)
 	sched_set_normal(t, realnice);
 }
 EXPORT_SYMBOL_GPL(torture_sched_set_normal);
+
+/*
+ * Sum the specified per-CPU atomic_long_t variable.
+ */
+s64 torture_sum_pcpu_atomic_long(atomic_long_t __percpu *pcp)
+{
+	int cpu;
+	s64 sum = 0;
+
+	for_each_possible_cpu(cpu) {
+		sum += atomic_long_read(per_cpu_ptr(pcp, cpu));
+	}
+	return sum;
+}
+EXPORT_SYMBOL_GPL(torture_sum_pcpu_atomic_long);
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index 5403a9f61178..cefed2a9caad 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -2345,11 +2345,11 @@ static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
 			set_current_state(TASK_INTERRUPTIBLE);
 			hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
 
-			if (likely(t.task))
+			if (likely(hrtimer_sleeper_task_get(&t)))
 				schedule();
 
 			hrtimer_cancel(&t.timer);
-		} while (t.task && pkt_dev->running && !signal_pending(current));
+		} while (hrtimer_sleeper_task_get(&t) && pkt_dev->running && !signal_pending(current));
 		__set_current_state(TASK_RUNNING);
 		end_time = ktime_get();
 	}
diff --git a/tools/testing/selftests/rcutorture/bin/kvm.sh b/tools/testing/selftests/rcutorture/bin/kvm.sh
index 65b04b832733..32c319997638 100755
--- a/tools/testing/selftests/rcutorture/bin/kvm.sh
+++ b/tools/testing/selftests/rcutorture/bin/kvm.sh
@@ -91,7 +91,7 @@ usage () {
 	echo "       --remote"
 	echo "       --results absolute-pathname"
 	echo "       --shutdown-grace seconds"
-	echo "       --torture lock|rcu|rcuscale|refscale|scf|X*"
+	echo "       --torture hazptr|lock|rcu|rcuscale|refscale|scf|X*"
 	echo "       --trust-make"
 	exit 1
 }
@@ -256,9 +256,9 @@ do
 		shift
 		;;
 	--torture)
-		checkarg --torture "(suite name)" "$#" "$2" '^\(lock\|rcu\|rcuscale\|refscale\|scf\|X.*\)$' '^--'
+		checkarg --torture "(suite name)" "$#" "$2" '^\(hazptr\|lock\|rcu\|rcuscale\|refscale\|scf\|X.*\)$' '^--'
 		TORTURE_SUITE=$2
-		TORTURE_MOD="`echo $TORTURE_SUITE | sed -e 's/^\(lock\|rcu\|scf\)$/\1torture/'`"
+		TORTURE_MOD="`echo $TORTURE_SUITE | sed -e 's/^\(hazptr\|lock\|rcu\|scf\)$/\1torture/'`"
 		shift
 		if test "$TORTURE_SUITE" = rcuscale || test "$TORTURE_SUITE" = refscale
 		then
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST b/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST
new file mode 100644
index 000000000000..4d62eb4a39f9
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/CFLIST
@@ -0,0 +1,2 @@
+NOPREEMPT
+PREEMPT
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon b/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon
new file mode 100644
index 000000000000..c440d227007d
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/CFcommon
@@ -0,0 +1,2 @@
+CONFIG_HAZPTR_TORTURE_TEST=y
+CONFIG_PRINTK_TIME=y
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT
new file mode 100644
index 000000000000..405941f3b50f
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT
@@ -0,0 +1,19 @@
+CONFIG_SMP=y
+CONFIG_NR_CPUS=16
+CONFIG_PREEMPT_LAZY=y
+CONFIG_PREEMPT_NONE=n
+CONFIG_PREEMPT_VOLUNTARY=n
+CONFIG_PREEMPT=n
+CONFIG_PREEMPT_DYNAMIC=n
+CONFIG_HZ_PERIODIC=n
+CONFIG_NO_HZ_IDLE=y
+CONFIG_NO_HZ_FULL=n
+CONFIG_HOTPLUG_CPU=y
+CONFIG_SUSPEND=n
+CONFIG_HIBERNATION=n
+CONFIG_DEBUG_LOCK_ALLOC=n
+CONFIG_PROVE_LOCKING=n
+CONFIG_KPROBES=n
+CONFIG_FTRACE=n
+CONFIG_DEBUG_KERNEL=y
+CONFIG_HAZPTR_DEBUG=y
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot
new file mode 100644
index 000000000000..1d09a6446080
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/NOPREEMPT.boot
@@ -0,0 +1 @@
+hazptrtorture.torture_type=hazptr-stack
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT
new file mode 100644
index 000000000000..c5b12f3bef74
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/PREEMPT
@@ -0,0 +1,16 @@
+CONFIG_SMP=y
+CONFIG_NR_CPUS=16
+CONFIG_PREEMPT_NONE=n
+CONFIG_PREEMPT_VOLUNTARY=n
+CONFIG_PREEMPT=y
+CONFIG_HZ_PERIODIC=n
+CONFIG_NO_HZ_IDLE=y
+CONFIG_NO_HZ_FULL=n
+CONFIG_HOTPLUG_CPU=y
+CONFIG_SUSPEND=n
+CONFIG_HIBERNATION=n
+CONFIG_DEBUG_LOCK_ALLOC=n
+CONFIG_PROVE_LOCKING=n
+CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
+CONFIG_DEBUG_KERNEL=y
+CONFIG_HAZPTR_DEBUG=y
diff --git a/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh b/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh
new file mode 100644
index 000000000000..a28ea2f292e4
--- /dev/null
+++ b/tools/testing/selftests/rcutorture/configs/hazptr/ver_functions.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+#
+# Kernel-version-dependent shell functions for the rest of the scripts.
+#
+# Claude created this file, and I quote:
+#
+#	"I created [this file] modeled on the lock torture
+#	version. It defines per_version_boot_params to pass
+#	hazptrtorture.shutdown_secs=$3, hazptrtorture.stat_interval=15,
+#	hazptrtorture.verbose=1, and optional CPU-hotplug parameters to
+#	the kernel command line."
+#
+# I therefore kept locktorture's ver_functions.sh copyright notice:
+#
+# Copyright (C) Meta Platforms, Inc. and affiliates.
+#
+# Authors: Paul E. McKenney <paulmck@kernel.org>
+
+# hazptrtorture_param_onoff bootparam-string config-file
+#
+# Adds onoff hazptrtorture module parameters to kernels having it.
+hazptrtorture_param_onoff () {
+	if ! bootparam_hotplug_cpu "$1" && configfrag_hotplug_cpu "$2"
+	then
+		echo CPU-hotplug kernel, adding hazptrtorture onoff. 1>&2
+		echo hazptrtorture.onoff_interval=3 hazptrtorture.onoff_holdoff=30
+	fi
+}
+
+# per_version_boot_params bootparam-string config-file seconds
+#
+# Adds per-version torture-module parameters to kernels supporting them.
+per_version_boot_params () {
+	echo	`hazptrtorture_param_onoff "$1" "$2"` \
+		hazptrtorture.stat_interval=15 \
+		hazptrtorture.shutdown_secs=$3 \
+		hazptrtorture.verbose=1 \
+		$1
+}