From a47dea1baa3b445365ad0757e6dee4ef8fde49e6 Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sat, 13 Jun 2026 09:40:07 +0300 Subject: srcu: make init_srcu_struct() consistently wrap __init_srcu_struct() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the SRCU initialization functions so it always follows one direction: init_srcu_struct() -> __init_srcu_struct() -> lockdep or generic This uses the same wrapper style as mutex. It avoids the old confusing style where init_srcu_struct() and __init_srcu_struct() called each other in different configs. It also helps Rust side to have simpler helper for SRCU initialization. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) (limited to 'include/linux') diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). -- cgit v1.2.3 From 91d5185187e97611e328d03ee5e00e54b5d6d2b0 Mon Sep 17 00:00:00 2001 From: Onur Özkan Date: Sat, 13 Jun 2026 09:40:09 +0300 Subject: srcu: expose srcu_readers_active() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed by rust/helpers/srcu.c which now adds rust_helper_srcu_readers_active() as a wrapper around the SRCU helper for Rust callers. To achive this: 1- Move the srcu_readers_active() implementation from "kernel/rcu/srcutree.c" to "include/linux/srcutree.h". 2- Implement a matching srcu_readers_active() in "include/linux/srcutiny.h" and use it on the existing open-coded WARN_ON() check in cleanup_srcu_struct(). Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 13 +++++++++++++ include/linux/srcutree.h | 24 ++++++++++++++++++++++++ kernel/rcu/srcutiny.c | 2 +- kernel/rcu/srcutree.c | 25 ------------------------- rust/helpers/srcu.c | 5 +++++ 5 files changed, 43 insertions(+), 26 deletions(-) (limited to 'include/linux') diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 47d48ed31848..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 56c5db5c29a0..4a00e90e17fc 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -599,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c index 225b3bf9334a..1a2f563640e0 100644 --- a/rust/helpers/srcu.c +++ b/rust/helpers/srcu.c @@ -9,6 +9,11 @@ __rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, return __init_srcu_struct(ssp, name, key); } +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + __rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) { return srcu_read_lock(ssp); -- cgit v1.2.3 From 736084507faa3596caad181a2ba6aa2a96197c52 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:43 -0700 Subject: rcu: Rename struct rcu_gp_oldstate to rcu_gp_seq The polled grace-period state structure rcu_gp_oldstate holds a snapshot of the normal (and, on SMP, expedited) grace-period sequence numbers. Upcoming changes store this structure in the callback segment list, where the "oldstate" name reads poorly: there it represents the grace period a segment is waiting on and is also compared against the current grace-period state. Rename struct rcu_gp_oldstate to the more neutral struct rcu_gp_seq, and shorten its members rgos_norm and rgos_exp to norm and exp. Local variables and parameters of this type are renamed from rgosp/rgos to gsp/gs accordingly. While at it, provide a single definition of the structure in rcupdate.h rather than separate Tiny-RCU and Tree-RCU definitions, and give it the ->exp field unconditionally. Tiny RCU does not track expedited grace periods and leaves ->exp unused, but a single definition that always has ->exp lets the shared callback code in rcu_segcblist.c reference it without CONFIG_SMP guards, including on !SMP builds. No functional change. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 13 +++++++-- include/linux/rcupdate_wait.h | 2 +- include/linux/rcutiny.h | 36 +++++++++++-------------- include/linux/rcutree.h | 29 +++++++++----------- kernel/rcu/rcutorture.c | 30 ++++++++++----------- kernel/rcu/tiny.c | 4 +-- kernel/rcu/tree.c | 62 +++++++++++++++++++++---------------------- kernel/rcu/tree_exp.h | 18 ++++++------- mm/slab_common.c | 6 ++--- 9 files changed, 100 insertions(+), 100 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ce00f1726e95 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -52,9 +52,18 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void synchronize_rcu(void); -struct rcu_gp_oldstate; +/* + * Grace-period sequence snapshot for the polled RCU APIs: ->norm for the + * normal grace period and ->exp for the expedited one. ->exp is unused by + * Tiny RCU, but is present unconditionally so that a single definition + * serves both Tiny RCU and Tree RCU. + */ +struct rcu_gp_seq { + unsigned long norm; + unsigned long exp; +}; unsigned long get_completed_synchronize_rcu(void); -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp); // Maximum number of unsigned long values corresponding to // not-yet-completed RCU grace periods. diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 4c92d4291cce..fa884704a3b7 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -18,7 +18,7 @@ struct rcu_synchronize { struct completion completion; /* This is for debugging. */ - struct rcu_gp_oldstate oldstate; + struct rcu_gp_seq oldstate; }; void wakeme_after_rcu(struct rcu_head *head); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index f519cd680228..e56ded733b1b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,11 +14,7 @@ #include /* for HZ */ -struct rcu_gp_oldstate { - unsigned long rgos_norm; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 @@ -26,31 +22,31 @@ struct rcu_gp_oldstate { * Are the two oldstate values the same? See the Tree RCU version for * docbook header. */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm; + return rgosp1->norm == rgosp2->norm; } unsigned long get_state_synchronize_rcu(void); -static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = get_state_synchronize_rcu(); + gsp->norm = get_state_synchronize_rcu(); } unsigned long start_poll_synchronize_rcu(void); -static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu(); + gsp->norm = start_poll_synchronize_rcu(); } bool poll_state_synchronize_rcu(unsigned long oldstate); -static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - return poll_state_synchronize_rcu(rgosp->rgos_norm); + return poll_state_synchronize_rcu(gsp->norm); } static inline void cond_synchronize_rcu(unsigned long oldstate) @@ -58,9 +54,9 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } -static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu(rgosp->rgos_norm); + cond_synchronize_rcu(gsp->norm); } static inline unsigned long start_poll_synchronize_rcu_expedited(void) @@ -68,9 +64,9 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) return start_poll_synchronize_rcu(); } -static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); + gsp->norm = start_poll_synchronize_rcu_expedited(); } static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) @@ -78,9 +74,9 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) cond_synchronize_rcu(oldstate); } -static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu_expedited(rgosp->rgos_norm); + cond_synchronize_rcu_expedited(gsp->norm); } extern void rcu_barrier(void); diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9d2d7bd251d4..16a04202888b 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -38,12 +38,7 @@ void synchronize_rcu_expedited(void); void rcu_barrier(void); void rcu_momentary_eqs(void); -struct rcu_gp_oldstate { - unsigned long rgos_norm; - unsigned long rgos_exp; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 @@ -60,29 +55,29 @@ struct rcu_gp_oldstate { * to a list header, allowing those structures to be slightly smaller. * * Note that equality is judged on a bitwise basis, so that an - * @rcu_gp_oldstate structure with an already-completed state in one field + * @rcu_gp_seq structure with an already-completed state in one field * will compare not-equal to a structure with an already-completed state - * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * in the other field. After all, the @rcu_gp_seq structure is opaque * so how did such a situation come to pass in the first place? */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; + return rgosp1->norm == rgosp2->norm && rgosp1->exp == rgosp2->exp; } unsigned long start_poll_synchronize_rcu_expedited(void); -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu_expedited(unsigned long oldstate); -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); unsigned long get_state_synchronize_rcu(void); -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); unsigned long start_poll_synchronize_rcu(void); -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp); bool poll_state_synchronize_rcu(unsigned long oldstate); -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu(unsigned long oldstate); -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp); #ifdef CONFIG_PROVE_RCU void rcu_irq_exit_check_preempt(void); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 882a158ada7b..d71911ac911a 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -392,23 +392,23 @@ struct rcu_torture_ops { void (*exp_current)(void); unsigned long (*get_gp_state_exp)(void); unsigned long (*start_gp_poll_exp)(void); - void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); + void (*start_gp_poll_exp_full)(struct rcu_gp_seq *gsp); bool (*poll_gp_state_exp)(unsigned long oldstate); void (*cond_sync_exp)(unsigned long oldstate); - void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); + void (*cond_sync_exp_full)(struct rcu_gp_seq *gsp); unsigned long (*get_comp_state)(void); - void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); + void (*get_comp_state_full)(struct rcu_gp_seq *gsp); bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); - bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); + bool (*same_gp_state_full)(struct rcu_gp_seq *rgosp1, struct rcu_gp_seq *rgosp2); unsigned long (*get_gp_state)(void); - void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); + void (*get_gp_state_full)(struct rcu_gp_seq *gsp); unsigned long (*start_gp_poll)(void); - void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); + void (*start_gp_poll_full)(struct rcu_gp_seq *gsp); bool (*poll_gp_state)(unsigned long oldstate); - bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); + bool (*poll_gp_state_full)(struct rcu_gp_seq *gsp); bool (*poll_need_2gp)(bool poll, bool poll_full); void (*cond_sync)(unsigned long oldstate); - void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); + void (*cond_sync_full)(struct rcu_gp_seq *gsp); int poll_active; int poll_active_full; call_rcu_func_t call; @@ -1605,7 +1605,7 @@ static void rcu_torture_write_types(void) static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) { unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; bool dopoll; bool dopoll_full; unsigned long r = torture_random(trsp); @@ -1653,18 +1653,18 @@ rcu_torture_writer(void *arg) bool booting_still = false; bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; int expediting = 0; unsigned long gp_snap; unsigned long gp_snap1; - struct rcu_gp_oldstate gp_snap_full; - struct rcu_gp_oldstate gp_snap1_full; + struct rcu_gp_seq gp_snap_full; + struct rcu_gp_seq gp_snap1_full; int i; int idx; unsigned long j; struct work_struct lazy_work; int oldnice = task_nice(current); - struct rcu_gp_oldstate *rgo = NULL; + struct rcu_gp_seq *rgo = NULL; int rgo_size = 0; struct rcu_torture *rp; struct rcu_torture *old_rp; @@ -1963,7 +1963,7 @@ static int rcu_torture_fakewriter(void *arg) { unsigned long gp_snap; - struct rcu_gp_oldstate gp_snap_full; + struct rcu_gp_seq gp_snap_full; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); @@ -2392,7 +2392,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct struct rcu_torture_one_read_state { bool checkpolling; unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; unsigned long started; struct rcu_torture *p; int readstate; diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 585cade21010..dccccd6be941 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -187,9 +187,9 @@ EXPORT_SYMBOL_GPL(call_rcu); * Store a grace-period-counter "cookie". For more information, * see the Tree RCU header comment. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ef553189ee24..0d2d877eb945 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3284,7 +3284,7 @@ EXPORT_SYMBOL_GPL(call_rcu); * Later on, this could in theory be the case for kernels built with * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this * is not a common case. Furthermore, this optimization would cause - * the rcu_gp_oldstate structure to expand by 50%, so this potential + * the rcu_gp_seq structure to expand by 50%, so this potential * grace-period optimization is ignored once the scheduler is running. */ static int rcu_blocking_is_gp(void) @@ -3413,16 +3413,16 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); /** * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie - * @rgosp: Place to put state cookie + * @gsp: Place to put state cookie * - * Stores into @rgosp a value that will always be treated by functions + * Stores into @gsp a value that will always be treated by functions * like poll_state_synchronize_rcu_full() as a cookie whose grace period * has already completed. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; - rgosp->rgos_exp = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; + gsp->exp = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); @@ -3446,13 +3446,13 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); /** * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited - * @rgosp: location to place combined normal/expedited grace-period state + * @gsp: location to place combined normal/expedited grace-period state * - * Places the normal and expedited grace-period states in @rgosp. This + * Places the normal and expedited grace-period states in @gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. - * The rcu_gp_oldstate structure takes up twice the memory of an unsigned + * The rcu_gp_seq structure takes up twice the memory of an unsigned * long, but is guaranteed to see all grace periods. In contrast, the * combined state occupies less memory, but can sometimes fail to take * grace periods into account. @@ -3460,7 +3460,7 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); * This does not guarantee that the needed grace period will actually * start. */ -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { /* * Any prior manipulation of RCU-protected data must happen @@ -3472,8 +3472,8 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) // in poll_state_synchronize_rcu_full() notwithstanding. Use of // the latter here would result in too-short grace periods due to // interactions with newly onlined CPUs. - rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq); - rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); + gsp->norm = rcu_seq_snap(&rcu_state.gp_seq); + gsp->exp = rcu_seq_snap(&rcu_state.expedited_sequence); } EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full); @@ -3524,18 +3524,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); /** * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * - * Places the normal and expedited grace-period states in *@rgos. This + * Places the normal and expedited grace-period states in *@gs. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed grace period is not already slated to start, notifies * RCU core of the need for that grace period. */ -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); start_poll_synchronize_rcu_common(); } @@ -3587,19 +3587,19 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); /** * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * * If a full RCU grace period has elapsed since the earlier call from - * which *rgosp was obtained, return @true, otherwise return @false. + * which *gsp was obtained, return @true, otherwise return @false. * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller - * can explicitly wait for a grace period, for example, by passing @rgosp + * can explicitly wait for a grace period, for example, by passing @gsp * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). * * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited * for more than a billion grace periods (and way more on a 64-bit - * system!). Those needing to keep rcu_gp_oldstate values for very + * system!). Those needing to keep rcu_gp_seq values for very * long time periods (many hours even on 32-bit systems) should check * them occasionally and either refresh them or set a flag indicating * that the grace period has completed. Alternatively, they can use @@ -3608,7 +3608,7 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * * This function provides the same memory-ordering guarantees that would * be provided by a synchronize_rcu() that was invoked at the call to - * the function that provided @rgosp, and that returned at the end of this + * the function that provided @gsp, and that returned at the end of this * function. And this guarantee requires that the root rcu_node structure's * ->gp_seq field be checked instead of that of the rcu_state structure. * The problem is that the just-ending grace-period's callbacks can be @@ -3618,15 +3618,15 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * cause a subsequent poll_state_synchronize_rcu_full() to return @true, * then the root rcu_node structure is the one that needs to be polled. */ -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { struct rcu_node *rnp = rcu_get_root(); smp_mb(); // Order against root rcu_node structure grace-period cleanup. - if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || - rgosp->rgos_exp == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { + if (gsp->norm == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || + gsp->exp == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } @@ -3661,11 +3661,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); /** * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu() to wait * for a full grace period. * @@ -3676,12 +3676,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..7024131574df 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -1047,18 +1047,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); /** * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period - * @rgosp: Place to put snapshot of grace-period state + * @gsp: Place to put snapshot of grace-period state * - * Places the normal and expedited grace-period states in rgosp. This + * Places the normal and expedited grace-period states in gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed expedited grace period is not already slated to start, * initiates that grace period. */ -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); (void)start_poll_synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full); @@ -1092,11 +1092,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); /** * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu_expedited() * to wait for a full grace period. * @@ -1107,12 +1107,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full); diff --git a/mm/slab_common.c b/mm/slab_common.c index b6426d7ceec9..d34743d1f3cb 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1322,7 +1322,7 @@ static struct workqueue_struct *rcu_reclaim_wq; */ struct kvfree_rcu_bulk_data { struct list_head list; - struct rcu_gp_oldstate gp_snap; + struct rcu_gp_seq gp_snap; unsigned long nr_records; void *records[] __counted_by(nr_records); }; @@ -1347,7 +1347,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; struct rcu_head *head_free; - struct rcu_gp_oldstate head_free_gp_snap; + struct rcu_gp_seq head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; @@ -1555,7 +1555,7 @@ static void kfree_rcu_work(struct work_struct *work) struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; - struct rcu_gp_oldstate head_gp_snap; + struct rcu_gp_seq head_gp_snap; int i; krwp = container_of(to_rcu_work(work), -- cgit v1.2.3 From fb5c815f12b5c17ed6c8c5879b3acb1407e5bac4 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:46 -0700 Subject: rcu/segcblist: Track segment grace periods with struct rcu_gp_seq Change the type of the per-segment ->gp_seq[] array in struct rcu_segcblist from unsigned long to struct rcu_gp_seq. This prepares the callback tracking infrastructure to record both normal and expedited grace periods per segment. The rcu_segcblist_nextgp(), rcu_segcblist_advance(), and rcu_segcblist_accelerate() helpers now take a struct rcu_gp_seq * instead of an unsigned long, and all callers use the .norm field for comparisons and assignments. The SRCU and Tasks RCU wrappers construct a struct rcu_gp_seq with only .norm set and forward to the core helpers. No functional change: only the .norm field is used. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 2 +- include/trace/events/rcu.h | 5 +++-- kernel/rcu/rcu_segcblist.c | 24 ++++++++++++++---------- kernel/rcu/rcu_segcblist.h | 6 +++--- kernel/rcu/tree.c | 25 ++++++++++++++----------- kernel/rcu/tree_nocb.h | 21 +++++++++++---------- 6 files changed, 46 insertions(+), 37 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 2fdc2208f1ca..137cc23b024c 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -190,7 +190,7 @@ struct rcu_cblist { struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; - unsigned long gp_seq[RCU_CBLIST_NSEGS]; + struct rcu_gp_seq gp_seq[RCU_CBLIST_NSEGS]; #ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len; #else diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..c84309c38834 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -547,10 +547,11 @@ TRACE_EVENT_RCU(rcu_segcb_stats, ), TP_fast_assign( + int i; __entry->ctx = ctx; memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); - memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); - + for (i = 0; i < RCU_CBLIST_NSEGS; i++) + __entry->gp_seq[i] = rs->gp_seq[i].norm; ), TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 421f1dadb5e5..4e3dfe42bc09 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -307,13 +307,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) /* * Return false if there are no CBs awaiting grace periods, otherwise, - * return true and store the nearest waited-upon grace period into *lp. + * return true and store the nearest waited-upon grace period state into *gsp. */ -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { if (!rcu_segcblist_pend_cbs(rsclp)) return false; - *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; + *gsp = rsclp->gp_seq[RCU_WAIT_TAIL]; return true; } @@ -496,7 +496,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { int i; @@ -509,7 +509,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) * are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { - if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) + if (ULONG_CMP_LT(gsp->norm, rsclp->gp_seq[i].norm)) break; WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); @@ -537,7 +537,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) * ready to invoke. Returns true if there are callbacks that won't be * ready to invoke until seq, false otherwise. */ -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { int i, j; @@ -555,7 +555,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (!rcu_segcblist_segempty(rsclp, i) && - ULONG_CMP_LT(rsclp->gp_seq[i], seq)) + ULONG_CMP_LT(rsclp->gp_seq[i].norm, gsp->norm)) break; /* @@ -595,7 +595,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) */ for (; i < RCU_NEXT_TAIL; i++) { WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); - rsclp->gp_seq[i] = seq; + rsclp->gp_seq[i].norm = gsp->norm; } return true; } @@ -637,10 +637,14 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { - rcu_segcblist_advance(rsclp, seq); + struct rcu_gp_seq gs = { .norm = seq }; + + rcu_segcblist_advance(rsclp, &gs); } bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { - return rcu_segcblist_accelerate(rsclp, seq); + struct rcu_gp_seq gs = { .norm = seq }; + + return rcu_segcblist_accelerate(rsclp, &gs); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 956f2967d9d2..16b0cb6b3250 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -124,7 +124,7 @@ bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, struct rcu_head *rhp); bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, @@ -139,8 +139,8 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); +void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp); void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0d2d877eb945..9ca811e534b5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1137,7 +1137,7 @@ static void rcu_gp_kthread_wake(void) */ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long gp_seq_req; + struct rcu_gp_seq gs; bool ret = false; rcu_lockdep_assert_cblist_protected(rdp); @@ -1159,15 +1159,15 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * accelerating callback invocation to an earlier grace-period * number. */ - gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); - if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) - ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); + gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + if (rcu_segcblist_accelerate(&rdp->cblist, &gs)) + ret = rcu_start_this_gp(rnp, rdp, gs.norm); /* Trace depending on how much we were able to accelerate. */ if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccWaitCB")); else - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccReadyCB")); trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); @@ -1184,14 +1184,14 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long c; + struct rcu_gp_seq gs; bool needwake; rcu_lockdep_assert_cblist_protected(rdp); - c = rcu_seq_snap(&rcu_state.gp_seq); - if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { + gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, gs.norm)) { /* Old request still live, so mark recent callbacks. */ - (void)rcu_segcblist_accelerate(&rdp->cblist, c); + (void)rcu_segcblist_accelerate(&rdp->cblist, &gs); return; } raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ @@ -1213,6 +1213,8 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, */ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { + struct rcu_gp_seq gs; + rcu_lockdep_assert_cblist_protected(rdp); raw_lockdep_assert_held_rcu_node(rnp); @@ -1224,7 +1226,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ - rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); + gs.norm = rnp->gp_seq; + rcu_segcblist_advance(&rdp->cblist, &gs); /* Classify any remaining callbacks. */ return rcu_accelerate_cbs(rnp, rdp); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index c4a5b4662b3a..eaa1ad5297e2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -433,7 +433,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool lazy) { unsigned long c; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long j = jiffies; long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); long lazy_len = READ_ONCE(rdp->lazy_len); @@ -502,7 +502,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } if (j != rdp->nocb_gp_adv_time && rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { + rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq.norm)) { rcu_advance_cbs_nowake(rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; } @@ -659,7 +659,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool gotcbs = false; unsigned long j = jiffies; @@ -731,7 +731,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL) || (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { + rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm))) { raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ needwake_gp = rcu_advance_cbs(rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist, @@ -744,8 +744,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) RCU_NEXT_READY_TAIL)); if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { if (!needwait_gp || - ULONG_CMP_LT(cur_gp_seq, wait_gp_seq)) - wait_gp_seq = cur_gp_seq; + ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq)) + wait_gp_seq = cur_gp_seq.norm; needwait_gp = true; trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("NeedWaitGP")); @@ -877,7 +877,7 @@ static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) static void nocb_cb_wait(struct rcu_data *rdp) { struct rcu_segcblist *cblist = &rdp->cblist; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool needwake_gp = false; struct rcu_node *rnp = rdp->mynode; @@ -919,7 +919,7 @@ static void nocb_cb_wait(struct rcu_data *rdp) lockdep_assert_irqs_enabled(); rcu_nocb_lock_irqsave(rdp, flags); if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && + rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm) && raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ @@ -1569,9 +1569,10 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp); sprintf(bufd, "%ld", rsclp->seglen[RCU_DONE_TAIL]); - sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], rsclp->gp_seq[RCU_WAIT_TAIL]); + sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], + rsclp->gp_seq[RCU_WAIT_TAIL].norm); sprintf(bufr, "%ld(%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], - rsclp->gp_seq[RCU_NEXT_READY_TAIL]); + rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm); sprintf(bufn, "%ld", rsclp->seglen[RCU_NEXT_TAIL]); sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n", -- cgit v1.2.3 From 573d76f7db5ba050bc7c7c73d30e1364fcee269b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 28 Jun 2026 09:00:01 -0700 Subject: rcu: Mark __rcu_access_pointer() as context_unsafe() A simple comparison of a pointer returned by rcu_access_pointer() results in a context-analysis warning for lockless inspection of the RCU-protected (also known as __rcu-protected) pointer. This can be suppressed by placing context_unsafe() calls around calls rcu_access_pointer(), but this is messy and distracting. This commit therefore wraps the underlying __rcu_access_pointer() macro with a call to context_unsafe(), thereby informing the context-analysis code that rcu_access_pointer() may safely be invoked outside of an RCU read-side critical section. Reported-by: Christoph Hellwig Suggested-by: Marco Elver Signed-off-by: Paul E. McKenney Tested-by: Nilay Shroff Reviewed-by: Marco Elver --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..e40dc2e20c5b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -490,12 +490,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ -- cgit v1.2.3 From 647dd6e09481daa23d93e927c2b7f46a3acf8a03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Jul 2026 10:50:08 -0700 Subject: rcu-tasks: Apply READ_ONCE() and WRITE_ONCE() to fix data race Now that rcutorture tests readers from interrupt handlers, KCSAN spotted an additional data race. This commit therefore fixes it by applying READ_ONCE() and WRITE_ONCE(). Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..fd3ddeb6aa3b 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,15 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; scp = t->trc_reader_scp; barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + WRITE_ONCE(t->trc_reader_nesting, n); + if (!n) { if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit v1.2.3 From 942b3e8f3f3f51011161cb728375481ebbc84e22 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:49 -0700 Subject: rcu: Update comments for gp_seq and expedited GP tracking Update documentation comments throughout the RCU callback infrastructure to reflect the transition from a single grace-period sequence number to the full struct rcu_gp_seq that tracks both normal and expedited grace periods. The ->gp_seq[] array documentation in rcu_segcblist.h is updated to describe dual (normal and expedited) GP tracking. The rcu_segcblist_advance(), rcu_segcblist_accelerate(), and rcu_advance_cbs() comments are updated to refer to the struct rcu_gp_seq state (gsp) instead of the old bare grace-period sequence number (seq). Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 14 ++++++++------ kernel/rcu/rcu_segcblist.c | 43 ++++++++++++++++++++++++++++--------------- kernel/rcu/tree.c | 6 +++--- 3 files changed, 39 insertions(+), 24 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 137cc23b024c..08b63ecf719b 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -50,12 +50,14 @@ struct rcu_cblist { * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * - * The ->gp_seq[] array contains the grace-period number at which the - * corresponding segment of callbacks will be ready to invoke. A given - * element of this array is meaningful only when the corresponding segment - * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks - * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have - * not yet been assigned a grace-period number). + * The ->gp_seq[] array contains the grace-period state at which the + * corresponding segment of callbacks will be ready to invoke. This tracks + * both normal and expedited grace periods, allowing callbacks to complete + * when either type of GP finishes. A given element of this array is + * meaningful only when the corresponding segment is non-empty, and it is + * never valid for RCU_DONE_TAIL (whose callbacks are already ready to + * invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned + * a grace-period state). */ #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index cf8951d33e76..dd770006e7f8 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -495,7 +495,8 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) /* * Advance the callbacks in the specified rcu_segcblist structure based - * on the current value of the grace-period counter. + * on the current grace-period state. Checks both normal and expedited + * grace periods, advancing callbacks when either GP type completes. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) { @@ -506,8 +507,10 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) return; /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL segment. + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL segment. + * We check against the current global GP state, which includes + * proper memory barriers and handles special completion values. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { if (!poll_state_synchronize_rcu_full(&rsclp->gp_seq[i])) @@ -534,9 +537,9 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) * them to complete at the end of the earlier grace period. * * This function operates on an rcu_segcblist structure, and also the - * grace-period sequence number seq at which new callbacks would become + * grace-period state gsp at which new callbacks would become * ready to invoke. Returns true if there are callbacks that won't be - * ready to invoke until seq, false otherwise. + * ready to invoke until the grace period represented by gsp, false otherwise. */ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { @@ -548,11 +551,11 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * Find the segment preceding the oldest segment of callbacks - * whose ->gp_seq[] completion is at or after that passed in via - * "seq", skipping any empty segments. This oldest segment, along + * whose grace period completion is at or after that passed in via + * "gsp", skipping any empty segments. This oldest segment, along * with any later segments, can be merged in with any newly arrived - * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq" - * as their ->gp_seq[] grace-period completion sequence number. + * callbacks in the RCU_NEXT_TAIL segment, and assigned "gsp" + * as their grace-period completion state. */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (!rcu_segcblist_segempty(rsclp, i) && @@ -561,7 +564,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * If all the segments contain callbacks that correspond to - * earlier grace-period sequence numbers than "seq", leave. + * earlier grace-period sequence numbers than "gsp", leave. * Assuming that the rcu_segcblist structure has enough * segments in its arrays, this can only happen if some of * the non-done segments contain callbacks that really are @@ -569,15 +572,15 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs * out by the next call to rcu_segcblist_advance(). * * Also advance to the oldest segment of callbacks whose - * ->gp_seq[] completion is at or after that passed in via "seq", + * ->gp_seq[] completion is at or after that passed in via "gsp", * skipping any empty segments. * * Note that segment "i" (and any lower-numbered segments * containing older callbacks) will be unaffected, and their - * grace-period numbers remain unchanged. For example, if i == + * grace-period states remain unchanged. For example, if i == * WAIT_TAIL, then neither WAIT_TAIL nor DONE_TAIL will be touched. * Instead, the CBs in NEXT_TAIL will be merged with those in - * NEXT_READY_TAIL and the grace-period number of NEXT_READY_TAIL + * NEXT_READY_TAIL and the grace-period state of NEXT_READY_TAIL * would be updated. NEXT_TAIL would then be empty. */ if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL) @@ -589,8 +592,8 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * Merge all later callbacks, including newly arrived callbacks, - * into the segment located by the for-loop above. Assign "seq" - * as the ->gp_seq[] value in order to correctly handle the case + * into the segment located by the for-loop above. Assign "gsp" + * as the grace-period state in order to correctly handle the case * where there were no pending callbacks in the rcu_segcblist * structure other than in the RCU_NEXT_TAIL segment. */ @@ -644,6 +647,10 @@ void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) return; + /* + * Find all callbacks whose normal GP sequence numbers indicate + * that they are ready to invoke. For SRCU, we only check norm. + */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { if (ULONG_CMP_LT(seq, rsclp->gp_seq[i].norm)) break; @@ -658,6 +665,12 @@ void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) rcu_segcblist_advance_compact(rsclp, i); } +/* + * SRCU wrapper for rcu_segcblist_accelerate() - converts SRCU's unsigned + * long GP sequence to rcu_gp_seq format with exp set to + * RCU_GET_STATE_NOT_TRACKED (since SRCU does not use expedited GPs) + * and calls the core rcu_segcblist_accelerate(). + */ bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { struct rcu_gp_seq gs = { .norm = seq, .exp = RCU_GET_STATE_NOT_TRACKED }; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f63ce75c5b4f..aa11e9a04830 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1204,7 +1204,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, /* * Move any callbacks whose grace period has completed to the * RCU_DONE_TAIL sublist, then compact the remaining sublists and - * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL + * assign ->gp_seq[] state to any callbacks in the RCU_NEXT_TAIL * sublist. This function is idempotent, so it does not hurt to * invoke it repeatedly. As long as it is not invoked -too- often... * Returns true if the RCU grace-period kthread needs to be awakened. @@ -1221,8 +1221,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) return false; /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL sublist. */ rcu_segcblist_advance(&rdp->cblist); -- cgit v1.2.3 From f99dc9e288c5a2e191cca8a9af59113a8bc2768a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 08:27:59 -0700 Subject: rcu-tasks: Fix IRQ read lock/unlock data race As noted by Marco Elver: rcu_read_lock_trace() .... t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); rcu_read_unlock_trace() < ... var decls only ... > scp = t->trc_reader_scp; This constitutes a data race between these two accesses to t->trc_reader_scp. If rcu_read_lock_trace() were to tear its store, this value would be corrupted. This commit therefore defers the rcu_read_lock_untrace() function's load from t->trc_reader_scp until after it has verified that this is the outermost rcu_read_unlock_trace(). With this change, the interrupt handler increments and decrements t->trc_reader_nesting and does not access t->trc_reader_scp, thus avoiding the data race. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index fd3ddeb6aa3b..70decf877348 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -126,11 +126,13 @@ static inline void rcu_read_unlock_trace(void) struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. n = READ_ONCE(t->trc_reader_nesting) - 1; - WRITE_ONCE(t->trc_reader_nesting, n); - if (!n) { + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); -- cgit v1.2.3 From 0f5fe218d9d4e23d47a49873c9bfacd717b9c145 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Jul 2026 15:11:04 -0700 Subject: rcu-tasks: Convert cond_resched_tasks_rcu_qs() to static inline In order to make "cc -E" output less annoying, this commit converts cond_resched_tasks_rcu_qs() to static inline. You know, the READ_ONCE() and WRITE_ONCE() macros used to be *so* simple. ;-) Reported-by: Andrew Morton Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..5e8770fd7b72 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -208,15 +208,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states -- cgit v1.2.3 From 86fa5387b473c160c26b781dff55014c4e1b3db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 17:34:07 -0700 Subject: rcutorture: Make RCU Tasks Trace track Reader Batches This commit adds the ->get_sp_seq and ->gp_diff fields to the tasks_tracing_ops structure so that RCU Tasks Trace rcutorture runs will track Reader Batch. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 3 +++ kernel/rcu/rcutorture.c | 3 ++- kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..f2c4173bf14f 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -198,10 +198,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c8118c38812a..8514fef3ec10 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1252,7 +1252,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..496acac8796f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1606,4 +1606,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ -- cgit v1.2.3