config PREEMPT_RCU

bool

select TREE_RCU

help

This option selects the RCU implementation that is

help

This option selects the full-fledged version of SRCU.

config NEED_SRCU_NMI_SAFE

def_bool HAVE_NMI && !ARCH_HAS_NMI_SAFE_THIS_CPU_OPS && !TINY_SRCU

config TASKS_RCU

bool

select IRQ_WORK

config FORCE_TASKS_RUDE_RCU

depends on RCU_NOCB_CPU

default n

help

config RCU_LAZY_DEFAULT_OFF

bool "Turn RCU lazy invocation off by default"

depends on RCU_LAZY

default n

help

config RCU_DOUBLE_CHECK_CB_TIME

bool "RCU callback-batch backup time check"

Say N if you are unsure.

config RCU_TORTURE_TEST_CHK_RDR_STATE

depends on RCU_TORTURE_TEST

default n

help

Say N if you are unsure.

config RCU_TORTURE_TEST_LOG_CPU

depends on RCU_TORTURE_TEST

default n

help

Say Y here if you want CPU IDs logged.

Say N if you are unsure.

config RCU_REF_SCALE_TEST

tristate "Scalability tests for read-side synchronization (RCU and others)"

depends on DEBUG_KERNEL

{

unsigned long cur_s = READ_ONCE(*sp);

}

/*

#endif

static inline void rcu_gp_set_torture_wait(int duration) { }

#endif

#ifdef CONFIG_TINY_SRCU

static inline bool rcu_watching_zero_in_eqs(int cpu, int *vp) { return false; }

static inline unsigned long rcu_get_gp_seq(void) { return 0; }

static inline unsigned long rcu_exp_batches_completed(void) { return 0; }

static inline void rcu_force_quiescent_state(void) { }

static inline bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) { return true; }

static inline void show_rcu_gp_kthreads(void) { }

bool rcu_watching_zero_in_eqs(int cpu, int *vp);

unsigned long rcu_get_gp_seq(void);

unsigned long rcu_exp_batches_completed(void);

bool rcu_check_boost_fail(unsigned long gp_state, int *cpup);

void show_rcu_gp_kthreads(void);

int rcu_get_gp_kthreads_prio(void);

void rcu_gp_slow_unregister(atomic_t *rgssp);

#endif /* #else #ifdef CONFIG_TINY_RCU */

#ifdef CONFIG_RCU_NOCB_CPU

void rcu_bind_current_to_nocb(void);

#else

torture_param(int, stutter, 5, "Number of seconds to run/halt test");

torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");

torture_param(int, test_boost_duration, 4, "Duration of each boost test, seconds.");

torture_param(int, test_boost_interval, 7, "Interval between boost tests, seconds.");

torture_param(int, test_nmis, 0, "End-test NMI tests, 0 to disable.");

torture_param(bool, test_no_idle_hz, true, "Test support for tickless idle CPUs");

static int nrealnocbers;

static int nrealreaders;

static struct task_struct *writer_task;

static struct task_struct **fakewriter_tasks;

static struct task_struct **reader_tasks;

bool rt_preempted;

int rt_cpu;

int rt_end_cpu;

};

static int err_segs_recorded;

static struct rt_read_seg err_segs[RCUTORTURE_RDR_MAX_SEGS];

void (*gp_slow_register)(atomic_t *rgssp);

void (*gp_slow_unregister)(atomic_t *rgssp);

bool (*reader_blocked)(void);

long cbflood_max;

int irq_capable;

int can_boost;

.reader_blocked = IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)

? has_rcu_reader_blocked

: NULL,

.irq_capable = 1,

.can_boost = IS_ENABLED(CONFIG_RCU_BOOST),

.extendables = RCUTORTURE_MAX_EXTEND,

.sync = synchronize_rcu_busted,

.exp_sync = synchronize_rcu_busted,

.call = call_rcu_busted,

.irq_capable = 1,

.extendables = RCUTORTURE_MAX_EXTEND,

.name = "busted"

static int srcu_torture_read_lock(void)

{

int idx;

int ret = 0;

if ((reader_flavor & SRCU_READ_FLAVOR_NORMAL) || !(reader_flavor & SRCU_READ_FLAVOR_ALL)) {

idx = srcu_read_lock(srcu_ctlp);

WARN_ON_ONCE(idx & ~0x1);

WARN_ON_ONCE(idx & ~0x1);

ret += idx << 2;

}

return ret;

}

static void srcu_torture_read_unlock(int idx)

{

WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1)));

if (reader_flavor & SRCU_READ_FLAVOR_LITE)

srcu_read_unlock_lite(srcu_ctlp, (idx & 0x4) >> 2);

if (reader_flavor & SRCU_READ_FLAVOR_NMI)

.readunlock = srcu_torture_read_unlock,

.readlock_held = torture_srcu_read_lock_held,

.get_gp_seq = srcu_torture_completed,

.deferred_free = srcu_torture_deferred_free,

.sync = srcu_torture_synchronize,

.exp_sync = srcu_torture_synchronize_expedited,

.readunlock = srcu_torture_read_unlock,

.readlock_held = torture_srcu_read_lock_held,

.get_gp_seq = srcu_torture_completed,

.deferred_free = srcu_torture_deferred_free,

.sync = srcu_torture_synchronize,

.exp_sync = srcu_torture_synchronize_expedited,

unsigned long gp_state;

unsigned long gp_state_time;

unsigned long oldstarttime;

/* Set real-time priority. */

sched_set_fifo_low(current);

sched_set_fifo_low(current);

} while (!torture_must_stop());

/* Clean up and exit. */

while (!kthread_should_stop()) {

torture_shutdown_absorb("rcu_torture_boost");

do {

torture_hrtimeout_jiffies(torture_random(&rand) % 10, &rand);

if (cur_ops->cb_barrier != NULL &&

cur_ops->cb_barrier();

} else {

switch (synctype[torture_random(&rand) % nsynctypes]) {

#define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count()

static void rcutorture_one_extend_check(char *s, int curstate, int new, int old, bool insoftirq)

{

if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE))

return;

WARN_ONCE(cur_ops->extendables &&

!(curstate & (RCUTORTURE_RDR_BH | RCUTORTURE_RDR_RBH)) &&

(preempt_count() & SOFTIRQ_MASK), ROEC_ARGS);

(preempt_count() & PREEMPT_MASK), ROEC_ARGS);

cur_ops->readlock_nesting() > 0, ROEC_ARGS);

}

rtrsp[-1].rt_preempted = cur_ops->reader_blocked();

}

}

/*

* Next, remove old protection, in decreasing order of strength

stutter_wait("rcu_torture_reader");

} while (!torture_must_stop());

if (irqreader && cur_ops->irq_capable) {

destroy_timer_on_stack(&t);

}

tick_dep_clear_task(current, TICK_DEP_BIT_RCU);

"shuffle_interval=%d stutter=%d irqreader=%d "

"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "

"test_boost=%d/%d test_boost_interval=%d "

"stall_cpu=%d stall_cpu_holdoff=%d stall_cpu_irqsoff=%d "

"stall_cpu_block=%d stall_cpu_repeat=%d "

"n_barrier_cbs=%d "

"nocbs_nthreads=%d nocbs_toggle=%d "

"test_nmis=%d "

"preempt_duration=%d preempt_interval=%d\n",

stat_interval, verbose, test_no_idle_hz, shuffle_interval,

stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,

test_boost, cur_ops->can_boost,

stall_cpu, stall_cpu_holdoff, stall_cpu_irqsoff,

stall_cpu_block, stall_cpu_repeat,

n_barrier_cbs,

int flags = 0;

unsigned long gp_seq = 0;

int i;

if (torture_cleanup_begin()) {

if (cur_ops->cb_barrier != NULL) {

rcu_torture_reader_mbchk = NULL;

if (fakewriter_tasks) {

torture_stop_kthread(rcu_torture_fakewriter,

fakewriter_tasks[i]);

kfree(fakewriter_tasks);

pr_alert("\t: No segments recorded!!!\n");

firsttime = 1;

for (i = 0; i < rt_read_nsegs; i++) {

if (err_segs[i].rt_delay_jiffies != 0) {

pr_cont("%s%ldjiffies", firsttime ? "" : "+",

err_segs[i].rt_delay_jiffies);

else

pr_cont(" ...");

}

if (err_segs[i].rt_delay_ms != 0) {

pr_cont(" %s%ldms", firsttime ? "" : "+",

err_segs[i].rt_delay_ms);

rcu_torture_init_srcu_lockdep();

if (nreaders >= 0) {

nrealreaders = nreaders;

} else {

writer_task);

if (torture_init_error(firsterr))

goto unwind;

sizeof(fakewriter_tasks[0]),

GFP_KERNEL);

if (fakewriter_tasks == NULL) {

goto unwind;

}

}

firsterr = torture_create_kthread(rcu_torture_fakewriter,

NULL, fakewriter_tasks[i]);

if (torture_init_error(firsterr))

.name = "srcu"

};

static void srcu_lite_ref_scale_read_section(const int nloops)

{

int i;

long i;

int firsterr = 0;

static const struct ref_scale_ops *scale_ops[] = {

&refcnt_ops, &rwlock_ops, &rwsem_ops, &lock_ops, &lock_irq_ops,

&acqrel_ops, &sched_clock_ops, &clock_ops, &jiffies_ops,

&typesafe_ref_ops, &typesafe_lock_ops, &typesafe_seqlock_ops,

#include "rcu_segcblist.h"

#include "rcu.h"

int rcu_scheduler_active __read_mostly;

static LIST_HEAD(srcu_boot_list);

static bool srcu_init_done;

{

int newval;

newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;

WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);

preempt_enable();

struct srcu_struct *ssp;

ssp = container_of(wp, struct srcu_struct, srcu_work);

if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max))) {

preempt_enable();

return; /* Already running or nothing to do. */

WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */

preempt_enable();

swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));

WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */

WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);

preempt_enable();

* at interrupt level, but the ->srcu_gp_running checks will

* straighten that out.

*/

WRITE_ONCE(ssp->srcu_gp_running, false);

idx = ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max));

preempt_enable();

{

unsigned long cookie;

cookie = get_state_synchronize_srcu(ssp);

if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie)) {

preempt_enable();

rhp->func = func;

rhp->next = NULL;

local_irq_save(flags);

*ssp->srcu_cb_tail = rhp;

ssp->srcu_cb_tail = &rhp->next;

{

unsigned long ret;

ret = get_state_synchronize_srcu(ssp);

srcu_gp_start_if_needed(ssp);

preempt_enable();

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);

/* Lockdep diagnostics. */

void __init rcu_scheduler_starting(void)

{

rcu_scheduler_active = RCU_SCHEDULER_RUNNING;

}

/*

* Queue work for srcu_struct structures with early boot callbacks.

/*

* Initialize SRCU per-CPU data. Note that statically allocated

* srcu_struct structures might already have srcu_read_lock() and

*/

static void init_srcu_struct_data(struct srcu_struct *ssp)

{

* Initialize the per-CPU srcu_data array, which feeds into the

* leaves of the srcu_node tree.

*/

for_each_possible_cpu(cpu) {

sdp = per_cpu_ptr(ssp->sda, cpu);

spin_lock_init(&ACCESS_PRIVATE(sdp, lock));

ssp->srcu_sup->node = NULL;

mutex_init(&ssp->srcu_sup->srcu_cb_mutex);

mutex_init(&ssp->srcu_sup->srcu_gp_mutex);

ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL;

ssp->srcu_sup->srcu_barrier_seq = 0;

mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);

atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);

INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);

ssp->srcu_sup->sda_is_static = is_static;

ssp->sda = alloc_percpu(struct srcu_data);

if (!ssp->sda)

goto err_free_sup;

init_srcu_struct_data(ssp);

}

/*

*/

static bool srcu_readers_lock_idx(struct srcu_struct *ssp, int idx, bool gp, unsigned long unlocks)

{

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

if (IS_ENABLED(CONFIG_PROVE_RCU))

mask = mask | READ_ONCE(sdp->srcu_reader_flavor);

}

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)),

"Mixed reader flavors for srcu_struct at %ps.\n", ssp);

return false;

return sum == unlocks;

}

/*

*/

static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx, unsigned long *rdm)

{

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

mask = mask | READ_ONCE(sdp->srcu_reader_flavor);

}

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && (mask & (mask - 1)),

unsigned long unlocks;

unlocks = srcu_readers_unlock_idx(ssp, idx, &rdm);

/*

* Make sure that a lock is always counted if the corresponding

* If the locks are the same as the unlocks, then there must have

* been no readers on this index at some point in this function.

* But there might be more readers, as a task might have read

* that most of the tasks have been preempted between fetching

*

*

* It can clearly do so once, given that it has already fetched

*

* However, it is important to note that a given smp_mb() takes

* effect not just for the task executing it, but also for any

* later task running on that same CPU.

*

*

* OK, but what about nesting? This does impose a limit on

* nesting of half of the size of the task_struct structure

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

}

return sum;

}

unsigned long jbase = SRCU_INTERVAL;

struct srcu_usage *sup = ssp->srcu_sup;

if (srcu_gp_is_expedited(ssp))

jbase = 0;

if (rcu_seq_state(READ_ONCE(sup->srcu_gp_seq))) {

void cleanup_srcu_struct(struct srcu_struct *ssp)

{

int cpu;

struct srcu_usage *sup = ssp->srcu_sup;

return; /* Just leak it! */

if (WARN_ON(srcu_readers_active(ssp)))

return; /* Just leak it! */

for_each_possible_cpu(cpu) {

struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);

flush_work(&sdp->work);

if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist)))

return; /* Forgot srcu_barrier(), so just leak it! */

*/

int __srcu_read_lock(struct srcu_struct *ssp)

{

smp_mb(); /* B */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_lock);

void __srcu_read_unlock(struct srcu_struct *ssp, int idx)

{

smp_mb(); /* C */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock);

*/

int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)

{

smp_mb__after_atomic(); /* B */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_lock_nmisafe);

*/

void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)

{

smp_mb__before_atomic(); /* C */ /* Avoid leaking the critical section. */

}

EXPORT_SYMBOL_GPL(__srcu_read_unlock_nmisafe);

/*

* Wait until all readers counted by array index idx complete, but

* loop an additional time if there is an expedited grace period pending.

*/

static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)

{

unsigned long curdelay;

curdelay = !srcu_get_delay(ssp);

for (;;) {

if (srcu_readers_active_idx_check(ssp, idx))

}

/*

* use the other rank of the ->srcu_(un)lock_count[] arrays. This allows

* us to wait for pre-existing readers in a starvation-free manner.

*/

static void srcu_flip(struct srcu_struct *ssp)

{

/*

* preceding call to srcu_readers_active_idx_check() found that

*

* This sum-equality check and ordering also ensures that if

* a given call to __srcu_read_lock() uses the new value of

* that reader's increments, which is all to the good, because

* this grace period need not wait on that reader. After all,

* if those earlier scans had seen that reader, there would have

*/

smp_mb(); /* E */ /* Pairs with B and C. */

/*

* Ensure that if the updater misses an __srcu_read_unlock()

check_init_srcu_struct(ssp);

/* If _lite() readers, don't do unsolicited expediting. */

return false;

/* If the local srcu_data structure has callbacks, not idle. */

sdp = raw_cpu_ptr(ssp->sda);

* read-side critical sections are delimited by srcu_read_lock() and

* srcu_read_unlock(), and may be nested.

*

*/

void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,

rcu_callback_t func)

*

* Wait for the count to drain to zero of both indexes. To avoid the

* possible starvation of synchronize_srcu(), it waits for the count of

*

* Can block; must be called from process context.

*

*/

unsigned long srcu_batches_completed(struct srcu_struct *ssp)

{

}

EXPORT_SYMBOL_GPL(srcu_batches_completed);

/*

* Because readers might be delayed for an extended period after

* might well be readers using both idx=0 and idx=1. We therefore

* need to wait for readers to clear from both index values before

* invoking a callback.

}

if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {

if (!try_check_zero(ssp, idx, 1)) {

mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);

return; /* readers present, retry later. */

* SRCU read-side critical sections are normally short,

* so check at least twice in quick succession after a flip.

*/

if (!try_check_zero(ssp, idx, 2)) {

mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);

return; /* readers present, retry later. */

ssp = sup->srcu_ssp;

srcu_advance_state(ssp);

curdelay = srcu_get_delay(ssp);

if (curdelay) {

WRITE_ONCE(sup->reschedule_count, 0);

} else {

int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);

int ss_state_idx = ss_state;

if (ss_state < 0 || ss_state >= ARRAY_SIZE(srcu_size_state_name))

ss_state_idx = ARRAY_SIZE(srcu_size_state_name) - 1;

pr_alert("%s%s Tree SRCU g%ld state %d (%s)",

struct srcu_data *sdp;

sdp = per_cpu_ptr(ssp->sda, cpu);

/*

* Make sure that a lock is always counted if the corresponding

*/

smp_rmb();

c0 = l0 - u0;

c1 = l1 - u1;

ssp->sda = alloc_percpu(struct srcu_data);

if (WARN_ON_ONCE(!ssp->sda))

return -ENOMEM;

}

return 0;

}

}

if (!IS_ENABLED(CONFIG_TINY_RCU))

}

/* See if tasks are still holding out, complain if so. */

#endif

}

{

#ifdef CONFIG_TASKS_RCU

rcu_spawn_tasks_kthread();

// Run the self-tests.

rcu_tasks_initiate_self_tests();

}

#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */

static inline void rcu_tasks_bootup_oddness(void) {}

static inline bool rcu_reclaim_tiny(struct rcu_head *head)

{

rcu_callback_t f;

rcu_lock_acquire(&rcu_callback_map);

trace_rcu_invoke_callback("", head);

f = head->func;

}

EXPORT_SYMBOL_GPL(synchronize_rcu);

/*

* Post an RCU callback to be invoked after the end of an RCU grace

* period. But since we have but one CPU, that would be after any

pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func);

mem_dump_obj(head);

}

return;

}

}

EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu);

{

}

#endif

void __init rcu_init(void)

}

EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);

#if defined(CONFIG_NO_HZ_FULL) && (!defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK))

/*

* An empty function that will trigger a reschedule on

/* Handle the ends of any preceding grace periods first. */

if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||

if (!offloaded)

ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */

rdp->core_needs_qs = false;

/* Now handle the beginnings of any new-to-this-CPU grace periods. */

if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||

/*

* If the current grace period is waiting for this CPU,

* set up to detect a quiescent state, otherwise don't

rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */

if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)

WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);

WRITE_ONCE(rdp->last_sched_clock, jiffies);

WRITE_ONCE(rdp->gpwrap, false);

rcu_gpnum_ovf(rnp, rdp);

{

struct rcu_synchronize *rs = container_of(

(struct rcu_head *) node, struct rcu_synchronize, head);

WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&

/* Finally. */

complete(&rs->completion);

/* Advance to a new grace period and initialize state. */

record_gp_stall_check_time();

/* Record GP times before starting GP, hence rcu_seq_start(). */

rcu_seq_start(&rcu_state.gp_seq);

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));

rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);

raw_spin_unlock_irq_rcu_node(rnp);

static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func)

{

rcu_segcblist_enqueue(&rdp->cblist, head);

trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued"));

}

* critical sections have completed.

*

* Use this API instead of call_rcu() if you don't want the callback to be

* memory pressure and on systems which are lightly loaded or mostly idle.

* This function will cause callbacks to be invoked sooner than later at the

* expense of extra power. Other than that, this function is identical to, and

* might well execute concurrently with RCU read-side critical sections

* that started after call_rcu() was invoked.

*

* RCU read-side critical sections are delimited by rcu_read_lock()

* and rcu_read_unlock(), and may be nested. In addition, but only in

* v5.0 and later, regions of code across which interrupts, preemption,

*

* Implementation of these memory-ordering guarantees is described here:

* Documentation/RCU/Design/Memory-Ordering/Tree-RCU-Memory-Ordering.rst.

*/

void call_rcu(struct rcu_head *head, rcu_callback_t func)

{

* snapshot before adding a request.

*/

if (IS_ENABLED(CONFIG_PROVE_RCU))

rcu_sr_normal_add_req(&rs);

*/

void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)

{

/*

* Any prior manipulation of RCU-protected data must happen

* before the loads from ->gp_seq and ->expedited_sequence.

*/

smp_mb(); /* ^^^ */

rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);

}

EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full);

* specified leaf rcu_node structure, which is acquired by the caller.

*/

static void rcu_report_exp_cpu_mult(struct rcu_node *rnp, unsigned long flags,

__releases(rnp->lock)

{

int cpu;

struct rcu_data *rdp;

raw_lockdep_assert_held_rcu_node(rnp);

raw_spin_unlock_irqrestore_rcu_node(rnp, flags);

return;

}

WRITE_ONCE(rnp->expmask, rnp->expmask & ~mask);

for_each_leaf_node_cpu_mask(rnp, cpu, mask) {

rdp = per_cpu_ptr(&rcu_data, cpu);

if (rdp_gp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {

WRITE_ONCE(rdp_gp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);

}

if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {

if (my_rdp->nocb_defer_wakeup > RCU_NOCB_WAKE_NOT) {

WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);

}

WRITE_ONCE(my_rdp->nocb_gp_sleep, true);

raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);

/* Dump out nocb kthread state for the specified rcu_data structure. */

static void show_rcu_nocb_state(struct rcu_data *rdp)

{

struct rcu_data *nocb_next_rdp;

struct rcu_segcblist *rsclp = &rdp->cblist;

bool waslocked;

typeof(*rdp),

nocb_entry_rdp);

rdp->cpu, rdp->nocb_gp_rdp->cpu,

nocb_next_rdp ? nocb_next_rdp->cpu : -1,

"kK"[!!rdp->nocb_cb_kthread],

jiffies - rdp->nocb_nobypass_last,

rdp->nocb_nobypass_count,

".D"[rcu_segcblist_ready_cbs(rsclp)],

".W"[!rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL)],

rcu_segcblist_segempty(rsclp, RCU_WAIT_TAIL) ? "" : bufw,

".R"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL)],

rcu_segcblist_segempty(rsclp, RCU_NEXT_READY_TAIL) ? "" : bufr,

".N"[!rcu_segcblist_segempty(rsclp, RCU_NEXT_TAIL)],

".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],

rcu_segcblist_n_cbs(&rdp->cblist),

rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',

rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,

{

struct rcu_data *rdp;

return;

rdp = this_cpu_ptr(&rcu_data);

rdp->cpu_no_qs.b.norm = false;

rcu_report_qs_rdp(rdp);

*/

static void rcu_flavor_sched_clock_irq(int user)

{

/*

* Get here if this CPU took its interrupt from user

*

* No memory barrier is required here because rcu_qs()

* references only CPU-local variables that other CPUs