summaryrefslogtreecommitdiff
path: root/kernel/rcu
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcu')
-rw-r--r--kernel/rcu/rcutorture.c29
-rw-r--r--kernel/rcu/tasks.h3
-rw-r--r--kernel/rcu/tree.c65
-rw-r--r--kernel/rcu/tree_nocb.h2
-rw-r--r--kernel/rcu/tree_stall.h7
5 files changed, 76 insertions, 30 deletions
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 5f2848b828dc..882a158ada7b 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -572,7 +572,7 @@ static unsigned long rcu_no_completed(void)
static void rcu_torture_deferred_free(struct rcu_torture *p)
{
- call_rcu_hurry(&p->rtort_rcu, rcu_torture_cb);
+ call_rcu(&p->rtort_rcu, rcu_torture_cb);
}
static void rcu_sync_torture_init(void)
@@ -619,7 +619,7 @@ static struct rcu_torture_ops rcu_ops = {
.poll_gp_state_exp = poll_state_synchronize_rcu,
.cond_sync_exp = cond_synchronize_rcu_expedited,
.cond_sync_exp_full = cond_synchronize_rcu_expedited_full,
- .call = call_rcu_hurry,
+ .call = call_rcu,
.cb_barrier = rcu_barrier,
.fqs = rcu_force_quiescent_state,
.gp_kthread_dbg = show_rcu_gp_kthreads,
@@ -1145,7 +1145,7 @@ static void rcu_tasks_torture_deferred_free(struct rcu_torture *p)
static void synchronize_rcu_mult_test(void)
{
- synchronize_rcu_mult(call_rcu_tasks, call_rcu_hurry);
+ synchronize_rcu_mult(call_rcu_tasks, call_rcu);
}
static struct rcu_torture_ops tasks_ops = {
@@ -1632,6 +1632,17 @@ static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void))
}
/*
+ * Do an rcu_barrier() to motivate lazy callbacks during a stutter
+ * pause. Without this, we can get false-positives rtort_pipe_count
+ * splats.
+ */
+static void rcu_torture_writer_work(struct work_struct *work)
+{
+ if (cur_ops->cb_barrier)
+ cur_ops->cb_barrier();
+}
+
+/*
* RCU torture writer kthread. Repeatedly substitutes a new structure
* for that pointed to by rcu_torture_current, freeing the old structure
* after a series of grace periods (the "pipeline").
@@ -1651,6 +1662,7 @@ rcu_torture_writer(void *arg)
int i;
int idx;
unsigned long j;
+ struct work_struct lazy_work;
int oldnice = task_nice(current);
struct rcu_gp_oldstate *rgo = NULL;
int rgo_size = 0;
@@ -1703,6 +1715,9 @@ rcu_torture_writer(void *arg)
pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n",
torture_type, jiffies - j);
+ if (IS_ENABLED(CONFIG_RCU_LAZY))
+ INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work);
+
do {
rcu_torture_writer_state = RTWS_FIXED_DELAY;
torture_hrtimeout_us(500, 1000, &rand);
@@ -1895,6 +1910,8 @@ rcu_torture_writer(void *arg)
!rcu_gp_is_normal();
}
rcu_torture_writer_state = RTWS_STUTTER;
+ if (IS_ENABLED(CONFIG_RCU_LAZY))
+ queue_work(system_percpu_wq, &lazy_work);
stutter_waited = stutter_wait("rcu_torture_writer");
if (stutter_waited &&
!atomic_read(&rcu_fwd_cb_nodelay) &&
@@ -1925,6 +1942,12 @@ rcu_torture_writer(void *arg)
pr_alert("%s" TORTURE_FLAG
" Dynamic grace-period expediting was disabled.\n",
torture_type);
+
+ if (IS_ENABLED(CONFIG_RCU_LAZY)) {
+ cancel_work_sync(&lazy_work);
+ destroy_work_on_stack(&lazy_work);
+ }
+
kfree(ulo);
kfree(rgo);
rcu_torture_writer_state = RTWS_STOPPING;
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 48f0d803c8e2..f4da5fad70f5 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
// Queuing callbacks before initialization not yet supported.
if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))
rcu_segcblist_init(&rtpcp->cblist);
- needwake = (func == wakeme_after_rcu) ||
+ needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) ||
+ (func == wakeme_after_rcu) ||
(rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);
if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {
if (rtp->lazy_jiffies)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 55df6d37145e..03a43d3d2616 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param
int ret = kstrtoul(val, 0, &j);
if (!ret) {
- WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
+ WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ));
adjust_jiffies_till_sched_qs();
}
return ret;
@@ -969,14 +969,11 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp)
if (rcu_cpu_stall_cputime && rdp->snap_record.gp_seq != rdp->gp_seq) {
int cpu = rdp->cpu;
struct rcu_snap_record *rsrp;
- struct kernel_cpustat *kcsp;
-
- kcsp = &kcpustat_cpu(cpu);
rsrp = &rdp->snap_record;
- rsrp->cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu);
- rsrp->cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu);
- rsrp->cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu);
+ rsrp->cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu);
+ rsrp->cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu);
+ rsrp->cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu);
rsrp->nr_hardirqs = kstat_cpu_irqs_sum(cpu) + arch_irq_stat_cpu(cpu);
rsrp->nr_softirqs = kstat_cpu_softirqs_sum(cpu);
rsrp->nr_csw = nr_context_switches_cpu(cpu);
@@ -1632,17 +1629,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node)
atomic_set_release(&sr_wn->inuse, 0);
}
-/* Enable rcu_normal_wake_from_gp automatically on small systems. */
-#define WAKE_FROM_GP_CPU_THRESHOLD 16
-
-static int rcu_normal_wake_from_gp = -1;
+static int rcu_normal_wake_from_gp = 1;
module_param(rcu_normal_wake_from_gp, int, 0644);
static struct workqueue_struct *sync_wq;
+#define RCU_SR_NORMAL_LATCH_THR 64
+
+/* Number of in-flight synchronize_rcu() calls queued on srs_next. */
+static atomic_long_t rcu_sr_normal_count;
+static int rcu_sr_normal_latched; /* 0/1 */
+
static void rcu_sr_normal_complete(struct llist_node *node)
{
struct rcu_synchronize *rs = container_of(
(struct rcu_head *) node, struct rcu_synchronize, head);
+ long nr;
WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&
!poll_state_synchronize_rcu_full(&rs->oldstate),
@@ -1650,6 +1651,15 @@ static void rcu_sr_normal_complete(struct llist_node *node)
/* Finally. */
complete(&rs->completion);
+ nr = atomic_long_dec_return(&rcu_sr_normal_count);
+ WARN_ON_ONCE(nr < 0);
+
+ /*
+ * Unlatch: switch back to normal path when fully
+ * drained and if it has been latched.
+ */
+ if (nr == 0)
+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0);
}
static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
@@ -1795,6 +1805,24 @@ static bool rcu_sr_normal_gp_init(void)
static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
{
+ /*
+ * Increment before publish to avoid a complete
+ * vs enqueue race on latch.
+ */
+ long nr = atomic_long_inc_return(&rcu_sr_normal_count);
+
+ /*
+ * Latch when threshold is reached. Checking for an exact match
+ * restricts cmpxchg() to a single context.
+ *
+ * This latch is intentionally relaxed and best-effort. Concurrent
+ * set/clear can race and temporarily lose the latch, which is OK
+ * because it only selects between the fast and fallback paths.
+ */
+ if (nr == RCU_SR_NORMAL_LATCH_THR)
+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1);
+
+ /* Publish for the GP kthread/worker. */
llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
}
@@ -2584,7 +2612,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
const long npj = NSEC_PER_SEC / HZ;
long rrn = READ_ONCE(rcu_resched_ns);
- rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
+ rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC);
tlimit = local_clock() + rrn;
jlimit = jiffies + (rrn + npj + 1) / npj;
jlimit_check = true;
@@ -3278,14 +3306,15 @@ static void synchronize_rcu_normal(void)
{
struct rcu_synchronize rs;
+ init_rcu_head_on_stack(&rs.head);
trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));
- if (READ_ONCE(rcu_normal_wake_from_gp) < 1) {
+ if (READ_ONCE(rcu_normal_wake_from_gp) < 1 ||
+ READ_ONCE(rcu_sr_normal_latched)) {
wait_rcu_gp(call_rcu_hurry);
goto trace_complete_out;
}
- init_rcu_head_on_stack(&rs.head);
init_completion(&rs.completion);
/*
@@ -3302,10 +3331,10 @@ static void synchronize_rcu_normal(void)
/* Now we can wait. */
wait_for_completion(&rs.completion);
- destroy_rcu_head_on_stack(&rs.head);
trace_complete_out:
trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));
+ destroy_rcu_head_on_stack(&rs.head);
}
/**
@@ -4904,12 +4933,6 @@ void __init rcu_init(void)
sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
WARN_ON(!sync_wq);
- /* Respect if explicitly disabled via a boot parameter. */
- if (rcu_normal_wake_from_gp < 0) {
- if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD)
- rcu_normal_wake_from_gp = 1;
- }
-
/* Fill in default value for rcutree.qovld boot parameter. */
/* -After- the rcu_node ->lock fields are initialized! */
if (qovld < 0)
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index 1047b30cd46b..373b877cf171 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu)
* No-CBs GP kthreads come here to wait for additional callbacks to show up
* or for grace periods to end.
*/
-static void nocb_gp_wait(struct rcu_data *my_rdp)
+static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp)
{
bool bypass = false;
int __maybe_unused cpu = my_rdp->cpu;
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index b67532cb8770..cf7ae51cba40 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -479,7 +479,6 @@ static void print_cpu_stat_info(int cpu)
{
struct rcu_snap_record rsr, *rsrp;
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
- struct kernel_cpustat *kcsp = &kcpustat_cpu(cpu);
if (!rcu_cpu_stall_cputime)
return;
@@ -488,9 +487,9 @@ static void print_cpu_stat_info(int cpu)
if (rsrp->gp_seq != rdp->gp_seq)
return;
- rsr.cputime_irq = kcpustat_field(kcsp, CPUTIME_IRQ, cpu);
- rsr.cputime_softirq = kcpustat_field(kcsp, CPUTIME_SOFTIRQ, cpu);
- rsr.cputime_system = kcpustat_field(kcsp, CPUTIME_SYSTEM, cpu);
+ rsr.cputime_irq = kcpustat_field(CPUTIME_IRQ, cpu);
+ rsr.cputime_softirq = kcpustat_field(CPUTIME_SOFTIRQ, cpu);
+ rsr.cputime_system = kcpustat_field(CPUTIME_SYSTEM, cpu);
pr_err("\t hardirqs softirqs csw/system\n");
pr_err("\t number: %8lld %10d %12lld\n",