summaryrefslogtreecommitdiff
path: root/kernel/sched/core.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched/core.c')
-rw-r--r--kernel/sched/core.c81
1 files changed, 59 insertions, 22 deletions
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 96226707c2f6..f78275192036 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -40,7 +40,6 @@
#include <linux/sched/rseq_api.h>
#include <linux/sched/rt.h>
-#include <linux/blkdev.h>
#include <linux/context_tracking.h>
#include <linux/cpuset.h>
#include <linux/delayacct.h>
@@ -443,6 +442,17 @@ static void __sched_core_flip(bool enabled)
sched_core_lock(cpu, &flags);
+ /*
+ * A core-wide selection may have the shared rq lock temporarily
+ * released by a lock-dropping ->pick_task(). Flipping would
+ * rebind rq_lockp() under it. Wait it out.
+ */
+ while (cpu_rq(cpu)->core->core_pick_in_flight) {
+ sched_core_unlock(cpu, &flags);
+ cpu_relax();
+ sched_core_lock(cpu, &flags);
+ }
+
for_each_cpu(t, smt_mask)
cpu_rq(t)->core_enabled = enabled;
@@ -4638,7 +4648,7 @@ void set_numabalancing_state(bool enabled)
__set_numabalancing_state(enabled);
}
-#ifdef CONFIG_PROC_SYSCTL
+#ifdef CONFIG_SYSCTL
static void reset_memory_tiering(void)
{
struct pglist_data *pgdat;
@@ -4674,7 +4684,7 @@ static int sysctl_numa_balancing(const struct ctl_table *table, int write,
}
return err;
}
-#endif /* CONFIG_PROC_SYSCTL */
+#endif /* CONFIG_SYSCTL */
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_SCHEDSTATS
@@ -4718,7 +4728,7 @@ out:
}
__setup("schedstats=", setup_schedstats);
-#ifdef CONFIG_PROC_SYSCTL
+#ifdef CONFIG_SYSCTL
static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
@@ -4738,7 +4748,7 @@ static int sysctl_schedstats(const struct ctl_table *table, int write, void *buf
set_schedstats(state);
return err;
}
-#endif /* CONFIG_PROC_SYSCTL */
+#endif /* CONFIG_SYSCTL */
#endif /* CONFIG_SCHEDSTATS */
#ifdef CONFIG_SYSCTL
@@ -5657,11 +5667,8 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
*/
static inline void prefetch_curr_exec_start(struct task_struct *p)
{
-#ifdef CONFIG_FAIR_GROUP_SCHED
- struct sched_entity *curr = p->se.cfs_rq->curr;
-#else
struct sched_entity *curr = task_rq(p)->cfs.curr;
-#endif
+
prefetch(curr);
prefetch(&curr->exec_start);
}
@@ -5973,8 +5980,13 @@ void preempt_count_add(int val)
#ifdef CONFIG_DEBUG_PREEMPT
/*
* Underflow?
+ *
+ * Cannot detect underflow based on the current preempt_count() value
+ * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
+ * bits.
*/
- if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+ if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
+ DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
return;
#endif
__preempt_count_add(val);
@@ -6006,7 +6018,10 @@ void preempt_count_sub(int val)
/*
* Underflow?
*/
- if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+ unsigned int uval = val;
+ unsigned int pc = preempt_count();
+
+ if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
return;
/*
* Is the spinlock portion underflowing?
@@ -6223,7 +6238,7 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
unsigned long cookie;
int i, cpu, occ = 0;
struct rq *rq_i;
- bool need_sync;
+ bool need_sync = false;
if (!sched_core_enabled(rq))
return __pick_next_task(rq, rf);
@@ -6242,6 +6257,8 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
return __pick_next_task(rq, rf);
}
+ rq->core->core_pick_in_flight++;
+
/*
* If there were no {en,de}queues since we picked (IOW, the task
* pointers are all still valid), and we haven't scheduled the last
@@ -6266,7 +6283,9 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
prev_balance(rq, rf);
smt_mask = cpu_smt_mask(cpu);
- need_sync = !!rq->core->core_cookie;
+
+restart:
+ need_sync |= !!rq->core->core_cookie;
/* reset state */
rq->core->core_cookie = 0UL;
@@ -6301,10 +6320,15 @@ pick_next_task(struct rq *rq, struct rq_flags *rf)
* and there are no cookied tasks running on siblings.
*/
if (!need_sync) {
-restart_single:
next = pick_task(rq, rf);
- if (unlikely(next == RETRY_TASK))
- goto restart_single;
+ if (unlikely(next == RETRY_TASK)) {
+ /* rq lock may have been dropped, clocks invalidated */
+ core_clock_updated = false;
+ if (!(rq->clock_update_flags & RQCF_UPDATED))
+ update_rq_clock(rq);
+ goto restart;
+ }
+
if (!next->core_cookie) {
rq->core_pick = NULL;
rq->core_dl_server = NULL;
@@ -6324,7 +6348,6 @@ restart_single:
*
* Tie-break prio towards the current CPU
*/
-restart_multi:
max = NULL;
for_each_cpu_wrap(i, smt_mask, cpu) {
rq_i = cpu_rq(i);
@@ -6338,8 +6361,13 @@ restart_multi:
update_rq_clock(rq_i);
p = pick_task(rq_i, rf);
- if (unlikely(p == RETRY_TASK))
- goto restart_multi;
+ if (unlikely(p == RETRY_TASK)) {
+ /* rq lock may have been dropped, clocks invalidated */
+ core_clock_updated = false;
+ if (!(rq->clock_update_flags & RQCF_UPDATED))
+ update_rq_clock(rq);
+ goto restart;
+ }
rq_i->core_pick = p;
rq_i->core_dl_server = rq_i->dl_server;
@@ -6445,6 +6473,7 @@ restart_multi:
}
out_set_next:
+ rq->core->core_pick_in_flight--;
put_prev_set_next_task(rq, rq->donor, next);
if (rq->core->core_forceidle_count && next == rq->idle)
queue_core_balance(rq);
@@ -6640,6 +6669,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu)
core_rq->core_forceidle_occupation = rq->core_forceidle_occupation;
/*
+ * A stale leftover would bias the count forever if this CPU later
+ * returns as its own leader. Move, don't copy.
+ */
+ core_rq->core_pick_in_flight = rq->core_pick_in_flight;
+ rq->core_pick_in_flight = 0;
+
+ /*
* Accounting edge for forced idle is handled in pick_next_task().
* Don't need another one here, since the hotplug thread shouldn't
* have a cookie.
@@ -9053,6 +9089,7 @@ void __init sched_init(void)
rq->core_forceidle_count = 0;
rq->core_forceidle_occupation = 0;
rq->core_forceidle_start = 0;
+ rq->core_pick_in_flight = 0;
rq->core_cookie = 0UL;
#endif
@@ -9199,7 +9236,7 @@ void __might_resched(const char *file, int line, unsigned int offsets)
}
EXPORT_SYMBOL(__might_resched);
-void __cant_sleep(const char *file, int line, int preempt_offset)
+void __cant_sleep(const char *file, int line)
{
static unsigned long prev_jiffy;
@@ -9209,7 +9246,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > preempt_offset)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
@@ -9241,7 +9278,7 @@ void __cant_migrate(const char *file, int line)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > 0)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)