From b42e0c41a422a212ddea0666d5a3a0e3c35206db Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:15:38 +0100 Subject: sched: Remove avg_wakeup Testing the load which led to this heuristic (nfs4 kbuild) shows that it has outlived it's usefullness. With intervening load balancing changes, I cannot see any difference with/without, so recover there fastpath cycles. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301062.6785.29.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched_features.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched_features.h b/kernel/sched_features.h index d5059fd761d9..96ef5dbc66e1 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -30,12 +30,6 @@ SCHED_FEAT(START_DEBIT, 1) */ SCHED_FEAT(WAKEUP_PREEMPT, 1) -/* - * Compute wakeup_gran based on task behaviour, clipped to - * [0, sched_wakeup_gran_ns] - */ -SCHED_FEAT(ADAPTIVE_GRAN, 1) - /* * When converting the wakeup granularity to virtual time, do it such * that heavier tasks preempting a lighter task have an edge. -- cgit v1.2.3 From e12f31d3e5d36328c7fbd0fce40a95e70b59152c Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:15:51 +0100 Subject: sched: Remove avg_overlap Both avg_overlap and avg_wakeup had an inherent problem in that their accuracy was detrimentally affected by cross-cpu wakeups, this because we are missing the necessary call to update_curr(). This can't be fixed without increasing overhead in our already too fat fastpath. Additionally, with recent load balancing changes making us prefer to place tasks in an idle cache domain (which is good for compute bound loads), communicating tasks suffer when a sync wakeup, which would enable affine placement, is turned into a non-sync wakeup by SYNC_LESS. With one task on the runqueue, wake_affine() rejects the affine wakeup request, leaving the unfortunate where placed, taking frequent cache misses. Remove it, and recover some fastpath cycles. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301121.6785.30.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- include/linux/sched.h | 3 --- kernel/sched.c | 33 --------------------------------- kernel/sched_debug.c | 1 - kernel/sched_fair.c | 18 ------------------ kernel/sched_features.h | 16 ---------------- 5 files changed, 71 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/include/linux/sched.h b/include/linux/sched.h index 70c560f5ada0..8604884cee87 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1180,9 +1180,6 @@ struct sched_entity { u64 vruntime; u64 prev_sum_exec_runtime; - u64 last_wakeup; - u64 avg_overlap; - u64 nr_migrations; #ifdef CONFIG_SCHEDSTATS diff --git a/kernel/sched.c b/kernel/sched.c index 35a8626ace7d..68ed6f4f3c13 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1887,11 +1887,6 @@ enqueue_task(struct rq *rq, struct task_struct *p, int wakeup, bool head) static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep) { - if (sleep && p->se.last_wakeup) { - update_avg(&p->se.avg_overlap, - p->se.sum_exec_runtime - p->se.last_wakeup); - p->se.last_wakeup = 0; - } sched_info_dequeued(p); p->sched_class->dequeue_task(rq, p, sleep); p->se.on_rq = 0; @@ -2452,15 +2447,6 @@ out_activate: activate_task(rq, p, 1); success = 1; - /* - * Only attribute actual wakeups done by this task. - */ - if (!in_interrupt()) { - struct sched_entity *se = ¤t->se; - - se->last_wakeup = se->sum_exec_runtime; - } - out_running: trace_sched_wakeup(rq, p, success); check_preempt_curr(rq, p, wake_flags); @@ -2522,8 +2508,6 @@ static void __sched_fork(struct task_struct *p) p->se.sum_exec_runtime = 0; p->se.prev_sum_exec_runtime = 0; p->se.nr_migrations = 0; - p->se.last_wakeup = 0; - p->se.avg_overlap = 0; #ifdef CONFIG_SCHEDSTATS memset(&p->se.statistics, 0, sizeof(p->se.statistics)); @@ -3594,23 +3578,6 @@ static inline void schedule_debug(struct task_struct *prev) static void put_prev_task(struct rq *rq, struct task_struct *prev) { - if (prev->state == TASK_RUNNING) { - u64 runtime = prev->se.sum_exec_runtime; - - runtime -= prev->se.prev_sum_exec_runtime; - runtime = min_t(u64, runtime, 2*sysctl_sched_migration_cost); - - /* - * In order to avoid avg_overlap growing stale when we are - * indeed overlapping and hence not getting put to sleep, grow - * the avg_overlap on preemption. - * - * We use the average preemption runtime because that - * correlates to the amount of cache footprint a task can - * build up. - */ - update_avg(&prev->se.avg_overlap, runtime); - } prev->sched_class->put_prev_task(rq, prev); } diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 20b95a420fec..8a46a719f367 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -407,7 +407,6 @@ void proc_sched_show_task(struct task_struct *p, struct seq_file *m) PN(se.exec_start); PN(se.vruntime); PN(se.sum_exec_runtime); - PN(se.avg_overlap); nr_switches = p->nvcsw + p->nivcsw; diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 6fc62854422c..c3b69d4b5d65 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1241,7 +1241,6 @@ static inline unsigned long effective_load(struct task_group *tg, int cpu, static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) { - struct task_struct *curr = current; unsigned long this_load, load; int idx, this_cpu, prev_cpu; unsigned long tl_per_task; @@ -1256,18 +1255,6 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, int sync) load = source_load(prev_cpu, idx); this_load = target_load(this_cpu, idx); - if (sync) { - if (sched_feat(SYNC_LESS) && - (curr->se.avg_overlap > sysctl_sched_migration_cost || - p->se.avg_overlap > sysctl_sched_migration_cost)) - sync = 0; - } else { - if (sched_feat(SYNC_MORE) && - (curr->se.avg_overlap < sysctl_sched_migration_cost && - p->se.avg_overlap < sysctl_sched_migration_cost)) - sync = 1; - } - /* * If sync wakeup then subtract the (maximum possible) * effect of the currently running task from the load @@ -1711,11 +1698,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ if (sched_feat(WAKEUP_SYNC) && sync) goto preempt; - if (sched_feat(WAKEUP_OVERLAP) && - se->avg_overlap < sysctl_sched_migration_cost && - pse->avg_overlap < sysctl_sched_migration_cost) - goto preempt; - if (!sched_feat(WAKEUP_PREEMPT)) return; diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 96ef5dbc66e1..c545e048dfed 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -41,12 +41,6 @@ SCHED_FEAT(ASYM_GRAN, 1) */ SCHED_FEAT(WAKEUP_SYNC, 0) -/* - * Wakeup preempt based on task behaviour. Tasks that do not overlap - * don't get preempted. - */ -SCHED_FEAT(WAKEUP_OVERLAP, 0) - /* * Use the SYNC wakeup hint, pipes and the likes use this to indicate * the remote end is likely to consume the data we just wrote, and @@ -63,16 +57,6 @@ SCHED_FEAT(SYNC_WAKEUPS, 1) */ SCHED_FEAT(AFFINE_WAKEUPS, 1) -/* - * Weaken SYNC hint based on overlap - */ -SCHED_FEAT(SYNC_LESS, 1) - -/* - * Add SYNC hint based on overlap - */ -SCHED_FEAT(SYNC_MORE, 0) - /* * Prefer to schedule the task we woke last (assuming it failed * wakeup-preemption), since its likely going to consume data we -- cgit v1.2.3 From 6bc6cf2b61336ed0c55a615eb4c0c8ed5daf3f08 Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:17:17 +0100 Subject: sched: Remove NORMALIZED_SLEEPER This feature hasn't been enabled in a long time, remove effectively dead code. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301447.6785.38.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 10 ---------- kernel/sched_features.h | 7 ------- 2 files changed, 17 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 0008cc4a1199..de98e2e9d6e1 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -741,16 +741,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) if (!initial && sched_feat(FAIR_SLEEPERS)) { unsigned long thresh = sysctl_sched_latency; - /* - * Convert the sleeper threshold into virtual time. - * SCHED_IDLE is a special sub-class. We care about - * fairness only relative to other SCHED_IDLE tasks, - * all of which have the same weight. - */ - if (sched_feat(NORMALIZED_SLEEPER) && (!entity_is_task(se) || - task_of(se)->policy != SCHED_IDLE)) - thresh = calc_delta_fair(thresh, se); - /* * Halve their sleep time's effect, to allow * for a gentler effect of sleepers: diff --git a/kernel/sched_features.h b/kernel/sched_features.h index c545e048dfed..404288354aee 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -12,13 +12,6 @@ SCHED_FEAT(FAIR_SLEEPERS, 1) */ SCHED_FEAT(GENTLE_FAIR_SLEEPERS, 1) -/* - * By not normalizing the sleep time, heavy tasks get an effective - * longer period, and lighter task an effective shorter period they - * are considered running. - */ -SCHED_FEAT(NORMALIZED_SLEEPER, 0) - /* * Place new tasks ahead so that they do not starve already running * tasks -- cgit v1.2.3 From 5ca9880c6f4ba4c84b517bc2fed5366adf63d191 Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:17:17 +0100 Subject: sched: Remove FAIR_SLEEPERS feature Our preemption model relies too heavily on sleeper fairness to disable it without dire consequences. Remove the feature, and save a branch or two. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301520.6785.40.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 2 +- kernel/sched_features.h | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index de98e2e9d6e1..97682f925ed5 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -738,7 +738,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) vruntime += sched_vslice(cfs_rq, se); /* sleeps up to a single latency don't count. */ - if (!initial && sched_feat(FAIR_SLEEPERS)) { + if (!initial) { unsigned long thresh = sysctl_sched_latency; /* diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 404288354aee..850f9809cf81 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -1,10 +1,3 @@ -/* - * Disregards a certain amount of sleep time (sched_latency_ns) and - * considers the task to be running during that period. This gives it - * a service deficit on wakeup, allowing it to run sooner. - */ -SCHED_FEAT(FAIR_SLEEPERS, 1) - /* * Only give sleepers 50% of their service deficit. This allows * them to run sooner, but does not allow tons of sleepers to -- cgit v1.2.3 From f2e74eeac03ffb779d64b66a643c5e598145a28b Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:17:18 +0100 Subject: sched: Remove WAKEUP_SYNC feature This feature never earned its keep, remove it. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301591.6785.42.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 4 ---- kernel/sched_features.h | 5 ----- 2 files changed, 9 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 97682f925ed5..1d99535b0928 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1658,7 +1658,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ struct task_struct *curr = rq->curr; struct sched_entity *se = &curr->se, *pse = &p->se; struct cfs_rq *cfs_rq = task_cfs_rq(curr); - int sync = wake_flags & WF_SYNC; int scale = cfs_rq->nr_running >= sched_nr_latency; if (unlikely(rt_prio(p->prio))) @@ -1691,9 +1690,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ if (unlikely(curr->policy == SCHED_IDLE)) goto preempt; - if (sched_feat(WAKEUP_SYNC) && sync) - goto preempt; - if (!sched_feat(WAKEUP_PREEMPT)) return; diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 850f9809cf81..1cb7c4701bf3 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -22,11 +22,6 @@ SCHED_FEAT(WAKEUP_PREEMPT, 1) */ SCHED_FEAT(ASYM_GRAN, 1) -/* - * Always wakeup-preempt SYNC wakeups, see SYNC_WAKEUPS. - */ -SCHED_FEAT(WAKEUP_SYNC, 0) - /* * Use the SYNC wakeup hint, pipes and the likes use this to indicate * the remote end is likely to consume the data we just wrote, and -- cgit v1.2.3 From c6ee36c423c3ed1fb86bb3eabba9fc256a300d16 Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:16:43 +0100 Subject: sched: Remove SYNC_WAKEUPS feature Sync wakeups are critical functionality with a long history. Remove it, we don't need the branch or icache footprint. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301817.6785.47.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched.c | 3 --- kernel/sched_features.h | 8 -------- 2 files changed, 11 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched.c b/kernel/sched.c index 16559de4edea..cc6dc8caa380 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2369,9 +2369,6 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, unsigned long flags; struct rq *rq; - if (!sched_feat(SYNC_WAKEUPS)) - wake_flags &= ~WF_SYNC; - this_cpu = get_cpu(); smp_wmb(); diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 1cb7c4701bf3..f54b6f9cc3dd 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -22,14 +22,6 @@ SCHED_FEAT(WAKEUP_PREEMPT, 1) */ SCHED_FEAT(ASYM_GRAN, 1) -/* - * Use the SYNC wakeup hint, pipes and the likes use this to indicate - * the remote end is likely to consume the data we just wrote, and - * therefore has cache benefit from being placed on the same cpu, see - * also AFFINE_WAKEUPS. - */ -SCHED_FEAT(SYNC_WAKEUPS, 1) - /* * Based on load and program behaviour, see if it makes sense to place * a newly woken task on the same cpu as the task that woke it -- -- cgit v1.2.3 From 13814d42e45dfbe845a0bbe5184565d9236896ae Mon Sep 17 00:00:00 2001 From: Mike Galbraith Date: Thu, 11 Mar 2010 17:17:04 +0100 Subject: sched: Remove ASYM_GRAN feature This features has been enabled for quite a while, after testing showed that easing preemption for light tasks was harmful to high priority threads. Remove the feature flag. Signed-off-by: Mike Galbraith Signed-off-by: Peter Zijlstra LKML-Reference: <1268301675.6785.44.camel@marge.simson.net> Signed-off-by: Ingo Molnar --- kernel/sched_fair.c | 28 +++++++++++----------------- kernel/sched_features.h | 6 ------ 2 files changed, 11 insertions(+), 23 deletions(-) (limited to 'kernel/sched_features.h') diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 1d99535b0928..9357ecdb7f6b 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -1583,24 +1583,18 @@ wakeup_gran(struct sched_entity *curr, struct sched_entity *se) /* * Since its curr running now, convert the gran from real-time * to virtual-time in his units. + * + * By using 'se' instead of 'curr' we penalize light tasks, so + * they get preempted easier. That is, if 'se' < 'curr' then + * the resulting gran will be larger, therefore penalizing the + * lighter, if otoh 'se' > 'curr' then the resulting gran will + * be smaller, again penalizing the lighter task. + * + * This is especially important for buddies when the leftmost + * task is higher priority than the buddy. */ - if (sched_feat(ASYM_GRAN)) { - /* - * By using 'se' instead of 'curr' we penalize light tasks, so - * they get preempted easier. That is, if 'se' < 'curr' then - * the resulting gran will be larger, therefore penalizing the - * lighter, if otoh 'se' > 'curr' then the resulting gran will - * be smaller, again penalizing the lighter task. - * - * This is especially important for buddies when the leftmost - * task is higher priority than the buddy. - */ - if (unlikely(se->load.weight != NICE_0_LOAD)) - gran = calc_delta_fair(gran, se); - } else { - if (unlikely(curr->load.weight != NICE_0_LOAD)) - gran = calc_delta_fair(gran, curr); - } + if (unlikely(se->load.weight != NICE_0_LOAD)) + gran = calc_delta_fair(gran, se); return gran; } diff --git a/kernel/sched_features.h b/kernel/sched_features.h index f54b6f9cc3dd..83c66e8ad3ee 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -16,12 +16,6 @@ SCHED_FEAT(START_DEBIT, 1) */ SCHED_FEAT(WAKEUP_PREEMPT, 1) -/* - * When converting the wakeup granularity to virtual time, do it such - * that heavier tasks preempting a lighter task have an edge. - */ -SCHED_FEAT(ASYM_GRAN, 1) - /* * Based on load and program behaviour, see if it makes sense to place * a newly woken task on the same cpu as the task that woke it -- -- cgit v1.2.3