CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer

endif

#

# Build efficiency:

#

#ifdef CONFIG_SCHED_CLASS_EXT

# include "ext.c"

#endif

#include "syscalls.c"

# include "cpufreq_schedutil.c"

#endif

#ifdef CONFIG_SCHEDSTATS

# include "stats.c"

#include "autogroup.h"

#include "pelt.h"

#include "smp.h"

#include "../workqueue_internal.h"

#include "../../io_uring/io-wq.h"

DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);

/*

* Debugging: various feature bits

*

*/

#define SCHED_FEAT(name, enabled) \

(1UL << __SCHED_FEAT_##name) * enabled |

#include "features.h"

0;

#undef SCHED_FEAT

*/

__read_mostly int sysctl_resched_latency_warn_ms = 100;

__read_mostly int sysctl_resched_latency_warn_once = 1;

/*

* Number of tasks to iterate in a single balance run.

* Limited because this is done with IRQs disabled.

*/

__read_mostly int scheduler_running;

if (rq->clock_update_flags & RQCF_ACT_SKIP)

return;

if (sched_feat(WARN_DOUBLE_CLOCK))

rq->clock_update_flags |= RQCF_UPDATED;

clock = sched_clock_cpu(cpu_of(rq));

scx_rq_clock_update(rq, clock);

bucket = &uc_rq->bucket[uc_se->bucket_id];

if (likely(bucket->tasks))

bucket->tasks--;

* Defensive programming: this should never happen. If it happens,

* e.g. due to future modification, warn and fix up the expected value.

*/

if (bucket->value >= rq_clamp) {

bkt_clamp = uclamp_rq_max_value(rq, clamp_id, uc_se->value);

uclamp_rq_set(rq, clamp_id, bkt_clamp);

* The condition is constructed such that a NOP is generated when

* sched_uclamp_used is disabled.

*/

return;

if (unlikely(!p->sched_class->uclamp_enabled))

* The condition is constructed such that a NOP is generated when

* sched_uclamp_used is disabled.

*/

return;

if (unlikely(!p->sched_class->uclamp_enabled))

}

if (update_root_tg) {

uclamp_update_root_tg();

}

if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {

uclamp_sync_util_min_rt_default();

}

void deactivate_task(struct rq *rq, struct task_struct *p, int flags)

{

WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);

ASSERT_EXCLUSIVE_WRITER(p->on_rq);

* XXX do further audits, this smells like something putrid.

*/

if (ctx->flags & SCA_MIGRATE_DISABLE)

else

lockdep_assert_held(&p->pi_lock);

void set_task_cpu(struct task_struct *p, unsigned int new_cpu)

{

unsigned int state = READ_ONCE(p->__state);

/*

WARN_ON_ONCE(!cpu_online(new_cpu));

WARN_ON_ONCE(is_migration_disabled(p));

trace_sched_migrate_task(p, new_cpu);

static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)

{

return false;

/*

* - we're serialized against set_special_state() by virtue of

* it disabling IRQs (this allows not taking ->pi_lock).

*/

if (!ttwu_state_match(p, state, &success))

goto out;

INIT_LIST_HEAD(&p->se.group_node);

/* A delayed task cannot be in clone(). */

#ifdef CONFIG_FAIR_GROUP_SCHED

p->se.cfs_rq = NULL;

return ns;

}

static u64 cpu_resched_latency(struct rq *rq)

{

int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms);

return 1;

}

__setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms);

/*

* This function gets called by the timer code, with HZ frequency.

* we are always sure that there is no proxy (only a

* single task is running).

*/

update_rq_clock(rq);

if (!is_idle_task(curr)) {

preempt_count_set(PREEMPT_DISABLED);

}

rcu_sleep_check();

profile_hit(SCHED_PROFILING, __builtin_return_address(0));

picked:

clear_tsk_need_resched(prev);

clear_preempt_need_resched();

rq->last_seen_need_resched_ns = 0;

if (likely(prev != next)) {

rq->nr_switches++;

* deadlock if the callback attempts to acquire a lock which is

* already acquired.

*/

/*

* If we are going to sleep and we have plugged IO queued,

int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,

void *key)

{

return try_to_wake_up(curr->private, mode, wake_flags);

}

EXPORT_SYMBOL(default_wake_function);

return 1;

}

/*

* whether the current CPU is in an RCU read-side critical section,

* so the tick can report quiescent states even for CPUs looping

* in kernel context. In contrast, in non-preemptible kernels,

* RCU quiescent state. Therefore, the following code causes

* cond_resched() to report a quiescent state, but only when RCU

* is in urgent need of one.

*/

#ifndef CONFIG_PREEMPT_RCU

rcu_all_qs();

#else /* !CONFIG_PREEMPT_DYNAMIC: */

static inline void preempt_dynamic_init(void) { }

#endif /* CONFIG_PREEMPT_DYNAMIC */

int io_schedule_prepare(void)

{

int old_iowait = current->in_iowait;

sched_show_task(p);

}

if (!state_filter)

sysrq_sched_debug_show();

rcu_read_unlock();

/*

* Only show locks if all tasks are dumped:

* operation in the resume sequence, just build a single sched

* domain, ignoring cpusets.

*/

if (--num_cpus_frozen)

return;

/*

cpuset_update_active_cpus();

} else {

num_cpus_frozen++;

}

}

* CPU masks are stable and all blatant races in the below code cannot

* happen.

*/

sched_init_domains(cpu_active_mask);

/* Move init over to a non-isolated CPU */

if (set_cpus_allowed_ptr(current, housekeeping_cpumask(HK_TYPE_DOMAIN)) < 0)

spin_unlock_irqrestore(&task_group_lock, flags);

}

{

struct task_group *tg;

tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),

struct task_group, css);

tg = autogroup_task_group(tsk, tg);

#ifdef CONFIG_FAIR_GROUP_SCHED

if (tsk->sched_class->task_change_group)

{

int queued, running, queue_flags =

DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;

struct rq *rq;

CLASS(task_rq_lock, rq_guard)(tsk);

rq = rq_guard.rq;

update_rq_clock(rq);

running = task_current_donor(rq, tsk);

if (running)

put_prev_task(rq, tsk);

if (!for_autogroup)

scx_cgroup_move_task(tsk);

unsigned int clamps;

lockdep_assert_held(&uclamp_mutex);

css_for_each_descendant_pre(css, top_css) {

uc_parent = css_tg(css)->parent

if (req.ret)

return req.ret;

guard(mutex)(&uclamp_mutex);

guard(rcu)();

struct mm_struct *mm;

int weight, cpu;

work->next = work; /* Prevent double-add */

if (t->flags & PF_EXITING)

* a cookie until after we've removed it, we must have core scheduling

* enabled here.

*/

if (sched_core_enqueued(p))

sched_core_dequeue(rq, p, DEQUEUE_SAVE);

}

}

{

struct root_domain *rd = cpu_rq(cpu)->rd;

return true;

return false;

}

return SCHED_CAPACITY_SCALE;

}

{

return false;

}

lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

dl_rq->running_bw += dl_bw;

/* kick cpufreq (see the comment in kernel/sched/sched.h). */

cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);

}

lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

dl_rq->running_bw -= dl_bw;

if (dl_rq->running_bw > old)

dl_rq->running_bw = 0;

/* kick cpufreq (see the comment in kernel/sched/sched.h). */

lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

dl_rq->this_bw += dl_bw;

}

static inline

lockdep_assert_rq_held(rq_of_dl_rq(dl_rq));

dl_rq->this_bw -= dl_bw;

if (dl_rq->this_bw > old)

dl_rq->this_bw = 0;

}

static inline

struct dl_bw *dl_b;

raw_spin_lock_irqsave(&p->pi_lock, rf.flags);

raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);

return;

}

rd->dl_bw.total_bw = 0;

/*

*/

for_each_cpu(i, rd->span) {

struct sched_dl_entity *dl_se = &cpu_rq(i)->fair_server;

if (dl_server(dl_se) && cpu_active(i))

}

}

#endif /* CONFIG_SMP */

static void switched_from_dl(struct rq *rq, struct task_struct *p)

#endif

};

int sched_dl_global_validate(void)

{

u64 runtime = global_rt_runtime();

u64 period = global_rt_period();

u64 new_bw = to_ratio(period, runtime);

struct dl_bw *dl_b;

int cpu, cpus, ret = 0;

unsigned long flags;

for_each_online_cpu(cpu) {

rcu_read_lock_sched();

goto next;

dl_b = dl_bw_of(cpu);

void sched_dl_do_global(void)

{

u64 new_bw = -1;

struct dl_bw *dl_b;

int cpu;

unsigned long flags;

for_each_possible_cpu(cpu) {

rcu_read_lock_sched();

rcu_read_unlock_sched();

continue;

}

}

#endif

void print_dl_stats(struct seq_file *m, int cpu)

{

print_dl_rq(m, cpu, &cpu_rq(cpu)->dl);

}

static int sched_dynamic_show(struct seq_file *m, void *v)

{

int i = IS_ENABLED(CONFIG_PREEMPT_RT) * 2;

for (; i < j; i++) {

if (preempt_dynamic_mode == i)

bool orig;

cpus_read_lock();

orig = sched_debug_verbose;

result = debugfs_write_file_bool(filp, ubuf, cnt, ppos);

sd_dentry = NULL;

}

cpus_read_unlock();

return result;

debugfs_create_u32("migration_cost_ns", 0644, debugfs_sched, &sysctl_sched_migration_cost);

debugfs_create_u32("nr_migrate", 0644, debugfs_sched, &sysctl_sched_nr_migrate);

update_sched_domain_debugfs();

#endif

#ifdef CONFIG_NUMA_BALANCING

* Copyright (c) 2022 Tejun Heo <tj@kernel.org>

* Copyright (c) 2022 David Vernet <dvernet@meta.com>

*/

#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))

enum scx_consts {

/*

* Keep built-in idle tracking even if ops.update_idle() is implemented.

*/

/*

* By default, if there are no other task to run on the CPU, ext core

* flag is specified, such tasks are passed to ops.enqueue() with

* %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.

*/

/*

* An exiting task may schedule after PF_EXITING is set. In such cases,

* depend on pid lookups and wants to handle these tasks directly, the

* following flag can be used.

*/

/*

* If set, only tasks with policy set to SCHED_EXT are attached to

* sched_ext. If clear, SCHED_NORMAL tasks are also included.

*/

/*

* A migration disabled task can only execute on its current CPU. By

* current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr

* and thus may disagree with cpumask_weight(p->cpus_ptr).

*/

/*

* CPU cgroup support flags

SCX_OPS_ENQ_LAST |

SCX_OPS_ENQ_EXITING |

SCX_OPS_ENQ_MIGRATION_DISABLED |

SCX_OPS_SWITCH_PARTIAL |

SCX_OPS_HAS_CGROUP_WEIGHT,

};

enum scx_pick_idle_cpu_flags {

SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */

};

enum scx_kick_flags {

static struct sched_ext_ops scx_ops;

static bool scx_warned_zero_slice;

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);

static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_migration_disabled);

static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);

static struct static_key_false scx_has_op[SCX_OPI_END] =

{ [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT };

static struct delayed_work scx_watchdog_work;

/* for %SCX_KICK_WAIT */

static unsigned long __percpu *scx_kick_cpus_pnt_seqs;

return p;

}

static enum scx_ops_enable_state scx_ops_enable_state(void)

{

return atomic_read(&scx_ops_enable_state_var);

if (!scx_rq_online(rq))

goto local;

goto global;

if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)

goto direct;

/* see %SCX_OPS_ENQ_EXITING */

if (!static_branch_unlikely(&scx_ops_enq_exiting) &&

goto local;

/* see %SCX_OPS_ENQ_MIGRATION_DISABLED */

if (!static_branch_unlikely(&scx_ops_enq_migration_disabled) &&

goto local;

if (!SCX_HAS_OP(enqueue))

goto global;

*/

touch_core_sched(rq, p);

p->scx.slice = SCX_SLICE_DFL;

local_norefill:

dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags);

return;

global:

touch_core_sched(rq, p); /* see the comment in local: */

p->scx.slice = SCX_SLICE_DFL;

dispatch_enqueue(find_global_dsq(p), p, enq_flags);

}

do_enqueue_task(rq, p, enq_flags, sticky_cpu);

out:

rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;

}

static void ops_dequeue(struct task_struct *p, u64 deq_flags)

* The caller must ensure that @p and @rq are on different CPUs.

*/

static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq,

{

int cpu = cpu_of(rq);

/*

* If @p has migration disabled, @p->cpus_ptr is updated to contain only

* easily be masked if task_allowed_on_cpu() is done first.

*/

if (unlikely(is_migration_disabled(p))) {

scx_ops_error("SCX_DSQ_LOCAL[_ON] cannot move migration disabled %s[%d] from CPU %d to %d",

p->comm, p->pid, task_cpu(p), cpu);

return false;

* picked CPU is outside the allowed mask.

*/

if (!task_allowed_on_cpu(p, cpu)) {

scx_ops_error("SCX_DSQ_LOCAL[_ON] target CPU %d not allowed for %s[%d]",

cpu, p->comm, p->pid);

return false;

}

return false;

return true;

}

}

#else /* CONFIG_SMP */

static inline void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, struct rq *src_rq, struct rq *dst_rq) { WARN_ON_ONCE(1); }

static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; }

#endif /* CONFIG_SMP */

if (prev_on_rq && (!static_branch_unlikely(&scx_ops_enq_last) ||

scx_rq_bypassing(rq))) {

rq->scx.flags |= SCX_RQ_BAL_KEEP;

goto has_tasks;

}

rq->scx.flags &= ~SCX_RQ_IN_BALANCE;

*/

if (keep_prev) {

p = prev;

p->scx.slice = SCX_SLICE_DFL;

} else {

p = first_local_task(rq);

if (!p) {

scx_warned_zero_slice = true;

}

p->scx.slice = SCX_SLICE_DFL;

}

}

#ifdef CONFIG_SMP

static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)

{

/*

* sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it

* can be a good migration opportunity with low cache and memory

if (unlikely(wake_flags & WF_EXEC))

return prev_cpu;

s32 cpu;

struct task_struct **ddsp_taskp;

cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU,

select_cpu, p, prev_cpu, wake_flags);

*ddsp_taskp = NULL;

if (ops_cpu_valid(cpu, "from ops.select_cpu()"))

return cpu;

else

return prev_cpu;

} else {

s32 cpu;

p->scx.slice = SCX_SLICE_DFL;

p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;

}

return cpu;

}

}

(struct cpumask *)p->cpus_ptr);

}

static void handle_hotplug(struct rq *rq, bool online)

{

int cpu = cpu_of(rq);

atomic_long_inc(&scx_hotplug_seq);

if (scx_enabled())

if (online && SCX_HAS_OP(cpu_online))

SCX_CALL_OP(SCX_KF_UNLOCKED, cpu_online, cpu);

rq->scx.flags &= ~SCX_RQ_ONLINE;

}

#endif /* CONFIG_SMP */

static bool check_rq_for_timeouts(struct rq *rq)

}

SCX_ATTR(ops);

static struct attribute *scx_sched_attrs[] = {

&scx_attr_ops.attr,

NULL,

};

ATTRIBUTE_GROUPS(scx_sched);

static void scx_ops_bypass(bool bypass)

{

static DEFINE_RAW_SPINLOCK(bypass_lock);

int cpu;

unsigned long flags;

WARN_ON_ONCE(scx_ops_bypass_depth <= 0);

if (scx_ops_bypass_depth != 1)

goto unlock;

} else {

scx_ops_bypass_depth--;

WARN_ON_ONCE(scx_ops_bypass_depth < 0);

if (scx_ops_bypass_depth != 0)

goto unlock;

}

atomic_inc(&scx_ops_breather_depth);

static_branch_disable(&__scx_ops_enabled);

for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)

static_branch_disable(&scx_has_op[i]);

static_branch_disable(&scx_ops_enq_last);

static_branch_disable(&scx_ops_enq_exiting);

static_branch_disable(&scx_ops_enq_migration_disabled);

static_branch_disable(&scx_ops_cpu_preempt);

synchronize_rcu();

if (ei->kind >= SCX_EXIT_ERROR) {

.at_jiffies = jiffies,

};

struct seq_buf s;

unsigned long flags;

char *buf;

int cpu;

rq_unlock(rq, &rf);

}

if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))

memcpy(ei->dump + dump_len - sizeof(trunc_marker),

trunc_marker, sizeof(trunc_marker));

return -EINVAL;

}

return 0;

}

mutex_lock(&scx_ops_enable_mutex);

if (!scx_ops_helper) {

WRITE_ONCE(scx_ops_helper,

scx_create_rt_helper("sched_ext_ops_helper"));

static_branch_enable_cpuslocked(&scx_has_op[i]);

check_hotplug_seq(ops);

cpus_read_unlock();

ret = validate_ops(ops);

if (((void (**)(void))ops)[i])

static_branch_enable(&scx_has_op[i]);

if (ops->flags & SCX_OPS_ENQ_LAST)

static_branch_enable(&scx_ops_enq_last);

if (ops->flags & SCX_OPS_ENQ_EXITING)

static_branch_enable(&scx_ops_enq_exiting);

if (ops->flags & SCX_OPS_ENQ_MIGRATION_DISABLED)

if (scx_ops.cpu_acquire || scx_ops.cpu_release)

static_branch_enable(&scx_ops_cpu_preempt);

/*

* Lock out forks, cgroup on/offlining and moves before opening the

SCX_TG_ONLINE);

BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params));

scx_kick_cpus_pnt_seqs =

__alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids,

__alignof__(scx_kick_cpus_pnt_seqs[0]));

for_each_possible_cpu(cpu) {

struct rq *rq = cpu_rq(cpu);

init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL);

INIT_LIST_HEAD(&rq->scx.runnable_list);

INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);

init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn);

init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn);

/********************************************************************************

* Helpers that can be called from the BPF scheduler.

*/

static bool scx_dsq_insert_preamble(struct task_struct *p, u64 enq_flags)

{

if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH))

* CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to

* the current local DSQ for running tasks and thus are not

* visible to the BPF scheduler.

*/

continue;

dispatch_dequeue(rq, p);

}

/**

* scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs

*

* All valid CPU IDs in the system are smaller than the returned value.

}

/**

* scx_bpf_task_running - Is task currently running?

* @p: task of interest

*/

return clock;

}

__bpf_kfunc_end_defs();

BTF_KFUNCS_START(scx_kfunc_ids_any)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur)

BTF_ID_FLAGS(func, scx_bpf_cpuperf_set)

BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)

BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)

BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)

BTF_ID_FLAGS(func, scx_bpf_task_cgroup, KF_RCU | KF_ACQUIRE)

#endif

BTF_ID_FLAGS(func, scx_bpf_now)

BTF_KFUNCS_END(scx_kfunc_ids_any)

static const struct btf_kfunc_id_set scx_kfunc_set_any = {

* check using scx_kf_allowed().

*/

if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,

&scx_kfunc_set_enqueue_dispatch)) ||

(ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,

&scx_kfunc_set_dispatch)) ||

return ret;

}

ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);

if (ret) {

pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);

*/

#ifdef CONFIG_SCHED_CLASS_EXT

void scx_tick(struct rq *rq);

void init_scx_entity(struct sched_ext_entity *scx);

void scx_pre_fork(struct task_struct *p);

return scx_enabled() && p->sched_class == &ext_sched_class;

}

#ifdef CONFIG_SCHED_CORE

bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,

bool in_fi);

static inline void scx_rq_deactivate(struct rq *rq) {}

static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; }

static inline bool task_on_scx(const struct task_struct *p) { return false; }

static inline void init_sched_ext_class(void) {}

#endif /* CONFIG_SCHED_CLASS_EXT */

/*

* Minimal preemption granularity for CPU-bound tasks:

*

*/

static int __init setup_sched_thermal_decay_shift(char *str)

{

static inline void assert_list_leaf_cfs_rq(struct rq *rq)

{

}

/* Iterate through all leaf cfs_rq's on a runqueue */

{

s64 vlag, limit;

vlag = avg_vruntime(cfs_rq) - se->vruntime;

limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);

}

/*

* Earliest Eligible Virtual Deadline First

*

* In order to provide latency guarantees for different request sizes

if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr)))

curr = NULL;

return curr;

/* Pick the leftmost entity if it's eligible */

return best;

}

struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)

{

struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);

return 0;

}

#endif

static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);

bool vma_pids_skipped;

bool vma_pids_forced = false;

work->next = work;

/*

* Make sure that rounding and/or propagation of PELT values never

* break this.

*/

sa->util_avg ||

sa->runnable_avg);

clear_buddies(cfs_rq, se);

if (flags & DEQUEUE_DELAYED) {

} else {

bool delay = sleep;

/*

if (flags & DEQUEUE_SPECIAL)

delay = false;

if (sched_feat(DELAY_DEQUEUE) && delay &&

!entity_eligible(cfs_rq, se)) {

update_stats_wait_end_fair(cfs_rq, se);

__dequeue_entity(cfs_rq, se);

update_load_avg(cfs_rq, se, UPDATE_TG);

}

update_stats_curr_start(cfs_rq, se);

cfs_rq->curr = se;

/*

if (sched_feat(PICK_BUDDY) &&

cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) {

/* ->next will never be delayed */

return cfs_rq->next;

}

/* in !on_rq case, update occurred at dequeue */

update_load_avg(cfs_rq, prev, 0);

}

cfs_rq->curr = NULL;

}

cfs_rq->throttled_clock_self = 0;

delta = 0;

cfs_rq->throttled_clock_self_time += delta;

cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);

list_del_leaf_cfs_rq(cfs_rq);

if (cfs_rq->nr_queued)

cfs_rq->throttled_clock_self = rq_clock(rq);

}

* throttled-list. rq->lock protects completion.

*/

cfs_rq->throttled = 1;

if (cfs_rq->nr_queued)

cfs_rq->throttled_clock = rq_clock(rq);

return true;

}

/* Already enqueued */

return;

first = list_empty(&rq->cfsb_csd_list);

{

lockdep_assert_rq_held(rq_of(cfs_rq));

cfs_rq->runtime_remaining <= 0))

return;

goto next;

/* By the above checks, this should never be true */

raw_spin_lock(&cfs_b->lock);

runtime = -cfs_rq->runtime_remaining + 1;

* We currently only expect to be unthrottling

* a single cfs_rq locally.

*/

list_add_tail(&cfs_rq->throttled_csd_list,

&local_unthrottle);

}

rq_unlock_irqrestore(rq, &rf);

}

rcu_read_unlock();

{

struct sched_entity *se = &p->se;

if (rq->cfs.h_nr_queued > 1) {

u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;

* Because a delayed entity is one that is still on

* the runqueue competing until elegibility.

*/

if (sched_feat(DELAY_ZERO)) {

update_entity_lag(cfs_rq, se);

update_cfs_group(se);

se->slice = slice;

slice = cfs_rq_min_slice(cfs_rq);

cfs_rq->h_nr_runnable += h_nr_runnable;

update_cfs_group(se);

se->slice = slice;

slice = cfs_rq_min_slice(cfs_rq);

cfs_rq->h_nr_runnable -= h_nr_runnable;

rq->next_balance = jiffies;

if (p && task_delayed) {

/* Fix-up what dequeue_task_fair() skipped */

hrtick_update(rq);

static void set_next_buddy(struct sched_entity *se)

{

for_each_sched_entity(se) {

return;

if (se_is_idle(se))

return;

* Preempt an idle entity in favor of a non-idle entity (and don't preempt

* in the inverse case).

*/

goto preempt;

if (cse_is_idle != pse_is_idle)

return;

* Note that even if @p does not turn out to be the most eligible

* task at this moment, current's slice protection will be lost.

*/

/*

* If @p has become the most eligible task, force preemption.

return 0;

/* Prevent to re-select dst_cpu via env's CPUs: */

}

return 0;

void nohz_balance_exit_idle(struct rq *rq)

{

if (likely(!rq->nohz_tick_stopped))

return;

{

struct rq *rq = cpu_rq(cpu);

/* If this CPU is going down, then nothing needs to be done: */

if (!cpu_active(cpu))

int balance_cpu;

struct rq *rq;

/*

* We assume there will be no idle load after this update and clear

struct cfs_rq *cfs_rqb;

s64 delta;

#ifdef CONFIG_FAIR_GROUP_SCHED

/*

static void switched_to_fair(struct rq *rq, struct task_struct *p)

{

attach_task_cfs_rq(p);

if (!first)

return;

if (hrtick_enabled_fair(rq))

hrtick_start_fair(rq, p);

#endif

};

void print_cfs_stats(struct seq_file *m, int cpu)

{

struct cfs_rq *cfs_rq, *pos;

rcu_read_unlock();

}

#endif /* CONFIG_NUMA_BALANCING */

__init void init_sched_fair_class(void)

{

static inline struct task_struct *rt_task_of(struct sched_rt_entity *rt_se)

{

WARN_ON_ONCE(!rt_entity_is_task(rt_se));

return container_of(rt_se, struct task_struct, rt);

}

BUG_ON(idx >= MAX_RT_PRIO);

queue = array->queue + idx;

return NULL;

next = list_entry(queue->next, struct sched_rt_entity, run_list);

int ret;

mutex_lock(&mutex);

old_period = sysctl_sched_rt_period;

old_runtime = sysctl_sched_rt_runtime;

sysctl_sched_rt_period = old_period;

sysctl_sched_rt_runtime = old_runtime;

}

mutex_unlock(&mutex);

return ret;

}

#endif /* CONFIG_SYSCTL */

void print_rt_stats(struct seq_file *m, int cpu)

{

rt_rq_iter_t iter;

print_rt_rq(m, cpu, rt_rq);

rcu_read_unlock();

}

#include "cpupri.h"

#include "cpudeadline.h"

/* task_struct::on_rq states: */

#define TASK_ON_RQ_QUEUED 1

#define TASK_ON_RQ_MIGRATING 2

* Also, some corner cases, like 'wrap around' is dangerous, but given

* that u64 is 'big enough'. So that shouldn't be a concern.

*/

#ifdef HAVE_RT_PUSH_IPI

/*

atomic_t nr_iowait;

u64 last_seen_need_resched_ns;

int ticks_without_resched;

#ifdef CONFIG_MEMBARRIER

int membarrier_state;

static inline struct task_struct *task_of(struct sched_entity *se)

{

return container_of(se, struct task_struct, se);

}

* The only reason for not seeing a clock update since the

* last rq_pin_lock() is if we're currently skipping updates.

*/

}

static inline u64 rq_clock(struct rq *rq)

static inline void rq_clock_start_loop_update(struct rq *rq)

{

lockdep_assert_rq_held(rq);

rq->clock_update_flags |= RQCF_ACT_SKIP;

}

struct rq_flags {

unsigned long flags;

struct pin_cookie cookie;

/*

* A copy of (rq::clock_update_flags & RQCF_UPDATED) for the

* current pin context is stashed here in case it needs to be

* restored in rq_repin_lock().

*/

unsigned int clock_update_flags;

};

extern struct balance_callback balance_push_callback;

{

rf->cookie = lockdep_pin_lock(__rq_lockp(rq));

rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);

rf->clock_update_flags = 0;

#endif

}

static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)

{

if (rq->clock_update_flags > RQCF_ACT_SKIP)

rf->clock_update_flags = RQCF_UPDATED;

scx_rq_clock_invalidate(rq);

lockdep_unpin_lock(__rq_lockp(rq), rf->cookie);

}

{

lockdep_repin_lock(__rq_lockp(rq), rf->cookie);

/*

* Restore the value we stashed in @rf for this pin context.

*/

rq->clock_update_flags |= rf->clock_update_flags;

}

extern

unsigned long next_update;

int imbalance; /* XXX unrelated to capacity but shared group state */

int id;

unsigned long cpumask[]; /* Balance mask */

};

extern int group_balance_cpu(struct sched_group *sg);

extern void update_sched_domain_debugfs(void);

extern void dirty_sched_domain_sysctl(int cpu);

extern int sched_update_scaling(void);

}

/*

*/

#define SCHED_FEAT(name, enabled) \

__SCHED_FEAT_##name ,

#undef SCHED_FEAT

/*

* To support run-time toggling of sched features, all the translation units

* (but core.c) reference the sysctl_sched_features defined in core.c.

*/

#ifdef CONFIG_JUMP_LABEL

#endif /* !CONFIG_JUMP_LABEL */

extern struct static_key_false sched_numa_balancing;

extern struct static_key_false sched_schedstats;

static inline struct cpuidle_state *idle_get_state(struct rq *rq)

{

return rq->idle_state;

}

# define SCHED_NR_MIGRATE_BREAK 32

#endif

extern unsigned int sysctl_sched_base_slice;

extern int sysctl_resched_latency_warn_ms;

extern int sysctl_resched_latency_warn_once;

extern unsigned int sysctl_numa_balancing_scan_period_max;

extern unsigned int sysctl_numa_balancing_scan_size;

extern unsigned int sysctl_numa_balancing_hot_threshold;

#ifdef CONFIG_SCHED_HRTICK

}

#endif

/*

* In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to

* acquire rq lock instead of rq_lock(). So at the end of these two functions

rq2->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);

#endif

}

#define DEFINE_LOCK_GUARD_2(name, type, _lock, _unlock, ...) \

__DEFINE_UNLOCK_GUARD(name, type, _unlock, type *lock2; __VA_ARGS__) \

extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq);

extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq);

extern bool sched_debug_verbose;

extern void print_cfs_stats(struct seq_file *m, int cpu);

extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);

extern void resched_latency_warn(int cpu, u64 latency);

extern void show_numa_stats(struct task_struct *p, struct seq_file *m);

extern void

print_numa_stats(struct seq_file *m, int node, unsigned long tsf,

unsigned long tpf, unsigned long gsf, unsigned long gpf);

extern void init_cfs_rq(struct cfs_rq *cfs_rq);

extern void init_rt_rq(struct rt_rq *rt_rq);

unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id);

static inline unsigned long uclamp_rq_get(struct rq *rq,

enum uclamp_id clamp_id)

{

unsigned long rq_util;

unsigned long max_util;

return false;

rq_util = cpu_util_cfs(cpu_of(rq)) + cpu_util_rt(rq);

return max_util != SCHED_CAPACITY_SCALE && rq_util >= max_util;

}

#define for_each_clamp_id(clamp_id) \

for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)

return false;

}

static inline unsigned long

uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id)

{

extern int sched_dynamic_mode(const char *str);

extern void sched_dynamic_update(int mode);

#endif

#ifdef CONFIG_SCHED_MM_CID

if (p->se.sched_delayed) {

/* CPU migration of "sleeping" task */

if (p->in_memstall)

set |= TSK_MEMSTALL;

if (p->in_iowait)

* blocking operation which obviously cannot be done while holding

* scheduler locks.

*/

return 0;

}

{

struct sched_param lparam;

return -EINVAL;

if (copy_from_user(&lparam, param, sizeof(struct sched_param)))

return -EFAULT;

struct sched_attr attr;

int retval;

return -EINVAL;

retval = sched_copy_attr(uattr, &attr);

struct task_struct *p;

int retval;

return -EINVAL;

scoped_guard (rcu) {

struct task_struct *p;

int retval;

return -EINVAL;

scoped_guard (rcu) {

#include <linux/bsearch.h>

DEFINE_MUTEX(sched_domains_mutex);

/* Protected by sched_domains_mutex: */

static cpumask_var_t sched_domains_tmpmask;

static cpumask_var_t sched_domains_tmpmask2;

static int __init sched_debug_setup(char *str)

{

sched_debug_verbose = true;

break;

}

}

/* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */

#define SD_FLAG(name, mflags) (name * !!((mflags) & SDF_NEEDS_GROUPS)) |

rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func);

#endif

init_dl_bw(&rd->dl_bw);

if (cpudl_init(&rd->cpudl) != 0)

goto free_rto_mask;

if (!sgc)

return -ENOMEM;

sgc->id = j;

*per_cpu_ptr(sdd->sgc, j) = sgc;

}

*

* Call with hotplug lock and sched_domains_mutex held

*/

struct sched_domain_attr *dattr_new)

{

bool __maybe_unused has_eas = false;

for (i = 0; i < ndoms_cur; i++) {

for (j = 0; j < n && !new_topology; j++) {

if (cpumask_equal(doms_cur[i], doms_new[j]) &&

goto match1;

}

/* No match - a current sched domain not in new doms_new[] */

detach_destroy_domains(doms_cur[i]);

ndoms_cur = ndoms_new;

update_sched_domain_debugfs();

}

/*

void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],

struct sched_domain_attr *dattr_new)

{

partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);

}