#include "cpupri.h"

#include "cpudeadline.h"

/* task_struct::on_rq states: */

#define TASK_ON_RQ_QUEUED 1

#define TASK_ON_RQ_MIGRATING 2

extern bool __checkparam_dl(const struct sched_attr *attr);

extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);

extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);

extern s64 dl_scaled_delta_exec(struct rq *rq, struct sched_dl_entity *dl_se, s64 delta_exec);

/*

* SCHED_DEADLINE supports servers (nested scheduling) with the following

extern void sched_destroy_group(struct task_group *tg);

extern void sched_release_group(struct task_group *tg);

#ifdef CONFIG_FAIR_GROUP_SCHED

extern int sched_group_set_shares(struct task_group *tg, unsigned long shares);

/* CFS-related fields in a runqueue */

struct cfs_rq {

struct load_weight load;

s64 avg_vruntime;

u64 avg_load;

SCX_RQ_BAL_PENDING = 1 << 2, /* balance hasn't run yet */

SCX_RQ_BAL_KEEP = 1 << 3, /* balance decided to keep current */

SCX_RQ_BYPASSING = 1 << 4,

SCX_RQ_IN_WAKEUP = 1 << 16,

SCX_RQ_IN_BALANCE = 1 << 17,

unsigned long ops_qseq;

u64 extra_enq_flags; /* see move_task_to_local_dsq() */

u32 nr_running;

u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */

bool cpu_released;

cpumask_var_t cpus_to_kick;

cpumask_var_t cpus_to_kick_if_idle;

cpumask_var_t cpus_to_preempt;

static inline void se_update_runnable(struct sched_entity *se)

{

}

static inline long se_runnable(struct sched_entity *se)

* Also, some corner cases, like 'wrap around' is dangerous, but given

* that u64 is 'big enough'. So that shouldn't be a concern.

*/

#ifdef HAVE_RT_PUSH_IPI

/*

atomic_t nr_iowait;

u64 last_seen_need_resched_ns;

int ticks_without_resched;

#ifdef CONFIG_MEMBARRIER

int membarrier_state;

static inline struct task_struct *task_of(struct sched_entity *se)

{

return container_of(se, struct task_struct, se);

}

* The only reason for not seeing a clock update since the

* last rq_pin_lock() is if we're currently skipping updates.

*/

}

static inline u64 rq_clock(struct rq *rq)

static inline void rq_clock_start_loop_update(struct rq *rq)

{

lockdep_assert_rq_held(rq);

rq->clock_update_flags |= RQCF_ACT_SKIP;

}

struct rq_flags {

unsigned long flags;

struct pin_cookie cookie;

/*

* A copy of (rq::clock_update_flags & RQCF_UPDATED) for the

* current pin context is stashed here in case it needs to be

* restored in rq_repin_lock().

*/

unsigned int clock_update_flags;

};

extern struct balance_callback balance_push_callback;

/*

* Lockdep annotation that avoids accidental unlocks; it's like a

* sticky/continuous lockdep_assert_held().

{

rf->cookie = lockdep_pin_lock(__rq_lockp(rq));

rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);

rf->clock_update_flags = 0;

#endif

}

static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf)

{

if (rq->clock_update_flags > RQCF_ACT_SKIP)

rf->clock_update_flags = RQCF_UPDATED;

lockdep_unpin_lock(__rq_lockp(rq), rf->cookie);

}

{

lockdep_repin_lock(__rq_lockp(rq), rf->cookie);

/*

* Restore the value we stashed in @rf for this pin context.

*/

rq->clock_update_flags |= rf->clock_update_flags;

}

extern

unsigned long next_update;

int imbalance; /* XXX unrelated to capacity but shared group state */

int id;

unsigned long cpumask[]; /* Balance mask */

};

extern int group_balance_cpu(struct sched_group *sg);

extern void update_sched_domain_debugfs(void);

extern void dirty_sched_domain_sysctl(int cpu);

extern int sched_update_scaling(void);

}

/*

*/

#define SCHED_FEAT(name, enabled) \

__SCHED_FEAT_##name ,

#undef SCHED_FEAT

/*

* To support run-time toggling of sched features, all the translation units

* (but core.c) reference the sysctl_sched_features defined in core.c.

*/

#ifdef CONFIG_JUMP_LABEL

#endif /* !CONFIG_JUMP_LABEL */

extern struct static_key_false sched_numa_balancing;

extern struct static_key_false sched_schedstats;

static inline int task_on_rq_queued(struct task_struct *p)

{

}

static inline int task_on_rq_migrating(struct task_struct *p)

extern const struct sched_class fair_sched_class;

extern const struct sched_class idle_sched_class;

/*

* Iterate only active classes. SCX can take over all fair tasks or be

* completely disabled. If the former, skip fair. If the latter, skip SCX.

static inline bool sched_fair_runnable(struct rq *rq)

{

}

extern struct task_struct *pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);

static inline struct cpuidle_state *idle_get_state(struct rq *rq)

{

return rq->idle_state;

}

# define SCHED_NR_MIGRATE_BREAK 32

#endif

extern unsigned int sysctl_sched_base_slice;

extern int sysctl_resched_latency_warn_ms;

extern int sysctl_resched_latency_warn_once;

extern unsigned int sysctl_numa_balancing_scan_period_max;

extern unsigned int sysctl_numa_balancing_scan_size;

extern unsigned int sysctl_numa_balancing_hot_threshold;

#ifdef CONFIG_SCHED_HRTICK

}

#endif

/*

* In double_lock_balance()/double_rq_lock(), we use raw_spin_rq_lock() to

* acquire rq lock instead of rq_lock(). So at the end of these two functions

rq2->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP);

#endif

}

#define DEFINE_LOCK_GUARD_2(name, type, _lock, _unlock, ...) \

__DEFINE_UNLOCK_GUARD(name, type, _unlock, type *lock2; __VA_ARGS__) \

extern struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq);

extern struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq);

extern bool sched_debug_verbose;

extern void print_cfs_stats(struct seq_file *m, int cpu);

extern void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq);

extern void resched_latency_warn(int cpu, u64 latency);

extern void show_numa_stats(struct task_struct *p, struct seq_file *m);

extern void

print_numa_stats(struct seq_file *m, int node, unsigned long tsf,

unsigned long tpf, unsigned long gsf, unsigned long gpf);

extern void init_cfs_rq(struct cfs_rq *cfs_rq);

extern void init_rt_rq(struct rt_rq *rt_rq);

};

DECLARE_PER_CPU(struct irqtime, cpu_irqtime);

/*

* Returns the irqtime minus the softirq time computed by ksoftirqd.

return total;

}

#endif /* CONFIG_IRQ_TIME_ACCOUNTING */

#ifdef CONFIG_CPU_FREQ

unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id);

static inline unsigned long uclamp_rq_get(struct rq *rq,

enum uclamp_id clamp_id)

{

unsigned long rq_util;

unsigned long max_util;

return false;

rq_util = cpu_util_cfs(cpu_of(rq)) + cpu_util_rt(rq);

return max_util != SCHED_CAPACITY_SCALE && rq_util >= max_util;

}

#define for_each_clamp_id(clamp_id) \

for ((clamp_id) = 0; (clamp_id) < UCLAMP_CNT; (clamp_id)++)

return false;

}

static inline unsigned long

uclamp_rq_get(struct rq *rq, enum uclamp_id clamp_id)

{

#endif /* !CONFIG_HAVE_SCHED_AVG_IRQ */

#if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)

#define perf_domain_span(pd) (to_cpumask(((pd)->em_pd->cpus)))

extern int sched_dynamic_mode(const char *str);

extern void sched_dynamic_update(int mode);

#endif

#ifdef CONFIG_SCHED_MM_CID

{

struct cpumask *cidmask = mm_cidmask(mm);

struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;

/* Try to re-use recent cid. This improves cache locality. */

return cid;

/*

* Expand cid allocation if the maximum number of concurrency

* and number of threads. Expanding cid allocation as much as

* possible improves cache locality.

*/

while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {

if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))

continue;

if (!cpumask_test_and_set_cpu(cid, cidmask))