diff options
Diffstat (limited to 'kernel')
37 files changed, 1497 insertions, 765 deletions
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks index 4198f0273ecd..1c6423aafcd4 100644 --- a/kernel/Kconfig.locks +++ b/kernel/Kconfig.locks @@ -243,6 +243,26 @@ config QUEUED_SPINLOCKS def_bool y if ARCH_USE_QUEUED_SPINLOCKS depends on SMP +config QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE + bool "Trace contended_release on queued spinlocks" + depends on QUEUED_SPINLOCKS && TRACEPOINTS + help + Fire the lock:contended_release tracepoint when a contended queued + spinlock is released, so it is possible to attribute a contended + spinlock to its holder. + + Architectures that can patch the unlock site do this at no cost and + do not need this option. + + Everywhere else the check is compiled into queued_spin_unlock() and + a small cost is paid on every unlock even when the tracepoint is + disabled: a static-branch NOP and possibly a few more instructions + to manage a stack frame. + + Sleeping locks fire lock:contended_release regardless of this option. + + If unsure, say N. + config BPF_ARCH_SPINLOCK bool diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt index 88c594c6d7fc..35f546a042b1 100644 --- a/kernel/Kconfig.preempt +++ b/kernel/Kconfig.preempt @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK config PREEMPT_COUNT bool +config HAS_SEPARATE_PREEMPT_RESCHED_BITS + bool + depends on PREEMPT_COUNT && 64BIT + config PREEMPTION bool select PREEMPT_COUNT diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 45944b3e31ca..e42f81a61311 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -4129,6 +4129,28 @@ bool cpuset_cpus_allowed_fallback(struct task_struct *tsk) return changed; } +/* + * Returns the number of CPUs available for this cgroup. + * + * This only really works for cgroup-v2 where all the controllers are mounted + * in the same hierarchy. If not cgroup-v2 or no cpuset controller is + * configured it reverts to num_online_cpus(). + */ +int cpuset_num_cpus(struct cgroup *cgrp) +{ + int nr = num_online_cpus(); + struct cpuset *cs; + + if (is_in_v2_mode()) { + guard(rcu)(); + cs = css_cs(cgroup_e_css(cgrp, &cpuset_cgrp_subsys)); + if (cs) + nr = cpumask_weight(cs->effective_cpus); + } + + return nr; +} + void __init cpuset_init_current_mems_allowed(void) { nodes_setall(current->mems_allowed); diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 4f21fc3b108b..2b36aa9fade0 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -168,9 +168,6 @@ static inline resource_size_t crash_resource_size(const struct resource *res) return !res->end ? 0 : resource_size(res); } - - - int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz) { @@ -272,6 +269,92 @@ int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, return 0; } +static struct crash_mem *alloc_cmem(unsigned int nr_ranges) +{ + struct crash_mem *cmem; + + cmem = kvzalloc_flex(*cmem, ranges, nr_ranges); + if (!cmem) + return NULL; + + cmem->max_nr_ranges = nr_ranges; + return cmem; +} + +unsigned int __weak arch_get_system_nr_ranges(void) { return 0; } +int __weak arch_crash_populate_cmem(struct crash_mem *cmem) { return -1; } +int __weak arch_crash_exclude_ranges(struct crash_mem *cmem) { return 0; } + +int __weak arch_crash_exclude_mem_range(struct crash_mem **mem, + unsigned long long mstart, + unsigned long long mend) +{ + return crash_exclude_mem_range(*mem, mstart, mend); +} + +int crash_exclude_core_ranges(struct crash_mem **cmem) +{ + int ret, i; + + /* Exclude crashkernel region */ + ret = arch_crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); + if (ret) + return ret; + + if (crashk_low_res.end) { + ret = arch_crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); + if (ret) + return ret; + } + + for (i = 0; i < crashk_cma_cnt; ++i) { + ret = arch_crash_exclude_mem_range(cmem, crashk_cma_ranges[i].start, + crashk_cma_ranges[i].end); + if (ret) + return ret; + } + + return 0; +} + +int crash_prepare_headers(int need_kernel_map, void **addr, unsigned long *sz, + unsigned long *nr_mem_ranges) +{ + unsigned int max_nr_ranges; + struct crash_mem *cmem; + int ret; + + max_nr_ranges = arch_get_system_nr_ranges(); + if (!max_nr_ranges) + return -ENOMEM; + + cmem = alloc_cmem(max_nr_ranges); + if (!cmem) + return -ENOMEM; + + ret = arch_crash_populate_cmem(cmem); + if (ret) + goto out; + + ret = crash_exclude_core_ranges(&cmem); + if (ret) + goto out; + + ret = arch_crash_exclude_ranges(cmem); + if (ret) + goto out; + + /* Return the computed number of memory ranges, for hotplug usage */ + if (nr_mem_ranges) + *nr_mem_ranges = cmem->nr_ranges; + + ret = crash_prepare_elf64_headers(cmem, need_kernel_map, addr, sz); + +out: + kvfree(cmem); + return ret; +} + /** * crash_exclude_mem_range - exclude a mem range for existing ranges * @mem: mem->range contains an array of ranges sorted in ascending order diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index cb875ddb6ba6..c685497cd470 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -81,7 +81,6 @@ static int add_key_to_keyring(struct dm_crypt_key *dm_key, kexec_dprintk("Error when adding key"); } - key_ref_put(keyring_ref); return r; } @@ -104,6 +103,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) struct dm_crypt_key *key; size_t keys_header_size; key_ref_t keyring_ref; + int ret = 0; u64 addr; /* find the target keyring (which must be writable) */ @@ -118,7 +118,8 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr); if (key_count > KEY_NUM_MAX) { kexec_dprintk("Failed to read the number of dm-crypt keys\n"); - return -1; + ret = -1; + goto out; } kexec_dprintk("There are %u keys\n", key_count); @@ -126,8 +127,10 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) keys_header_size = get_keys_header_size(key_count); keys_header = kzalloc(keys_header_size, GFP_KERNEL); - if (!keys_header) - return -ENOMEM; + if (!keys_header) { + ret = -ENOMEM; + goto out; + } dm_crypt_keys_read((char *)keys_header, keys_header_size, &addr); @@ -137,7 +140,9 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) add_key_to_keyring(key, keyring_ref); } - return 0; +out: + key_ref_put(keyring_ref); + return ret; } static int read_key_from_user_keyring(struct dm_crypt_key *dm_key) diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a73ba0747df..2eee83cdb43d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7800,10 +7800,20 @@ unsigned long perf_misc_flags(struct perf_event *event, unsigned long perf_instruction_pointer(struct perf_event *event, struct pt_regs *regs) { - if (should_sample_guest(event)) - return perf_guest_get_ip(); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (should_sample_guest(event)) { + return event->attr.exclude_kernel && + !(perf_guest_state() & PERF_GUEST_USER) ? + 0 : perf_guest_get_ip(); + } - return perf_arch_instruction_pointer(regs); + return event->attr.exclude_kernel && !user_mode(regs) ? + 0 : perf_arch_instruction_pointer(regs); } static void @@ -7837,10 +7847,22 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, } static void perf_sample_regs_intr(struct perf_regs *regs_intr, - struct pt_regs *regs) + struct pt_regs *regs, + bool exclude_kernel) { - regs_intr->regs = regs; - regs_intr->abi = perf_reg_abi(current); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (exclude_kernel && !user_mode(regs)) { + regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE; + regs_intr->regs = NULL; + } else { + regs_intr->regs = regs; + regs_intr->abi = perf_reg_abi(current); + } } @@ -8731,7 +8753,8 @@ void perf_prepare_sample(struct perf_sample_data *data, /* regs dump ABI info */ int size = sizeof(u64); - perf_sample_regs_intr(&data->regs_intr, regs); + perf_sample_regs_intr(&data->regs_intr, regs, + event->attr.exclude_kernel); if (data->regs_intr.regs) { u64 mask = event->attr.sample_regs_intr; @@ -13918,7 +13941,9 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel) { + if (!attr.exclude_kernel || + ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && + !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); if (err) return err; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012c..a18529ab2b87 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -54,7 +54,7 @@ static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); /* Covers return_instance's uprobe lifetime. */ -DEFINE_STATIC_SRCU(uretprobes_srcu); +DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu); /* Have a copy of original instruction */ #define UPROBE_COPY_INSN 0 @@ -707,12 +707,13 @@ static void put_uprobe(struct uprobe *uprobe) } /* Initialize hprobe as SRCU-protected "leased" uprobe */ -static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx) +static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, + struct srcu_ctr __percpu *srcu_scp) { WARN_ON(!uprobe); hprobe->state = HPROBE_LEASED; hprobe->uprobe = uprobe; - hprobe->srcu_idx = srcu_idx; + hprobe->srcu_scp = srcu_scp; } /* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */ @@ -720,7 +721,7 @@ static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe) { hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE; hprobe->uprobe = uprobe; - hprobe->srcu_idx = -1; + hprobe->srcu_scp = NULL; } /* @@ -757,7 +758,7 @@ static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate) { switch (hstate) { case HPROBE_LEASED: - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); break; case HPROBE_STABLE: put_uprobe(hprobe->uprobe); @@ -829,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get) */ if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) { /* We won the race, we are the ones to unlock SRCU */ - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); return get && uprobe ? get_uprobe(uprobe) : uprobe; } @@ -1806,14 +1807,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1818,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; @@ -2045,7 +2036,7 @@ static void ri_timer(struct timer_list *timer) struct return_instance *ri; /* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); /* RCU protects return_instance from freeing. */ guard(rcu)(); @@ -2142,7 +2133,7 @@ static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) t->utask = n_utask; /* protect uprobes from freeing, we'll need try_get_uprobe() them */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); p = &n_utask->return_instances; for (o = o_utask->return_instances; o; o = o->next) { @@ -2254,8 +2245,8 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, { struct uprobe_task *utask = current->utask; unsigned long orig_ret_vaddr, trampoline_vaddr; + struct srcu_ctr __percpu *srcu_scp; bool chained; - int srcu_idx; if (!get_xol_area()) goto free; @@ -2293,8 +2284,12 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; } - /* __srcu_read_lock() because SRCU lock survives switch to user space */ - srcu_idx = __srcu_read_lock(&uretprobes_srcu); + /* + * Use srcu_down_read_fast() because the SRCU lock survives a switch to + * user space and can be unlocked from a different context by ri_timer() + * or dup_utask(). + */ + srcu_scp = srcu_down_read_fast(&uretprobes_srcu); ri->func = instruction_pointer(regs); ri->stack = user_stack_pointer(regs); @@ -2303,7 +2298,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, utask->depth++; - hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx); + hprobe_init_leased(&ri->hprobe, uprobe, srcu_scp); ri->next = utask->return_instances; rcu_assign_pointer(utask->return_instances, ri); diff --git a/kernel/fork.c b/kernel/fork.c index 94e021eabf1d..175c73bbe2bf 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 128c5752f225..8fef2f806cef 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -45,26 +45,23 @@ #include <linux/rseq.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#include <linux/kmemleak.h> #include <vdso/futex.h> +#include <asm/runtime-const.h> + #include "futex.h" #include "../locking/rtmutex_common.h" -/* - * The base of the bucket array and its size are always used together - * (after initialization only in futex_hash()), so ensure that they - * reside in the same cacheline. - */ -static struct { - unsigned long hashmask; - unsigned int hashshift; - struct futex_hash_bucket *queues[MAX_NUMNODES]; -} __futex_data __read_mostly __aligned(2*sizeof(long)); +static u32 __futex_mask __ro_after_init; +static u32 __futex_shift __ro_after_init; +static struct futex_hash_bucket **__futex_queues __ro_after_init; -#define futex_hashmask (__futex_data.hashmask) -#define futex_hashshift (__futex_data.hashshift) -#define futex_queues (__futex_data.queues) +static __always_inline struct futex_hash_bucket **futex_queues(void) +{ + return runtime_const_ptr(__futex_queues); +} struct futex_private_hash { int state; @@ -143,8 +140,14 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (fph && futex_ref_put(fph)) - wake_up_var(fph->mm); + struct mm_struct *mm; + + if (!fph) + return; + + mm = fph->mm; + if (futex_ref_put(fph)) + wake_up_var(mm); } static struct futex_hash_bucket * @@ -395,13 +398,13 @@ __futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_ * NOTE: this isn't perfectly uniform, but it is fast and * handles sparse node masks. */ - node = (hash >> futex_hashshift) % nr_node_ids; + node = runtime_const_shift_right_32(hash, __futex_shift) % nr_node_ids; if (!node_possible(node)) { node = find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } - return &futex_queues[node][hash & futex_hashmask]; + return &futex_queues()[node][runtime_const_mask_32(hash, __futex_mask)]; } /** @@ -520,7 +523,7 @@ int get_futex_key(u32 __user *uaddr, unsigned int flags, union futex_key *key, * The futex address must be "naturally" aligned. */ key->both.offset = address % PAGE_SIZE; - if (unlikely((address % size) != 0)) + if (unlikely((address & (size-1)) != 0)) return -EINVAL; address -= key->both.offset; @@ -1954,7 +1957,7 @@ int futex_hash_allocate_default(void) * 16 <= threads * 4 <= global hash size */ buckets = roundup_pow_of_two(4 * threads); - buckets = clamp(buckets, 16, futex_hashmask + 1); + buckets = clamp(buckets, 16, __futex_mask + 1); if (current_buckets >= buckets) return 0; @@ -2052,10 +2055,22 @@ static int __init futex_init(void) hashsize = max(4, hashsize); hashsize = roundup_pow_of_two(hashsize); #endif - futex_hashshift = ilog2(hashsize); + __futex_mask = hashsize - 1; + __futex_shift = ilog2(hashsize); size = sizeof(struct futex_hash_bucket) * hashsize; order = get_order(size); + __futex_queues = kcalloc(nr_node_ids, sizeof(*__futex_queues), GFP_KERNEL); + kmemleak_not_leak(__futex_queues); + + runtime_const_init(shift, __futex_shift); + runtime_const_init(mask, __futex_mask); + runtime_const_init(ptr, __futex_queues); + + barrier(); + + BUG_ON(!futex_queues()); + for_each_node(n) { struct futex_hash_bucket *table; @@ -2069,10 +2084,9 @@ static int __init futex_init(void) for (i = 0; i < hashsize; i++) futex_hash_bucket_init(&table[i]); - futex_queues[n] = table; + futex_queues()[n] = table; } - futex_hashmask = hashsize - 1; pr_info("futex hash table entries: %lu (%lu bytes on %d NUMA nodes, total %lu KiB, %s).\n", hashsize, size, num_possible_nodes(), size * num_possible_nodes() / 1024, order > MAX_PAGE_ORDER ? "vmalloc" : "linear"); diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 86a2e5ae08f9..44c4d6fc502a 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile @@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o +obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c new file mode 100644 index 000000000000..ca904dba24b9 --- /dev/null +++ b/kernel/irq/refcount_interrupt_test.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for refcounted interrupt enable/disables. + */ + +#include <kunit/test.h> +#include <linux/interrupt_rc.h> + +#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled()) +#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled()) + +/* ===== Test cases ===== */ +static void test_single_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); +} + +static void test_nested_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_multiple_irq_change(struct kunit *test) +{ + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static void test_irq_save(struct kunit *test) +{ + unsigned long flags; + + local_irq_save(flags); + TEST_IRQ_OFF(); + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_ON(); + + local_interrupt_disable(); + TEST_IRQ_OFF(); + local_irq_save(flags); + TEST_IRQ_OFF(); + local_irq_restore(flags); + TEST_IRQ_OFF(); + local_interrupt_enable(); + TEST_IRQ_ON(); +} + +static struct kunit_case test_cases[] = { + KUNIT_CASE(test_single_irq_change), + KUNIT_CASE(test_nested_irq_change), + KUNIT_CASE(test_multiple_irq_change), + KUNIT_CASE(test_irq_save), + {}, +}; + +/* init and exit are the same. */ +static int test_init(struct kunit *test) +{ + TEST_IRQ_ON(); + + return 0; +} + +static void test_exit(struct kunit *test) +{ + TEST_IRQ_ON(); +} + +static struct kunit_suite refcount_interrupt_test_suite = { + .name = "refcount_interrupt", + .test_cases = test_cases, + .init = test_init, + .exit = test_exit, +}; + +kunit_test_suite(refcount_interrupt_test_suite); +MODULE_AUTHOR("Lyude Paul <lyude@redhat.com>"); +MODULE_DESCRIPTION("Refcounted interrupt unit test suite"); +MODULE_LICENSE("GPL"); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index 2bfbb2d144e6..59fb9d71e9d8 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -27,6 +27,7 @@ #include <linux/syscalls.h> #include <linux/vmalloc.h> #include <linux/dma-map-ops.h> +#include <linux/kexec_handover.h> #include "kexec_internal.h" #ifdef CONFIG_KEXEC_SIG @@ -798,6 +799,16 @@ int kexec_add_buffer(struct kexec_buf *kbuf) return 0; } +static bool kexec_only_cma_segments(struct kimage *image) +{ + for (int i = 0; i < image->nr_segments; i++) { + if (!image->segment_cma[i]) + return false; + } + + return true; +} + /* Calculate and store the digest of segments */ static int kexec_calculate_store_digests(struct kimage *image) { @@ -822,6 +833,21 @@ static int kexec_calculate_store_digests(struct kimage *image) sha256_init(&sctx); + /* + * If KHO is enabled, the destinations are located in KHO scratch. + * KHO scratch can only contain early boot allocations and movable + * allocations. That means there is no risk of memory corruption by + * uncancelled DMA. + * + * If all segments were loaded into contiguous memory, there will be no + * relocations at all, so also no risk of corruption. + */ + if (image->type != KEXEC_TYPE_CRASH && + (kho_is_enabled() || kexec_only_cma_segments(image))) { + pr_debug("disabling checksum verification in purgatory\n"); + goto skip_checksum; + } + for (j = i = 0; i < image->nr_segments; i++) { struct kexec_segment *ksegment; @@ -867,6 +893,7 @@ static int kexec_calculate_store_digests(struct kimage *image) j++; } +skip_checksum: sha256_final(&sctx, digest); ret = kexec_purgatory_get_set_symbol(image, "purgatory_sha_regions", diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 175c08a6e41e..6fad9152387a 100644 --- a/kernel/liveupdate/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -501,7 +501,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys, struct page *page; u64 sz; - sz = 1 << (order + PAGE_SHIFT); + sz = 1UL << (order + PAGE_SHIFT); page = kho_get_preserved_page(phys, order); /* Reserve the memory preserved in KHO in memblock */ diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c index 5c27134ce7ba..cd715a7c1d99 100644 --- a/kernel/liveupdate/luo_flb.c +++ b/kernel/liveupdate/luo_flb.c @@ -133,7 +133,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb) return 0; } -static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb) +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) { struct luo_flb_private *private = luo_flb_get_private(flb); @@ -168,7 +168,10 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) if (private->incoming.finished) return -ENODATA; - if (private->incoming.retrieved) + if (private->incoming.retrieve_status < 0) + return private->incoming.retrieve_status; + + if (private->incoming.retrieve_status > 0) return 0; if (!fh->active) @@ -194,12 +197,13 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb) err = flb->ops->retrieve(&args); if (err) { + private->incoming.retrieve_status = err; module_put(flb->ops->owner); return err; } private->incoming.obj = args.obj; - private->incoming.retrieved = true; + private->incoming.retrieve_status = 1; return 0; } @@ -213,7 +217,7 @@ void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) if (!refcount_dec_and_test(&private->incoming.count)) return; - if (!private->incoming.retrieved) { + if (private->incoming.retrieve_status <= 0) { int err = luo_flb_retrieve_one(flb); if (WARN_ON(err)) @@ -264,7 +268,7 @@ int luo_flb_file_preserve(struct liveupdate_file_handler *fh) exit_err: list_for_each_entry_continue_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); up_read(&luo_register_rwlock); return err; @@ -289,7 +293,7 @@ void luo_flb_file_unpreserve(struct liveupdate_file_handler *fh) guard(rwsem_read)(&luo_register_rwlock); list_for_each_entry_reverse(iter, flb_list, list) - luo_flb_file_unpreserve_one(iter->flb); + liveupdate_flb_put_outgoing(iter->flb); } /** @@ -544,6 +548,10 @@ int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) return -EOPNOTSUPP; guard(mutex)(&private->outgoing.lock); + if (!private->outgoing.obj) + return -ENOENT; + + refcount_inc(&private->outgoing.count); *objp = private->outgoing.obj; return 0; diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c index f38b5b18f3f8..f48e9a4185f9 100644 --- a/kernel/liveupdate/luo_session.c +++ b/kernel/liveupdate/luo_session.c @@ -154,7 +154,6 @@ static struct luo_session *luo_session_alloc(const char *name) return ERR_PTR(-ENOMEM); strscpy(session->name, name, sizeof(session->name)); - INIT_LIST_HEAD(&session->file_set.files_list); luo_file_set_init(&session->file_set); INIT_LIST_HEAD(&session->list); mutex_init(&session->mutex); @@ -316,8 +315,12 @@ static int luo_session_finish(struct luo_session *session, struct luo_ucmd *ucmd) { struct liveupdate_session_finish *argp = ucmd->cmd; - int err = luo_session_finish_one(session); + int err; + + if (argp->reserved) + return -EINVAL; + err = luo_session_finish_one(session); if (err) return err; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af8..25d77d4a1061 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -787,17 +787,33 @@ static void lockdep_print_held_locks(struct task_struct *p) { int i, depth = READ_ONCE(p->lockdep_depth); - if (!depth) - printk("no locks held by %s/%d.\n", p->comm, task_pid_nr(p)); - else - printk("%d lock%s held by %s/%d:\n", depth, - str_plural(depth), p->comm, task_pid_nr(p)); /* - * It's not reliable to print a task's held locks if it's not sleeping - * and it's not the current task. + * Note that it's always somewhat unreliable to print held locks + * of a task that is running on another CPU, but we cannot guarantee + * the stability of ->held_locks without actually stopping all active + * remote CPUs, which we absolutely do not want to do because it's + * very intrusive and thus slow. + * + * So we do the next best thing here: we print out the held lock + * array on a best-effort basis, without crashing even if the + * fields are being modified on another CPU. Note the careful + * construction of print_lock() so that it never crashes. + * + * We also print out the CPU the task is or was last running on, with + * the message saying 'on CPU...' if the task is running, and + * 'last CPU' if it's not. + * + * Also note that the task_is_running(p) information is fundamentally + * racy: even if the message says the task is 'on CPU', the task may + * have scheduled out already, or if it says 'last CPU', it may just + * have scheduled in on another CPU. But even with these limitations + * it's still useful debuggining information. */ - if (p != current && task_is_running(p)) - return; + printk("locks held by %s/%d: %d, %s CPU#%d%s\n", + p->comm, task_pid_nr(p), depth, + task_is_running(p) ? "last" : "on", task_cpu(p), + depth > 0 ? ":" : ""); + for (i = 0; i < depth; i++) { printk(" #%d: ", i); print_lock(p->held_locks + i); @@ -5437,6 +5453,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c index 8a85912d7ee6..942a939cee95 100644 --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -1272,6 +1272,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(contention_begin); EXPORT_TRACEPOINT_SYMBOL_GPL(contention_end); EXPORT_TRACEPOINT_SYMBOL_GPL(contended_release); +__weak int arch_contended_release_trace_reg(void) { return 0; } + +__weak void arch_contended_release_trace_unreg(void) { } + /** * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 * @cnt: the atomic which we are to dec diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index f7e152c40d6d..6c78961fe753 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -211,7 +211,7 @@ EXPORT_SYMBOL_GPL(percpu_is_read_locked); */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { - if (per_cpu_sum(*sem->read_count) != 0) + if (data_race(per_cpu_sum(*sem->read_count)) != 0) return false; /* diff --git a/kernel/locking/qspinlock.c b/kernel/locking/qspinlock.c index af8d122bb649..33fe6d437c8f 100644 --- a/kernel/locking/qspinlock.c +++ b/kernel/locking/qspinlock.c @@ -104,6 +104,28 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock, #define queued_spin_lock_slowpath native_queued_spin_lock_slowpath #endif +#if !defined(queued_spin_unlock) && \ + IS_ENABLED(CONFIG_QUEUED_SPINLOCKS_TRACE_CONTENDED_RELEASE) +/* + * Out-of-line trace-and-release path for queued_spin_unlock(), used when + * the contended_release tracepoint is enabled. + * + * queued_spin_release() is duplicated here on purpose: doing the release + * in this function (rather than tracing here and releasing in the caller) + * lets queued_spin_unlock() return right after the call, so the + * tracepoint-disabled hot path never has to keep lock live across a call + * in a callee-saved register. Keep this release in sync with the one in + * queued_spin_unlock(). + */ +void __lockfunc queued_spin_release_traced(struct qspinlock *lock) +{ + if (queued_spin_is_contended(lock)) + trace_call__contended_release(lock); + queued_spin_release(lock); +} +EXPORT_SYMBOL(queued_spin_release_traced); +#endif + #endif /* _GEN_PV_LOCK_SLOWPATH */ /** diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index b42d293da38b..83a17eaf5717 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ */ BUILD_LOCK_OPS(spin, raw_spinlock, __acquires); +/* No rwlock_t variants for now, so just build this function by hand */ +static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + for (;;) { + preempt_disable(); + local_interrupt_disable(); + if (likely(do_raw_spin_trylock(lock))) + break; + local_interrupt_enable(); + preempt_enable(); + + arch_spin_relax(&lock->raw_lock); + } +} + #ifndef CONFIG_PREEMPT_RT BUILD_LOCK_OPS(read, rwlock, __acquires_shared); BUILD_LOCK_OPS(write, rwlock, __acquires); @@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_lock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ +noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) +{ + __raw_spin_lock_irq_disable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable); +#endif + #ifndef CONFIG_INLINE_SPIN_LOCK_BH noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) { @@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) EXPORT_SYMBOL(_raw_spin_unlock_irq); #endif +#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ +noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) +{ + __raw_spin_unlock_irq_enable(lock); +} +EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable); +#endif + #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) { diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig index 530c897311d4..71165e7f04f4 100644 --- a/kernel/power/Kconfig +++ b/kernel/power/Kconfig @@ -268,6 +268,16 @@ config DPM_WATCHDOG captured in pstore device for inspection in subsequent boot session. +config DPM_WATCHDOG_ENABLED + bool "Enable DPM watchdog by default" + depends on DPM_WATCHDOG + default y + help + If you say Y here, the DPM watchdog will be enabled by default. + If you say N, it will be compiled in but disabled. It can be + enabled at boot time via the "pm.dpm_watchdog_enabled" kernel + parameter or at runtime via sysfs. + config DPM_WATCHDOG_TIMEOUT int "Watchdog timeout to panic in seconds" range 1 120 diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index d933b5b2c05d..b209712cb2c3 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -458,7 +458,7 @@ static struct rtree_node *alloc_rtree_node(gfp_t gfp_mask, int safe_needed, return node; } -/** +/* * add_rtree_block - Add a new leave node to the radix tree. * * The leave nodes need to be allocated in order to keep the leaves @@ -528,7 +528,7 @@ static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask, static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone, int clear_nosave_free); -/** +/* * create_zone_bm_rtree - Create a radix tree for one zone. * * Allocated the mem_zone_bm_rtree structure and initializes it. @@ -566,7 +566,7 @@ static struct mem_zone_bm_rtree *create_zone_bm_rtree(gfp_t gfp_mask, return zone; } -/** +/* * free_zone_bm_rtree - Free the memory of the radix tree. * * Free all node pages of the radix tree. The mem_zone_bm_rtree @@ -678,7 +678,7 @@ static int create_mem_extents(struct list_head *list, gfp_t gfp_mask) return 0; } -/** +/* * memory_bm_create - Allocate memory for a memory bitmap. */ static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, @@ -720,7 +720,7 @@ static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, goto Exit; } -/** +/* * memory_bm_free - Free memory occupied by the memory bitmap. * @bm: Memory bitmap. */ @@ -736,7 +736,7 @@ static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free) INIT_LIST_HEAD(&bm->zones); } -/** +/* * memory_bm_find_bit - Find the bit for a given PFN in a memory bitmap. * * Find the bit in memory bitmap @bm that corresponds to the given PFN. @@ -988,7 +988,7 @@ static void memory_bm_recycle(struct memory_bitmap *bm) } } -/** +/* * register_nosave_region - Register a region of unsaveable memory. * * Register a range of page frames the contents of which should not be saved @@ -1305,7 +1305,7 @@ static unsigned int count_free_highmem_pages(void) return cnt; } -/** +/* * saveable_highmem_page - Check if a highmem page is saveable. * * Determine whether a highmem page should be included in a hibernation image. @@ -1362,7 +1362,7 @@ static unsigned int count_highmem_pages(void) } #endif /* CONFIG_HIGHMEM */ -/** +/* * saveable_page - Check if the given page is saveable. * * Determine whether a non-highmem page should be included in a hibernation @@ -1440,7 +1440,7 @@ static inline bool do_copy_page(long *dst, long *src) return !z; } -/** +/* * safe_copy_page - Copy a page in a safe way. * * Check if the page we are going to copy is marked as present in the kernel @@ -1687,7 +1687,7 @@ static unsigned long preallocate_image_highmem(unsigned long nr_pages) return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM); } -/** +/* * __fraction - Compute (an approximation of) x * (multiplier / base). */ static unsigned long __fraction(u64 x, u64 multiplier, u64 base) @@ -1982,7 +1982,7 @@ int hibernate_preallocate_memory(void) } #ifdef CONFIG_HIGHMEM -/** +/* * count_pages_for_highmem - Count non-highmem pages needed for copying highmem. * * Compute the number of non-highmem pages that will be necessary for creating @@ -2003,7 +2003,7 @@ static unsigned int count_pages_for_highmem(unsigned int nr_highmem) static unsigned int count_pages_for_highmem(unsigned int nr_highmem) { return 0; } #endif /* CONFIG_HIGHMEM */ -/** +/* * enough_free_mem - Check if there is enough free memory for the image. */ static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) @@ -2023,7 +2023,7 @@ static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem) } #ifdef CONFIG_HIGHMEM -/** +/* * get_highmem_buffer - Allocate a buffer for highmem pages. * * If there are some highmem pages in the hibernation image, we may need a @@ -2035,7 +2035,7 @@ static inline int get_highmem_buffer(int safe_needed) return buffer ? 0 : -ENOMEM; } -/** +/* * alloc_highmem_pages - Allocate some highmem pages for the image. * * Try to allocate as many pages as needed, but if the number of free highmem @@ -2065,7 +2065,7 @@ static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; } #endif /* CONFIG_HIGHMEM */ -/** +/* * swsusp_alloc - Allocate memory for hibernation image. * * We first try to allocate as many highmem pages as there are @@ -2292,7 +2292,7 @@ static void duplicate_memory_bitmap(struct memory_bitmap *dst, } } -/** +/* * mark_unsafe_pages - Mark pages that were used before hibernation. * * Mark the pages that cannot be used for storing the image during restoration, @@ -2330,7 +2330,7 @@ static int check_header(struct swsusp_info *info) return 0; } -/** +/* * load_header - Check the image header and copy the data from it. */ static int load_header(struct swsusp_info *info) @@ -2483,7 +2483,7 @@ static int prepare_highmem_image(struct memory_bitmap *bm, static struct page *last_highmem_page; -/** +/* * get_highmem_page_buffer - Prepare a buffer to store a highmem image page. * * For a given highmem image page get a buffer that suspend_write_next() should @@ -2706,7 +2706,7 @@ static int prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm, return error; } -/** +/* * get_buffer - Get the address to store the next image data page. * * Get the address that snapshot_write_next() should return to its caller to @@ -2797,9 +2797,10 @@ next: return error; error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY); - if (error) + if (error) { + memory_bm_free(©_bm, PG_UNSAFE_CLEAR); return error; - + } nr_zero_pages = 0; hibernate_restore_protection_begin(); @@ -2843,7 +2844,7 @@ next: return PAGE_SIZE; } -/** +/* * snapshot_write_finalize - Complete the loading of a hibernation image. * * Must be called after the last call to snapshot_write_next() in case the last diff --git a/kernel/power/swap.c b/kernel/power/swap.c index c626e9dc3c1c..c78f1593600b 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c @@ -430,19 +430,22 @@ static int swap_write_page(struct swap_map_handle *handle, void *buf, if (!handle->cur) return -EINVAL; - offset = alloc_swapdev_block(root_swap); - error = write_page(buf, offset, hb); - if (error) - return error; - handle->cur->entries[handle->k++] = offset; + + /* + * If the current map page is full, allocate and link next one first. + * Delaying this until here avoids writing an empty swap map page when + * the image size is an exact MAP_PAGE_ENTRIES multiple. + */ if (handle->k >= MAP_PAGE_ENTRIES) { offset = alloc_swapdev_block(root_swap); if (!offset) return -ENOSPC; + handle->cur->next_swap = offset; error = write_page(handle->cur, handle->cur_swap, hb); if (error) - goto out; + return error; + clear_page(handle->cur); handle->cur_swap = offset; handle->k = 0; @@ -450,7 +453,7 @@ static int swap_write_page(struct swap_map_handle *handle, void *buf, if (hb && low_free_pages() <= handle->reqd_free_pages) { error = hib_wait_io(hb); if (error) - goto out; + return error; /* * Recalculate the number of required free pages, to * make sure we never take more than half. @@ -458,14 +461,21 @@ static int swap_write_page(struct swap_map_handle *handle, void *buf, handle->reqd_free_pages = reqd_free_pages(); } } - out: - return error; + + offset = alloc_swapdev_block(root_swap); + error = write_page(buf, offset, hb); + if (error) + return error; + handle->cur->entries[handle->k++] = offset; + return 0; } static int flush_swap_writer(struct swap_map_handle *handle) { - if (handle->cur && handle->cur_swap) + if (handle->cur && handle->cur_swap && handle->k) return write_page(handle->cur, handle->cur_swap, NULL); + else if (handle->cur && handle->cur_swap) + return 0; else return -EINVAL; } @@ -739,7 +749,7 @@ static int save_compressed_image(struct swap_map_handle *handle, data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); if (IS_ERR_OR_NULL(data[thr].cc)) { - pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + pr_err("Could not allocate comp stream %pe\n", data[thr].cc); ret = -EFAULT; goto out_clean; } @@ -1243,7 +1253,7 @@ static int load_compressed_image(struct swap_map_handle *handle, data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); if (IS_ERR_OR_NULL(data[thr].cc)) { - pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); + pr_err("Could not allocate comp stream %pe\n", data[thr].cc); ret = -EFAULT; goto out_clean; } diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c index fd763da06a87..a8b6bd5ec46b 100644 --- a/kernel/power/wakelock.c +++ b/kernel/power/wakelock.c @@ -63,7 +63,7 @@ static unsigned int number_of_wakelocks; static inline bool wakelocks_limit_exceeded(void) { - return number_of_wakelocks > CONFIG_PM_WAKELOCKS_LIMIT; + return number_of_wakelocks >= CONFIG_PM_WAKELOCKS_LIMIT; } static inline void increment_wakelocks_number(void) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..f5f7ff8c680a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5657,11 +5657,8 @@ EXPORT_PER_CPU_SYMBOL(kernel_cpustat); */ static inline void prefetch_curr_exec_start(struct task_struct *p) { -#ifdef CONFIG_FAIR_GROUP_SCHED - struct sched_entity *curr = p->se.cfs_rq->curr; -#else struct sched_entity *curr = task_rq(p)->cfs.curr; -#endif + prefetch(curr); prefetch(&curr->exec_start); } @@ -5973,8 +5970,13 @@ void preempt_count_add(int val) #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? + * + * Cannot detect underflow based on the current preempt_count() value + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32 + * bits. */ - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) && + DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) return; #endif __preempt_count_add(val); @@ -6006,7 +6008,10 @@ void preempt_count_sub(int val) /* * Underflow? */ - if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) + unsigned int uval = val; + unsigned int pc = preempt_count(); + + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc)) return; /* * Is the spinlock portion underflowing? @@ -9199,7 +9204,7 @@ void __might_resched(const char *file, int line, unsigned int offsets) } EXPORT_SYMBOL(__might_resched); -void __cant_sleep(const char *file, int line, int preempt_offset) +void __cant_sleep(const char *file, int line) { static unsigned long prev_jiffy; @@ -9209,7 +9214,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; - if (preempt_count() > preempt_offset) + if (preempt_count()) return; if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) @@ -9241,7 +9246,7 @@ void __cant_migrate(const char *file, int line) if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) return; - if (preempt_count() > 0) + if (preempt_count()) return; if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index dff4ee04694c..a1782755efcc 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -50,6 +50,7 @@ struct sugov_cpu { unsigned long util; unsigned long bw_min; + unsigned long bw_max; /* The field below is for single-CPU policies only: */ #ifdef CONFIG_NO_HZ_COMMON @@ -232,6 +233,7 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost) util = effective_cpu_util(sg_cpu->cpu, util, &min, &max); util = max(util, boost); sg_cpu->bw_min = min; + sg_cpu->bw_max = max; sg_cpu->util = sugov_effective_cpu_perf(sg_cpu->cpu, util, min, max); } @@ -314,7 +316,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, * A CPU running a task which woken up after an IO operation can have its * utilization boosted to speed up the completion of those IO operations. * The IO boost value is increased each time a task wakes up from IO, in - * sugov_iowait_apply(), and it's instead decreased by this function, + * sugov_iowait_boost(), and it's instead decreased by this function, * each time an increase has not been requested (!iowait_boost_pending). * * A CPU which also appears to have been idle for at least one tick has also @@ -484,7 +486,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time, sg_cpu->util = prev_util; cpufreq_driver_adjust_perf(sg_policy->policy, sg_cpu->bw_min, - sg_cpu->util, max_cap); + sg_cpu->util, sg_cpu->bw_max, max_cap); sg_policy->need_freq_update = false; sg_policy->last_freq_update_time = time; @@ -590,7 +592,7 @@ static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf) { struct sugov_tunables *tunables = to_sugov_tunables(attr_set); - return sprintf(buf, "%u\n", tunables->rate_limit_us); + return sysfs_emit(buf, "%u\n", tunables->rate_limit_us); } static ssize_t diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index 200300043fa5..857dbe3519a8 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -2734,15 +2734,17 @@ static int balance_dl(struct rq *rq, struct rq_flags *rf) */ static void wakeup_preempt_dl(struct rq *rq, struct task_struct *p, int flags) { + struct task_struct *donor = rq->donor; /* * Can only get preempted by stop-class, and those should be * few and short lived, doesn't really make sense to push * anything away for that. */ - if (p->sched_class != &dl_sched_class) + if (p->sched_class != &dl_sched_class || + donor->sched_class != &dl_sched_class) return; - if (dl_entity_preempt(&p->dl, &rq->donor->dl)) { + if (dl_entity_preempt(&p->dl, &donor->dl)) { resched_curr(rq); return; } diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 40584b27ea0c..72236db67983 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -633,6 +633,82 @@ static void debugfs_fair_server_init(void) } } +#ifdef CONFIG_FAIR_GROUP_SCHED +static int cgroup_mode = 2; + +/* See __sched_cgroup_mode_update(). */ +static const char *cgroup_mode_str[] = { + "up", + "smp", + "concur", + "max", + "tasks", +}; + +static int sched_cgroup_mode(const char *str) +{ + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (!strcmp(str, cgroup_mode_str[i])) + return i; + } + return -EINVAL; +} + +static ssize_t sched_cgroup_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + char buf[16]; + int mode; + + if (cnt > 15) + cnt = 15; + + if (copy_from_user(buf, ubuf, cnt)) + return -EFAULT; + + buf[cnt] = 0; + mode = sched_cgroup_mode(strstrip(buf)); + if (mode < 0) + return mode; + + __sched_cgroup_mode_update(mode); + WRITE_ONCE(cgroup_mode, mode); + + *ppos += cnt; + return cnt; +} + +static int sched_cgroup_show(struct seq_file *m, void *v) +{ + int mode = READ_ONCE(cgroup_mode); + + for (int i = 0; i < ARRAY_SIZE(cgroup_mode_str); i++) { + if (mode == i) + seq_puts(m, "("); + seq_puts(m, cgroup_mode_str[i]); + if (mode == i) + seq_puts(m, ")"); + + seq_puts(m, " "); + } + seq_puts(m, "\n"); + return 0; +} + +static int sched_cgroup_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_cgroup_show, NULL); +} + +static const struct file_operations sched_cgroup_fops = { + .open = sched_cgroup_open, + .write = sched_cgroup_write, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; +#endif + static __init int sched_init_debug(void) { struct dentry __maybe_unused *numa, *llc; @@ -686,6 +762,10 @@ static __init int sched_init_debug(void) debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); +#ifdef CONFIG_FAIR_GROUP_SCHED + debugfs_create_file("cgroup_mode", 0644, debugfs_sched, NULL, &sched_cgroup_fops); +#endif + debugfs_fair_server_init(); #ifdef CONFIG_SCHED_CLASS_EXT debugfs_ext_server_init(); @@ -895,10 +975,11 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, " %15s %5d %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %10ld %9Ld.%06ld %c %9Ld.%06ld %c %9Ld.%06ld %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), + p->se.h_load.weight, SPLIT_NS(p->se.vruntime), - entity_eligible(cfs_rq_of(&p->se), &p->se) ? 'E' : 'N', + entity_eligible(&rq->cfs, &p->se) ? 'E' : 'N', SPLIT_NS(p->se.deadline), p->se.custom_slice ? 'S' : ' ', SPLIT_NS(p->se.slice), @@ -927,7 +1008,7 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID vruntime eligible " + SEQ_printf(m, " S task PID weight vruntime eligible " "deadline slice sum-exec switches " "prio wait-time sum-sleep sum-block" #ifdef CONFIG_NUMA_BALANCING @@ -1035,6 +1116,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) cfs_rq->tg_load_avg_contrib); SEQ_printf(m, " .%-30s: %ld\n", "tg_load_avg", atomic_long_read(&cfs_rq->tg->load_avg)); + SEQ_printf(m, " .%-30s: %lu\n", "h_load", + cfs_rq->h_load); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_CFS_BANDWIDTH SEQ_printf(m, " .%-30s: %d\n", "throttled", @@ -1359,7 +1442,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(wait_count); PN_SCHEDSTAT(iowait_sum); P_SCHEDSTAT(iowait_count); - P_SCHEDSTAT(nr_migrations_cold); P_SCHEDSTAT(nr_failed_migrations_affine); P_SCHEDSTAT(nr_failed_migrations_running); P_SCHEDSTAT(nr_failed_migrations_hot); @@ -1371,8 +1453,6 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, P_SCHEDSTAT(nr_wakeups_remote); P_SCHEDSTAT(nr_wakeups_affine); P_SCHEDSTAT(nr_wakeups_affine_attempts); - P_SCHEDSTAT(nr_wakeups_passive); - P_SCHEDSTAT(nr_wakeups_idle); avg_atom = p->se.sum_exec_runtime; if (nr_switches) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index f15f5764818e..635f5775cc13 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -38,6 +38,7 @@ #include <linux/sched/isolation.h> #include <linux/sched/nohz.h> #include <linux/sched/prio.h> +#include <linux/static_call.h> #include <linux/cpuidle.h> #include <linux/interrupt.h> @@ -72,7 +73,10 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG; /* - * Minimal preemption granularity for CPU-bound tasks: + * Default base time slice (request size r_i) for SCHED_NORMAL/SCHED_BATCH: + * + * Under EEVDF this is the request size used to compute the virtual + * deadline; see update_deadline(). * * (default: 0.70 msec * (1 + ilog(ncpus)), units: nanoseconds) */ @@ -296,8 +300,8 @@ static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight */ static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se) { - if (unlikely(se->load.weight != NICE_0_LOAD)) - delta = __calc_delta(delta, NICE_0_LOAD, &se->load); + if (se->h_load.weight != NICE_0_LOAD) + delta = __calc_delta(delta, NICE_0_LOAD, &se->h_load); return delta; } @@ -427,38 +431,6 @@ static inline struct sched_entity *parent_entity(const struct sched_entity *se) return se->parent; } -static void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ - int se_depth, pse_depth; - - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ - - /* First walk up until both entities are at same depth */ - se_depth = (*se)->depth; - pse_depth = (*pse)->depth; - - while (se_depth > pse_depth) { - se_depth--; - *se = parent_entity(*se); - } - - while (pse_depth > se_depth) { - pse_depth--; - *pse = parent_entity(*pse); - } - - while (!is_same_group(*se, *pse)) { - *se = parent_entity(*se); - *pse = parent_entity(*pse); - } -} - static int tg_is_idle(struct task_group *tg) { return tg->idle > 0; @@ -502,11 +474,6 @@ static inline struct sched_entity *parent_entity(struct sched_entity *se) return NULL; } -static inline void -find_matching_se(struct sched_entity **se, struct sched_entity **pse) -{ -} - static inline int tg_is_idle(struct task_group *tg) { return 0; @@ -685,7 +652,7 @@ static inline unsigned long avg_vruntime_weight(struct cfs_rq *cfs_rq, unsigned static inline void __sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 w_vruntime, key = entity_key(cfs_rq, se); w_vruntime = key * weight; @@ -702,7 +669,7 @@ sum_w_vruntime_add_paranoid(struct cfs_rq *cfs_rq, struct sched_entity *se) s64 key, tmp; again: - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); key = entity_key(cfs_rq, se); if (check_mul_overflow(key, weight, &key)) @@ -748,7 +715,7 @@ sum_w_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se) static void sum_w_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se) { - unsigned long weight = avg_vruntime_weight(cfs_rq, se->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); s64 key = entity_key(cfs_rq, se); cfs_rq->sum_w_vruntime -= key * weight; @@ -790,7 +757,7 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) s64 runtime = cfs_rq->sum_w_vruntime; if (curr) { - unsigned long w = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long w = avg_vruntime_weight(cfs_rq, curr->h_load.weight); runtime += entity_key(cfs_rq, curr) * w; weight += w; @@ -813,6 +780,67 @@ u64 avg_vruntime(struct cfs_rq *cfs_rq) return cfs_rq->zero_vruntime; } +/* + * \Sum (v_i - v0)*w_i + * V = ------------------- + v0 + * \Sum w_i + * + * Let W = \Sum w_i, and move v_j such that 'v_j == V', thus: + * + * V = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * {(v_j - v0)*w_j + \Sum_i!=j (v_i - v0)*w_i} + v0 + * + * v_j = 1/W * (v_j - v0)*w_j + 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j - 1/W * (v_j - v0)*w_j = 1/W * \Sum_i!=j (v_i - v0)*w_i + v0 + * + * v_j*W - (v_j - v0)*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) + v0*w_j = \Sum_i!=j (v_i - v0)*w_i + v0*W + * + * v_j*(W - w_j) = \Sum_i!=j (v_i - v0)*w_i + v0*(W - w_j) + * + * \Sum_i!=j (v_i - v0)*w_i + * v_j = ------------------------ + v0 + * W - w_j + * + * When v_j happens to be curr, then '\Sum_i!=j (v_i - v0)*w_i' + * is cfs_rq->sum_w_runtime, and 'W - w_j' is cfs_rq->sum_weight, since curr + * is not included in the sum. + */ +static u64 ineligible_vruntime(struct cfs_rq *cfs_rq) +{ + struct sched_entity *curr = cfs_rq->curr; + long weight = cfs_rq->sum_weight; + s64 delta = 0; + + if (curr && !curr->on_rq) + curr = NULL; + + /* + * This is called from set_next_task_fair(.first=true) / + * set_protect_slice() so curr had better be set and on_rq. + */ + WARN_ON_ONCE(!curr); + + if (weight) { + s64 runtime = cfs_rq->sum_w_vruntime; + + /* + * Do not add @curr to obtain the effective '- w_j' terms. + */ + + /* sign flips effective floor / ceiling */ + if (runtime < 0) + runtime -= (weight - 1); + + delta = div64_long(runtime, weight); + } + + return cfs_rq->zero_vruntime + delta + 1; +} + static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); /* @@ -861,8 +889,6 @@ bool update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) u64 avruntime = avg_vruntime(cfs_rq); s64 vlag = entity_lag(cfs_rq, se, avruntime); - WARN_ON_ONCE(!se->on_rq); - if (se->sched_delayed) { /* previous vlag < 0 otherwise se would not be delayed */ vlag = max(vlag, se->vlag); @@ -898,7 +924,7 @@ static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime) long load = cfs_rq->sum_weight; if (curr && curr->on_rq) { - unsigned long weight = avg_vruntime_weight(cfs_rq, curr->load.weight); + unsigned long weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight); avg += entity_key(cfs_rq, curr) * weight; load += weight; @@ -1039,6 +1065,9 @@ RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, */ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; @@ -1048,6 +1077,9 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(&rq_of(cfs_rq)->cfs != cfs_rq); + WARN_ON_ONCE(!entity_is_task(se)); + rb_erase_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, &min_vruntime_cb); sum_w_vruntime_sub(cfs_rq, se); @@ -1090,8 +1122,14 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity slice = cfs_rq_min_slice(cfs_rq); slice = min(slice, se->slice); - if (slice != se->slice) - vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + + /* If there are shorter slices than se's one */ + if (slice != se->slice) { + if (sched_feat(PREEMPT_SHORT)) + vprot = min_vruntime(vprot, ineligible_vruntime(cfs_rq)); + else + vprot = min_vruntime(vprot, se->vruntime + calc_delta_fair(slice, se)); + } se->vprot = vprot; } @@ -1099,8 +1137,9 @@ static inline void set_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity static inline void update_protect_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { u64 slice = cfs_rq_min_slice(cfs_rq); + u64 vruntime = min_vruntime(se->vruntime, avg_vruntime(cfs_rq)); - se->vprot = min_vruntime(se->vprot, se->vruntime + calc_delta_fair(slice, se)); + se->vprot = min_vruntime(se->vprot, vruntime + calc_delta_fair(slice, se)); } static inline bool protect_slice(struct sched_entity *se) @@ -1144,7 +1183,7 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, bool protect) * We can safely skip eligibility check if there is only one entity * in this cfs_rq, saving some cycles. */ - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return curr && curr->on_rq ? curr : se; /* @@ -1394,8 +1433,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) return delta_exec; } -static void set_next_buddy(struct sched_entity *se); - #ifdef CONFIG_SCHED_CACHE /* @@ -1990,7 +2027,7 @@ static void update_curr(struct cfs_rq *cfs_rq) * not necessarily be the actual task running * (rq->curr.se). This is easy to confuse! */ - struct sched_entity *curr = cfs_rq->curr; + struct sched_entity *curr = cfs_rq->h_curr; struct rq *rq = rq_of(cfs_rq); s64 delta_exec; bool resched; @@ -2002,26 +2039,29 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; + account_cfs_rq_runtime(cfs_rq, delta_exec); + + if (!entity_is_task(curr)) + return; + + cfs_rq = &rq->cfs; + curr->vruntime += calc_delta_fair(delta_exec, curr); resched = update_deadline(cfs_rq, curr); - if (entity_is_task(curr)) { - /* - * If the fair_server is active, we need to account for the - * fair_server time whether or not the task is running on - * behalf of fair_server or not: - * - If the task is running on behalf of fair_server, we need - * to limit its time based on the assigned runtime. - * - Fair task that runs outside of fair_server should account - * against fair_server such that it can account for this time - * and possibly avoid running this period. - */ - dl_server_update(&rq->fair_server, delta_exec); - } - - account_cfs_rq_runtime(cfs_rq, delta_exec); + /* + * If the fair_server is active, we need to account for the + * fair_server time whether or not the task is running on + * behalf of fair_server or not: + * - If the task is running on behalf of fair_server, we need + * to limit its time based on the assigned runtime. + * - Fair task that runs outside of fair_server should account + * against fair_server such that it can account for this time + * and possibly avoid running this period. + */ + dl_server_update(&rq->fair_server, delta_exec); - if (cfs_rq->nr_queued == 1) + if (cfs_rq->h_nr_queued == 1) return; if (resched || !protect_slice(curr)) { @@ -2032,7 +2072,10 @@ static void update_curr(struct cfs_rq *cfs_rq) static void update_curr_fair(struct rq *rq) { - update_curr(cfs_rq_of(&rq->donor->se)); + struct sched_entity *se = &rq->donor->se; + + for_each_sched_entity(se) + update_curr(cfs_rq_of(se)); } static inline void @@ -2108,7 +2151,7 @@ update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Are we enqueueing a waiting task? (for current tasks * a dequeue/enqueue event is a NOP) */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_start_fair(cfs_rq, se); if (flags & ENQUEUE_WAKEUP) @@ -2126,7 +2169,7 @@ update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int fl * Mark the end of the wait period if dequeueing a * waiting task: */ - if (se != cfs_rq->curr) + if (se != cfs_rq->h_curr) update_stats_wait_end_fair(cfs_rq, se); if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) { @@ -4467,6 +4510,7 @@ static inline void update_scan_period(struct task_struct *p, int new_cpu) static void account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_add(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4482,6 +4526,7 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) static void account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); update_load_sub(&cfs_rq->load, se->load.weight); if (entity_is_task(se)) { struct task_struct *p = task_of(se); @@ -4563,7 +4608,7 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) static void rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) { - unsigned long old_weight = se->load.weight; + long old_weight = se->h_load.weight; /* * VRUNTIME @@ -4663,16 +4708,17 @@ rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) se->vprot = div64_long(se->vprot * old_weight, weight); } -static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, - unsigned long weight) +static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight, bool on_rq) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; u64 avruntime = 0; - if (se->on_rq) { - /* commit outstanding execution time */ - update_curr(cfs_rq); + if (se->h_load.weight == weight) + return; + + if (on_rq) { avruntime = avg_vruntime(cfs_rq); se->vlag = entity_lag(cfs_rq, se, avruntime); se->deadline -= avruntime; @@ -4682,46 +4728,90 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, rel_vprot = true; } - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (!curr) __dequeue_entity(cfs_rq, se); - update_load_sub(&cfs_rq->load, se->load.weight); } - dequeue_load_avg(cfs_rq, se); rescale_entity(se, weight, rel_vprot); - update_load_set(&se->load, weight); + update_load_set(&se->h_load, weight); - do { - u32 divider = get_pelt_divider(&se->avg); - se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); - } while (0); - - enqueue_load_avg(cfs_rq, se); - if (se->on_rq) { + if (on_rq) { if (rel_vprot) se->vprot += avruntime; se->deadline += avruntime; se->rel_deadline = 0; se->vruntime = avruntime - se->vlag; - update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } } +static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + if (se->load.weight == weight) + return; + + if (se->on_rq) { + WARN_ON_ONCE(cfs_rq != cfs_rq_of(se)); + update_load_sub(&cfs_rq->load, se->load.weight); + } + dequeue_load_avg(cfs_rq, se); + + update_load_set(&se->load, weight); + + do { + u32 divider = get_pelt_divider(&se->avg); + se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); + } while (0); + + enqueue_load_avg(cfs_rq, se); + + if (se->on_rq) + update_load_add(&cfs_rq->load, se->load.weight); +} + +/* + * weight = NICE_0_LOAD; + * for_each_entity_se(se) + * weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + */ +static __always_inline +unsigned long __calc_prop_weight(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) +{ + weight *= se->load.weight; + if (parent_entity(se)) + weight /= cfs_rq->load.weight; + else + weight /= NICE_0_LOAD; + + return max(weight, MIN_SHARES); +} + static void reweight_task_fair(struct rq *rq, struct task_struct *p, const struct load_weight *lw) { struct sched_entity *se = &p->se; - struct cfs_rq *cfs_rq = cfs_rq_of(se); - struct load_weight *load = &se->load; + unsigned long weight = NICE_0_LOAD; + + if (se->on_rq) + update_curr_fair(rq); - reweight_entity(cfs_rq, se, lw->weight); - load->inv_weight = lw->inv_weight; + reweight_entity(cfs_rq_of(se), se, lw->weight); + se->load.inv_weight = lw->inv_weight; + + if (!se->on_rq) + return; + + for_each_sched_entity(se) + weight = __calc_prop_weight(cfs_rq_of(se), se, weight); + + reweight_eevdf(&rq->cfs, &p->se, weight, p->se.on_rq); } static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); @@ -4800,12 +4890,10 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); * * hence icky! */ -static long calc_group_shares(struct cfs_rq *cfs_rq) +static long __calc_smp_shares(struct cfs_rq *cfs_rq, long tg_shares, long shares_max) { - long tg_weight, tg_shares, load, shares; struct task_group *tg = cfs_rq->tg; - - tg_shares = READ_ONCE(tg->shares); + long tg_weight, load, shares; load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg); @@ -4831,7 +4919,115 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) * case no task is runnable on a CPU MIN_SHARES=2 should be returned * instead of 0. */ - return clamp_t(long, shares, MIN_SHARES, tg_shares); + return clamp_t(long, shares, MIN_SHARES, shares_max); +} + +static int tg_cpus(struct task_group *tg) +{ + int nr = num_online_cpus(); + + if (cpusets_enabled()) { + struct cgroup *cgrp = tg->css.cgroup; + if (cgrp) + nr = cpuset_num_cpus(cgrp); + } + + return nr; +} + +static inline int tg_tasks(struct task_group *tg) +{ + return max(1, atomic_long_read(&tg->runnable_avg) >> SCHED_CAPACITY_SHIFT); +} + +/* + * Func: fraction(nr_tasks * tg->shares) + * + * Scale tg->shares by the number of tasks. + */ +static long calc_tasks_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_tasks(tg); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: min(fraction(nr_cpus * tg->shares), nice -20) + * + * Scale tg->shares by the maximal number of CPUs; but clip the max shares at + * nice -20, otherwise a single spinner on a 512 CPU machine would result in + * 512*NICE_0_LOAD, which is also crazy. + */ +static long calc_max_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = tg_cpus(tg); + long tg_shares = READ_ONCE(tg->shares); + long max_shares = scale_load(sched_prio_to_weight[0]); + return __calc_smp_shares(cfs_rq, tg_shares * nr, max_shares); +} + +/* + * Func: fraction(nr * tg->shares); nr = min(nr_tasks, nr_cpus) + * + * Scales between "smp" and "max" in a natural way. No longer needs clipping + * since there are no unnatural inflations like with "max". + */ +static long calc_concur_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + int nr = min(tg_tasks(tg), tg_cpus(tg)); + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, nr * tg_shares, nr * tg_shares); +} + +/* + * Func: fraction(tg->shares) + * + * This infamously results in tiny shares when you have many CPUs. + */ +static long calc_smp_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + long tg_shares = READ_ONCE(tg->shares); + return __calc_smp_shares(cfs_rq, tg_shares, tg_shares); +} + +/* + * Ignore this pesky SMP stuff, use (4). + */ +static long calc_up_shares(struct cfs_rq *cfs_rq) +{ + struct task_group *tg = cfs_rq->tg; + return READ_ONCE(tg->shares); +} + +DEFINE_STATIC_CALL(calc_group_shares, calc_concur_shares); + +void __sched_cgroup_mode_update(int mode) +{ + long (*func)(struct cfs_rq *); + switch (mode) { + case 0: + func = &calc_up_shares; + break; + case 1: + func = &calc_smp_shares; + break; + case 2: + default: + func = &calc_concur_shares; + break; + case 3: + func = &calc_max_shares; + break; + case 4: + func = &calc_tasks_shares; + break; + } + static_call_update(calc_group_shares, func); } /* @@ -4850,9 +5046,8 @@ static void update_cfs_group(struct sched_entity *se) if (!gcfs_rq || !gcfs_rq->load.weight) return; - shares = calc_group_shares(gcfs_rq); - if (unlikely(se->load.weight != shares)) - reweight_entity(cfs_rq_of(se), se, shares); + shares = static_call(calc_group_shares)(gcfs_rq); + reweight_entity(cfs_rq_of(se), se, shares); } #else /* !CONFIG_FAIR_GROUP_SCHED: */ @@ -4970,11 +5165,11 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) * differential update where we store the last value we propagated. This in * turn allows skipping updates if the differential is 'small'. * - * Updating tg's load_avg is necessary before update_cfs_share(). + * Updating tg's load_avg is necessary before update_cfs_group(). */ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -4995,17 +5190,21 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC) return; - delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; - if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) { - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib; + dr = cfs_rq->avg.runnable_avg - cfs_rq->tg_runnable_avg_contrib; + if (abs(dl) > cfs_rq->tg_load_avg_contrib / 64 || + abs(dr) > cfs_rq->tg_runnable_avg_contrib / 64) { + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg; + cfs_rq->tg_runnable_avg_contrib = cfs_rq->avg.runnable_avg; cfs_rq->last_update_tg_load_avg = now; } } static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) { - long delta; + long dl, dr; u64 now; /* @@ -5015,9 +5214,12 @@ static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq) return; now = rq_clock(rq_of(cfs_rq)); - delta = 0 - cfs_rq->tg_load_avg_contrib; - atomic_long_add(delta, &cfs_rq->tg->load_avg); + dl = 0 - cfs_rq->tg_load_avg_contrib; + dr = 0 - cfs_rq->tg_runnable_avg_contrib; + atomic_long_add(dl, &cfs_rq->tg->load_avg); + atomic_long_add(dr, &cfs_rq->tg->runnable_avg); cfs_rq->tg_load_avg_contrib = 0; + cfs_rq->tg_runnable_avg_contrib = 0; cfs_rq->last_update_tg_load_avg = now; } @@ -5174,7 +5376,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -5188,7 +5391,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -5430,7 +5633,7 @@ static void migrate_se_pelt_lag(struct sched_entity *se) {} * The cfs_rq avg is the direct sum of all its entities (blocked and runnable) * avg. The immediate corollary is that all (fair) tasks must be attached. * - * cfs_rq->avg is used for task_h_load() and update_cfs_share() for example. + * cfs_rq->avg is used for task_h_load() and update_cfs_group() for example. * * Return: true if the load decayed or we removed load. * @@ -5968,6 +6171,7 @@ static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { u64 vslice, vruntime = avg_vruntime(cfs_rq); + unsigned int nr_queued = cfs_rq->h_nr_queued; bool update_zero = false; s64 lag = 0; @@ -5975,6 +6179,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) se->slice = sysctl_sched_base_slice; vslice = calc_delta_fair(se->slice, se); + if (flags & ENQUEUE_QUEUED) + nr_queued -= 1; + /* * Due to how V is constructed as the weighted average of entities, * adding tasks with positive lag, or removing tasks with negative lag @@ -5983,7 +6190,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ - if (sched_feat(PLACE_LAG) && cfs_rq->nr_queued && se->vlag) { + if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) { struct sched_entity *curr = cfs_rq->curr; long load, weight; @@ -6043,9 +6250,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ load = cfs_rq->sum_weight; if (curr && curr->on_rq) - load += avg_vruntime_weight(cfs_rq, curr->load.weight); + load += avg_vruntime_weight(cfs_rq, curr->h_load.weight); - weight = avg_vruntime_weight(cfs_rq, se->load.weight); + weight = avg_vruntime_weight(cfs_rq, se->h_load.weight); lag *= load + weight; if (WARN_ON_ONCE(!load)) load = 1; @@ -6104,22 +6311,8 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq); static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq); static void -requeue_delayed_entity(struct sched_entity *se); - -static void enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool curr = cfs_rq->curr == se; - - /* - * If we're the current task, we must renormalise before calling - * update_curr(). - */ - if (curr) - place_entity(cfs_rq, se, flags); - - update_curr(cfs_rq); - /* * When enqueuing a sched_entity, we must: * - Update loads to have both entity and cfs_rq synced with now. @@ -6138,13 +6331,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) */ update_cfs_group(se); - /* - * XXX now that the entity has been re-weighted, and it's lag adjusted, - * we can place the entity. - */ - if (!curr) - place_entity(cfs_rq, se, flags); - account_entity_enqueue(cfs_rq, se); /* Entity has migrated, no longer consider this task hot */ @@ -6153,8 +6339,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) check_schedstat_required(); update_stats_enqueue_fair(cfs_rq, se, flags); - if (!curr) - __enqueue_entity(cfs_rq, se); se->on_rq = 1; if (cfs_rq->nr_queued == 1) { @@ -6172,21 +6356,19 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } } -static void __clear_buddies_next(struct sched_entity *se) +static void set_next_buddy(struct cfs_rq *cfs_rq, struct sched_entity *se) { - for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (cfs_rq->next != se) - break; - - cfs_rq->next = NULL; - } + if (WARN_ON_ONCE(!se->on_rq || se->sched_delayed)) + return; + if (se_is_idle(se)) + return; + cfs_rq->next = se; } static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (cfs_rq->next == se) - __clear_buddies_next(se); + cfs_rq->next = NULL; } static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); @@ -6197,7 +6379,7 @@ static void set_delayed(struct sched_entity *se) /* * Delayed se of cfs_rq have no tasks queued on them. - * Do not adjust h_nr_runnable since dequeue_entities() + * Do not adjust h_nr_runnable since __dequeue_task() * will account it for blocked tasks. */ if (!entity_is_task(se)) @@ -6230,45 +6412,16 @@ static void clear_delayed(struct sched_entity *se) } } -static bool +static void dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) { - bool sleep = flags & DEQUEUE_SLEEP; - int action = 0; + int action = UPDATE_TG; - update_curr(cfs_rq); - clear_buddies(cfs_rq, se); - - if (flags & DEQUEUE_DELAYED) { - WARN_ON_ONCE(!se->sched_delayed); - } else { - bool delay = sleep; - /* - * DELAY_DEQUEUE relies on spurious wakeups, special task - * states must not suffer spurious wakeups, excempt them. - */ - if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) - delay = false; - - WARN_ON_ONCE(delay && se->sched_delayed); - - if (sched_feat(DELAY_DEQUEUE) && delay && - !entity_eligible(cfs_rq, se)) { - if (entity_is_task(se)) - action |= UPDATE_UTIL_EST; - update_load_avg(cfs_rq, se, action); - update_entity_lag(cfs_rq, se); - set_delayed(se); - return false; - } - } - - action = UPDATE_TG; if (entity_is_task(se)) { if (task_on_rq_migrating(task_of(se))) action |= DO_DETACH; - if (sleep && !(flags & DEQUEUE_DELAYED)) + if ((flags & DEQUEUE_SLEEP) && !(flags & DEQUEUE_DELAYED)) action |= UPDATE_UTIL_EST; } @@ -6286,14 +6439,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_stats_dequeue_fair(cfs_rq, se, flags); - update_entity_lag(cfs_rq, se); - if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { - se->deadline -= se->vruntime; - se->rel_deadline = 1; - } - - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); se->on_rq = 0; account_entity_dequeue(cfs_rq, se); @@ -6302,9 +6447,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) update_cfs_group(se); - if (flags & DEQUEUE_DELAYED) - clear_delayed(se); - if (cfs_rq->nr_queued == 0) { update_idle_cfs_rq_clock_pelt(cfs_rq); #ifdef CONFIG_CFS_BANDWIDTH @@ -6317,15 +6459,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) } #endif } - - return true; } static void -set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) +set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - clear_buddies(cfs_rq, se); - /* 'current' is not kept within the tree. */ if (se->on_rq) { /* @@ -6334,16 +6472,12 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) * runqueue. */ update_stats_wait_end_fair(cfs_rq, se); - __dequeue_entity(cfs_rq, se); update_load_avg(cfs_rq, se, UPDATE_TG); - - if (first) - set_protect_slice(cfs_rq, se); } update_stats_curr_start(cfs_rq, se); - WARN_ON_ONCE(cfs_rq->curr); - cfs_rq->curr = se; + WARN_ON_ONCE(cfs_rq->h_curr); + cfs_rq->h_curr = se; /* * Track our maximum slice length, if the CPU's load is at @@ -6363,23 +6497,17 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool first) se->prev_sum_exec_runtime = se->sum_exec_runtime; } -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags); +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags); -/* - * Pick the next process, keeping these things in mind, in this order: - * 1) keep things fair between processes/task groups - * 2) pick the "next" process, since someone really wants that to run - * 3) pick the "last" process, for cache locality - * 4) do not run the "skip" process, if something else is available - */ static struct sched_entity * -pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq, bool protect) +pick_next_entity(struct rq *rq, bool protect) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; se = pick_eevdf(cfs_rq, protect); if (se->sched_delayed) { - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); + __dequeue_task(rq, task_of(se), DEQUEUE_SLEEP | DEQUEUE_DELAYED); /* * Must not reference @se again, see __block_task(). */ @@ -6399,13 +6527,11 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) if (prev->on_rq) { update_stats_wait_start_fair(cfs_rq, prev); - /* Put 'current' back into the tree. */ - __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ update_load_avg(cfs_rq, prev, 0); } - WARN_ON_ONCE(cfs_rq->curr != prev); - cfs_rq->curr = NULL; + WARN_ON_ONCE(cfs_rq->h_curr != prev); + cfs_rq->h_curr = NULL; } static void @@ -6960,7 +7086,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) assert_list_leaf_cfs_rq(rq); /* Determine whether we need to wake up potentially idle CPU: */ - if (rq->curr == rq->idle && rq->cfs.nr_queued) + if (rq->curr == rq->idle && rq->cfs.h_nr_queued) resched_curr(rq); } @@ -7120,7 +7246,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b) * period the timer is deactivated until scheduling resumes; cfs_b->idle is * used to track this state. */ -static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) +static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun) __must_hold(&cfs_b->lock) { int throttled; @@ -7155,10 +7281,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u * This check is repeated as we release cfs_b->lock while we unthrottle. */ while (throttled && cfs_b->runtime > 0) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); + raw_spin_unlock_irq_enable(&cfs_b->lock); /* we can't nest cfs_b->lock while distributing bandwidth */ throttled = distribute_cfs_runtime(cfs_b); - raw_spin_lock_irqsave(&cfs_b->lock, flags); + raw_spin_lock_irq_disable(&cfs_b->lock); } /* @@ -7266,7 +7392,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) { /* confirm we're still not at a refresh boundary */ - scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) { + scoped_guard(raw_spinlock_irq, &cfs_b->lock) { u64 runtime = 0, slice = sched_cfs_bandwidth_slice(); cfs_b->slack_started = false; @@ -7295,7 +7421,7 @@ static void check_enqueue_throttle(struct cfs_rq *cfs_rq) return; /* an active group must be handled by the update_curr() path */ - if (!cfs_rq->runtime_enabled || cfs_rq->curr) + if (!cfs_rq->runtime_enabled || cfs_rq->h_curr) return; /* ensure the group is not already throttled */ @@ -7351,14 +7477,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) int idle = 0; int count = 0; - CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock); + guard(raw_spinlock_irq)(&cfs_b->lock); for (;;) { overrun = hrtimer_forward_now(timer, cfs_b->period); if (!overrun) break; - idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags); + idle = do_sched_cfs_period_timer(cfs_b, overrun); if (++count > 3) { u64 new, old = ktime_to_ns(cfs_b->period); @@ -7667,7 +7793,7 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p) resched_curr(rq); return; } - delta = (se->load.weight * vdelta) / NICE_0_LOAD; + delta = (se->h_load.weight * vdelta) / NICE_0_LOAD; /* * Correct for instantaneous load of other classes. @@ -7767,10 +7893,8 @@ static int choose_idle_cpu(int cpu, struct task_struct *p) } static void -requeue_delayed_entity(struct sched_entity *se) +requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); - /* * se->sched_delayed should imply: se->on_rq == 1. * Because a delayed entity is one that is still on @@ -7779,22 +7903,68 @@ requeue_delayed_entity(struct sched_entity *se) WARN_ON_ONCE(!se->sched_delayed); WARN_ON_ONCE(!se->on_rq); - update_curr(cfs_rq); - if (update_entity_lag(cfs_rq, se)) { - cfs_rq->nr_queued--; + cfs_rq->h_nr_queued--; if (se != cfs_rq->curr) __dequeue_entity(cfs_rq, se); place_entity(cfs_rq, se, 0); if (se != cfs_rq->curr) __enqueue_entity(cfs_rq, se); - cfs_rq->nr_queued++; + cfs_rq->h_nr_queued++; } update_load_avg(cfs_rq, se, 0); clear_delayed(se); } +static unsigned long enqueue_hierarchy(struct task_struct *p, int flags) +{ + unsigned long weight = NICE_0_LOAD; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + int h_nr_idle = task_has_idle_policy(p); + int h_nr_runnable = 1; + + if (task_new && se->sched_delayed) + h_nr_runnable = 0; + + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + + update_curr(cfs_rq); + + if (!se->on_rq) { + enqueue_entity(cfs_rq, se, flags); + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); + } + + cfs_rq->h_nr_runnable += h_nr_runnable; + cfs_rq->h_nr_queued++; + cfs_rq->h_nr_idle += h_nr_idle; + + if (cfs_rq_is_idle(cfs_rq)) + h_nr_idle = 1; + + weight = __calc_prop_weight(cfs_rq, se, weight); + + flags = ENQUEUE_WAKEUP; + } + + return weight; +} + +/* Update curr's vruntime before placing entity or updating lag */ +static inline void update_curr_eevdf(struct cfs_rq *cfs_rq) +{ + if (!cfs_rq->curr) + return; + + update_curr(cfs_rq_of(cfs_rq->curr)); +} + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -7803,13 +7973,12 @@ requeue_delayed_entity(struct sched_entity *se) static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) { - struct cfs_rq *cfs_rq; - struct sched_entity *se = &p->se; - int h_nr_idle = task_has_idle_policy(p); - int h_nr_runnable = 1; - int task_new = !(flags & ENQUEUE_WAKEUP); int rq_h_nr_queued = rq->cfs.h_nr_queued; - u64 slice = 0; + int task_new = !(flags & ENQUEUE_WAKEUP); + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight; + bool curr; if (task_is_throttled(p) && enqueue_throttled_task(p)) return; @@ -7821,10 +7990,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) * estimated utilization, before we update schedutil. */ if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED)) - util_est_enqueue(&rq->cfs, p); + util_est_enqueue(cfs_rq, p); + + update_curr_eevdf(cfs_rq); if (flags & ENQUEUE_DELAYED) { - requeue_delayed_entity(se); + requeue_delayed_entity(cfs_rq, se); return; } @@ -7836,57 +8007,22 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (p->in_iowait) cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT); - if (task_new && se->sched_delayed) - h_nr_runnable = 0; - - for_each_sched_entity(se) { - if (se->on_rq) { - if (se->sched_delayed) - requeue_delayed_entity(se); - break; - } - cfs_rq = cfs_rq_of(se); - - /* - * Basically set the slice of group entries to the min_slice of - * their respective cfs_rq. This ensures the group can service - * its entities in the desired time-frame. - */ - if (slice) { - se->slice = slice; - se->custom_slice = 1; - } - enqueue_entity(cfs_rq, se, flags); - slice = cfs_rq_min_slice(cfs_rq); - - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; - - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; - - flags = ENQUEUE_WAKEUP; - } - - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + /* + * XXX comment on the curr thing + */ + curr = (cfs_rq->curr == se); + if (curr) + place_entity(cfs_rq, se, flags); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + if (se->on_rq && se->sched_delayed) + requeue_delayed_entity(cfs_rq, se); - cfs_rq->h_nr_runnable += h_nr_runnable; - cfs_rq->h_nr_queued++; - cfs_rq->h_nr_idle += h_nr_idle; + weight = enqueue_hierarchy(p, flags); - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = 1; + if (!curr) { + reweight_eevdf(cfs_rq, se, weight, false); + place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED); + __enqueue_entity(cfs_rq, se); } if (!rq_h_nr_queued && rq->cfs.h_nr_queued) @@ -7917,105 +8053,109 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) hrtick_update(rq); } -/* - * Basically dequeue_task_fair(), except it can deal with dequeue_entity() - * failing half-way through and resume the dequeue later. - * - * Returns: - * -1 - dequeue delayed - * 0 - dequeue throttled - * 1 - dequeue complete - */ -static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) +static void dequeue_hierarchy(struct task_struct *p, int flags) { - bool was_sched_idle = sched_idle_rq(rq); + struct sched_entity *se = &p->se; bool task_sleep = flags & DEQUEUE_SLEEP; bool task_delayed = flags & DEQUEUE_DELAYED; bool task_throttled = flags & DEQUEUE_THROTTLE; - struct task_struct *p = NULL; - int h_nr_idle = 0; - int h_nr_queued = 0; int h_nr_runnable = 0; - struct cfs_rq *cfs_rq; - u64 slice = 0; + int h_nr_idle = task_has_idle_policy(p); + bool dequeue = true; - if (entity_is_task(se)) { - p = task_of(se); - h_nr_queued = 1; - h_nr_idle = task_has_idle_policy(p); - if (task_sleep || task_delayed || !se->sched_delayed) - h_nr_runnable = 1; - } + if (task_sleep || task_delayed || !se->sched_delayed) + h_nr_runnable = 1; for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); + struct cfs_rq *cfs_rq = cfs_rq_of(se); - if (!dequeue_entity(cfs_rq, se, flags)) { - if (p && &p->se == se) - return -1; + update_curr(cfs_rq); - slice = cfs_rq_min_slice(cfs_rq); - break; + if (dequeue) { + dequeue_entity(cfs_rq, se, flags); + /* Don't dequeue parent if it has other entities besides us */ + if (cfs_rq->load.weight) + dequeue = false; + } else { + update_load_avg(cfs_rq, se, UPDATE_TG); + se_update_runnable(se); + update_cfs_group(se); } cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; + cfs_rq->h_nr_queued--; cfs_rq->h_nr_idle -= h_nr_idle; if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + h_nr_idle = 1; if (throttled_hierarchy(cfs_rq) && task_throttled) record_throttle_clock(cfs_rq); - /* Don't dequeue parent if it has other entities besides us */ - if (cfs_rq->load.weight) { - slice = cfs_rq_min_slice(cfs_rq); - - /* Avoid re-evaluating load for this entity: */ - se = parent_entity(se); - /* - * Bias pick_next to pick a task from this cfs_rq, as - * p is sleeping when it is within its sched_slice. - */ - if (task_sleep && se) - set_next_buddy(se); - break; - } flags |= DEQUEUE_SLEEP; flags &= ~(DEQUEUE_DELAYED | DEQUEUE_SPECIAL); } +} - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); +/* + * The part of dequeue_task_fair() that is needed to dequeue delayed tasks. + * + * Returns: + * true - dequeued + * false - delayed + */ +static bool __dequeue_task(struct rq *rq, struct task_struct *p, int flags) +{ + struct sched_entity *se = &p->se; + struct cfs_rq *cfs_rq = &rq->cfs; + bool was_sched_idle = sched_idle_rq(rq); + bool task_sleep = flags & DEQUEUE_SLEEP; + bool task_delayed = flags & DEQUEUE_DELAYED; - update_load_avg(cfs_rq, se, UPDATE_TG); - se_update_runnable(se); - update_cfs_group(se); + clear_buddies(cfs_rq, se); - se->slice = slice; - if (se != cfs_rq->curr) - min_vruntime_cb_propagate(&se->run_node, NULL); - slice = cfs_rq_min_slice(cfs_rq); + update_curr_eevdf(cfs_rq); + update_entity_lag(cfs_rq, se); - cfs_rq->h_nr_runnable -= h_nr_runnable; - cfs_rq->h_nr_queued -= h_nr_queued; - cfs_rq->h_nr_idle -= h_nr_idle; + if (flags & DEQUEUE_DELAYED) { + WARN_ON_ONCE(!se->sched_delayed); + } else { + bool delay = task_sleep; + /* + * DELAY_DEQUEUE relies on spurious wakeups, special task + * states must not suffer spurious wakeups, excempt them. + */ + if (flags & (DEQUEUE_SPECIAL | DEQUEUE_THROTTLE)) + delay = false; - if (cfs_rq_is_idle(cfs_rq)) - h_nr_idle = h_nr_queued; + WARN_ON_ONCE(delay && se->sched_delayed); - if (throttled_hierarchy(cfs_rq) && task_throttled) - record_throttle_clock(cfs_rq); + if (sched_feat(DELAY_DEQUEUE) && delay && + !entity_eligible(cfs_rq, se)) { + update_load_avg(cfs_rq_of(se), se, UPDATE_UTIL_EST); + set_delayed(se); + return false; + } } - sub_nr_running(rq, h_nr_queued); + dequeue_hierarchy(p, flags); + + if (sched_feat(PLACE_REL_DEADLINE) && !task_sleep) { + se->deadline -= se->vruntime; + se->rel_deadline = 1; + } + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + + sub_nr_running(rq, 1); /* balance early to pull high priority tasks */ if (unlikely(!was_sched_idle && sched_idle_rq(rq))) rq->next_balance = jiffies; - if (p && task_delayed) { + if (task_delayed) { + clear_delayed(se); + WARN_ON_ONCE(!task_sleep); WARN_ON_ONCE(p->on_rq != 1); @@ -8027,7 +8167,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) __block_task(rq, p); } - return 1; + return true; } /* @@ -8045,11 +8185,11 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (!p->se.sched_delayed) util_est_dequeue(&rq->cfs, p); - if (dequeue_entities(rq, &p->se, flags) < 0) + if (!__dequeue_task(rq, p, flags)) return false; /* - * Must not reference @p after dequeue_entities(DEQUEUE_DELAYED). + * Must not reference @p after __dequeue_task(DEQUEUE_DELAYED). */ return true; } @@ -9644,19 +9784,6 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu) static void task_dead_fair(struct task_struct *p) { struct sched_entity *se = &p->se; - - if (se->sched_delayed) { - struct rq_flags rf; - struct rq *rq; - - rq = task_rq_lock(p, &rf); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - task_rq_unlock(rq, p, &rf); - } - remove_entity_load_avg(se); } @@ -9690,27 +9817,14 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context set_task_max_allowed_capacity(p); } -static void set_next_buddy(struct sched_entity *se) -{ - for_each_sched_entity(se) { - if (WARN_ON_ONCE(!se->on_rq)) - return; - if (se_is_idle(se)) - return; - cfs_rq_of(se)->next = se; - } -} - enum preempt_wakeup_action { PREEMPT_WAKEUP_NONE, /* No preemption. */ PREEMPT_WAKEUP_SHORT, /* Ignore slice protection. */ - PREEMPT_WAKEUP_PICK, /* Let __pick_eevdf() decide. */ + PREEMPT_WAKEUP_PICK, /* Let pick_eevdf() decide. */ PREEMPT_WAKEUP_RESCHED, /* Force reschedule. */ }; -static inline bool -set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, - struct sched_entity *pse, struct sched_entity *se) +static inline bool set_preempt_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) { /* * Keep existing buddy if the deadline is sooner than pse. @@ -9721,7 +9835,16 @@ set_preempt_buddy(struct cfs_rq *cfs_rq, int wake_flags, if (cfs_rq->next && entity_before(cfs_rq->next, pse)) return false; - set_next_buddy(pse); + set_next_buddy(cfs_rq, pse); + return true; +} + +static inline bool set_short_buddy(struct cfs_rq *cfs_rq, struct sched_entity *pse) +{ + if (cfs_rq->next && cfs_rq->next->slice < pse->slice) + return false; + + set_next_buddy(cfs_rq, pse); return true; } @@ -9774,13 +9897,14 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f enum preempt_wakeup_action preempt_action = PREEMPT_WAKEUP_PICK; struct task_struct *donor = rq->donor; struct sched_entity *nse, *se = &donor->se, *pse = &p->se; - struct cfs_rq *cfs_rq = task_cfs_rq(donor); + struct cfs_rq *cfs_rq = &rq->cfs; int cse_is_idle, pse_is_idle; /* * XXX Getting preempted by higher class, try and find idle CPU? */ - if (p->sched_class != &fair_sched_class) + if (p->sched_class != &fair_sched_class || + donor->sched_class != &fair_sched_class) return; if (unlikely(se == pse)) @@ -9805,42 +9929,43 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f * prevents us from potentially nominating it as a false LAST_BUDDY * below. */ - if (test_tsk_need_resched(rq->curr)) + if (!sched_feat(PREEMPT_SHORT) && test_tsk_need_resched(rq->curr)) return; if (!sched_feat(WAKEUP_PREEMPTION)) return; - find_matching_se(&se, &pse); WARN_ON_ONCE(!pse); cse_is_idle = se_is_idle(se); pse_is_idle = se_is_idle(pse); + nse = se; /* * Preempt an idle entity in favor of a non-idle entity (and don't preempt * in the inverse case). */ - if (cse_is_idle && !pse_is_idle) { - /* - * When non-idle entity preempt an idle entity, - * don't give idle entity slice protection. - */ - preempt_action = PREEMPT_WAKEUP_SHORT; + if (cse_is_idle && !pse_is_idle) goto preempt; - } + + update_curr_fair(rq); if (cse_is_idle != pse_is_idle) - return; + goto update; /* * BATCH and IDLE tasks do not preempt others. */ if (unlikely(!normal_policy(p->policy))) - return; + goto update; + + /* + * Do not preempt for tasks that are sched_delayed as it would violate + * EEVDF to forcibly queue an ineligible task. + */ + if (pse->sched_delayed) + goto update; - cfs_rq = cfs_rq_of(se); - update_curr(cfs_rq); /* * If @p has a shorter slice than current and @p is eligible, override * current's slice protection in order to allow preemption. @@ -9852,17 +9977,13 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f /* * Ignore wakee preemption on WF_FORK as it is less likely that - * there is shared data as exec often follow fork. Do not - * preempt for tasks that are sched_delayed as it would violate - * EEVDF to forcibly queue an ineligible task. + * there is shared data as exec often follow fork. */ - if ((wake_flags & WF_FORK) || pse->sched_delayed) - return; + if (wake_flags & WF_FORK) + goto update; /* Prefer picking wakee soon if appropriate. */ - if (sched_feat(NEXT_BUDDY) && - set_preempt_buddy(cfs_rq, wake_flags, pse, se)) { - + if (sched_feat(NEXT_BUDDY) && set_preempt_buddy(cfs_rq, pse)) { /* * Decide whether to obey WF_SYNC hint for a new buddy. Old * buddies are ignored as they may not be relevant to the @@ -9884,29 +10005,33 @@ static void wakeup_preempt_fair(struct rq *rq, struct task_struct *p, int wake_f } pick: - nse = pick_next_entity(rq, cfs_rq, preempt_action != PREEMPT_WAKEUP_SHORT); - /* If @p has become the most eligible task, force preemption */ - if (nse == pse) - goto preempt; + if (cfs_rq->h_nr_queued) { + nse = pick_next_entity(rq, preempt_action != PREEMPT_WAKEUP_SHORT); + if (unlikely(!nse)) + goto pick; + + /* If @p has become the most eligible task, force preemption */ + if (nse == pse) + goto preempt; + } /* - * Because p is enqueued, nse being null can only mean that we - * dequeued a delayed task. If there are still entities queued in - * cfs, check if the next one will be p. + * If @p is eligible but not the next task to run then cancel protection + * to prevent large scheduling latency */ - if (!nse && cfs_rq->nr_queued) - goto pick; - + if (preempt_action == PREEMPT_WAKEUP_SHORT && entity_eligible(cfs_rq, pse)) + goto preempt; +update: if (sched_feat(RUN_TO_PARITY)) update_protect_slice(cfs_rq, se); return; preempt: - if (preempt_action == PREEMPT_WAKEUP_SHORT) { - cancel_protect_slice(se); - clear_buddies(cfs_rq, se); - } + cancel_protect_slice(se); + + if (preempt_action == PREEMPT_WAKEUP_SHORT) + set_short_buddy(cfs_rq, pse); resched_curr_lazy(rq); } @@ -9914,33 +10039,24 @@ preempt: struct task_struct *pick_task_fair(struct rq *rq, struct rq_flags *rf) __must_hold(__rq_lockp(rq)) { + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *se; - struct cfs_rq *cfs_rq; struct task_struct *p; - bool throttled; int new_tasks; again: - cfs_rq = &rq->cfs; - if (!cfs_rq->nr_queued) + if (!cfs_rq->h_nr_queued) goto idle; - throttled = false; - - do { - /* Might not have done put_prev_entity() */ - if (cfs_rq->curr && cfs_rq->curr->on_rq) - update_curr(cfs_rq); + /* Might not have done put_prev_entity() */ + if (cfs_rq->curr && cfs_rq->curr->on_rq) + update_curr(cfs_rq); - se = pick_next_entity(rq, cfs_rq, true); - if (!se) - goto again; - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + se = pick_next_entity(rq, true); + if (!se) + goto again; p = task_of(se); - if (unlikely(throttled)) - task_throttle_setup_work(p); return p; idle: @@ -9977,7 +10093,7 @@ void fair_server_init(struct rq *rq) static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct task_struct *next) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; + struct cfs_rq *cfs_rq = &rq->cfs; struct sched_entity *nse = NULL; #ifdef CONFIG_FAIR_GROUP_SCHED @@ -9987,7 +10103,7 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t while (se) { cfs_rq = cfs_rq_of(se); - if (!nse || cfs_rq->curr) + if (!nse || cfs_rq->h_curr) put_prev_entity(cfs_rq, se); #ifdef CONFIG_FAIR_GROUP_SCHED if (nse) { @@ -10006,6 +10122,14 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t #endif se = parent_entity(se); } + + /* Put 'current' back into the tree. */ + cfs_rq = &rq->cfs; + se = &prev->se; + WARN_ON_ONCE(cfs_rq->curr != se); + cfs_rq->curr = NULL; + if (se->on_rq) + __enqueue_entity(cfs_rq, se); } /* @@ -10014,8 +10138,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t static void yield_task_fair(struct rq *rq) { struct task_struct *curr = rq->donor; - struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se; + struct cfs_rq *cfs_rq = &rq->cfs; /* * Are we the only task in the tree? @@ -10056,11 +10180,11 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p) struct sched_entity *se = &p->se; /* !se->on_rq also covers throttled task */ - if (!se->on_rq) + if (!se->on_rq || se->sched_delayed) return false; /* Tell the scheduler that we'd really like se to run next. */ - set_next_buddy(se); + set_next_buddy(&task_rq(p)->cfs, se); yield_task_fair(rq); @@ -10399,15 +10523,10 @@ static inline long migrate_degrades_locality(struct task_struct *p, */ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_cpu) { - struct cfs_rq *dst_cfs_rq; + struct cfs_rq *dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#ifdef CONFIG_FAIR_GROUP_SCHED - dst_cfs_rq = tg_cfs_rq(task_group(p), dest_cpu); -#else - dst_cfs_rq = &cpu_rq(dest_cpu)->cfs; -#endif - if (sched_feat(PLACE_LAG) && dst_cfs_rq->nr_queued && - !entity_eligible(task_cfs_rq(p), &p->se)) + if (sched_feat(PLACE_LAG) && dst_cfs_rq->h_nr_queued && + !entity_eligible(&task_rq(p)->cfs, &p->se)) return 1; return 0; @@ -11190,7 +11309,7 @@ static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq) while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) { load = cfs_rq->h_load; load = div64_ul(load * se->avg.load_avg, - cfs_rq_load_avg(cfs_rq) + 1); + cfs_rq_load_avg(cfs_rq) + 1); cfs_rq = group_cfs_rq(se); cfs_rq->h_load = load; cfs_rq->last_h_load_update = now; @@ -11862,10 +11981,25 @@ static inline void update_sg_lb_stats(struct lb_env *env, continue; if (sd_flags & SD_ASYM_CPUCAPACITY) { - /* Check for a misfit task on the cpu */ - if (sgs->group_misfit_task_load < rq->misfit_task_load) { - sgs->group_misfit_task_load = rq->misfit_task_load; - *sg_overloaded = 1; + if (rq->misfit_task_load) { + /* + * Always mark the root domain overloaded so big + * CPUs can pick up misfit tasks via newly idle + * balance. + */ + if (balancing_at_rd) + *sg_overloaded = 1; + + /* + * Only account misfit load if @dst_cpu can + * help; otherwise, the group may be classified + * as misfit_task and update_sd_pick_busiest() + * will skip it. + */ + if (capacity_greater(capacity_of(env->dst_cpu), + group->sgc->max_capacity) && + (sgs->group_misfit_task_load < rq->misfit_task_load)) + sgs->group_misfit_task_load = rq->misfit_task_load; } } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { /* Check for a task running on a CPU with reduced capacity */ @@ -11944,6 +12078,17 @@ static bool update_sd_pick_busiest(struct lb_env *env, sds->local_stat.group_type != group_has_spare)) return false; + /* + * Candidate sg has no more than one task per CPU and has higher + * per-CPU capacity. Migrating tasks to less capable CPUs may harm + * throughput. Maximize throughput, power/energy consequences are not + * considered. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + (sgs->group_type <= group_fully_busy) && + (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) + return false; + if (sgs->group_type > busiest->group_type) return true; @@ -12050,17 +12195,6 @@ has_spare: break; } - /* - * Candidate sg has no more than one task per CPU and has higher - * per-CPU capacity. Migrating tasks to less capable CPUs may harm - * throughput. Maximize throughput, power/energy consequences are not - * considered. - */ - if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && - (sgs->group_type <= group_fully_busy) && - (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) - return false; - return true; } @@ -12978,9 +13112,24 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, * average load. */ if (env->sd->flags & SD_ASYM_CPUCAPACITY && - !capacity_greater(capacity_of(env->dst_cpu), capacity) && - nr_running == 1) - continue; + nr_running == 1) { + bool cluster_equal_cap = static_branch_unlikely(&sched_cluster_active) && + (get_actual_cpu_capacity(env->dst_cpu) == + get_actual_cpu_capacity(i)); + bool smt_degraded_cap = sched_smt_active() && !is_core_idle(i); + + /* + * Busy SMT siblings reduce the capacity of CPU @i. Do + * not skip it in this case. + * + * CONFIG_SCHED_CLUSTER requires balancing load across + * clusters of identical capacity, accounting for + * hardware and cpufreq pressure. + */ + if (!smt_degraded_cap && !cluster_equal_cap && + !capacity_greater(capacity_of(env->dst_cpu), capacity)) + continue; + } /* * Make sure we only pull tasks from a CPU of lower priority @@ -13439,75 +13588,78 @@ more_balance: } } - if (!ld_moved) { - schedstat_inc(sd->lb_failed[idle]); - /* - * Increment the failure counter only on periodic balance. - * We do not want newidle balance, which can be very - * frequent, pollute the failure counter causing - * excessive cache_hot migrations and active balances. - * - * Similarly for migration_misfit which is not related to - * load/util migration, don't pollute nr_balance_failed. - * - * The same for cache aware scheduling's allowance for - * load imbalance. If regular load balance does not - * migrate task due to LLC locality, it is a expected - * behavior and don't pollute nr_balance_failed. - * See can_migrate_task(). - */ - if (idle != CPU_NEWLY_IDLE && - env.migration_type != migrate_misfit && - !(env.flags & LBF_LLC_PINNED)) - sd->nr_balance_failed++; + if (ld_moved) { + sd->nr_balance_failed = 0; + goto out_unbalanced; + } - if (need_active_balance(&env)) { - unsigned long flags; + schedstat_inc(sd->lb_failed[idle]); + /* + * Increment the failure counter only on periodic balance. + * We do not want newidle balance, which can be very + * frequent, pollute the failure counter causing + * excessive cache_hot migrations and active balances. + * + * Similarly for migration_misfit which is not related to + * load/util migration, don't pollute nr_balance_failed. + * + * The same for cache aware scheduling's allowance for + * load imbalance. If regular load balance does not + * migrate task due to LLC locality, it is a expected + * behavior and don't pollute nr_balance_failed. + * See can_migrate_task(). + */ + if (idle != CPU_NEWLY_IDLE && + env.migration_type != migrate_misfit && + !(env.flags & LBF_LLC_PINNED)) + sd->nr_balance_failed++; - raw_spin_rq_lock_irqsave(busiest, flags); + if (!need_active_balance(&env)) + goto out_unbalanced; - /* - * Don't kick the active_load_balance_cpu_stop, - * if the curr task on busiest CPU can't be - * moved to this_cpu: - */ - if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) { - raw_spin_rq_unlock_irqrestore(busiest, flags); - goto out_one_pinned; - } + scoped_guard (raw_spin_rq_lock_irqsave, busiest) { + /* + * Don't kick the active_load_balance_cpu_stop, + * if the curr task on busiest CPU can't be + * moved to this_cpu: + */ + if (!cpumask_test_cpu(this_cpu, busiest->curr->cpus_ptr)) + goto out_one_pinned; - /* Record that we found at least one task that could run on this_cpu */ - env.flags &= ~LBF_ALL_PINNED; + /* Record that we found at least one task that could run on this_cpu */ + env.flags &= ~LBF_ALL_PINNED; - /* - * ->active_balance synchronizes accesses to - * ->active_balance_work. Once set, it's cleared - * only after active load balance is finished. - */ - if (!busiest->active_balance) { - busiest->active_balance = 1; - busiest->push_cpu = this_cpu; - active_balance = 1; - } + /* + * ->active_balance synchronizes accesses to + * ->active_balance_work. Once set, it's cleared + * only after active load balance is finished. + */ + if (busiest->active_balance) + goto out_unbalanced; - preempt_disable(); - raw_spin_rq_unlock_irqrestore(busiest, flags); - if (active_balance) { - stop_one_cpu_nowait(cpu_of(busiest), - active_load_balance_cpu_stop, busiest, - &busiest->active_balance_work); - } - preempt_enable(); - } - } else { - sd->nr_balance_failed = 0; - } + /* + * @busiest dropped its rq_lock in the middle of + * scheduling out its ->curr task (->on_rq := 0), no + * need to forcefully punt it away with active balance. + */ + if (!busiest->curr->on_rq) + goto out_unbalanced; - if (likely(!active_balance) || need_active_balance(&env)) { - /* We were unbalanced, so reset the balancing interval */ - sd->balance_interval = sd->min_interval; + busiest->active_balance = 1; + busiest->push_cpu = this_cpu; + active_balance = 1; + preempt_disable(); } + if (active_balance) { + stop_one_cpu_nowait(cpu_of(busiest), + active_load_balance_cpu_stop, busiest, + &busiest->active_balance_work); + } + preempt_enable(); +out_unbalanced: + /* We were unbalanced, so reset the balancing interval */ + sd->balance_interval = sd->min_interval; goto out; out_balanced: @@ -13851,29 +14003,62 @@ static inline int on_null_domain(struct rq *rq) */ static inline int find_new_ilb(void) { - int this_cpu = smp_processor_id(); - const struct cpumask *hk_mask; - int ilb_cpu; + struct cpumask *ilb_cpus; + int ilb_cpu, fallback = -1; + + lockdep_assert_irqs_disabled(); - hk_mask = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE); + /* + * Reuse the per-CPU select_rq_mask, which is protected from concurrent + * use on this CPU by having interrupts disabled. + */ + ilb_cpus = this_cpu_cpumask_var_ptr(select_rq_mask); + cpumask_and(ilb_cpus, nohz.idle_cpus_mask, + housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)); - for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) { - if (ilb_cpu == this_cpu) + for_each_cpu(ilb_cpu, ilb_cpus) { + if (!idle_cpu(ilb_cpu)) { + /* + * Once an idle fallback exists, a busy CPU proves that + * this core cannot be fully idle. Skip its siblings. + */ + if (sched_smt_active() && fallback >= 0) + cpumask_andnot(ilb_cpus, ilb_cpus, cpu_smt_mask(ilb_cpu)); continue; + } - if (idle_cpu(ilb_cpu)) - return ilb_cpu; + /* + * Running the idle load balancer on an idle sibling of a busy + * SMT core can reduce the capacity available to its sibling. Prefer + * a CPU whose entire core is idle, but retain the first idle CPU as + * a fallback so idle balancing can still make progress when no fully + * idle core exists. + */ + if (sched_smt_active() && !is_core_idle(ilb_cpu)) { + if (fallback < 0) + fallback = ilb_cpu; + + /* + * The core is not idle, so there is no need to check + * any of its other SMT siblings. + */ + cpumask_andnot(ilb_cpus, ilb_cpus, + cpu_smt_mask(ilb_cpu)); + continue; + } + + return ilb_cpu; } - return -1; + return fallback; } /* * Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU * SMP function call (IPI). * - * We pick the first idle CPU in the HK_TYPE_KERNEL_NOISE housekeeping set - * (if there is one). + * Prefer a CPU on a fully idle core in the HK_TYPE_KERNEL_NOISE housekeeping + * set. Fall back to the first idle CPU when no fully idle core exists. */ static void kick_ilb(unsigned int flags) { @@ -14573,7 +14758,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * if the sibling is forced idle, then trigger schedule to * give forced idle task a chance. * - * sched_slice() considers only this active rq and it gets the + * __entity_slice_used() considers only this active rq and it gets the * whole slice. But during force idle, we have siblings acting * like a single runqueue and hence we need to consider runnable * tasks on this CPU and the forced idle CPU. Ideally, we should @@ -14582,7 +14767,7 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) * MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check * if we need to give up the CPU. */ - if (rq->core->core_forceidle_count && rq->cfs.nr_queued == 1 && + if (rq->core->core_forceidle_count && rq->cfs.h_nr_queued == 1 && __entity_slice_used(&curr->se, MIN_NR_TASKS_DURING_FORCEIDLE)) resched_curr(rq); } @@ -14791,30 +14976,8 @@ bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b, WARN_ON_ONCE(task_rq(b)->core != rq->core); -#ifdef CONFIG_FAIR_GROUP_SCHED - /* - * Find an se in the hierarchy for tasks a and b, such that the se's - * are immediate siblings. - */ - while (sea->cfs_rq->tg != seb->cfs_rq->tg) { - int sea_depth = sea->depth; - int seb_depth = seb->depth; - - if (sea_depth >= seb_depth) - sea = parent_entity(sea); - if (sea_depth <= seb_depth) - seb = parent_entity(seb); - } - - se_fi_update(sea, rq->core->core_forceidle_seq, in_fi); - se_fi_update(seb, rq->core->core_forceidle_seq, in_fi); - - cfs_rqa = sea->cfs_rq; - cfs_rqb = seb->cfs_rq; -#else /* !CONFIG_FAIR_GROUP_SCHED: */ cfs_rqa = &task_rq(a)->cfs; cfs_rqb = &task_rq(b)->cfs; -#endif /* !CONFIG_FAIR_GROUP_SCHED */ /* * Find delta after normalizing se's vruntime with its cfs_rq's @@ -14853,11 +15016,20 @@ static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {} static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued) { struct sched_entity *se = &curr->se; - struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - entity_tick(cfs_rq, se, queued); + if (se->on_rq) { + unsigned long weight = NICE_0_LOAD; + struct cfs_rq *cfs_rq; + + for_each_sched_entity(se) { + cfs_rq = cfs_rq_of(se); + entity_tick(cfs_rq, se, queued); + + weight = __calc_prop_weight(cfs_rq, se, weight); + } + + se = &curr->se; + reweight_eevdf(cfs_rq, se, weight, se->on_rq); } if (queued) @@ -14897,7 +15069,7 @@ prio_changed_fair(struct rq *rq, struct task_struct *p, u64 oldprio) if (p->prio == oldprio) return; - if (rq->cfs.nr_queued == 1) + if (rq->cfs.h_nr_queued == 1) return; /* @@ -15026,33 +15198,44 @@ static void switched_to_fair(struct rq *rq, struct task_struct *p) } } -/* - * Account for a task changing its policy or group. - * - * This routine is mostly called to set cfs_rq->curr field when a task - * migrates between groups/classes. - */ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) { struct sched_entity *se = &p->se; bool throttled = false; + struct cfs_rq *cfs_rq = &rq->cfs; + unsigned long weight = NICE_0_LOAD; + bool on_rq = se->on_rq; + + clear_buddies(cfs_rq, se); + + if (on_rq) + __dequeue_entity(cfs_rq, se); for_each_sched_entity(se) { - struct cfs_rq *cfs_rq = cfs_rq_of(se); + cfs_rq = cfs_rq_of(se); - if (IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) && - first && cfs_rq->curr) - break; + if (!IS_ENABLED(CONFIG_FAIR_GROUP_SCHED) || + !first || !cfs_rq->h_curr) + set_next_entity(cfs_rq, se); - set_next_entity(cfs_rq, se, first); /* ensure bandwidth has been allocated on our new cfs_rq */ throttled |= account_cfs_rq_runtime(cfs_rq, 0); + + if (on_rq) + weight = __calc_prop_weight(cfs_rq, se, weight); } if (throttled) task_throttle_setup_work(p); se = &p->se; + cfs_rq->curr = se; + + if (on_rq) { + reweight_eevdf(cfs_rq, se, weight, se->on_rq); + if (first) + set_protect_slice(cfs_rq, se); + } if (task_on_rq_queued(p)) { /* @@ -15165,17 +15348,8 @@ void unregister_fair_sched_group(struct task_group *tg) struct sched_entity *se = tg_se(tg, cpu); struct rq *rq = cpu_rq(cpu); - if (se) { - if (se->sched_delayed) { - guard(rq_lock_irqsave)(rq); - if (se->sched_delayed) { - update_rq_clock(rq); - dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); - } - list_del_leaf_cfs_rq(cfs_rq); - } + if (se) remove_entity_load_avg(se); - } /* * Only empty task groups can be destroyed; so we can speculatively diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index ef152d401fe2..156025ef81b7 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -8,6 +8,7 @@ * */ #include <linux/sched/isolation.h> +#include <linux/llist.h> #include <linux/pci.h> #include "sched.h" @@ -27,6 +28,7 @@ struct housekeeping { }; static struct housekeeping housekeeping; +static __initdata LLIST_HEAD(memblock_freelist); bool housekeeping_enabled(enum hk_type type) { @@ -189,10 +191,22 @@ void __init housekeeping_init(void) WARN_ON_ONCE(cpumask_empty(omask)); cpumask_copy(nmask, omask); RCU_INIT_POINTER(housekeeping.cpumasks[type], nmask); - memblock_free(omask, cpumask_size()); + __llist_add((struct llist_node *)omask, &memblock_freelist); } } +static int __init housekeeping_late_init(void) +{ + struct llist_node *llnode, *pos, *t; + + /* Free allocated memblock memory, if any */ + llnode = __llist_del_all(&memblock_freelist); + llist_for_each_safe(pos, t, llnode) + memblock_free(pos, cpumask_size()); + return 0; +} +pure_initcall(housekeeping_late_init); + static void __init housekeeping_setup_type(enum hk_type type, cpumask_var_t housekeeping_staging) { diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index 897790889ba3..779eb58a4261 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -206,7 +206,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa, /* * running is a subset of runnable (weight) so running can't be set if * runnable is clear. But there are some corner cases where the current - * se has been already dequeued but cfs_rq->curr still points to it. + * se has been already dequeued but cfs_rq->h_curr still points to it. * This means that weight will be 0 but not running for a sched_entity * but also for a cfs_rq if the latter becomes idle. As an example, * this happens during sched_balance_newidle() which calls @@ -307,7 +307,7 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se) { if (___update_load_sum(now, &se->avg, !!se->on_rq, se_runnable(se), - cfs_rq->curr == se)) { + cfs_rq->h_curr == se)) { ___update_load_avg(&se->avg, se_weight(se)); cfs_se_util_change(&se->avg); @@ -323,7 +323,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) if (___update_load_sum(now, &cfs_rq->avg, scale_load_down(cfs_rq->load.weight), cfs_rq->h_nr_runnable, - cfs_rq->curr != NULL)) { + cfs_rq->h_curr != NULL)) { ___update_load_avg(&cfs_rq->avg, 1); trace_pelt_cfs_tp(cfs_rq); diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index e2e825dcd088..4e152410653d 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1023,7 +1023,7 @@ void psi_account_irqtime(struct rq *rq, struct task_struct *curr, struct task_st irq = irq_time_read(cpu); delta = (s64)(irq - rq->psi_irq_time); - if (delta < 0) + if (delta <= 0) return; rq->psi_irq_time = irq; diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index e474c31d8fe6..e6e5f8a2caaf 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1629,7 +1629,8 @@ static void wakeup_preempt_rt(struct rq *rq, struct task_struct *p, int flags) /* * XXX If we're preempted by DL, queue a push? */ - if (p->sched_class != &rt_sched_class) + if (p->sched_class != &rt_sched_class || + donor->sched_class != &rt_sched_class) return; if (p->prio < donor->prio) { diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba26..26ae13c86b69 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -493,6 +493,8 @@ struct task_group { * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; + atomic_long_t runnable_avg; + #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED @@ -528,21 +530,8 @@ struct task_group { }; -#ifdef CONFIG_GROUP_SCHED_WEIGHT #define ROOT_TASK_GROUP_LOAD NICE_0_LOAD -/* - * A weight of 0 or 1 can cause arithmetics problems. - * A weight of a cfs_rq is the sum of weights of which entities - * are queued on this cfs_rq, so a weight of a entity should not be - * too large, so as the shares value of a task group. - * (The default weight is 1024 - so there's no practical - * limitation from this.) - */ -#define MIN_SHARES (1UL << 1) -#define MAX_SHARES (1UL << 18) -#endif - typedef int (*tg_visitor)(struct task_group *, void *); extern int walk_tg_tree_from(struct task_group *from, @@ -571,6 +560,7 @@ extern void free_fair_sched_group(struct task_group *tg); extern int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent); extern void online_fair_sched_group(struct task_group *tg); extern void unregister_fair_sched_group(struct task_group *tg); +extern void __sched_cgroup_mode_update(int mode); #else /* !CONFIG_FAIR_GROUP_SCHED: */ static inline void free_fair_sched_group(struct task_group *tg) { } static inline int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) @@ -628,6 +618,17 @@ static inline bool cfs_task_bw_constrained(struct task_struct *p) { return false #endif /* !CONFIG_CGROUP_SCHED */ +/* + * A weight of 0 or 1 can cause arithmetics problems. + * A weight of a cfs_rq is the sum of weights of which entities + * are queued on this cfs_rq, so a weight of a entity should not be + * too large, so as the shares value of a task group. + * (The default weight is 1024 - so there's no practical + * limitation from this.) + */ +#define MIN_SHARES (1UL << 1) +#define MAX_SHARES (1UL << 18) + extern void unregister_rt_sched_group(struct task_group *tg); extern void free_rt_sched_group(struct task_group *tg); extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent); @@ -706,6 +707,7 @@ struct cfs_rq { /* * CFS load tracking */ + struct sched_entity *h_curr; struct sched_avg avg; #ifndef CONFIG_64BIT u64 last_update_time_copy; @@ -721,6 +723,7 @@ struct cfs_rq { #ifdef CONFIG_FAIR_GROUP_SCHED u64 last_update_tg_load_avg; unsigned long tg_load_avg_contrib; + unsigned long tg_runnable_avg_contrib; long propagate; long prop_runnable_sum; @@ -2018,7 +2021,8 @@ DEFINE_LOCK_GUARD_1(rq_lock, struct rq, rq_unlock(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock, _T) DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, @@ -2026,7 +2030,8 @@ DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, rq_unlock_irq(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irq, _T) DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, @@ -2034,9 +2039,20 @@ DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, rq_unlock_irqrestore(_T->lock, &_T->rf), struct rq_flags rf) -DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); #define class_rq_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, _T) +DEFINE_LOCK_GUARD_1(raw_spin_rq_lock_irqsave, struct rq, + raw_spin_rq_lock_irqsave(_T->lock, _T->flags), + raw_spin_rq_unlock_irqrestore(_T->lock, _T->flags), + unsigned long flags) + +DECLARE_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, __acquires(__rq_lockp(_T)), + __releases(__rq_lockp(*(struct rq **)_T))); +#define class_raw_spin_rq_lock_irqsave_constructor(_T) \ + WITH_LOCK_GUARD_1_ATTRS(raw_spin_rq_lock_irqsave, _T) + #define this_rq_lock_irq(...) __acquire_ret(_this_rq_lock_irq(__VA_ARGS__), __rq_lockp(__ret)) static inline struct rq *_this_rq_lock_irq(struct rq_flags *rf) __acquires_ret { @@ -2571,6 +2587,7 @@ extern const u32 sched_prio_to_wmult[40]; #define ENQUEUE_MIGRATED 0x00040000 #define ENQUEUE_INITIAL 0x00080000 #define ENQUEUE_RQ_SELECTED 0x00100000 +#define ENQUEUE_QUEUED 0x00200000 #define RETRY_TASK ((void *)-1UL) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 622e2e01974c..21e816ad23ee 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1995,10 +1995,6 @@ sd_init(struct sched_domain_topology_level *tl, /* * Convert topological properties into behaviour. */ - /* Don't attempt to spread across CPUs of different capacities. */ - if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child) - sd->child->flags &= ~SD_PREFER_SIBLING; - if (sd->flags & SD_SHARE_CPUCAPACITY) { sd->imbalance_pct = 110; diff --git a/kernel/softirq.c b/kernel/softirq.c index 4425d8dce44b..7980a4a232f9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE #include <linux/export.h> #include <linux/kernel_stat.h> #include <linux/interrupt.h> @@ -88,6 +89,28 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled); EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context); #endif +DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state); + +void _local_interrupt_disable(void) +{ + __local_interrupt_disable(); +} +EXPORT_SYMBOL(_local_interrupt_disable); + +void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +EXPORT_SYMBOL(_local_interrupt_enable); + +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +/* + * Any 32bit architecture that still cares about performance should + * probably ensure this is near preempt_count. + */ +DEFINE_PER_CPU(unsigned int, nmi_nesting); +#endif + /* * SOFTIRQ_OFFSET usage: * @@ -726,10 +749,19 @@ static inline void __irq_exit_rcu(void) #endif account_hardirq_exit(current); preempt_count_sub(HARDIRQ_OFFSET); - if (!in_interrupt() && local_softirq_pending()) { + /* + * Interrupts may happen between hardirq_disable_enter() and + * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes + * softirq here, we may have a softirq handler calling + * local_interrupt_disable() but it won't disable the IRQ because + * hardirq disabling count is already 1, hence we need to prevent + * invoking softirq when a local_interrupt_disable() is ongoing. + */ + if (!in_interrupt() && !hardirq_disable_count() && + local_softirq_pending()) { /* * If we left hrtimers unarmed, make sure to arm them now, - * before enabling interrupts to run SoftIRQ. + * before enabling interrupts to run softirq. */ hrtimer_rearm_deferred(); invoke_softirq(); diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index 773d8e9ae30c..d085ba1f4b44 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -7,6 +7,7 @@ * Copyright (C) 2010 SUSE Linux Products GmbH * Copyright (C) 2010 Tejun Heo <tj@kernel.org> */ +#include <linux/bug.h> #include <linux/compiler.h> #include <linux/completion.h> #include <linux/cpu.h> @@ -376,17 +377,14 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * * and will remain untouched until stopper starts executing @fn. * * CONTEXT: - * Don't care. - * - * RETURNS: - * true if cpu_stop_work was queued successfully and @fn will be called, - * false otherwise. + * Don't care, but the caller must ensure @cpu's stopper stays enabled + * until the work is queued, e.g. by preempt_disable(). */ -bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, - struct cpu_stop_work *work_buf) +void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, + struct cpu_stop_work *work_buf) { *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, }; - return cpu_stop_queue_work(cpu, work_buf); + WARN_ON_ONCE(!cpu_stop_queue_work(cpu, work_buf)); } static bool queue_stop_cpus_work(const struct cpumask *cpumask, |