diff options
Diffstat (limited to 'init')
| -rw-r--r-- | init/Kconfig | 52 | ||||
| -rw-r--r-- | init/do_mounts.c | 21 | ||||
| -rw-r--r-- | init/init_task.c | 15 | ||||
| -rw-r--r-- | init/initramfs.c | 68 | ||||
| -rw-r--r-- | init/initramfs_test.c | 97 | ||||
| -rw-r--r-- | init/main.c | 45 |
6 files changed, 180 insertions, 118 deletions
diff --git a/init/Kconfig b/init/Kconfig index 2937c4d308ae..5230d4879b1c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -118,10 +118,7 @@ config GCC_ASM_GOTO_OUTPUT_BROKEN config CC_HAS_ASM_GOTO_OUTPUT def_bool y depends on !GCC_ASM_GOTO_OUTPUT_BROKEN - # Detect basic support depends on $(success,echo 'int foo(int x) { asm goto ("": "=r"(x) ::: bar); return x; bar: return 0; }' | $(CC) -x c - -c -o /dev/null) - # Detect clang (< v17) scoped label issues - depends on $(success,echo 'void b(void **);void* c(void);int f(void){{asm goto(""::::l0);return 0;l0:return 1;}void *x __attribute__((cleanup(b)))=c();{asm goto(""::::l1);return 2;l1:return 3;}}' | $(CC) -x c - -c -o /dev/null) config CC_HAS_ASM_GOTO_TIED_OUTPUT depends on CC_HAS_ASM_GOTO_OUTPUT @@ -170,6 +167,9 @@ config CC_HAS_BROKEN_COUNTED_BY_REF # https://github.com/llvm/llvm-project/issues/182575 default y if CC_IS_CLANG && CLANG_VERSION < 220100 +config CC_HAS_ALLOC_TOKEN + def_bool $(cc-option,-falloc-token-max=123) + config CC_HAS_MULTIDIMENSIONAL_NONSTRING def_bool $(success,echo 'char tag[][4] __attribute__((__nonstring__)) = { };' | $(CC) $(CLANG_FLAGS) -x c - -c -o /dev/null -Werror) @@ -1019,6 +1019,17 @@ config NUMA_BALANCING This system will be inactive on UMA systems. +config SCHED_CACHE + bool "Cache aware load balance" + default y + depends on SMP + help + When enabled, the scheduler will attempt to aggregate tasks from + the same process onto a single Last Level Cache (LLC) domain when + possible. This improves cache locality by keeping tasks that share + resources within the same cache domain, reducing cache misses and + lowering data access latency. + config NUMA_BALANCING_DEFAULT_ENABLED bool "Automatically enable NUMA aware memory/task placement" default y @@ -1050,14 +1061,14 @@ config PAGE_COUNTER bool config CGROUP_FAVOR_DYNMODS - bool "Favor dynamic modification latency reduction by default" - help - This option enables the "favordynmods" mount option by default - which reduces the latencies of dynamic cgroup modifications such - as task migrations and controller on/offs at the cost of making - hot path operations such as forks and exits more expensive. + bool "Favor dynamic modification latency reduction by default" + help + This option enables the "favordynmods" mount option by default + which reduces the latencies of dynamic cgroup modifications such + as task migrations and controller on/offs at the cost of making + hot path operations such as forks and exits more expensive. - Say N if unsure. + Say N if unsure. config MEMCG bool "Memory controller" @@ -1139,7 +1150,7 @@ config GROUP_SCHED_WEIGHT def_bool n config GROUP_SCHED_BANDWIDTH - def_bool n + def_bool n config FAIR_GROUP_SCHED bool "Group scheduling for SCHED_OTHER" @@ -1645,10 +1656,10 @@ config LD_ORPHAN_WARN depends on $(ld-option,--orphan-handling=error) config LD_ORPHAN_WARN_LEVEL - string - depends on LD_ORPHAN_WARN - default "error" if WERROR - default "warn" + string + depends on LD_ORPHAN_WARN + default "error" if WERROR + default "warn" config SYSCTL bool @@ -1842,6 +1853,12 @@ config FUTEX_MPOL depends on FUTEX && NUMA default y +config HAVE_FUTEX_ROBUST_UNLOCK + bool + +config FUTEX_ROBUST_UNLOCK + def_bool FUTEX && HAVE_GENERIC_VDSO && GENERIC_IRQ_ENTRY && RSEQ && HAVE_FUTEX_ROBUST_UNLOCK + config EPOLL bool "Enable eventpoll support" if EXPERT default y @@ -2198,7 +2215,8 @@ config RUST depends on !DEBUG_INFO_BTF || (PAHOLE_HAS_LANG_EXCLUDE && !LTO) depends on !CFI || HAVE_CFI_ICALL_NORMALIZE_INTEGERS_RUSTC select CFI_ICALL_NORMALIZE_INTEGERS if CFI - depends on !KASAN_SW_TAGS + depends on !KASAN || CC_IS_CLANG + depends on !KASAN_SW_TAGS || RUSTC_VERSION >= 109600 help Enables Rust support in the kernel. @@ -2212,6 +2230,8 @@ config RUST If unsure, say N. +source "rust/kernel/Kconfig.test" + config RUSTC_VERSION_TEXT string depends on RUST diff --git a/init/do_mounts.c b/init/do_mounts.c index 55ed3ac0b70f..95e0b3a0f711 100644 --- a/init/do_mounts.c +++ b/init/do_mounts.c @@ -143,16 +143,14 @@ static int __init do_mount_root(const char *name, const char *fs, const int flags, const void *data) { struct super_block *s; - struct page *p = NULL; char *data_page = NULL; int ret; if (data) { /* init_mount() requires a full page as fifth argument */ - p = alloc_page(GFP_KERNEL); - if (!p) + data_page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!data_page) return -ENOMEM; - data_page = page_address(p); strscpy_pad(data_page, data, PAGE_SIZE); } @@ -170,19 +168,20 @@ static int __init do_mount_root(const char *name, const char *fs, MAJOR(ROOT_DEV), MINOR(ROOT_DEV)); out: - if (p) - put_page(p); + kfree(data_page); return ret; } void __init mount_root_generic(char *name, char *pretty_name, int flags) { - struct page *page = alloc_page(GFP_KERNEL); - char *fs_names = page_address(page); + char *fs_names = kmalloc(PAGE_SIZE, GFP_KERNEL); char *p; char b[BDEVNAME_SIZE]; int num_fs, i; + if (!fs_names) + panic("VFS: Unable to mount root fs: not enough memory"); + scnprintf(b, BDEVNAME_SIZE, "unknown-block(%u,%u)", MAJOR(ROOT_DEV), MINOR(ROOT_DEV)); if (root_fs_names) @@ -242,7 +241,7 @@ retry: printk("\n"); panic("VFS: Unable to mount root fs on \"%s\" or %s", pretty_name, b); out: - put_page(page); + kfree(fs_names); } #ifdef CONFIG_ROOT_NFS @@ -343,7 +342,7 @@ static int __init mount_nodev_root(char *root_device_name) int err = -EINVAL; int num_fs, i; - fs_names = (void *)__get_free_page(GFP_KERNEL); + fs_names = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!fs_names) return -EINVAL; num_fs = split_fs_names(fs_names, PAGE_SIZE); @@ -360,7 +359,7 @@ static int __init mount_nodev_root(char *root_device_name) break; } - free_page((unsigned long)fs_names); + kfree(fs_names); return err; } diff --git a/init/init_task.c b/init/init_task.c index b5f48ebdc2b6..b67ef6040a65 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -7,6 +7,8 @@ #include <linux/sched/rt.h> #include <linux/sched/task.h> #include <linux/sched/ext.h> +#include <linux/sched/exec_state.h> +#include <linux/user_namespace.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/mm.h> @@ -56,6 +58,13 @@ static struct sighand_struct init_sighand = { .signalfd_wqh = __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh), }; +/* init to 2 - one for init_task, one to ensure it is never freed */ +struct task_exec_state init_task_exec_state = { + .count = REFCOUNT_INIT(2), + .dumpable = TASK_DUMPABLE_OWNER, + .user_ns = &init_user_ns, +}; + #ifdef CONFIG_SHADOW_CALL_STACK unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)] = { [(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC @@ -113,6 +122,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .nr_cpus_allowed= NR_CPUS, .mm = NULL, .active_mm = &init_mm, + .exec_state = &init_task_exec_state, .restart_block = { .fn = do_no_restart_syscall, }, @@ -200,6 +210,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq, &init_task.alloc_lock), #endif + .blocked_donor = NULL, #ifdef CONFIG_RT_MUTEXES .pi_waiters = RB_ROOT_CACHED, .pi_top_task = NULL, @@ -215,6 +226,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { .numa_group = NULL, .numa_faults = NULL, #endif +#ifdef CONFIG_SCHED_CACHE + .preferred_llc = -1, + .pref_llc_queued = 0, +#endif #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) .kasan_depth = 1, #endif diff --git a/init/initramfs.c b/init/initramfs.c index 58db15fb18fd..20a18fcda48e 100644 --- a/init/initramfs.c +++ b/init/initramfs.c @@ -1,25 +1,28 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/init.h> #include <linux/async.h> -#include <linux/export.h> -#include <linux/fs.h> -#include <linux/slab.h> -#include <linux/types.h> -#include <linux/fcntl.h> #include <linux/delay.h> -#include <linux/string.h> #include <linux/dirent.h> -#include <linux/syscalls.h> -#include <linux/utime.h> +#include <linux/export.h> +#include <linux/fcntl.h> #include <linux/file.h> +#include <linux/fs.h> +#include <linux/hex.h> +#include <linux/init.h> +#include <linux/init_syscalls.h> #include <linux/kstrtox.h> #include <linux/memblock.h> #include <linux/mm.h> #include <linux/namei.h> -#include <linux/init_syscalls.h> -#include <linux/umh.h> -#include <linux/security.h> #include <linux/overflow.h> +#include <linux/security.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/syscalls.h> +#include <linux/types.h> +#include <linux/umh.h> +#include <linux/utime.h> + +#include <asm/byteorder.h> #include "do_mounts.h" #include "initramfs_internal.h" @@ -190,26 +193,30 @@ static __initdata gid_t gid; static __initdata unsigned rdev; static __initdata u32 hdr_csum; -static void __init parse_header(char *s) +static int __init parse_header(char *s) { - unsigned long parsed[13]; - int i; + __be32 header[13]; + int ret; - for (i = 0, s += 6; i < 13; i++, s += 8) - parsed[i] = simple_strntoul(s, NULL, 16, 8); + ret = hex2bin((u8 *)header, s + 6, sizeof(header)); + if (ret) { + error("damaged header"); + return ret; + } - ino = parsed[0]; - mode = parsed[1]; - uid = parsed[2]; - gid = parsed[3]; - nlink = parsed[4]; - mtime = parsed[5]; /* breaks in y2106 */ - body_len = parsed[6]; - major = parsed[7]; - minor = parsed[8]; - rdev = new_encode_dev(MKDEV(parsed[9], parsed[10])); - name_len = parsed[11]; - hdr_csum = parsed[12]; + ino = be32_to_cpu(header[0]); + mode = be32_to_cpu(header[1]); + uid = be32_to_cpu(header[2]); + gid = be32_to_cpu(header[3]); + nlink = be32_to_cpu(header[4]); + mtime = be32_to_cpu(header[5]); /* breaks in y2106 */ + body_len = be32_to_cpu(header[6]); + major = be32_to_cpu(header[7]); + minor = be32_to_cpu(header[8]); + rdev = new_encode_dev(MKDEV(be32_to_cpu(header[9]), be32_to_cpu(header[10]))); + name_len = be32_to_cpu(header[11]); + hdr_csum = be32_to_cpu(header[12]); + return 0; } /* Finite-state machine */ @@ -289,7 +296,8 @@ static int __init do_header(void) error("no cpio magic"); return 1; } - parse_header(collected); + if (parse_header(collected)) + return 1; next_header = this_header + N_ALIGN(name_len) + body_len; next_header = (next_header + 3) & ~3; state = SkipIt; diff --git a/init/initramfs_test.c b/init/initramfs_test.c index 2ce38d9a8fd0..bc55306d226d 100644 --- a/init/initramfs_test.c +++ b/init/initramfs_test.c @@ -3,7 +3,9 @@ #include <linux/fcntl.h> #include <linux/file.h> #include <linux/fs.h> +#include <linux/init.h> #include <linux/init_syscalls.h> +#include <linux/initrd.h> #include <linux/stringify.h> #include <linux/timekeeping.h> #include "initramfs_internal.h" @@ -27,7 +29,18 @@ struct initramfs_test_cpio { char *data; }; -static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, char *out) +/* regular newc header format */ +#define CPIO_HDR_FMT "%s%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%s" +/* + * Bogus newc header with "0x" prefixes on the uid, gid, and namesize values. + * parse_header()/simple_str[n]toul() accepted this, contrary to the initramfs + * specification. hex2bin() now fails. + */ +#define CPIO_HDR_OX_INJECT \ + "%s%08x%08x0x%06x0X%06x%08x%08x%08x%08x%08x%08x%08x0x%06x%08x%s" + +static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, + bool inject_ox, char *out) { int i; size_t off = 0; @@ -38,9 +51,8 @@ static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, char *out) size_t thislen; /* +1 to account for nulterm */ - thislen = sprintf(pos, "%s" - "%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x" - "%s", + thislen = sprintf(pos, + inject_ox ? CPIO_HDR_OX_INJECT : CPIO_HDR_FMT, c->magic, c->ino, c->mode, c->uid, c->gid, c->nlink, c->mtime, c->filesize, c->devmajor, c->devminor, c->rdevmajor, c->rdevminor, c->namesize, c->csum, @@ -102,7 +114,7 @@ static void __init initramfs_test_extract(struct kunit *test) /* +3 to cater for any 4-byte end-alignment */ cpio_srcbuf = kzalloc(ARRAY_SIZE(c) * (CPIO_HDRLEN + PATH_MAX + 3), GFP_KERNEL); - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); ktime_get_real_ts64(&ts_before); err = unpack_to_rootfs(cpio_srcbuf, len); @@ -177,7 +189,7 @@ static void __init initramfs_test_fname_overrun(struct kunit *test) /* limit overrun to avoid crashes / filp_open() ENAMETOOLONG */ cpio_srcbuf[CPIO_HDRLEN + strlen(c[0].fname) + 20] = '\0'; - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); /* overwrite trailing fname terminator and padding */ suffix_off = len - 1; while (cpio_srcbuf[suffix_off] == '\0') { @@ -219,7 +231,7 @@ static void __init initramfs_test_data(struct kunit *test) cpio_srcbuf = kmalloc(CPIO_HDRLEN + c[0].namesize + c[0].filesize + 6, GFP_KERNEL); - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); err = unpack_to_rootfs(cpio_srcbuf, len); KUNIT_EXPECT_NULL(test, err); @@ -274,7 +286,7 @@ static void __init initramfs_test_csum(struct kunit *test) cpio_srcbuf = kmalloc(8192, GFP_KERNEL); - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); err = unpack_to_rootfs(cpio_srcbuf, len); KUNIT_EXPECT_NULL(test, err); @@ -284,7 +296,7 @@ static void __init initramfs_test_csum(struct kunit *test) /* mess up the csum and confirm that unpack fails */ c[0].csum--; - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); err = unpack_to_rootfs(cpio_srcbuf, len); KUNIT_EXPECT_NOT_NULL(test, err); @@ -306,7 +318,7 @@ static void __init initramfs_test_hardlink(struct kunit *test) { char *err, *cpio_srcbuf; size_t len; - struct kstat st0, st1; + struct kstat st0 = {}, st1 = {}; struct initramfs_test_cpio c[] = { { .magic = "070701", .ino = 1, @@ -330,7 +342,7 @@ static void __init initramfs_test_hardlink(struct kunit *test) cpio_srcbuf = kmalloc(8192, GFP_KERNEL); - len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf); err = unpack_to_rootfs(cpio_srcbuf, len); KUNIT_EXPECT_NULL(test, err); @@ -371,7 +383,7 @@ static void __init initramfs_test_many(struct kunit *test) }; c.namesize = 1 + sprintf(thispath, "initramfs_test_many-%d", i); - p += fill_cpio(&c, 1, p); + p += fill_cpio(&c, 1, false, p); } len = p - cpio_srcbuf; @@ -425,7 +437,7 @@ static void __init initramfs_test_fname_pad(struct kunit *test) } }; memcpy(tbufs->padded_fname, "padded_fname", sizeof("padded_fname")); - len = fill_cpio(c, ARRAY_SIZE(c), tbufs->cpio_srcbuf); + len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_srcbuf); err = unpack_to_rootfs(tbufs->cpio_srcbuf, len); KUNIT_EXPECT_NULL(test, err); @@ -451,7 +463,7 @@ static void __init initramfs_test_fname_path_max(struct kunit *test) { char *err; size_t len; - struct kstat st0, st1; + struct kstat st0 = {}, st1 = {}; char fdata[] = "this file data will not be unpacked"; struct test_fname_path_max { char fname_oversize[PATH_MAX + 1]; @@ -481,7 +493,7 @@ static void __init initramfs_test_fname_path_max(struct kunit *test) memcpy(tbufs->fname_oversize, "fname_oversize", sizeof("fname_oversize") - 1); memcpy(tbufs->fname_ok, "fname_ok", sizeof("fname_ok") - 1); - len = fill_cpio(c, ARRAY_SIZE(c), tbufs->cpio_src); + len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_src); /* unpack skips over fname_oversize instead of returning an error */ err = unpack_to_rootfs(tbufs->cpio_src, len); @@ -494,6 +506,45 @@ static void __init initramfs_test_fname_path_max(struct kunit *test) kfree(tbufs); } +static void __init initramfs_test_hdr_hex(struct kunit *test) +{ + char *err; + size_t len; + char fdata[] = "this file data will not be unpacked"; + struct initramfs_test_bufs { + char cpio_src[(CPIO_HDRLEN + PATH_MAX + 3 + sizeof(fdata)) * 2]; + } *tbufs = kzalloc(sizeof(struct initramfs_test_bufs), GFP_KERNEL); + struct initramfs_test_cpio c[] = { { + .magic = "070701", + .ino = 1, + .mode = S_IFREG | 0777, + .uid = 0x123456, + .gid = 0x123457, + .nlink = 1, + .namesize = sizeof("initramfs_test_hdr_hex_0"), + .fname = "initramfs_test_hdr_hex_0", + .filesize = sizeof(fdata), + .data = fdata, + }, { + .magic = "070701", + .ino = 2, + .mode = S_IFDIR | 0777, + .uid = 0x000056, + .gid = 0x000057, + .nlink = 1, + .namesize = sizeof("initramfs_test_hdr_hex_1"), + .fname = "initramfs_test_hdr_hex_1", + } }; + + /* inject_ox=true to add "0x" cpio field prefixes */ + len = fill_cpio(c, ARRAY_SIZE(c), true, tbufs->cpio_src); + + err = unpack_to_rootfs(tbufs->cpio_src, len); + KUNIT_EXPECT_NOT_NULL(test, err); + + kfree(tbufs); +} + /* * The kunit_case/_suite struct cannot be marked as __initdata as this will be * used in debugfs to retrieve results after test has run. @@ -507,11 +558,25 @@ static struct kunit_case __refdata initramfs_test_cases[] = { KUNIT_CASE(initramfs_test_many), KUNIT_CASE(initramfs_test_fname_pad), KUNIT_CASE(initramfs_test_fname_path_max), + KUNIT_CASE(initramfs_test_hdr_hex), {}, }; -static struct kunit_suite initramfs_test_suite = { +static int __init initramfs_test_init(struct kunit_suite *suite) +{ + /* + * unpack_to_rootfs() uses module-static state (victim, byte_count, + * state, ...). The boot-time async do_populate_rootfs() may still be + * running, so wait for it to finish before we call unpack_to_rootfs() + * from the test thread, otherwise the two writers race and crash. + */ + wait_for_initramfs(); + return 0; +} + +static struct kunit_suite __refdata initramfs_test_suite = { .name = "initramfs", + .suite_init = initramfs_test_init, .test_cases = initramfs_test_cases, }; kunit_test_init_section_suites(&initramfs_test_suite); diff --git a/init/main.c b/init/main.c index 96f93bb06c49..e363232b428b 100644 --- a/init/main.c +++ b/init/main.c @@ -324,51 +324,6 @@ static void * __init get_boot_config_from_initrd(size_t *_size) #ifdef CONFIG_BOOT_CONFIG -static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; - -#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) - -static int __init xbc_snprint_cmdline(char *buf, size_t size, - struct xbc_node *root) -{ - struct xbc_node *knode, *vnode; - char *end = buf + size; - const char *val, *q; - int ret; - - xbc_node_for_each_key_value(root, knode, val) { - ret = xbc_node_compose_key_after(root, knode, - xbc_namebuf, XBC_KEYLEN_MAX); - if (ret < 0) - return ret; - - vnode = xbc_node_get_child(knode); - if (!vnode) { - ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf); - if (ret < 0) - return ret; - buf += ret; - continue; - } - xbc_array_for_each_value(vnode, val) { - /* - * For prettier and more readable /proc/cmdline, only - * quote the value when necessary, i.e. when it contains - * whitespace. - */ - q = strpbrk(val, " \t\r\n") ? "\"" : ""; - ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", - xbc_namebuf, q, val, q); - if (ret < 0) - return ret; - buf += ret; - } - } - - return buf - (end - size); -} -#undef rest - /* Make an extra command line under given key word */ static char * __init xbc_make_cmdline(const char *key) { |