diff options
Diffstat (limited to 'lib')
125 files changed, 3529 insertions, 2267 deletions
diff --git a/lib/.gitignore b/lib/.gitignore index 101a4aa92fb5..42aee3ddb374 100644 --- a/lib/.gitignore +++ b/lib/.gitignore @@ -5,3 +5,4 @@ /gen_crc32table /gen_crc64table /oid_registry_data.c +/raid6 diff --git a/lib/Kconfig b/lib/Kconfig index 00a9509636c1..55748b68714e 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -8,17 +8,6 @@ config BINARY_PRINTF menu "Library routines" -config RAID6_PQ - tristate - -config RAID6_PQ_BENCHMARK - bool "Automatically choose fastest RAID6 PQ functions" - depends on RAID6_PQ - default y - help - Benchmark all available RAID6 PQ functions on init and choose the - fastest one. - config LINEAR_RANGES tristate @@ -54,14 +43,27 @@ config PACKING_KUNIT_TEST config BITREVERSE tristate + select GENERIC_BITREVERSE if !HAVE_ARCH_BITREVERSE config HAVE_ARCH_BITREVERSE bool default n + depends on BITREVERSE help This option enables the use of hardware bit-reversal instructions on architectures which support such operations. +config GENERIC_BITREVERSE + tristate + depends on BITREVERSE + help + Generic bit reversal implementation. Drivers should never enable + it explicitly. Instead, enable BITREVERSE. + + Architectures may want to select it as a fall-back option for + HAVE_ARCH_BITREVERSE, when the hardware-accelerated bit reverse + instruction set is optional, like RISC-V ZBKB extension. + config ARCH_HAS_STRNCPY_FROM_USER bool @@ -590,7 +592,7 @@ config OBJAGG config LWQ_TEST bool "Boot-time test for lwq queuing" help - Run boot-time test of light-weight queuing. + Run boot-time test of light-weight queuing. endmenu diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8ff5adcfe1e0..1244dcac2294 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -630,7 +630,7 @@ config DEBUG_FORCE_WEAK_PER_CPU config WARN_CONTEXT_ANALYSIS bool "Compiler context-analysis warnings" - depends on CC_IS_CLANG && CLANG_VERSION >= 220100 + depends on CC_IS_CLANG && CLANG_VERSION >= 230000 # Branch profiling re-defines "if", which messes with the compiler's # ability to analyze __cond_acquires(..), resulting in false positives. depends on !TRACE_BRANCH_PROFILING @@ -641,7 +641,7 @@ config WARN_CONTEXT_ANALYSIS and releasing user-definable "context locks". Clang's name of the feature is "Thread Safety Analysis". Requires - Clang 22.1.0 or later. + Clang 23 or later. Produces warnings by default. Select CONFIG_WERROR if you wish to turn these warnings into errors. @@ -2649,12 +2649,17 @@ config TEST_VMALLOC config TEST_WORKQUEUE tristate "Test module for stress/performance analysis of workqueue" + depends on m default n help This builds the "test_workqueue" module for benchmarking workqueue throughput under contention. Useful for evaluating affinity scope changes (e.g., cache_shard vs cache). + The test drives sysfs to switch affinity scopes, so it must be + loaded after userspace has mounted sysfs; building it in (=y) + would run module_init before /sys is available. + If unsure, say N. config TEST_BPF @@ -2971,6 +2976,16 @@ config BITS_TEST If unsure, say N. +config SHDI3_KUNIT_TEST + tristate "KUnit test for __ashldi3(), __ashrdi3(), and __lshrdi3()" + depends on KUNIT + depends on ARM || XTENSA || MICROBLAZE || ((RISCV || SPARC) && !64BIT) + help + This builds the unit test for __ashldi3(), __ashrdi3(), and + __lshrdi3() helper functions used to implement 64-bit arithmetic + shift left, arithmetic shift right and logical shift right, + respectively, on a 32-bit CPUs. + config SLUB_KUNIT_TEST tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS depends on SLUB_DEBUG && KUNIT @@ -3070,7 +3085,7 @@ config FORTIFY_KUNIT_TEST config LONGEST_SYM_KUNIT_TEST tristate "Test the longest symbol possible" if !KUNIT_ALL_TESTS depends on KUNIT && KPROBES - depends on !PREFIX_SYMBOLS && !CFI && !GCOV_KERNEL + depends on !CALL_PADDING && !CFI && !GCOV_KERNEL default KUNIT_ALL_TESTS help Tests the longest symbol possible diff --git a/lib/Makefile b/lib/Makefile index f33a24bf1c19..7f75cc6edf94 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -145,7 +145,7 @@ obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o obj-$(CONFIG_LIST_HARDENED) += list_debug.o obj-$(CONFIG_DEBUG_OBJECTS) += debugobjects.o -obj-$(CONFIG_BITREVERSE) += bitrev.o +obj-$(CONFIG_GENERIC_BITREVERSE) += bitrev.o obj-$(CONFIG_LINEAR_RANGES) += linear_ranges.o obj-$(CONFIG_PACKING) += packing.o obj-$(CONFIG_PACKING_KUNIT_TEST) += packing_test.o @@ -167,7 +167,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ obj-$(CONFIG_XZ_DEC) += xz/ -obj-$(CONFIG_RAID6_PQ) += raid6/ lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index ed1bdcf1f8ab..d9be1cf5187d 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -45,6 +45,7 @@ int alloc_tag_ref_offs; struct allocinfo_private { struct codetag_iterator iter; + struct codetag_iterator reported_iter; bool print_header; }; @@ -54,20 +55,24 @@ static void *allocinfo_start(struct seq_file *m, loff_t *pos) loff_t node = *pos; priv = (struct allocinfo_private *)m->private; - codetag_lock_module_list(alloc_tag_cttype, true); + codetag_lock_module_list(alloc_tag_cttype); if (node == 0) { priv->print_header = true; priv->iter = codetag_get_ct_iter(alloc_tag_cttype); - codetag_next_ct(&priv->iter); + } else { + priv->iter = priv->reported_iter; } + codetag_next_ct(&priv->iter); return priv->iter.ct ? priv : NULL; } static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) { struct allocinfo_private *priv = (struct allocinfo_private *)arg; - struct codetag *ct = codetag_next_ct(&priv->iter); + struct codetag *ct; + priv->reported_iter = priv->iter; + ct = codetag_next_ct(&priv->iter); (*pos)++; if (!ct) return NULL; @@ -77,7 +82,7 @@ static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos) static void allocinfo_stop(struct seq_file *m, void *arg) { - codetag_lock_module_list(alloc_tag_cttype, false); + codetag_unlock_module_list(alloc_tag_cttype); } static void print_allocinfo_header(struct seq_buf *buf) @@ -136,7 +141,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl return 0; if (can_sleep) - codetag_lock_module_list(alloc_tag_cttype, true); + codetag_lock_module_list(alloc_tag_cttype); else if (!codetag_trylock_module_list(alloc_tag_cttype)) return 0; @@ -161,7 +166,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl } } - codetag_lock_module_list(alloc_tag_cttype, false); + codetag_unlock_module_list(alloc_tag_cttype); return nr; } @@ -767,50 +772,82 @@ static __init bool need_page_alloc_tagging(void) * their codetag uninitialized. Track these early PFNs so we can clear * their codetag refs later to avoid warnings when they are freed. * - * Early allocations include: - * - Base allocations independent of CPU count - * - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init, - * such as trace ring buffers, scheduler per-cpu data) - * - * For simplicity, we fix the size to 8192. - * If insufficient, a warning will be triggered to alert the user. + * Each page is cast to a pfn_pool: the first few bytes hold metadata + * (next pointer and slot count), the remainder stores PFNs. + */ +struct pfn_pool { + struct pfn_pool *next; + atomic_t count; + unsigned long pfns[]; +}; + +#define PFN_POOL_SIZE ((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \ + sizeof(unsigned long)) + +/* + * Skip early PFN recording for a page allocation. Reuses the + * %__GFP_NO_OBJ_EXT bit. Used by __alloc_tag_add_early_pfn() to avoid + * recursion when allocating pages for the early PFN tracking list + * itself. * - * TODO: Replace fixed-size array with dynamic allocation using - * a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion. + * Codetags of the pages allocated with __GFP_NO_CODETAG should be + * cleared (via clear_page_tag_ref()) before freeing the pages to prevent + * alloc_tag_sub_check() from triggering a warning. */ -#define EARLY_ALLOC_PFN_MAX 8192 +#define __GFP_NO_CODETAG __GFP_NO_OBJ_EXT -static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX] __initdata; -static atomic_t early_pfn_count __initdata = ATOMIC_INIT(0); +static struct pfn_pool *current_pfn_pool __initdata; static void __init __alloc_tag_add_early_pfn(unsigned long pfn) { - int old_idx, new_idx; + struct pfn_pool *pool; + int idx; do { - old_idx = atomic_read(&early_pfn_count); - if (old_idx >= EARLY_ALLOC_PFN_MAX) { - pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n", - EARLY_ALLOC_PFN_MAX); - return; + pool = READ_ONCE(current_pfn_pool); + if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) { + struct page *new_page = alloc_page(__GFP_HIGH | __GFP_NO_CODETAG); + struct pfn_pool *new; + + if (!new_page) { + pr_warn_once("early PFN tracking page allocation failed\n"); + return; + } + new = page_address(new_page); + new->next = pool; + atomic_set(&new->count, 0); + if (cmpxchg(¤t_pfn_pool, pool, new) != pool) { + clear_page_tag_ref(new_page); + __free_page(new_page); + continue; + } + pool = new; } - new_idx = old_idx + 1; - } while (!atomic_try_cmpxchg(&early_pfn_count, &old_idx, new_idx)); + idx = atomic_read(&pool->count); + if (idx >= PFN_POOL_SIZE) + continue; + if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx) + break; + } while (1); - early_pfns[old_idx] = pfn; + pool->pfns[idx] = pfn; } typedef void alloc_tag_add_func(unsigned long pfn); static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata = RCU_INITIALIZER(__alloc_tag_add_early_pfn); -void alloc_tag_add_early_pfn(unsigned long pfn) +void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) { alloc_tag_add_func *alloc_tag_add; if (static_key_enabled(&mem_profiling_compressed)) return; + /* Skip allocations for the tracking list itself to avoid recursion. */ + if (gfp_flags & __GFP_NO_CODETAG) + return; + rcu_read_lock(); alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr); if (alloc_tag_add) @@ -820,7 +857,9 @@ void alloc_tag_add_early_pfn(unsigned long pfn) static void __init clear_early_alloc_pfn_tag_refs(void) { - unsigned int i; + struct pfn_pool *pool, *next; + struct page *page; + int i; if (static_key_enabled(&mem_profiling_compressed)) return; @@ -829,37 +868,45 @@ static void __init clear_early_alloc_pfn_tag_refs(void) /* Make sure we are not racing with __alloc_tag_add_early_pfn() */ synchronize_rcu(); - for (i = 0; i < atomic_read(&early_pfn_count); i++) { - unsigned long pfn = early_pfns[i]; - - if (pfn_valid(pfn)) { - struct page *page = pfn_to_page(pfn); - union pgtag_ref_handle handle; - union codetag_ref ref; - - if (get_page_tag_ref(page, &ref, &handle)) { - /* - * An early-allocated page could be freed and reallocated - * after its page_ext is initialized but before we clear it. - * In that case, it already has a valid tag set. - * We should not overwrite that valid tag with CODETAG_EMPTY. - * - * Note: there is still a small race window between checking - * ref.ct and calling set_codetag_empty(). We accept this - * race as it's unlikely and the extra complexity of atomic - * cmpxchg is not worth it for this debug-only code path. - */ - if (ref.ct) { + for (pool = current_pfn_pool; pool; pool = next) { + int nr_pfns = atomic_read(&pool->count); + + for (i = 0; i < nr_pfns; i++) { + unsigned long pfn = pool->pfns[i]; + + if (pfn_valid(pfn)) { + union pgtag_ref_handle handle; + union codetag_ref ref; + + if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) { + /* + * An early-allocated page could be freed and reallocated + * after its page_ext is initialized but before we clear it. + * In that case, it already has a valid tag set. + * We should not overwrite that valid tag + * with CODETAG_EMPTY. + * + * Note: there is still a small race window between checking + * ref.ct and calling set_codetag_empty(). We accept this + * race as it's unlikely and the extra complexity of atomic + * cmpxchg is not worth it for this debug-only code path. + */ + if (ref.ct) { + put_page_tag_ref(handle); + continue; + } + + set_codetag_empty(&ref); + update_page_tag_ref(handle, &ref); put_page_tag_ref(handle); - continue; } - - set_codetag_empty(&ref); - update_page_tag_ref(handle, &ref); - put_page_tag_ref(handle); } } + next = pool->next; + page = virt_to_page(pool); + clear_page_tag_ref(page); + __free_page(page); } } #else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */ diff --git a/lib/base64.c b/lib/base64.c index 41961a444028..325c7332b049 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -122,7 +122,7 @@ EXPORT_SYMBOL_GPL(base64_encode); * @src: the string to decode. Doesn't need to be NUL-terminated. * @srclen: the length of @src in bytes * @dst: (output) the decoded binary data - * @padding: whether to append '=' padding characters + * @padding: whether the input is expected to include '=' padding characters * @variant: which base64 variant to use * * Decodes a string using the selected Base64 variant. @@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6 return -1; val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); - *bp++ = val >> 10; if (srclen == 2) { if (val & 0x800003ff) return -1; + *bp++ = val >> 10; } else { val |= base64_rev_tables[s[2]]; if (val & 0x80000003) return -1; + *bp++ = val >> 10; *bp++ = val >> 2; } return bp - dst; diff --git a/lib/bitmap-str.c b/lib/bitmap-str.c index be745209507a..26d36c938c6a 100644 --- a/lib/bitmap-str.c +++ b/lib/bitmap-str.c @@ -40,32 +40,6 @@ int bitmap_parse_user(const char __user *ubuf, EXPORT_SYMBOL(bitmap_parse_user); /** - * bitmap_print_to_pagebuf - convert bitmap to list or hex format ASCII string - * @list: indicates whether the bitmap must be list - * @buf: page aligned buffer into which string is placed - * @maskp: pointer to bitmap to convert - * @nmaskbits: size of bitmap, in bits - * - * Output format is a comma-separated list of decimal numbers and - * ranges if list is specified or hex digits grouped into comma-separated - * sets of 8 digits/set. Returns the number of characters written to buf. - * - * It is assumed that @buf is a pointer into a PAGE_SIZE, page-aligned - * area and that sufficient storage remains at @buf to accommodate the - * bitmap_print_to_pagebuf() output. Returns the number of characters - * actually printed to @buf, excluding terminating '\0'. - */ -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - int nmaskbits) -{ - ptrdiff_t len = PAGE_SIZE - offset_in_page(buf); - - return list ? scnprintf(buf, len, "%*pbl\n", nmaskbits, maskp) : - scnprintf(buf, len, "%*pb\n", nmaskbits, maskp); -} -EXPORT_SYMBOL(bitmap_print_to_pagebuf); - -/** * bitmap_print_to_buf - convert bitmap to list or hex format ASCII string * @list: indicates whether the bitmap must be list * true: print in decimal list format @@ -101,7 +75,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * @off: in the string from which we are copying, We copy to @buf * @count: the maximum number of bytes to print * - * The bitmap_print_to_pagebuf() is used indirectly via its cpumap wrapper + * The sprintf("%*pb[l]") is used indirectly via its cpumap wrapper * cpumap_print_to_pagebuf() or directly by drivers to export hexadecimal * bitmask and decimal list to userspace by sysfs ABI. * Drivers might be using a normal attribute for this kind of ABIs. A @@ -111,18 +85,11 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * struct device_attribute *attr, char *buf) * { * ... - * return bitmap_print_to_pagebuf(true, buf, &mask, nr_trig_max); + * return scnprintf(buf, PAGE_SIZE - offset_in_page(buf), nr_trig_max, &mask); * } * * show entry of attribute has no offset and count parameters and this * means the file is limited to one page only. - * bitmap_print_to_pagebuf() API works terribly well for this kind of - * normal attribute with buf parameter and without offset, count:: - * - * bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, - * int nmaskbits) - * { - * } * * The problem is once we have a large bitmap, we have a chance to get a * bitmask or list more than one page. Especially for list, it could be @@ -149,7 +116,7 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * * The role of cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf() * is similar with cpumap_print_to_pagebuf(), the difference is that - * bitmap_print_to_pagebuf() mainly serves sysfs attribute with the assumption + * scnprintf("%*pb[l]") mainly serves sysfs attribute with the assumption * the destination buffer is exactly one page and won't be more than one page. * cpumap_print_bitmask_to_buf() and cpumap_print_list_to_buf(), on the other * hand, mainly serves bin_attribute which doesn't work with exact one page, @@ -158,7 +125,8 @@ static int bitmap_print_to_buf(bool list, char *buf, const unsigned long *maskp, * * WARNING! * - * This function is not a replacement for sprintf() or bitmap_print_to_pagebuf(). + * This function is not a replacement for sprintf(). + * * It is intended to workaround sysfs limitations discussed above and should be * used carefully in general case for the following reasons: * diff --git a/lib/bitrev.c b/lib/bitrev.c index 81b56e0a7f32..05088231f31f 100644 --- a/lib/bitrev.c +++ b/lib/bitrev.c @@ -1,5 +1,4 @@ // SPDX-License-Identifier: GPL-2.0-only -#ifndef CONFIG_HAVE_ARCH_BITREVERSE #include <linux/types.h> #include <linux/module.h> #include <linux/bitrev.h> @@ -43,5 +42,3 @@ const u8 byte_rev_table[256] = { 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, }; EXPORT_SYMBOL_GPL(byte_rev_table); - -#endif /* CONFIG_HAVE_ARCH_BITREVERSE */ diff --git a/lib/bootconfig.c b/lib/bootconfig.c index c470b93d5dbc..2ed9ee3dc81c 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -408,6 +408,71 @@ const char * __init xbc_node_find_next_key_value(struct xbc_node *root, return ""; /* No value key */ } +static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; + +#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) + +/** + * xbc_snprint_cmdline() - Render bootconfig keys under @root as a cmdline string + * @buf: Destination buffer (may be NULL when @size is 0 to query the length) + * @size: Size of @buf in bytes + * @root: Subtree root whose key=value pairs should be rendered + * + * Walk all key/value pairs under @root and emit them as a space-separated + * cmdline string into @buf. Values containing whitespace are quoted with + * double quotes. Returns the number of bytes that would be written if @buf + * were large enough (matching snprintf semantics), or a negative errno on + * failure. + */ +int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +{ + struct xbc_node *knode, *vnode; + const char *val, *q; + size_t len = 0; + int ret; + + /* + * Track the running written length rather than advancing @buf, so we + * never form "buf + size" or "buf += ret" while @buf is NULL (the + * size-probe call passes buf=NULL, size=0). NULL pointer arithmetic + * is undefined behavior and trips host UBSan / FORTIFY_SOURCE when + * this renderer runs at kernel build time. snprintf(NULL, 0, ...) + * itself is well defined and returns the would-be length. + */ + xbc_node_for_each_key_value(root, knode, val) { + ret = xbc_node_compose_key_after(root, knode, + xbc_namebuf, XBC_KEYLEN_MAX); + if (ret < 0) + return ret; + + vnode = xbc_node_get_child(knode); + if (!vnode) { + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s ", xbc_namebuf); + if (ret < 0) + return ret; + len += ret; + continue; + } + xbc_array_for_each_value(vnode, val) { + /* + * For prettier and more readable /proc/cmdline, only + * quote the value when necessary, i.e. when it contains + * whitespace. + */ + q = strpbrk(val, " \t\r\n") ? "\"" : ""; + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s=%s%s%s ", xbc_namebuf, q, val, q); + if (ret < 0) + return ret; + len += ret; + } + } + + return len; +} +#undef rest + /* XBC parse and tree build */ static int __init xbc_init_node(struct xbc_node *node, char *data, uint16_t flag) diff --git a/lib/bug.c b/lib/bug.c index 224f4cfa4aa3..7c1c2c27f58e 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -1,41 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 /* - Generic support for BUG() - - This respects the following config options: - - CONFIG_BUG - emit BUG traps. Nothing happens without this. - CONFIG_GENERIC_BUG - enable this code. - CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file - CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG - - CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable - (though they're generally always on). - - CONFIG_GENERIC_BUG is set by each architecture using this code. - - To use this, your architecture must: - - 1. Set up the config options: - - Enable CONFIG_GENERIC_BUG if CONFIG_BUG - - 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) - - Define HAVE_ARCH_BUG - - Implement BUG() to generate a faulting instruction - - NOTE: struct bug_entry does not have "file" or "line" entries - when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate - the values accordingly. - - 3. Implement the trap - - In the illegal instruction trap handler (typically), verify - that the fault was in kernel mode, and call report_bug() - - report_bug() will return whether it was a false alarm, a warning, - or an actual bug. - - You must implement the is_valid_bugaddr(bugaddr) callback which - returns true if the eip is a real kernel address, and it points - to the expected BUG trap instruction. - - Jeremy Fitzhardinge <jeremy@goop.org> 2006 + * Generic support for BUG() + * + * This respects the following config options: + * + * CONFIG_BUG - emit BUG traps. Nothing happens without this. + * CONFIG_GENERIC_BUG - enable this code. + * CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file + * CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG + * + * CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable + * (though they're generally always on). + * + * CONFIG_GENERIC_BUG is set by each architecture using this code. + * + * To use this, your architecture must: + * + * 1. Set up the config options: + * - Enable CONFIG_GENERIC_BUG if CONFIG_BUG + * + * 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) + * - Define HAVE_ARCH_BUG + * - Implement BUG() to generate a faulting instruction + * - NOTE: struct bug_entry does not have "file" or "line" entries + * when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate + * the values accordingly. + * + * 3. Implement the trap + * - In the illegal instruction trap handler (typically), verify + * that the fault was in kernel mode, and call report_bug() + * - report_bug() will return whether it was a false alarm, a warning, + * or an actual bug. + * - You must implement the is_valid_bugaddr(bugaddr) callback which + * returns true if the eip is a real kernel address, and it points + * to the expected BUG trap instruction. + * + * Jeremy Fitzhardinge <jeremy@goop.org> 2006 */ #define pr_fmt(fmt) fmt @@ -48,6 +48,7 @@ #include <linux/rculist.h> #include <linux/ftrace.h> #include <linux/context_tracking.h> +#include <kunit/test-bug.h> extern struct bug_entry __start___bug_table[], __stop___bug_table[]; @@ -71,7 +72,7 @@ static struct bug_entry *module_find_bug(unsigned long bugaddr) guard(rcu)(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { - unsigned i; + unsigned int i; bug = mod->bug_table; for (i = 0; i < mod->num_bugs; ++i, ++bug) @@ -191,14 +192,14 @@ void __warn_printf(const char *fmt, struct pt_regs *regs) } #endif - printk("%s", fmt); + pr_warn("%s", fmt); } static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) { bool warning, once, done, no_cut, has_args; const char *file, *fmt; - unsigned line; + unsigned int line; if (!bug) { if (!is_valid_bugaddr(bugaddr)) @@ -209,8 +210,6 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga return BUG_TRAP_TYPE_NONE; } - disable_trace_on_warning(); - bug_get_file_line(bug, &file, &line); fmt = bug_get_format(bug); @@ -220,6 +219,15 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga no_cut = bug->flags & BUGFLAG_NO_CUT_HERE; has_args = bug->flags & BUGFLAG_ARGS; + /* + * Before the once logic so suppressed warnings do not consume + * the single-fire budget of WARN_ON_ONCE(). + */ + if (warning && kunit_is_suppressed_warning(true)) + return BUG_TRAP_TYPE_WARN; + + disable_trace_on_warning(); + if (warning && once) { if (done) return BUG_TRAP_TYPE_WARN; @@ -237,7 +245,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga * extra debugging message it writes before triggering the handler. */ if (!no_cut) { - printk(KERN_DEFAULT CUT_HERE); + pr_info(CUT_HERE); __warn_printf(fmt, has_args ? regs : NULL); } diff --git a/lib/cmdline.c b/lib/cmdline.c index 90ed997d9570..16cce6621cec 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n) * When @pint is NULL the function can be used as a validator of * the current option in the string. * - * Return values: + * Return: * 0 - no int in string * 1 - int found, no subsequent comma * 2 - int found including a subsequent comma @@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options); * * Parses a string into a number. The number stored at @ptr is * potentially suffixed with K, M, G, T, P, E. + * + * Return: The value as recognized by simple_strtoull() multiplied + * by the value as specified by suffix, if any. */ unsigned long long memparse(const char *ptr, char **retptr) { char *endptr; /* local pointer to end of parsed string */ - unsigned long long ret = simple_strtoull(ptr, &endptr, 0); + unsigned int shl = 0; + /* Consume valid suffix even in case of overflow. */ switch (*endptr) { case 'E': case 'e': - ret <<= 10; + shl += 10; fallthrough; case 'P': case 'p': - ret <<= 10; + shl += 10; fallthrough; case 'T': case 't': - ret <<= 10; + shl += 10; fallthrough; case 'G': case 'g': - ret <<= 10; + shl += 10; fallthrough; case 'M': case 'm': - ret <<= 10; + shl += 10; fallthrough; case 'K': case 'k': - ret <<= 10; - endptr++; + shl += 10; fallthrough; default: break; } + if (shl && likely(ptr != endptr)) { + /* Have valid suffix with preceding number. */ + if (unlikely(check_shl_overflow(ret, shl, &ret))) + ret = ULLONG_MAX; + endptr++; + } + if (retptr) *retptr = endptr; @@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse); * This function parses a string containing a comma-separated list of * strings like a=b,c. * - * Return true if there's such option in the string, or return false. + * Return: True if there's such option in the string or false otherwise. */ bool parse_option_str(const char *str, const char *option) { diff --git a/lib/codetag.c b/lib/codetag.c index 304667897ad4..4001a7ea6675 100644 --- a/lib/codetag.c +++ b/lib/codetag.c @@ -35,12 +35,9 @@ struct codetag_module { static DEFINE_MUTEX(codetag_lock); static LIST_HEAD(codetag_types); -void codetag_lock_module_list(struct codetag_type *cttype, bool lock) +void codetag_lock_module_list(struct codetag_type *cttype) { - if (lock) - down_read(&cttype->mod_lock); - else - up_read(&cttype->mod_lock); + down_read(&cttype->mod_lock); } bool codetag_trylock_module_list(struct codetag_type *cttype) @@ -48,6 +45,11 @@ bool codetag_trylock_module_list(struct codetag_type *cttype) return down_read_trylock(&cttype->mod_lock) != 0; } +void codetag_unlock_module_list(struct codetag_type *cttype) +{ + up_read(&cttype->mod_lock); +} + struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype) { struct codetag_iterator iter = { diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 3d6b8996f027..fee1dfccd116 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -4,32 +4,32 @@ #include <linux/audit_arch.h> #include <asm/unistd32.h> -unsigned compat_dir_class[] = { +unsigned int compat_dir_class[] = { #include <asm-generic/audit_dir_write.h> ~0U }; -unsigned compat_read_class[] = { +unsigned int compat_read_class[] = { #include <asm-generic/audit_read.h> ~0U }; -unsigned compat_write_class[] = { +unsigned int compat_write_class[] = { #include <asm-generic/audit_write.h> ~0U }; -unsigned compat_chattr_class[] = { +unsigned int compat_chattr_class[] = { #include <asm-generic/audit_change_attr.h> ~0U }; -unsigned compat_signal_class[] = { +unsigned int compat_signal_class[] = { #include <asm-generic/audit_signal.h> ~0U }; -int audit_classify_compat_syscall(int abi, unsigned syscall) +int audit_classify_compat_syscall(int abi, unsigned int syscall) { switch (syscall) { #ifdef __NR_open diff --git a/lib/crc/Kconfig b/lib/crc/Kconfig index f47bb4c706fb..927fc6a6b2b9 100644 --- a/lib/crc/Kconfig +++ b/lib/crc/Kconfig @@ -82,6 +82,7 @@ config CRC64 config CRC64_ARCH bool depends on CRC64 && CRC_OPTIMIZATIONS + default y if ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN default y if ARM64 default y if RISCV && RISCV_ISA_ZBC && 64BIT default y if X86_64 diff --git a/lib/crc/Makefile b/lib/crc/Makefile index ff213590e4e3..386e9c175263 100644 --- a/lib/crc/Makefile +++ b/lib/crc/Makefile @@ -39,9 +39,12 @@ crc64-y := crc64-main.o ifeq ($(CONFIG_CRC64_ARCH),y) CFLAGS_crc64-main.o += -I$(src)/$(SRCARCH) -CFLAGS_REMOVE_arm64/crc64-neon-inner.o += $(CC_FLAGS_NO_FPU) -CFLAGS_arm64/crc64-neon-inner.o += $(CC_FLAGS_FPU) -march=armv8-a+crypto -crc64-$(CONFIG_ARM64) += arm64/crc64-neon-inner.o +crc64-cflags-$(CONFIG_ARM) += -march=armv8-a -mfpu=crypto-neon-fp-armv8 +crc64-cflags-$(CONFIG_ARM64) += -march=armv8-a+crypto +CFLAGS_REMOVE_crc64-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_crc64-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) $(crc64-cflags-y) +crc64-$(CONFIG_ARM) += crc64-neon.o +crc64-$(CONFIG_ARM64) += crc64-neon.o crc64-$(CONFIG_RISCV) += riscv/crc64_lsb.o riscv/crc64_msb.o crc64-$(CONFIG_X86) += x86/crc64-pclmul.o diff --git a/lib/crc/arm/crc64-neon.h b/lib/crc/arm/crc64-neon.h new file mode 100644 index 000000000000..645f553220ff --- /dev/null +++ b/lib/crc/arm/crc64-neon.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 0); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 1); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + uint64_t l = vgetq_lane_u64(a, 1); + uint64_t m = vgetq_lane_u64(b, 0); + uint64x2_t result; + + asm("vmull.p64 %q0, %P1, %P2" : "=w"(result) : "w"(l), "w"(m)); + + return result; +} diff --git a/lib/crc/arm/crc64.h b/lib/crc/arm/crc64.h new file mode 100644 index 000000000000..de274288af61 --- /dev/null +++ b/lib/crc/arm/crc64.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * CRC64 using ARM PMULL instructions + */ + +#include <asm/simd.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); + +#define crc64_be_arch crc64_be_generic + +static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) +{ + if (len >= 128 && static_branch_likely(&have_pmull) && + likely(may_use_simd())) { + do { + size_t chunk = min_t(size_t, len & ~15, SZ_4K); + + scoped_ksimd() + crc = crc64_nvme_neon(crc, p, chunk); + + p += chunk; + len -= chunk; + } while (len >= 128); + } + return crc64_nvme_generic(crc, p, len); +} + +#define crc64_mod_init_arch crc64_mod_init_arch +static void crc64_mod_init_arch(void) +{ + if (elf_hwcap2 & HWCAP2_PMULL) + static_branch_enable(&have_pmull); +} diff --git a/lib/crc/arm64/crc64-neon.h b/lib/crc/arm64/crc64-neon.h new file mode 100644 index 000000000000..fcd5b1e6f812 --- /dev/null +++ b/lib/crc/arm64/crc64-neon.h @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0-only + +static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), + vgetq_lane_u64(b, 0))); +} + +static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) +{ + poly64x2_t l = vreinterpretq_p64_u64(a); + poly64x2_t m = vreinterpretq_p64_u64(b); + + return vreinterpretq_u64_p128(vmull_high_p64(l, m)); +} + +static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) +{ + return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), + vgetq_lane_u64(b, 0))); +} diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h index 60151ec3035a..c7a69e1f3d8f 100644 --- a/lib/crc/arm64/crc64.h +++ b/lib/crc/arm64/crc64.h @@ -8,7 +8,7 @@ #include <linux/minmax.h> #include <linux/sizes.h> -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); #define crc64_be_arch crc64_be_generic @@ -19,7 +19,7 @@ static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len) size_t chunk = len & ~15; scoped_ksimd() - crc = crc64_nvme_arm64_c(crc, p, chunk); + crc = crc64_nvme_neon(crc, p, chunk); p += chunk; len &= 15; diff --git a/lib/crc/arm64/crc64-neon-inner.c b/lib/crc/crc64-neon.c index 28527e544ff6..4753fb94a4be 100644 --- a/lib/crc/arm64/crc64-neon-inner.c +++ b/lib/crc/crc64-neon.c @@ -6,7 +6,9 @@ #include <linux/types.h> #include <asm/neon-intrinsics.h> -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len); +#include "crc64-neon.h" + +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len); /* x^191 mod G, x^127 mod G */ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, @@ -15,27 +17,7 @@ static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL, static const u64 bconsts_val[2] = { 0x27ecfa329aef9f77ULL, 0x34d926535897936aULL }; -static inline uint64x2_t pmull64(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 0), - vgetq_lane_u64(b, 0))); -} - -static inline uint64x2_t pmull64_high(uint64x2_t a, uint64x2_t b) -{ - poly64x2_t l = vreinterpretq_p64_u64(a); - poly64x2_t m = vreinterpretq_p64_u64(b); - - return vreinterpretq_u64_p128(vmull_high_p64(l, m)); -} - -static inline uint64x2_t pmull64_hi_lo(uint64x2_t a, uint64x2_t b) -{ - return vreinterpretq_u64_p128(vmull_p64(vgetq_lane_u64(a, 1), - vgetq_lane_u64(b, 0))); -} - -u64 crc64_nvme_arm64_c(u64 crc, const u8 *p, size_t len) +u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len) { uint64x2_t fold_consts = vld1q_u64(fold_consts_val); uint64x2_t v0 = { crc, 0 }; diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index d3904b72dae7..591c1c2a7fb3 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -131,11 +131,6 @@ config CRYPTO_LIB_MD5 The MD5 and HMAC-MD5 library functions. Select this if your module uses any of the functions from <crypto/md5.h>. -config CRYPTO_LIB_MD5_ARCH - bool - depends on CRYPTO_LIB_MD5 && !UML - default y if PPC - config CRYPTO_LIB_MLDSA tristate select CRYPTO_LIB_SHA3 diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 4ad91f390038..f1e9bf89785f 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -187,10 +187,6 @@ clean-files += powerpc/ghashp8-ppc.S obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o -ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) -CFLAGS_md5.o += -I$(src)/$(SRCARCH) -libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o -endif # CONFIG_CRYPTO_LIB_MD5_ARCH ################################################################################ diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c index 2650603d8ba8..8dcdf5ec98be 100644 --- a/lib/crypto/gf128hash.c +++ b/lib/crypto/gf128hash.c @@ -109,7 +109,7 @@ static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) #else /* CONFIG_ARCH_SUPPORTS_INT128 */ /* Do a 32 x 32 => 64 bit carryless multiplication. */ -static u64 clmul32(u32 a, u32 b) +static noinline_for_stack u64 clmul32(u32 a, u32 b) { /* * With 32-bit multiplicands and one term every 4 bits, there are up to diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c index c4af57db0ea8..6bf130cfbbf9 100644 --- a/lib/crypto/md5.c +++ b/lib/crypto/md5.c @@ -2,8 +2,8 @@ /* * MD5 and HMAC-MD5 library functions * - * md5_block_generic() is derived from cryptoapi implementation, originally - * based on the public domain implementation written by Colin Plumb in 1993. + * md5_block() is derived from cryptoapi implementation, originally based on the + * public domain implementation written by Colin Plumb in 1993. * * Copyright (c) Cryptoapi developers. * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> @@ -31,8 +31,8 @@ static const struct md5_block_state md5_iv = { #define MD5STEP(f, w, x, y, z, in, s) \ (w += f(x, y, z) + in, w = rol32(w, s) + x) -static void md5_block_generic(struct md5_block_state *state, - const u8 data[MD5_BLOCK_SIZE]) +static void md5_block(struct md5_block_state *state, + const u8 data[MD5_BLOCK_SIZE]) { u32 in[MD5_BLOCK_WORDS]; u32 a, b, c, d; @@ -119,21 +119,15 @@ static void md5_block_generic(struct md5_block_state *state, state->h[3] += d; } -static void __maybe_unused md5_blocks_generic(struct md5_block_state *state, - const u8 *data, size_t nblocks) +static void md5_blocks(struct md5_block_state *state, + const u8 *data, size_t nblocks) { do { - md5_block_generic(state, data); + md5_block(state, data); data += MD5_BLOCK_SIZE; } while (--nblocks); } -#ifdef CONFIG_CRYPTO_LIB_MD5_ARCH -#include "md5.h" /* $(SRCARCH)/md5.h */ -#else -#define md5_blocks md5_blocks_generic -#endif - void md5_init(struct md5_ctx *ctx) { ctx->state = md5_iv; diff --git a/lib/crypto/powerpc/md5-asm.S b/lib/crypto/powerpc/md5-asm.S deleted file mode 100644 index fa6bc440cf4a..000000000000 --- a/lib/crypto/powerpc/md5-asm.S +++ /dev/null @@ -1,235 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Fast MD5 implementation for PPC - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> - -#define rHP r3 -#define rWP r4 - -#define rH0 r0 -#define rH1 r6 -#define rH2 r7 -#define rH3 r5 - -#define rW00 r8 -#define rW01 r9 -#define rW02 r10 -#define rW03 r11 -#define rW04 r12 -#define rW05 r14 -#define rW06 r15 -#define rW07 r16 -#define rW08 r17 -#define rW09 r18 -#define rW10 r19 -#define rW11 r20 -#define rW12 r21 -#define rW13 r22 -#define rW14 r23 -#define rW15 r24 - -#define rT0 r25 -#define rT1 r26 - -#define INITIALIZE \ - PPC_STLU r1,-INT_FRAME_SIZE(r1); \ - SAVE_GPRS(14, 26, r1) /* push registers onto stack */ - -#define FINALIZE \ - REST_GPRS(14, 26, r1); /* pop registers from stack */ \ - addi r1,r1,INT_FRAME_SIZE - -#ifdef __BIG_ENDIAN__ -#define LOAD_DATA(reg, off) \ - lwbrx reg,0,rWP; /* load data */ -#define INC_PTR \ - addi rWP,rWP,4; /* increment per word */ -#define NEXT_BLOCK /* nothing to do */ -#else -#define LOAD_DATA(reg, off) \ - lwz reg,off(rWP); /* load data */ -#define INC_PTR /* nothing to do */ -#define NEXT_BLOCK \ - addi rWP,rWP,64; /* increment per block */ -#endif - -#define R_00_15(a, b, c, d, w0, w1, p, q, off, k0h, k0l, k1h, k1l) \ - LOAD_DATA(w0, off) /* W */ \ - and rT0,b,c; /* 1: f = b and c */ \ - INC_PTR /* ptr++ */ \ - andc rT1,d,b; /* 1: f' = ~b and d */ \ - LOAD_DATA(w1, off+4) /* W */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - addis w1,w1,k1h; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - and rT0,a,b; /* 2: f = b and c */ \ - andc rT1,c,a; /* 2: f' = ~b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - INC_PTR /* ptr++ */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_16_31(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - andc rT0,c,d; /* 1: f = c and ~d */ \ - and rT1,b,d; /* 1: f' = b and d */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - or rT0,rT0,rT1; /* 1: f = f or f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - andc rT0,b,c; /* 2: f = c and ~d */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - add d,d,w1; /* 2: a = a + wk */ \ - and rT1,a,c; /* 2: f' = b and d */ \ - or rT0,rT0,rT1; /* 2: f = f or f' */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a +b */ - -#define R_32_47(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - xor rT0,b,c; /* 1: f' = b xor c */ \ - addi w0,w0,k0l; /* 1: wk = w + k */ \ - xor rT1,rT0,d; /* 1: f = f xor f' */ \ - addis w0,w0,k0h; /* 1: wk = w + k' */ \ - add a,a,rT1; /* 1: a = a + f */ \ - addi w1,w1,k1l; /* 2: wk = w + k */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addis w1,w1,k1h; /* 2: wk = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add d,d,w1; /* 2: a = a + wk */ \ - add a,a,b; /* 1: a = a + b */ \ - xor rT1,rT0,a; /* 2: f = b xor f' */ \ - add d,d,rT1; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -#define R_48_63(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ - addi w0,w0,k0l; /* 1: w = w + k */ \ - orc rT0,b,d; /* 1: f = b or ~d */ \ - addis w0,w0,k0h; /* 1: w = w + k' */ \ - xor rT0,rT0,c; /* 1: f = f xor c */ \ - add a,a,w0; /* 1: a = a + wk */ \ - addi w1,w1,k1l; /* 2: w = w + k */ \ - add a,a,rT0; /* 1: a = a + f */ \ - addis w1,w1,k1h; /* 2: w = w + k' */ \ - rotrwi a,a,p; /* 1: a = a rotl x */ \ - add a,a,b; /* 1: a = a + b */ \ - orc rT0,a,c; /* 2: f = b or ~d */ \ - add d,d,w1; /* 2: a = a + wk */ \ - xor rT0,rT0,b; /* 2: f = f xor c */ \ - add d,d,rT0; /* 2: a = a + f */ \ - rotrwi d,d,q; /* 2: a = a rotl x */ \ - add d,d,a; /* 2: a = a + b */ - -_GLOBAL(ppc_md5_transform) - INITIALIZE - - mtctr r5 - lwz rH0,0(rHP) - lwz rH1,4(rHP) - lwz rH2,8(rHP) - lwz rH3,12(rHP) - -ppc_md5_main: - R_00_15(rH0, rH1, rH2, rH3, rW00, rW01, 25, 20, 0, - 0xd76b, -23432, 0xe8c8, -18602) - R_00_15(rH2, rH3, rH0, rH1, rW02, rW03, 15, 10, 8, - 0x2420, 0x70db, 0xc1be, -12562) - R_00_15(rH0, rH1, rH2, rH3, rW04, rW05, 25, 20, 16, - 0xf57c, 0x0faf, 0x4788, -14806) - R_00_15(rH2, rH3, rH0, rH1, rW06, rW07, 15, 10, 24, - 0xa830, 0x4613, 0xfd47, -27391) - R_00_15(rH0, rH1, rH2, rH3, rW08, rW09, 25, 20, 32, - 0x6981, -26408, 0x8b45, -2129) - R_00_15(rH2, rH3, rH0, rH1, rW10, rW11, 15, 10, 40, - 0xffff, 0x5bb1, 0x895d, -10306) - R_00_15(rH0, rH1, rH2, rH3, rW12, rW13, 25, 20, 48, - 0x6b90, 0x1122, 0xfd98, 0x7193) - R_00_15(rH2, rH3, rH0, rH1, rW14, rW15, 15, 10, 56, - 0xa679, 0x438e, 0x49b4, 0x0821) - - R_16_31(rH0, rH1, rH2, rH3, rW01, rW06, 27, 23, - 0x0d56, 0x6e0c, 0x1810, 0x6d2d) - R_16_31(rH2, rH3, rH0, rH1, rW11, rW00, 18, 12, - 0x9d02, -32109, 0x124c, 0x2332) - R_16_31(rH0, rH1, rH2, rH3, rW05, rW10, 27, 23, - 0x8ea7, 0x4a33, 0x0245, -18270) - R_16_31(rH2, rH3, rH0, rH1, rW15, rW04, 18, 12, - 0x8eee, -8608, 0xf258, -5095) - R_16_31(rH0, rH1, rH2, rH3, rW09, rW14, 27, 23, - 0x969d, -10697, 0x1cbe, -15288) - R_16_31(rH2, rH3, rH0, rH1, rW03, rW08, 18, 12, - 0x3317, 0x3e99, 0xdbd9, 0x7c15) - R_16_31(rH0, rH1, rH2, rH3, rW13, rW02, 27, 23, - 0xac4b, 0x7772, 0xd8cf, 0x331d) - R_16_31(rH2, rH3, rH0, rH1, rW07, rW12, 18, 12, - 0x6a28, 0x6dd8, 0x219a, 0x3b68) - - R_32_47(rH0, rH1, rH2, rH3, rW05, rW08, 28, 21, - 0x29cb, 0x28e5, 0x4218, -7788) - R_32_47(rH2, rH3, rH0, rH1, rW11, rW14, 16, 9, - 0x473f, 0x06d1, 0x3aae, 0x3036) - R_32_47(rH0, rH1, rH2, rH3, rW01, rW04, 28, 21, - 0xaea1, -15134, 0x640b, -11295) - R_32_47(rH2, rH3, rH0, rH1, rW07, rW10, 16, 9, - 0x8f4c, 0x4887, 0xbc7c, -22499) - R_32_47(rH0, rH1, rH2, rH3, rW13, rW00, 28, 21, - 0x7eb8, -27199, 0x00ea, 0x6050) - R_32_47(rH2, rH3, rH0, rH1, rW03, rW06, 16, 9, - 0xe01a, 0x22fe, 0x4447, 0x69c5) - R_32_47(rH0, rH1, rH2, rH3, rW09, rW12, 28, 21, - 0xb7f3, 0x0253, 0x59b1, 0x4d5b) - R_32_47(rH2, rH3, rH0, rH1, rW15, rW02, 16, 9, - 0x4701, -27017, 0xc7bd, -19859) - - R_48_63(rH0, rH1, rH2, rH3, rW00, rW07, 26, 22, - 0x0988, -1462, 0x4c70, -19401) - R_48_63(rH2, rH3, rH0, rH1, rW14, rW05, 17, 11, - 0xadaf, -5221, 0xfc99, 0x66f7) - R_48_63(rH0, rH1, rH2, rH3, rW12, rW03, 26, 22, - 0x7e80, -16418, 0xba1e, -25587) - R_48_63(rH2, rH3, rH0, rH1, rW10, rW01, 17, 11, - 0x4130, 0x380d, 0xe0c5, 0x738d) - lwz rW00,0(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW08, rW15, 26, 22, - 0xe837, -30770, 0xde8a, 0x69e8) - lwz rW14,4(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW06, rW13, 17, 11, - 0x9e79, 0x260f, 0x256d, -27941) - lwz rW12,8(rHP) - R_48_63(rH0, rH1, rH2, rH3, rW04, rW11, 26, 22, - 0xab75, -20775, 0x4f9e, -28397) - lwz rW10,12(rHP) - R_48_63(rH2, rH3, rH0, rH1, rW02, rW09, 17, 11, - 0x662b, 0x7c56, 0x11b2, 0x0358) - - add rH0,rH0,rW00 - stw rH0,0(rHP) - add rH1,rH1,rW14 - stw rH1,4(rHP) - add rH2,rH2,rW12 - stw rH2,8(rHP) - add rH3,rH3,rW10 - stw rH3,12(rHP) - NEXT_BLOCK - - bdnz ppc_md5_main - - FINALIZE - blr diff --git a/lib/crypto/powerpc/md5.h b/lib/crypto/powerpc/md5.h deleted file mode 100644 index 540b08e34d1d..000000000000 --- a/lib/crypto/powerpc/md5.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * MD5 optimized for PowerPC - */ - -void ppc_md5_transform(u32 *state, const u8 *data, size_t nblocks); - -static void md5_blocks(struct md5_block_state *state, - const u8 *data, size_t nblocks) -{ - ppc_md5_transform(state->h, data, nblocks); -} diff --git a/lib/debugobjects.c b/lib/debugobjects.c index b18a682fe3da..877f7675ba26 100644 --- a/lib/debugobjects.c +++ b/lib/debugobjects.c @@ -720,6 +720,41 @@ static inline bool debug_objects_is_pi_blocked_on(void) #endif } +static inline bool can_fill_pool(void) +{ + /* + * On !RT enabled kernels there are no restrictions and spinlock_t and + * raw_spinlock_t are the same types. + */ + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) + return true; + + /* + * On RT enabled kernels, the task must not be blocked on a lock as + * that could corrupt the PI state when blocking on a lock in the + * allocation path. + */ + if (debug_objects_is_pi_blocked_on()) + return false; + + /* + * On RT enabled kernels the pool refill should happen in preemptible + * context. + */ + if (preemptible()) + return true; + + /* + * Though during system boot before scheduling is set up, preemption is + * disabled and the pool can get exhausted. Before scheduling is active + * a task cannot be blocked on a sleeping lock, but it might hold a lock + * and if interrupted then hard interrupt context might run into a lock + * inversion. So exclude hard interrupt context from allocations before + * scheduling is active. + */ + return system_state < SYSTEM_SCHEDULING && !in_hardirq(); +} + static void debug_objects_fill_pool(void) { if (!static_branch_likely(&obj_cache_enabled)) @@ -734,18 +769,11 @@ static void debug_objects_fill_pool(void) if (likely(!pool_should_refill(&pool_global))) return; - /* - * On RT enabled kernels the pool refill must happen in preemptible - * context and not enqueued on an rt_mutex -- for !RT kernels we rely - * on the fact that spinlock_t and raw_spinlock_t are basically the - * same type and this lock-type inversion works just fine. - */ - if (!IS_ENABLED(CONFIG_PREEMPT_RT) || system_state < SYSTEM_SCHEDULING || - (preemptible() && !debug_objects_is_pi_blocked_on())) { + if (can_fill_pool()) { /* * Annotate away the spinlock_t inside raw_spinlock_t warning * by temporarily raising the wait-type to LD_WAIT_CONFIG, matching - * the preemptible() condition above. + * the preemptible() condition in can_fill_pool(). */ static DEFINE_WAIT_OVERRIDE_MAP(fill_pool_map, LD_WAIT_CONFIG); lock_map_acquire_try(&fill_pool_map); @@ -866,6 +894,14 @@ int debug_object_activate(void *addr, const struct debug_obj_descr *descr) } raw_spin_unlock_irqrestore(&db->lock, flags); + + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. + */ + if (!debug_objects_enabled) + return 0; + debug_print_object(&o, "activate"); switch (o.state) { @@ -1043,6 +1079,15 @@ void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr) return; } + /* + * lookup_object_or_alloc() might have raced with a concurrent + * allocation failure which disabled debug objects. Don't run the fixup + * as it might turn a valid object useless. See for example + * hrtimer_fixup_assert_init(). + */ + if (!debug_objects_enabled) + return; + /* Object is neither tracked nor static. It's not initialized. */ debug_print_object(&o, "assert_init"); debug_object_fixup(descr->fixup_assert_init, addr, ODEBUG_STATE_NOTAVAILABLE); diff --git a/lib/error-inject.c b/lib/error-inject.c index f3d1b70be605..32f3d1ca9ea2 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -219,9 +219,9 @@ static int __init ei_debugfs_init(void) dir = debugfs_create_dir("error_injection", NULL); file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops); - if (!file) { + if (IS_ERR(file)) { debugfs_remove(dir); - return -ENOMEM; + return PTR_ERR(file); } return 0; diff --git a/lib/fonts/font_acorn_8x8.c b/lib/fonts/font_acorn_8x8.c index 36c51016769d..4ff52c79f8c4 100644 --- a/lib/fonts/font_acorn_8x8.c +++ b/lib/fonts/font_acorn_8x8.c @@ -5,7 +5,12 @@ #define FONTDATAMAX 2048 +#ifdef BOOTLOADER +/* The acorndata_8x8 symbol is needed by the ARM bootloader too. */ +const struct font_data acorndata_8x8 = { +#else static const struct font_data acorndata_8x8 = { +#endif { 0, 0, FONTDATAMAX, 0 }, { /* 00 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* ^@ */ /* 01 */ 0x7e, 0x81, 0xa5, 0x81, 0xbd, 0x99, 0x81, 0x7e, /* ^A */ diff --git a/lib/glob.c b/lib/glob.c index 7aca76c25bcb..c80d9dd736b4 100644 --- a/lib/glob.c +++ b/lib/glob.c @@ -11,6 +11,9 @@ MODULE_DESCRIPTION("glob(7) matching"); MODULE_LICENSE("Dual MIT/GPL"); +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end); + /** * glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0) * @pat: Shell-style pattern to match, e.g. "*.[ch]". @@ -41,6 +44,29 @@ MODULE_LICENSE("Dual MIT/GPL"); */ bool __pure glob_match(char const *pat, char const *str) { + return glob_match_str(pat, str, NULL); +} +EXPORT_SYMBOL(glob_match); + +/** + * glob_match_len - glob match against a length-bounded string + * @pat: Shell-style pattern to match. + * @str: String to match. Need not be NUL-terminated. + * @len: Number of bytes of @str that may be read. + * + * Like glob_match(), but @str is only read up to @len bytes, so it can be + * used on buffers that are not NUL-terminated (e.g. trace event fields). + * A NUL byte within @len still terminates the string. + */ +bool __pure glob_match_len(char const *pat, char const *str, size_t len) +{ + return glob_match_str(pat, str, str + len); +} +EXPORT_SYMBOL(glob_match_len); + +static bool __pure glob_match_str(char const *pat, char const *str, + char const *str_end) +{ /* * Backtrack to previous * on mismatch and retry starting one * character later in the string. Because * matches all characters @@ -55,9 +81,11 @@ bool __pure glob_match(char const *pat, char const *str) * on mismatch, or true after matching the trailing nul bytes. */ for (;;) { - unsigned char c = *str++; + unsigned char c = (str_end && str >= str_end) ? '\0' : *str; unsigned char d = *pat++; + str++; + switch (d) { case '?': /* Wildcard: anything but nul */ if (c == '\0') @@ -125,4 +153,3 @@ backtrack: } } } -EXPORT_SYMBOL(glob_match); diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index 16200feacbf3..eba2d3e28980 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -4,6 +4,7 @@ #include <linux/interval_tree.h> #include <linux/prandom.h> #include <linux/slab.h> +#include <linux/printk.h> #include <asm/timex.h> #include <linux/bitmap.h> #include <linux/maple_tree.h> @@ -311,6 +312,27 @@ static inline int span_iteration_check(void) {return 0; } static int interval_tree_test_init(void) { + if (nnodes <= 0) { + pr_warn("nnodes must be positive\n"); + return -EINVAL; + } + if (nsearches <= 0) { + pr_warn("nsearches must be positive\n"); + return -EINVAL; + } + if (perf_loops <= 0) { + pr_warn("perf_loops must be positive\n"); + return -EINVAL; + } + if (search_loops <= 0) { + pr_warn("search_loops must be positive\n"); + return -EINVAL; + } + if (max_endpoint < 2) { + pr_warn("max_endpoint must be at least 2\n"); + return -EINVAL; + } + nodes = kmalloc_objs(struct interval_tree_node, nnodes); if (!nodes) return -ENOMEM; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 243662af1af7..c2484551a4e8 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1224,13 +1224,13 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) { *new = *old; if (iov_iter_is_bvec(new)) - return new->bvec = kmemdup(new->bvec, - new->nr_segs * sizeof(struct bio_vec), + return new->bvec = kmemdup_array(new->bvec, + new->nr_segs, sizeof(struct bio_vec), flags); else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) /* iovec and kvec have identical layout */ - return new->__iov = kmemdup(new->__iov, - new->nr_segs * sizeof(struct iovec), + return new->__iov = kmemdup_array(new->__iov, + new->nr_segs, sizeof(struct iovec), flags); return NULL; } @@ -1568,6 +1568,7 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, struct folio *folio; unsigned int nr = 0, offset; loff_t pos = i->xarray_start + i->iov_offset; + bool will_alloc = !*pages; XA_STATE(xas, i->xarray, pos >> PAGE_SHIFT); offset = pos & ~PAGE_MASK; @@ -1595,6 +1596,14 @@ static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i, } rcu_read_unlock(); + if (!nr) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } + return 0; + } + maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize); iov_iter_advance(i, maxsize); return maxsize; @@ -1628,6 +1637,8 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, bi.bi_bvec_done = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); + if (!maxpages) + return -ENOMEM; while (bi.bi_size && bi.bi_idx < i->nr_segs) { struct bio_vec bv = bvec_iter_bvec(i->bvec, bi); @@ -1745,6 +1756,7 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, unsigned long addr; unsigned int gup_flags = 0; size_t offset; + bool will_alloc = !*pages; int res; if (i->data_source == ITER_DEST) @@ -1761,8 +1773,14 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i, if (!maxpages) return -ENOMEM; res = pin_user_pages_fast(addr, maxpages, gup_flags, *pages); - if (unlikely(res <= 0)) + if (unlikely(res <= 0)) { + if (will_alloc) { + kvfree(*pages); + *pages = NULL; + } return res; + } + maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset); iov_iter_advance(i, maxsize); return maxsize; diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 97be2a39f537..edc4eb7c1bca 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -39,25 +39,30 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) return s; } -/* - * Convert non-negative integer string representation in explicitly given radix - * to an integer. A maximum of max_chars characters will be converted. +/** + * _parse_integer_limit - Convert integer string representation to an integer + * @s: Integer string representation + * @base: Radix + * @p: Where to store result + * @max_chars: Maximum amount of characters to convert + * + * Convert non-negative integer string representation in explicitly given + * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX. * - * Return number of characters consumed maybe or-ed with overflow bit. - * If overflow occurs, result integer (incorrect) is still returned. + * This function is the workhorse of other string conversion functions and it + * is discouraged to use it explicitly. Consider kstrto*() family instead. * - * Don't you dare use this function. + * Return: Number of characters consumed, maybe ORed with overflow bit */ noinline unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, size_t max_chars) { + unsigned int rv, overflow = 0; unsigned long long res; - unsigned int rv; res = 0; - rv = 0; - while (max_chars--) { + for (rv = 0; rv < max_chars; rv++, s++) { unsigned int c = *s; unsigned int lc = _tolower(c); unsigned int val; @@ -76,15 +81,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon * it in the max base we support (16) */ if (unlikely(res & (~0ull << 60))) { - if (res > div_u64(ULLONG_MAX - val, base)) - rv |= KSTRTOX_OVERFLOW; + if (check_mul_overflow(res, base, &res) || + check_add_overflow(res, val, &res)) { + res = ULLONG_MAX; + overflow = KSTRTOX_OVERFLOW; + } + } else { + res = res * base + val; } - res = res * base + val; - rv++; - s++; } *p = res; - return rv; + return rv | overflow; } noinline diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile index 656f1fa35abc..2e8a6b71a2ab 100644 --- a/lib/kunit/Makefile +++ b/lib/kunit/Makefile @@ -10,7 +10,8 @@ kunit-objs += test.o \ executor.o \ attributes.o \ device.o \ - platform.o + platform.o \ + bug.o ifeq ($(CONFIG_KUNIT_DEBUGFS),y) kunit-objs += debugfs.o @@ -21,6 +22,7 @@ obj-$(if $(CONFIG_KUNIT),y) += hooks.o obj-$(CONFIG_KUNIT_TEST) += kunit-test.o obj-$(CONFIG_KUNIT_TEST) += platform-test.o +obj-$(CONFIG_KUNIT_TEST) += backtrace-suppression-test.o # string-stream-test compiles built-in only. ifeq ($(CONFIG_KUNIT_TEST),y) diff --git a/lib/kunit/backtrace-suppression-test.c b/lib/kunit/backtrace-suppression-test.c new file mode 100644 index 000000000000..7a2a59c6a780 --- /dev/null +++ b/lib/kunit/backtrace-suppression-test.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test for suppressing warning tracebacks. + * + * Copyright (C) 2024, Guenter Roeck + * Author: Guenter Roeck <linux@roeck-us.net> + */ + +#include <kunit/test.h> +#include <linux/bug.h> +#include <linux/completion.h> +#include <linux/kthread.h> + +static void backtrace_suppression_test_warn_direct(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + WARN(1, "This backtrace should be suppressed"); + /* + * Count must be checked inside the scope; the handle + * is not accessible after the block exits. + */ + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); +} + +static noinline void trigger_backtrace_warn(void) +{ + WARN(1, "This backtrace should be suppressed"); +} + +static void backtrace_suppression_test_warn_indirect(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + trigger_backtrace_warn(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static void backtrace_suppression_test_warn_multi(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + WARN(1, "This backtrace should be suppressed"); + trigger_backtrace_warn(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2); + } +} + +static void backtrace_suppression_test_warn_on_direct(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE) && !IS_ENABLED(CONFIG_KALLSYMS)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE or CONFIG_KALLSYMS"); + + kunit_warning_suppress(test) { + WARN_ON(1); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static noinline void trigger_backtrace_warn_on(void) +{ + WARN_ON(1); +} + +static void backtrace_suppression_test_warn_on_indirect(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE"); + + kunit_warning_suppress(test) { + trigger_backtrace_warn_on(); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + } +} + +static void backtrace_suppression_test_count(struct kunit *test) +{ + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 0); + + WARN(1, "suppressed"); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1); + + WARN(1, "suppressed again"); + KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2); + } +} + +static void backtrace_suppression_test_active_state(struct kunit *test) +{ + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning()); + } + + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); + + kunit_warning_suppress(test) { + KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning()); + } + + KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning()); +} + +static void backtrace_suppression_test_multi_scope(struct kunit *test) +{ + struct kunit_suppressed_warning *sw1, *sw2; + + if (!IS_ENABLED(CONFIG_BUG)) + kunit_skip(test, "requires CONFIG_BUG"); + if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE)) + kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE"); + + sw1 = kunit_start_suppress_warning(test); + trigger_backtrace_warn_on(); + WARN(1, "suppressed by sw1"); + kunit_end_suppress_warning(test, sw1); + + sw2 = kunit_start_suppress_warning(test); + WARN(1, "suppressed by sw2"); + kunit_end_suppress_warning(test, sw2); + + KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw1), 2); + KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw2), 1); +} + +struct cross_kthread_data { + bool was_active; + struct completion done; +}; + +static int cross_kthread_fn(void *data) +{ + struct cross_kthread_data *d = data; + + d->was_active = kunit_has_active_suppress_warning(); + complete(&d->done); + while (!kthread_should_stop()) + schedule(); + return 0; +} + +static void backtrace_suppression_test_cross_kthread(struct kunit *test) +{ + struct cross_kthread_data data; + struct task_struct *task; + + data.was_active = false; + init_completion(&data.done); + + kunit_warning_suppress(test) { + task = kthread_run(cross_kthread_fn, &data, "kunit-cross-test"); + KUNIT_ASSERT_FALSE(test, IS_ERR(task)); + wait_for_completion(&data.done); + kthread_stop(task); + } + + KUNIT_EXPECT_FALSE(test, data.was_active); +} + +static struct kunit_case backtrace_suppression_test_cases[] = { + KUNIT_CASE(backtrace_suppression_test_warn_direct), + KUNIT_CASE(backtrace_suppression_test_warn_indirect), + KUNIT_CASE(backtrace_suppression_test_warn_multi), + KUNIT_CASE(backtrace_suppression_test_warn_on_direct), + KUNIT_CASE(backtrace_suppression_test_warn_on_indirect), + KUNIT_CASE(backtrace_suppression_test_count), + KUNIT_CASE(backtrace_suppression_test_active_state), + KUNIT_CASE(backtrace_suppression_test_multi_scope), + KUNIT_CASE(backtrace_suppression_test_cross_kthread), + {} +}; + +static struct kunit_suite backtrace_suppression_test_suite = { + .name = "backtrace-suppression-test", + .test_cases = backtrace_suppression_test_cases, +}; +kunit_test_suites(&backtrace_suppression_test_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit test to verify warning backtrace suppression"); diff --git a/lib/kunit/bug.c b/lib/kunit/bug.c new file mode 100644 index 000000000000..8579235c9ca6 --- /dev/null +++ b/lib/kunit/bug.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit helpers for backtrace suppression + * + * Copyright (C) 2025 Alessandro Carminati <acarmina@redhat.com> + * Copyright (C) 2024 Guenter Roeck <linux@roeck-us.net> + */ + +#include <kunit/resource.h> +#include <linux/export.h> +#include <linux/rculist.h> +#include <linux/sched.h> +#include <linux/sched/task.h> +#include <linux/spinlock.h> + +#include "hooks-impl.h" + +struct kunit_suppressed_warning { + struct list_head node; + struct task_struct *task; + struct kunit *test; + atomic_t counter; +}; + +static LIST_HEAD(suppressed_warnings); +static DEFINE_SPINLOCK(suppressed_warnings_lock); + +static void kunit_suppress_warning_remove(struct kunit_suppressed_warning *w) +{ + unsigned long flags; + + spin_lock_irqsave(&suppressed_warnings_lock, flags); + list_del_rcu(&w->node); + spin_unlock_irqrestore(&suppressed_warnings_lock, flags); + put_task_struct(w->task); +} + +KUNIT_DEFINE_ACTION_WRAPPER(kunit_suppress_warning_cleanup, + kunit_suppress_warning_remove, + struct kunit_suppressed_warning *); + +bool kunit_has_active_suppress_warning(void) +{ + return __kunit_is_suppressed_warning_impl(false); +} +EXPORT_SYMBOL_GPL(kunit_has_active_suppress_warning); + +struct kunit_suppressed_warning * +kunit_start_suppress_warning(struct kunit *test) +{ + struct kunit_suppressed_warning *w; + unsigned long flags; + int ret; + + if (kunit_has_active_suppress_warning()) { + KUNIT_FAIL(test, "Another suppression block is already active"); + return NULL; + } + + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + if (!w) { + KUNIT_FAIL(test, "Failed to allocate suppression handle."); + return NULL; + } + + w->task = get_task_struct(current); + w->test = test; + + spin_lock_irqsave(&suppressed_warnings_lock, flags); + list_add_rcu(&w->node, &suppressed_warnings); + spin_unlock_irqrestore(&suppressed_warnings_lock, flags); + + ret = kunit_add_action_or_reset(test, + kunit_suppress_warning_cleanup, w); + if (ret) { + KUNIT_FAIL(test, "Failed to add suppression cleanup action."); + return NULL; + } + + return w; +} +EXPORT_SYMBOL_GPL(kunit_start_suppress_warning); + +void kunit_end_suppress_warning(struct kunit *test, + struct kunit_suppressed_warning *w) +{ + if (!w) + return; + kunit_release_action(test, kunit_suppress_warning_cleanup, w); +} +EXPORT_SYMBOL_GPL(kunit_end_suppress_warning); + +void __kunit_suppress_auto_cleanup(struct kunit_suppressed_warning **wp) +{ + if (*wp) + kunit_end_suppress_warning((*wp)->test, *wp); +} +EXPORT_SYMBOL_GPL(__kunit_suppress_auto_cleanup); + +int kunit_suppressed_warning_count(struct kunit_suppressed_warning *w) +{ + return w ? atomic_read(&w->counter) : 0; +} +EXPORT_SYMBOL_GPL(kunit_suppressed_warning_count); + +bool __kunit_is_suppressed_warning_impl(bool count) +{ + struct kunit_suppressed_warning *w; + + guard(rcu)(); + list_for_each_entry_rcu(w, &suppressed_warnings, node) { + if (w->task == current) { + if (count) + atomic_inc(&w->counter); + return true; + } + } + + return false; +} diff --git a/lib/kunit/hooks-impl.h b/lib/kunit/hooks-impl.h index 4e71b2d0143b..d8720f261692 100644 --- a/lib/kunit/hooks-impl.h +++ b/lib/kunit/hooks-impl.h @@ -19,6 +19,7 @@ void __printf(3, 4) __kunit_fail_current_test_impl(const char *file, int line, const char *fmt, ...); void *__kunit_get_static_stub_address_impl(struct kunit *test, void *real_fn_addr); +bool __kunit_is_suppressed_warning_impl(bool count); /* Code to set all of the function pointers. */ static inline void kunit_install_hooks(void) @@ -26,6 +27,7 @@ static inline void kunit_install_hooks(void) /* Install the KUnit hook functions. */ kunit_hooks.fail_current_test = __kunit_fail_current_test_impl; kunit_hooks.get_static_stub_address = __kunit_get_static_stub_address_impl; + kunit_hooks.is_suppressed_warning = __kunit_is_suppressed_warning_impl; } #endif /* _KUNIT_HOOKS_IMPL_H */ diff --git a/lib/kunit/platform.c b/lib/kunit/platform.c index 0b518de26065..737758d710b2 100644 --- a/lib/kunit/platform.c +++ b/lib/kunit/platform.c @@ -6,6 +6,7 @@ #include <linux/completion.h> #include <linux/device/bus.h> #include <linux/device/driver.h> +#include <linux/err.h> #include <linux/platform_device.h> #include <kunit/platform_device.h> @@ -130,6 +131,69 @@ int kunit_platform_device_add(struct kunit *test, struct platform_device *pdev) } EXPORT_SYMBOL_GPL(kunit_platform_device_add); +/** + * kunit_platform_device_register_full() - Register a KUnit test-managed platform + * device described by platform device info + * @test: test context + * @pdevinfo: platform device information describing the new device + * + * Register a test-managed platform device. The device is unregistered when the + * test completes. + * + * Return: New platform device on success, IS_ERR() on error. + */ +struct platform_device * +kunit_platform_device_register_full(struct kunit *test, + const struct platform_device_info *pdevinfo) +{ + struct platform_device *pdev; + int ret; + + pdev = platform_device_register_full(pdevinfo); + if (IS_ERR(pdev)) + return pdev; + + ret = kunit_add_action_or_reset(test, platform_device_unregister_wrapper, pdev); + if (ret) + return ERR_PTR(ret); + + return pdev; +} +EXPORT_SYMBOL_GPL(kunit_platform_device_register_full); + +static bool +kunit_platform_device_add_match(struct kunit *test, struct kunit_resource *res, + void *match_data) +{ + struct platform_device *pdev = match_data; + + return res->data == pdev && res->free == kunit_platform_device_add_exit; +} + +/** + * kunit_platform_device_unregister() - Unregister a KUnit-managed platform device + * @test: test context + * @pdev: platform device to unregister + * + * Unregister a test-managed platform device and cancel its release action. + */ +void kunit_platform_device_unregister(struct kunit *test, + struct platform_device *pdev) +{ + struct kunit_resource *res; + + res = kunit_find_resource(test, kunit_platform_device_add_match, pdev); + if (res) { + res->free = NULL; + kunit_put_resource(res); + } else { + kunit_remove_action(test, platform_device_unregister_wrapper, pdev); + } + + platform_device_unregister(pdev); +} +EXPORT_SYMBOL_GPL(kunit_platform_device_unregister); + struct kunit_platform_device_probe_nb { struct completion *x; struct device *dev; diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index d939403331b5..bfafe1204c7b 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1431,10 +1431,9 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { int saved_preempt_count = preempt_count(); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP int saved_mgd_count = current->migration_disabled; -#endif int saved_rcu_count = current->rcu_read_lock_nesting; + int saved_sched_rt_mutex = current->sched_rt_mutex; #endif WARN_ON(irqs_disabled()); @@ -1471,10 +1470,10 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) preempt_count_set(saved_preempt_count); #ifdef CONFIG_PREEMPT_RT -#ifdef CONFIG_SMP + current->sched_rt_mutex = saved_sched_rt_mutex; + while (current->migration_disabled > saved_mgd_count) migrate_enable(); -#endif while (current->rcu_read_lock_nesting > saved_rcu_count) rcu_read_unlock(); diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 60ae5e6fc1ee..e52876435b77 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -5727,13 +5727,16 @@ int mtree_store(struct maple_tree *mt, unsigned long index, void *entry, EXPORT_SYMBOL(mtree_store); /** - * mtree_insert_range() - Insert an entry at a given range if there is no value. + * mtree_insert_range() - Insert an entry from [first, last] at a given range + * if there is no value. * @mt: The maple tree * @first: The start of the range - * @last: The end of the range + * @last: The end of the range (inclusive) * @entry: The entry to store * @gfp: The GFP_FLAGS to use for allocations. * + * Note that @last is inclusive. That is, @last = @first + length - 1; + * * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid * request, -ENOMEM if memory could not be allocated. */ diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 33c154264bfe..a3bfa9360b23 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -16,6 +16,7 @@ #include <linux/cpumask.h> #include <linux/delay.h> #include <linux/kprobes.h> +#include <linux/stringify.h> #include <linux/nmi.h> #include <linux/cpu.h> #include <linux/sched/debug.h> @@ -27,6 +28,8 @@ static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; /* "in progress" flag of arch_trigger_cpumask_backtrace */ static unsigned long backtrace_flag; +#define NMI_BT_TIMEOUT_SEC 10 + /* * When raise() is called it will be passed a pointer to the * backtrace_mask. Architectures that call nmi_cpu_backtrace() @@ -68,14 +71,20 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, raise(to_cpumask(backtrace_mask)); } - /* Wait for up to 10 seconds for all CPUs to do the backtrace */ - for (i = 0; i < 10 * 1000; i++) { + /* Wait for up to NMI_BT_TIMEOUT_SEC seconds for all CPUs to do the backtrace */ + for (i = 0; i < NMI_BT_TIMEOUT_SEC * 1000; i++) { if (cpumask_empty(to_cpumask(backtrace_mask))) break; mdelay(1); touch_softlockup_watchdog(); } - nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + + if (!cpumask_empty(to_cpumask(backtrace_mask))) { + pr_warn("After " __stringify(NMI_BT_TIMEOUT_SEC) " seconds, these CPUS still haven't responded to the NMI: %*pbl\n", + cpumask_pr_args(to_cpumask(backtrace_mask))); + + nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + } /* * Force flush any remote buffers that might be stuck in IRQ context diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig index 5ab2b0a7be4c..978cd6ba08ac 100644 --- a/lib/raid/Kconfig +++ b/lib/raid/Kconfig @@ -28,3 +28,36 @@ config XOR_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. + +config RAID6_PQ + tristate + +# selected by architectures that provide an optimized PQ implementation +config RAID6_PQ_ARCH + depends on RAID6_PQ + default y if KERNEL_MODE_NEON # arm32/arm64 + default y if LOONGARCH + default y if ALTIVEC # powerpc + default y if RISCV_ISA_V + default y if S390 + default y if X86 + bool + +config RAID6_PQ_KUNIT_TEST + tristate "KUnit tests for RAID6 PQ functions" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RAID6_PQ + default KUNIT_ALL_TESTS + help + Unit tests for the RAID6 PQ library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. diff --git a/lib/raid/Makefile b/lib/raid/Makefile index 3540fe846dc4..6fc5eeb53df0 100644 --- a/lib/raid/Makefile +++ b/lib/raid/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += xor/ +obj-y += xor/ raid6/ diff --git a/lib/raid6/.gitignore b/lib/raid/raid6/.gitignore index 6be57745afd1..6be57745afd1 100644 --- a/lib/raid6/.gitignore +++ b/lib/raid/raid6/.gitignore diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile new file mode 100644 index 000000000000..038d6c74d1ba --- /dev/null +++ b/lib/raid/raid6/Makefile @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0 + +ccflags-y += -I $(src) + +ifeq ($(CONFIG_RAID6_PQ_ARCH),y) +CFLAGS_algos.o += -I$(src)/$(SRCARCH) +endif + +obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/ + +raid6_pq-y += algos.o tables.o + +# generic integer generation and recovery implementation +raid6_pq-y += int1.o int2.o int4.o int8.o +raid6_pq-y += recov.o + +# architecture-specific generation and recovery implementations: +raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \ + arm/neon1.o \ + arm/neon2.o \ + arm/neon4.o \ + arm/neon8.o \ + arm/recov_neon.o \ + arm/recov_neon_inner.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch/loongarch_simd.o \ + loongarch/recov_loongarch_simd.o +raid6_pq-$(CONFIG_ALTIVEC) += powerpc/altivec1.o \ + powerpc/altivec2.o \ + powerpc/altivec4.o \ + powerpc/altivec8.o \ + powerpc/vpermxor1.o \ + powerpc/vpermxor2.o \ + powerpc/vpermxor4.o \ + powerpc/vpermxor8.o +raid6_pq-$(CONFIG_RISCV_ISA_V) += riscv/rvv.o \ + riscv/recov_rvv.o +raid6_pq-$(CONFIG_S390) += s390/s390vx8.o \ + s390/recov_s390xc.o +ifeq ($(CONFIG_X86),y) +raid6_pq-$(CONFIG_X86_32) += x86/mmx.o \ + x86/sse1.o +endif +raid6_pq-$(CONFIG_X86) += x86/sse2.o \ + x86/avx2.o \ + x86/avx512.o \ + x86/recov_ssse3.o \ + x86/recov_avx2.o \ + x86/recov_avx512.o + +hostprogs += mktables + +CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU) +CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU) + +ifeq ($(CONFIG_ALTIVEC),y) +altivec_flags := -maltivec $(call cc-option,-mabi=altivec) +# Enable <altivec.h> +altivec_flags += -isystem $(shell $(CC) -print-file-name=include) + +CFLAGS_powerpc/altivec1.o += $(altivec_flags) +CFLAGS_powerpc/altivec2.o += $(altivec_flags) +CFLAGS_powerpc/altivec4.o += $(altivec_flags) +CFLAGS_powerpc/altivec8.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor1.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor2.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor4.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor8.o += $(altivec_flags) + +ifdef CONFIG_CC_IS_CLANG +# clang ppc port does not yet support -maltivec when -msoft-float is +# enabled. A future release of clang will resolve this +# https://llvm.org/pr31177 +CFLAGS_REMOVE_powerpc/altivec1.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec2.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec4.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec8.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor1.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor2.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor4.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor8.o += -msoft-float +endif # CONFIG_CC_IS_CLANG +endif # CONFIG_ALTIVEC + +quiet_cmd_mktable = TABLE $@ + cmd_mktable = $(obj)/mktables > $@ + +targets += tables.c +$(obj)/tables.c: $(obj)/mktables FORCE + $(call if_changed,mktable) + +quiet_cmd_unroll = UNROLL $@ + cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ + +targets += int1.c int2.c int4.c int8.c +$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c +$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/altivec1.c \ + powerpc/altivec2.c \ + powerpc/altivec4.c \ + powerpc/altivec8.c +$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/vpermxor1.c \ + powerpc/vpermxor2.c \ + powerpc/vpermxor4.c \ + powerpc/vpermxor8.c +$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += s390/s390vx8.c +$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c new file mode 100644 index 000000000000..6f5c89ab2b17 --- /dev/null +++ b/lib/raid/raid6/algos.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002 H. Peter Anvin - All Rights Reserved + * + * Algorithm list and algorithm selection for RAID-6 + */ + +#include <linux/module.h> +#include <linux/gfp.h> +#include <linux/raid/pq.h> +#include <linux/slab.h> +#include <linux/static_call.h> +#include <kunit/visibility.h> +#include "algos.h" + +#define RAID6_MAX_ALGOS 16 +static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; +static unsigned int raid6_nr_algos; +static const struct raid6_recov_calls *raid6_recov_algo; + +/* Selected algorithm */ +DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); +DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); + +/** + * raid6_gen_syndrome - generate RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to + * @ptrs[@disks - 3]. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. + * + * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying + * algorithm. + */ +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + + static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_gen_syndrome); + +/** + * raid6_xor_syndrome - update RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @start: first index into @disk to update + * @stop: last index into @disk to update + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively for the memory pointed to by + * @ptrs[@start..@stop]. + * + * This is used to update parity in place using the following sequence: + * + * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. + * 2) update the the data in @ptrs[@start..@stop]. + * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. + * + * Data between @start and @stop that is not changed should be filled + * with a pointer to the kernel zero page. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. @stop must be larger or equal to @start. + */ +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + WARN_ON_ONCE(stop < start); + + static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_xor_syndrome); + +/* + * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used + * + * Returns %true if raid6_can_xor_syndrome() can be used, else %false. + */ +bool raid6_can_xor_syndrome(void) +{ + return !!static_call_query(raid6_xor_syndrome_impl); +} +EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); + +/** + * raid6_recov_2data - recover two missing data disks + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: first failed data disk index + * @failb: second failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the + * data in the remaining disks and the two parities pointed to by the other + * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks + * and the two parities. @faila must be smaller than @failb. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + WARN_ON_ONCE(failb <= faila); + + static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_2data); + +/** + * raid6_recov_datap - recover a missing data disk and missing P-parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in + * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity + * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks + * includes the data disks and the two parities. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + + static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_datap); + +#define RAID6_TIME_JIFFIES_LG2 4 +#define RAID6_TEST_DISKS 8 + +static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], + const int disks) +{ + /* work on the second half of the disks */ + int start = (disks >> 1) - 1, stop = disks - 3; + const struct raid6_calls *best = NULL; + unsigned long bestgenperf = 0; + unsigned int i; + + for (i = 0; i < raid6_nr_algos; i++) { + const struct raid6_calls *algo = raid6_algos[i]; + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { + algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + if (perf > bestgenperf) { + bestgenperf = perf; + best = algo; + } + pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + } + + if (!best) { + pr_err("raid6: Yikes! No algorithm found!\n"); + return -EINVAL; + } + + static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); + + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, + (bestgenperf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + + if (best->xor_syndrome) { + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { + best->xor_syndrome(disks, start, stop, + PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", + (perf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + } + + return 0; +} + + +/* Try to pick the best algorithm */ +/* This code uses the gfmul table as convenient data set to abuse */ + +static int __init raid6_select_algo(void) +{ + const int disks = RAID6_TEST_DISKS; + char *disk_ptr, *p; + void *dptrs[RAID6_TEST_DISKS]; + int i, cycle; + int error; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { + pr_info("raid6: skipped pq benchmark and selected %s\n", + raid6_algos[raid6_nr_algos - 1]->name); + static_call_update(raid6_gen_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->xor_syndrome); + return 0; + } + + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = kmalloc(PAGE_SIZE * RAID6_TEST_DISKS, GFP_KERNEL); + if (!disk_ptr) { + pr_err("raid6: Yikes! No memory available.\n"); + return -ENOMEM; + } + + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + PAGE_SIZE * i; + + cycle = ((disks - 2) * PAGE_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * PAGE_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); + + /* select raid gen_syndrome function */ + error = raid6_choose_gen(&dptrs, disks); + + kfree(disk_ptr); + + return error; +} + +/* + * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled + * implementation should be registered last so it will be selected when the + * boot-time benchmark is disabled. + */ +void __init raid6_algo_add(const struct raid6_calls *algo) +{ + if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) + return; + raid6_algos[raid6_nr_algos++] = algo; +} + +void __init raid6_algo_add_default(void) +{ + raid6_algo_add(&raid6_intx1); + raid6_algo_add(&raid6_intx2); + raid6_algo_add(&raid6_intx4); + raid6_algo_add(&raid6_intx8); +} + +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) +{ + if (WARN_ON_ONCE(raid6_recov_algo)) + return; + raid6_recov_algo = algo; +} + +#ifdef CONFIG_RAID6_PQ_ARCH +#include "pq_arch.h" +#else +static inline void arch_raid6_init(void) +{ + raid6_algo_add_default(); +} +#endif /* CONFIG_RAID6_PQ_ARCH */ + +static int __init raid6_init(void) +{ + /* + * Architectures providing arch_raid6_init must add all PQ generation + * algorithms they want to consider in arch_raid6_init(), including + * the generic ones using raid6_algo_add_default() if wanted. + */ + arch_raid6_init(); + + /* + * Architectures don't have to set a recovery algorithm, we'll just pick + * the generic integer one if none was set. + */ + if (!raid6_recov_algo) + raid6_recov_algo = &raid6_recov_intx1; + static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); + static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); + pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); + + return raid6_select_algo(); +} + +static void __exit raid6_exit(void) +{ +} + +subsys_initcall(raid6_init); +module_exit(raid6_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); + +#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) +const struct raid6_calls *raid6_algo_find(unsigned int idx) +{ + if (idx >= raid6_nr_algos) { + /* + * Always include the simplest generic integer implementation in + * the unit tests as a baseline. + */ + if (idx == raid6_nr_algos && + raid6_algos[0] != &raid6_intx1) + return &raid6_intx1; + return NULL; + } + return raid6_algos[idx]; +} +EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); + +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) +{ + switch (idx) { + case 0: + /* always test the generic integer implementation */ + return &raid6_recov_intx1; + case 1: + /* test the optimized implementation if there is one */ + if (raid6_recov_algo != &raid6_recov_intx1) + return raid6_recov_algo; + return NULL; + default: + return NULL; + } +} +EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); +#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h new file mode 100644 index 000000000000..43f636be183f --- /dev/null +++ b/lib/raid/raid6/algos.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + */ +#ifndef _PQ_IMPL_H +#define _PQ_IMPL_H + +#include <linux/init.h> +#include <linux/raid/pq_tables.h> + +/* Routine choices */ +struct raid6_calls { + const char *name; + void (*gen_syndrome)(int disks, size_t bytes, void **ptrs); + void (*xor_syndrome)(int disks, int start, int stop, size_t bytes, + void **ptrs); +}; + +struct raid6_recov_calls { + const char *name; + void (*data2)(int disks, size_t bytes, int faila, int failb, + void **ptrs); + void (*datap)(int disks, size_t bytes, int faila, void **ptrs); +}; + +void __init raid6_algo_add(const struct raid6_calls *algo); +void __init raid6_algo_add_default(void); +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo); + +/* for the kunit test */ +const struct raid6_calls *raid6_algo_find(unsigned int idx); +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx); + +/* generic implementations */ +extern const struct raid6_calls raid6_intx1; +extern const struct raid6_calls raid6_intx2; +extern const struct raid6_calls raid6_intx4; +extern const struct raid6_calls raid6_intx8; +extern const struct raid6_recov_calls raid6_recov_intx1; + +#endif /* _PQ_IMPL_H */ diff --git a/lib/raid6/neon.c b/lib/raid/raid6/arm/neon.c index 6d9474ce6da9..af90869aaffc 100644 --- a/lib/raid6/neon.c +++ b/lib/raid/raid6/arm/neon.c @@ -1,18 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics + * RAID6 syndrome calculation using ARM NEON intrinsics * * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> */ -#include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif +#include "algos.h" /* * There are 2 reasons these wrappers are kept in a separate compilation unit @@ -46,18 +40,11 @@ start, stop, (unsigned long)bytes, ptrs);\ } \ struct raid6_calls const raid6_neonx ## _n = { \ - raid6_neon ## _n ## _gen_syndrome, \ - raid6_neon ## _n ## _xor_syndrome, \ - raid6_have_neon, \ - "neonx" #_n, \ - 0 \ + .gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \ + .name = "neonx" #_n, \ } -static int raid6_have_neon(void) -{ - return cpu_has_neon(); -} - RAID6_NEON_WRAPPER(1); RAID6_NEON_WRAPPER(2); RAID6_NEON_WRAPPER(4); diff --git a/lib/raid6/neon.h b/lib/raid/raid6/arm/neon.h index 2ca41ee9b499..2ca41ee9b499 100644 --- a/lib/raid6/neon.h +++ b/lib/raid/raid6/arm/neon.h diff --git a/lib/raid6/neon.uc b/lib/raid/raid6/arm/neon.uc index 355270af0cd6..14a9fc2c60fa 100644 --- a/lib/raid6/neon.uc +++ b/lib/raid/raid6/arm/neon.uc @@ -25,7 +25,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" typedef uint8x16_t unative_t; diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h new file mode 100644 index 000000000000..3f876ea6749c --- /dev/null +++ b/lib/raid/raid6/arm/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/neon.h> + +extern const struct raid6_calls raid6_neonx1; +extern const struct raid6_calls raid6_neonx2; +extern const struct raid6_calls raid6_neonx4; +extern const struct raid6_calls raid6_neonx8; +extern const struct raid6_recov_calls raid6_recov_neon; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (cpu_has_neon()) { + raid6_algo_add(&raid6_neonx1); + raid6_algo_add(&raid6_neonx2); + raid6_algo_add(&raid6_neonx4); + raid6_algo_add(&raid6_neonx8); + raid6_recov_algo_add(&raid6_recov_neon); + } +} diff --git a/lib/raid6/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c index 9d99aeabd31a..1524050d09b7 100644 --- a/lib/raid6/recov_neon.c +++ b/lib/raid/raid6/arm/recov_neon.c @@ -4,20 +4,11 @@ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> */ +#include <linux/mm.h> #include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#include "neon.h" -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif - -static int raid6_has_neon(void) -{ - return cpu_has_neon(); -} +#include "algos.h" +#include "arm/neon.h" static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -35,13 +26,13 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +63,10 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -91,7 +82,5 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_neon = { .data2 = raid6_2data_recov_neon, .datap = raid6_datap_recov_neon, - .valid = raid6_has_neon, .name = "neon", - .priority = 10, }; diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c index f9e7e8f5a151..53c355efa7ff 100644 --- a/lib/raid6/recov_neon_inner.c +++ b/lib/raid/raid6/arm/recov_neon_inner.c @@ -5,7 +5,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" #ifdef CONFIG_ARM /* diff --git a/lib/raid/raid6/arm64/pq_arch.h b/lib/raid/raid6/arm64/pq_arch.h new file mode 100644 index 000000000000..27ff564d7594 --- /dev/null +++ b/lib/raid/raid6/arm64/pq_arch.h @@ -0,0 +1 @@ +#include "arm/pq_arch.h" diff --git a/lib/raid6/int.uc b/lib/raid/raid6/int.uc index 1ba56c3fa482..e63bd5a9c2ed 100644 --- a/lib/raid6/int.uc +++ b/lib/raid/raid6/int.uc @@ -18,7 +18,7 @@ * This file is postprocessed using unroll.awk */ -#include <linux/raid/pq.h> +#include "algos.h" /* * This is the C data type to use @@ -139,9 +139,7 @@ static void raid6_int$#_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_intx$# = { - raid6_int$#_gen_syndrome, - raid6_int$#_xor_syndrome, - NULL, /* always valid */ - "int" NSTRING "x$#", - 0 + .gen_syndrome = raid6_int$#_gen_syndrome, + .xor_syndrome = raid6_int$#_xor_syndrome, + .name = "int" NSTRING "x$#", }; diff --git a/lib/raid6/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c index aa5d9f924ca3..c1eb53fafd27 100644 --- a/lib/raid6/loongarch_simd.c +++ b/lib/raid/raid6/loongarch/loongarch_simd.c @@ -9,8 +9,9 @@ * Copyright 2002-2004 H. Peter Anvin */ -#include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * The vector algorithms are currently priority 0, which means the generic @@ -25,11 +26,6 @@ #ifdef CONFIG_CPU_HAS_LSX #define NSIZE 16 -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -243,11 +239,9 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lsx = { - raid6_lsx_gen_syndrome, - raid6_lsx_xor_syndrome, - raid6_has_lsx, - "lsx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lsx_gen_syndrome, + .xor_syndrome = raid6_lsx_xor_syndrome, + .name = "lsx", }; #undef NSIZE @@ -256,11 +250,6 @@ const struct raid6_calls raid6_lsx = { #ifdef CONFIG_CPU_HAS_LASX #define NSIZE 32 -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -412,11 +401,9 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lasx = { - raid6_lasx_gen_syndrome, - raid6_lasx_xor_syndrome, - raid6_has_lasx, - "lasx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lasx_gen_syndrome, + .xor_syndrome = raid6_lasx_xor_syndrome, + .name = "lasx", }; #undef NSIZE #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h new file mode 100644 index 000000000000..ae443a4d7b69 --- /dev/null +++ b/lib/raid/raid6/loongarch/pq_arch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpu-features.h> + +extern const struct raid6_calls raid6_lsx; +extern const struct raid6_calls raid6_lasx; + +extern const struct raid6_recov_calls raid6_recov_lsx; +extern const struct raid6_recov_calls raid6_recov_lasx; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_algo_add(&raid6_lsx); + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_algo_add(&raid6_lasx); + + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_recov_algo_add(&raid6_recov_lasx); + else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_recov_algo_add(&raid6_recov_lsx); +} diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c index 93dc515997a1..87a2313bbb4f 100644 --- a/lib/raid6/recov_loongarch_simd.c +++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c @@ -10,8 +10,11 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * Unlike with the syndrome calculation algorithms, there's no boot-time @@ -21,11 +24,6 @@ */ #ifdef CONFIG_CPU_HAS_LSX -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -42,13 +40,13 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -197,10 +195,10 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -288,18 +286,11 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lsx = { .data2 = raid6_2data_recov_lsx, .datap = raid6_datap_recov_lsx, - .valid = raid6_has_lsx, .name = "lsx", - .priority = 1, }; #endif /* CONFIG_CPU_HAS_LSX */ #ifdef CONFIG_CPU_HAS_LASX -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -316,13 +307,13 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -436,10 +427,10 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -506,8 +497,6 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lasx = { .data2 = raid6_2data_recov_lasx, .datap = raid6_datap_recov_lasx, - .valid = raid6_has_lasx, .name = "lasx", - .priority = 2, }; #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/mktables.c b/lib/raid/raid6/mktables.c index 3be03793237c..b6327b562fdb 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid/raid6/mktables.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * mktables.c + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved * - * Make RAID-6 tables. This is a host user space program to be run at - * compile time. + * Make RAID-6 tables. This is a host user space program to be run at compile + * time. */ #include <stdio.h> @@ -56,10 +50,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#ifdef __KERNEL__\n"); printf("#include <linux/export.h>\n"); - printf("#endif\n"); - printf("#include <linux/raid/pq.h>\n"); + printf("#include \"algos.h\"\n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -76,9 +68,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfmul);\n"); - printf("#endif\n"); /* Compute vector multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -101,9 +91,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_vgfmul);\n"); - printf("#endif\n"); /* Compute power-of-2 table (exponent) */ v = 1; @@ -120,9 +108,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexp);\n"); - printf("#endif\n"); /* Compute log-of-2 table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -140,9 +126,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gflog);\n"); - printf("#endif\n"); /* Compute inverse table x^-1 == x^254 */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -155,9 +139,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfinv);\n"); - printf("#endif\n"); /* Compute inv(2^x + 1) (exponent-xor-inverse) table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -169,9 +151,7 @@ int main(int argc, char *argv[]) (j == 7) ? '\n' : ' '); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexi);\n"); - printf("#endif\n"); return 0; } diff --git a/lib/raid6/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc index d20ed0d11411..c5429fb71dd6 100644 --- a/lib/raid6/altivec.uc +++ b/lib/raid/raid6/powerpc/altivec.uc @@ -22,15 +22,11 @@ * bracked this with preempt_disable/enable or in a lock) */ -#include <linux/raid/pq.h> - -#ifdef CONFIG_ALTIVEC +#include "algos.h" #include <altivec.h> -#ifdef __KERNEL__ -# include <asm/cputable.h> -# include <asm/switch_to.h> -#endif /* __KERNEL__ */ +#include <asm/cputable.h> +#include <asm/switch_to.h> /* * This is the C data type to use. We use a vector of @@ -108,25 +104,7 @@ static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec(void); -#if $# == 1 -int raid6_have_altivec(void) -{ - /* This assumes either all CPUs have Altivec or none does */ -# ifdef __KERNEL__ - return cpu_has_feature(CPU_FTR_ALTIVEC); -# else - return 1; -# endif -} -#endif - const struct raid6_calls raid6_altivec$# = { - raid6_altivec$#_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_altivec, - "altivecx$#", - 0 + .gen_syndrome = raid6_altivec$#_gen_syndrome, + .name = "altivecx$#", }; - -#endif /* CONFIG_ALTIVEC */ diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h new file mode 100644 index 000000000000..ea1878777ff2 --- /dev/null +++ b/lib/raid/raid6/powerpc/pq_arch.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cputable.h> + +extern const struct raid6_calls raid6_altivec1; +extern const struct raid6_calls raid6_altivec2; +extern const struct raid6_calls raid6_altivec4; +extern const struct raid6_calls raid6_altivec8; +extern const struct raid6_calls raid6_vpermxor1; +extern const struct raid6_calls raid6_vpermxor2; +extern const struct raid6_calls raid6_vpermxor4; +extern const struct raid6_calls raid6_vpermxor8; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + + /* This assumes either all CPUs have Altivec or none does */ + if (cpu_has_feature(CPU_FTR_ALTIVEC)) { + raid6_algo_add(&raid6_altivec1); + raid6_algo_add(&raid6_altivec2); + raid6_algo_add(&raid6_altivec4); + raid6_algo_add(&raid6_altivec8); + } + if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && + cpu_has_feature(CPU_FTR_ARCH_207S)) { + raid6_algo_add(&raid6_vpermxor1); + raid6_algo_add(&raid6_vpermxor2); + raid6_algo_add(&raid6_vpermxor4); + raid6_algo_add(&raid6_vpermxor8); + } +} diff --git a/lib/raid6/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc index 1bfb127fbfe8..e8964361aaef 100644 --- a/lib/raid6/vpermxor.uc +++ b/lib/raid/raid6/powerpc/vpermxor.uc @@ -20,15 +20,11 @@ * This instruction was introduced in POWER8 - ISA v2.07. */ -#include <linux/raid/pq.h> -#ifdef CONFIG_ALTIVEC - #include <altivec.h> #include <asm/ppc-opcode.h> -#ifdef __KERNEL__ #include <asm/cputable.h> #include <asm/switch_to.h> -#endif +#include "algos.h" typedef vector unsigned char unative_t; #define NSIZE sizeof(unative_t) @@ -80,26 +76,7 @@ static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec_vpermxor(void); -#if $# == 1 -int raid6_have_altivec_vpermxor(void) -{ - /* Check if arch has both altivec and the vpermxor instructions */ -# ifdef __KERNEL__ - return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && - cpu_has_feature(CPU_FTR_ARCH_207S)); -# else - return 1; -#endif - -} -#endif - const struct raid6_calls raid6_vpermxor$# = { - raid6_vpermxor$#_gen_syndrome, - NULL, - raid6_have_altivec_vpermxor, - "vpermxor$#", - 0 + .gen_syndrome = raid6_vpermxor$#_gen_syndrome, + .name = "vpermxor$#", }; -#endif diff --git a/lib/raid6/recov.c b/lib/raid/raid6/recov.c index b5e47c008b41..3fa53bc3fde4 100644 --- a/lib/raid6/recov.c +++ b/lib/raid/raid6/recov.c @@ -1,19 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/recov.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * RAID-6 data recovery in dual failure mode. In single failure mode, - * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct - * the syndrome.) + * RAID-6 data recovery in dual failure mode. In single failure mode, use the + * RAID-5 algorithm (or, in the case of Q failure, just reconstruct the + * syndrome.) */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" /* Recover two failed data blocks. */ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, @@ -31,13 +27,13 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +68,10 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -95,41 +91,5 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_intx1 = { .data2 = raid6_2data_recov_intx1, .datap = raid6_datap_recov_intx1, - .valid = NULL, .name = "intx1", - .priority = 0, }; - -#ifndef __KERNEL__ -/* Testing only */ - -/* Recover two failed blocks. */ -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs) -{ - if ( faila > failb ) { - int tmp = faila; - faila = failb; - failb = tmp; - } - - if ( failb == disks-1 ) { - if ( faila == disks-2 ) { - /* P+Q failure. Just rebuild the syndrome. */ - raid6_call.gen_syndrome(disks, bytes, ptrs); - } else { - /* data+Q failure. Reconstruct data from P, - then rebuild syndrome. */ - /* NOT IMPLEMENTED - equivalent to RAID-5 */ - } - } else { - if ( failb == disks-2 ) { - /* data+P failure. */ - raid6_datap_recov(disks, bytes, faila, ptrs); - } else { - /* data+data failure. */ - raid6_2data_recov(disks, bytes, faila, failb, ptrs); - } - } -} - -#endif diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h new file mode 100644 index 000000000000..82f1a188f8c4 --- /dev/null +++ b/lib/raid/raid6/riscv/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/vector.h> + +extern const struct raid6_calls raid6_rvvx1; +extern const struct raid6_calls raid6_rvvx2; +extern const struct raid6_calls raid6_rvvx4; +extern const struct raid6_calls raid6_rvvx8; +extern const struct raid6_recov_calls raid6_recov_rvv; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (has_vector()) { + raid6_algo_add(&raid6_rvvx1); + raid6_algo_add(&raid6_rvvx2); + raid6_algo_add(&raid6_rvvx4); + raid6_algo_add(&raid6_rvvx8); + raid6_recov_algo_add(&raid6_recov_rvv); + } +} diff --git a/lib/raid6/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c index 40c393206b6a..78e158a3e332 100644 --- a/lib/raid6/recov_rvv.c +++ b/lib/raid/raid6/riscv/recov_rvv.c @@ -4,8 +4,11 @@ * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" #include "rvv.h" +#include "pq_arch.h" static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, u8 *dq, const u8 *pbmul, @@ -158,13 +161,13 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +199,10 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -216,7 +219,5 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_rvv = { .data2 = raid6_2data_recov_rvv, .datap = raid6_datap_recov_rvv, - .valid = rvv_has_vector, .name = "rvv", - .priority = 1, }; diff --git a/lib/raid6/rvv.c b/lib/raid/raid6/riscv/rvv.c index 75c9dafedb28..4ac50606f3dc 100644 --- a/lib/raid6/rvv.c +++ b/lib/raid/raid6/riscv/rvv.c @@ -10,6 +10,7 @@ */ #include "rvv.h" +#include "pq_arch.h" #ifdef __riscv_vector #error "This code must be built without compiler support for vector" diff --git a/lib/raid6/rvv.h b/lib/raid/raid6/riscv/rvv.h index 6d0708a2c8a4..df0e3637cae8 100644 --- a/lib/raid6/rvv.h +++ b/lib/raid/raid6/riscv/rvv.h @@ -2,27 +2,11 @@ /* * Copyright 2024 Institute of Software, CAS. * - * raid6/rvv.h - * * Definitions for RISC-V RAID-6 code */ -#ifdef __KERNEL__ #include <asm/vector.h> -#else -#define kernel_vector_begin() -#define kernel_vector_end() -#include <sys/auxv.h> -#include <asm/hwcap.h> -#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V) -#endif - -#include <linux/raid/pq.h> - -static int rvv_has_vector(void) -{ - return has_vector(); -} +#include "algos.h" #define RAID6_RVV_WRAPPER(_n) \ static void raid6_rvv ## _n ## _gen_syndrome(int disks, \ @@ -48,9 +32,7 @@ static int rvv_has_vector(void) kernel_vector_end(); \ } \ struct raid6_calls const raid6_rvvx ## _n = { \ - raid6_rvv ## _n ## _gen_syndrome, \ - raid6_rvv ## _n ## _xor_syndrome, \ - rvv_has_vector, \ - "rvvx" #_n, \ - 0 \ + .gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \ + .name = "rvvx" #_n, \ } diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h new file mode 100644 index 000000000000..95d14c342306 --- /dev/null +++ b/lib/raid/raid6/s390/pq_arch.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <linux/cpufeature.h> + +extern const struct raid6_calls raid6_s390vx8; +extern const struct raid6_recov_calls raid6_recov_s390xc; + +static __always_inline void __init arch_raid6_init(void) +{ + if (cpu_has_vx()) + raid6_algo_add(&raid6_s390vx8); + else + raid6_algo_add_default(); + raid6_recov_algo_add(&raid6_recov_s390xc); +} diff --git a/lib/raid6/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c index 487018f81192..08d56896e5ea 100644 --- a/lib/raid6/recov_s390xc.c +++ b/lib/raid/raid6/s390/recov_s390xc.c @@ -6,7 +6,9 @@ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" static inline void xor_block(u8 *p1, u8 *p2) { @@ -34,13 +36,13 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -81,10 +83,10 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -110,7 +112,5 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_s390xc = { .data2 = raid6_2data_recov_s390xc, .datap = raid6_datap_recov_s390xc, - .valid = NULL, .name = "s390xc", - .priority = 1, }; diff --git a/lib/raid6/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc index 8aa53eb2f395..e5cf9054be2a 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid/raid6/s390/s390vx.uc @@ -12,8 +12,8 @@ */ #include <linux/cpufeature.h> -#include <linux/raid/pq.h> #include <asm/fpu.h> +#include "algos.h" #define NSIZE 16 @@ -121,15 +121,8 @@ static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(&vxstate, KERNEL_VXR); } -static int raid6_s390vx$#_valid(void) -{ - return cpu_has_vx(); -} - const struct raid6_calls raid6_s390vx$# = { - raid6_s390vx$#_gen_syndrome, - raid6_s390vx$#_xor_syndrome, - raid6_s390vx$#_valid, - "vx128x$#", - 1 + .gen_syndrome = raid6_s390vx$#_gen_syndrome, + .xor_syndrome = raid6_s390vx$#_xor_syndrome, + .name = "vx128x$#", }; diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile new file mode 100644 index 000000000000..87a001b22847 --- /dev/null +++ b/lib/raid/raid6/tests/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += raid6_kunit.o diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c new file mode 100644 index 000000000000..9f3e671a1224 --- /dev/null +++ b/lib/raid/raid6/tests/raid6_kunit.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved + * + * Test RAID-6 recovery algorithms. + */ + +#include <kunit/test.h> +#include <linux/prandom.h> +#include <linux/vmalloc.h> +#include <linux/raid/pq.h> +#include "../algos.h" + +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + +#define RAID6_KUNIT_SEED 42 +#define RAID6_KUNIT_NUM_TEST_ITERS 10 +#define RAID6_KUNIT_MAX_BUFFERS 64 /* Including P and Q */ +#define RAID6_KUNIT_MAX_FAILURES 2 +#define RAID6_KUNIT_MAX_BYTES PAGE_SIZE + +static struct rnd_state rng; +static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *aligned_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *test_recov_buffers[RAID6_KUNIT_MAX_FAILURES]; +static size_t test_buflen; + +struct test_args { + unsigned int recov_idx; + const struct raid6_recov_calls *recov; + unsigned int gen_idx; + const struct raid6_calls *gen; +}; + +static struct test_args args; + +static u32 rand32(void) +{ + return prandom_u32_state(&rng); +} + +/* Generate a random length that is a multiple of 512. */ +static unsigned int random_length(unsigned int max_length) +{ + return round_up((rand32() % max_length) + 1, 512); +} + +static unsigned int random_nr_buffers(void) +{ + return (rand32() % (RAID6_KUNIT_MAX_BUFFERS - (RAID6_MIN_DISKS - 1))) + + RAID6_MIN_DISKS; +} + +/* Generate a random alignment that is a multiple of 64. */ +static unsigned int random_alignment(unsigned int max_alignment) +{ + if (max_alignment == 0) + return 0; + return (rand32() % (max_alignment + 1)) & ~63; +} + +static void makedata(int start, int stop) +{ + int i; + + for (i = start; i <= stop; i++) + prandom_bytes_state(&rng, test_buffers[i], test_buflen); +} + +static char member_type(unsigned int nr_buffers, int d) +{ + if (d == nr_buffers - 2) + return 'P'; + if (d == nr_buffers - 1) + return 'Q'; + return 'D'; +} + +static void test_recover_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int faila, int failb) +{ + const struct test_args *ta = test->param_value; + void *dataptrs[RAID6_KUNIT_MAX_BUFFERS]; + int i; + + if (faila > failb) + swap(faila, failb); + + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + memset(test_recov_buffers[i], 0xf0, test_buflen); + + memcpy(dataptrs, aligned_buffers, sizeof(dataptrs)); + dataptrs[faila] = test_recov_buffers[0]; + dataptrs[failb] = test_recov_buffers[1]; + + if (failb == nr_buffers - 1) { + /* + * We don't implement the data+Q failure scenario, since it + * is equivalent to a RAID-5 failure (XOR, then recompute Q). + */ + if (WARN_ON_ONCE(faila != nr_buffers - 2)) + return; + + /* P+Q failure. Just rebuild the syndrome. */ + ta->gen->gen_syndrome(nr_buffers, len, dataptrs); + } else if (failb == nr_buffers - 2) { + /* data+P failure. */ + ta->recov->datap(nr_buffers, len, faila, dataptrs); + } else { + /* data+data failure. */ + ta->recov->data2(nr_buffers, len, faila, failb, dataptrs); + } + + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[faila], dataptrs[faila], + len, + "faila miscompared: %3d[%c] buffers %u len %u (failb=%3d[%c])\n", + faila, member_type(nr_buffers, faila), + nr_buffers, len, + failb, member_type(nr_buffers, failb)); + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[failb], dataptrs[failb], + len, + "failb miscompared: %3d[%c] buffers %u len %u (faila=%3d[%c])\n", + failb, member_type(nr_buffers, failb), + nr_buffers, len, + faila, member_type(nr_buffers, faila)); +} + +static void test_recover(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + unsigned int nr_data = nr_buffers - 2; + int iterations, i; + + /* Test P+Q recovery */ + test_recover_one(test, nr_buffers, len, nr_data, nr_buffers - 1); + + /* Test data+P recovery */ + for (i = 0; i < nr_buffers - 2; i++) + test_recover_one(test, nr_buffers, len, i, nr_data); + + /* Double data failure is impossible with a single data disk */ + if (nr_data == 1) + return; + + /* Test data+data recovery using random sampling */ + iterations = nr_buffers * 2; /* should provide good enough coverage */ + for (i = 0; i < iterations; i++) { + int faila = rand32() % nr_data, failb; + + do { + failb = rand32() % nr_data; + } while (failb == faila); + + test_recover_one(test, nr_buffers, len, faila, failb); + } +} + +/* Simulate rmw run */ +static void test_rmw_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int p1, int p2) +{ + const struct test_args *ta = test->param_value; + + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + makedata(p1, p2); + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + test_recover(test, nr_buffers, len); +} + +static void test_rmw(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + int iterations = nr_buffers / 2, i; + + for (i = 0; i < iterations; i++) { + int p1 = rand32() % (nr_buffers - 2); + int p2 = rand32() % (nr_buffers - 2); + + if (p2 < p1) + swap(p1, p2); + test_rmw_one(test, nr_buffers, len, p1, p2); + } +} + +static void raid6_test_one(struct kunit *test) +{ + const struct test_args *ta = test->param_value; + unsigned int nr_buffers = random_nr_buffers(); + unsigned int len = random_length(RAID6_KUNIT_MAX_BYTES); + unsigned int max_alignment; + int i; + + /* Nuke syndromes */ + memset(test_buffers[nr_buffers - 2], 0xee, test_buflen); + memset(test_buffers[nr_buffers - 1], 0xee, test_buflen); + + /* + * If we're not using the entire buffer size, inject randomize alignment + * into the buffer. + */ + max_alignment = RAID6_KUNIT_MAX_BYTES - len; + if (rand32() % 2 == 0) { + /* Use random alignments mod 64 */ + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + + random_alignment(max_alignment); + } else { + /* Go up to the guard page, to catch buffer overreads */ + unsigned int align = test_buflen - len; + + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + align; + } + + /* Generate assumed good syndrome */ + ta->gen->gen_syndrome(nr_buffers, len, aligned_buffers); + + test_recover(test, nr_buffers, len); + + if (ta->gen->xor_syndrome) + test_rmw(test, nr_buffers, len); +} + +static void raid6_test(struct kunit *test) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_NUM_TEST_ITERS; i++) + raid6_test_one(test); +} + +static const void *raid6_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + if (!prev) { + memset(&args, 0, sizeof(args)); +next_algo: + args.recov_idx = 0; + args.gen = raid6_algo_find(args.gen_idx); + if (!args.gen) + return NULL; + } + + if (args.recov) + args.recov_idx++; + args.recov = raid6_recov_algo_find(args.recov_idx); + if (!args.recov) { + args.gen_idx++; + goto next_algo; + } + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "gen=%s recov=%s", + args.gen->name, args.recov->name); + return &args; +} + +static struct kunit_case raid6_test_cases[] = { + KUNIT_CASE_PARAM(raid6_test, raid6_gen_params), + {}, +}; + +static int raid6_suite_init(struct kunit_suite *suite) +{ + int i; + + prandom_seed_state(&rng, RAID6_KUNIT_SEED); + + /* + * Allocate the test buffer using vmalloc() with a page-aligned length + * so that it is immediately followed by a guard page. This allows + * buffer overreads to be detected, even in assembly code. + */ + test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) { + test_recov_buffers[i] = vmalloc(test_buflen); + if (!test_recov_buffers[i]) + goto out_free_recov_buffers; + } + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) { + test_buffers[i] = vmalloc(test_buflen); + if (!test_buffers[i]) + goto out_free_buffers; + } + + makedata(0, RAID6_KUNIT_MAX_BUFFERS - 1); + + return 0; + +out_free_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); +out_free_recov_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); + return -ENOMEM; +} + +static void raid6_suite_exit(struct kunit_suite *suite) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); +} + +static struct kunit_suite raid6_test_suite = { + .name = "raid6", + .test_cases = raid6_test_cases, + .suite_init = raid6_suite_init, + .suite_exit = raid6_suite_exit, +}; +kunit_test_suite(raid6_test_suite); + +MODULE_DESCRIPTION("Unit test for the RAID P/Q library functions"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid6/unroll.awk b/lib/raid/raid6/unroll.awk index 0809805a7e23..0809805a7e23 100644 --- a/lib/raid6/unroll.awk +++ b/lib/raid/raid6/unroll.awk diff --git a/lib/raid6/avx2.c b/lib/raid/raid6/x86/avx2.c index 059024234dce..7d829c669ea7 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid/raid6/x86/avx2.c @@ -1,20 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright (C) 2012 Intel Corporation - * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> +/* + * Copyright (C) 2012 Intel Corporation + * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> * - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- */ - -/* * AVX2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx2_constants { u64 x1d[4]; @@ -23,11 +19,6 @@ static const struct raid6_avx2_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); -} - /* * Plain AVX2 implementation */ @@ -128,11 +119,9 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x1 = { - raid6_avx21_gen_syndrome, - raid6_avx21_xor_syndrome, - raid6_have_avx2, - "avx2x1", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx21_gen_syndrome, + .xor_syndrome = raid6_avx21_xor_syndrome, + .name = "avx2x1", }; /* @@ -258,11 +247,9 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x2 = { - raid6_avx22_gen_syndrome, - raid6_avx22_xor_syndrome, - raid6_have_avx2, - "avx2x2", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx22_gen_syndrome, + .xor_syndrome = raid6_avx22_xor_syndrome, + .name = "avx2x2", }; #ifdef CONFIG_X86_64 @@ -461,10 +448,8 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x4 = { - raid6_avx24_gen_syndrome, - raid6_avx24_xor_syndrome, - raid6_have_avx2, - "avx2x4", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx24_gen_syndrome, + .xor_syndrome = raid6_avx24_xor_syndrome, + .name = "avx2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/avx512.c b/lib/raid/raid6/x86/avx512.c index 009bd0adeebf..e671eb5bde63 100644 --- a/lib/raid6/avx512.c +++ b/lib/raid/raid6/x86/avx512.c @@ -1,24 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- -------------------------------------------------------- - * - * Copyright (C) 2016 Intel Corporation +/* + * Copyright (C) 2016 Intel Corporation * - * Author: Gayatri Kammela <gayatri.kammela@intel.com> - * Author: Megha Dey <megha.dey@linux.intel.com> + * Author: Gayatri Kammela <gayatri.kammela@intel.com> + * Author: Megha Dey <megha.dey@linux.intel.com> * - * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- - */ - -/* * AVX512 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx512_constants { u64 x1d[8]; @@ -29,16 +24,6 @@ static const struct raid6_avx512_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} - static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -156,11 +141,9 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x1 = { - raid6_avx5121_gen_syndrome, - raid6_avx5121_xor_syndrome, - raid6_have_avx512, - "avx512x1", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5121_gen_syndrome, + .xor_syndrome = raid6_avx5121_xor_syndrome, + .name = "avx512x1", }; /* @@ -313,11 +296,9 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x2 = { - raid6_avx5122_gen_syndrome, - raid6_avx5122_xor_syndrome, - raid6_have_avx512, - "avx512x2", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5122_gen_syndrome, + .xor_syndrome = raid6_avx5122_xor_syndrome, + .name = "avx512x2", }; #ifdef CONFIG_X86_64 @@ -551,10 +532,8 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(); } const struct raid6_calls raid6_avx512x4 = { - raid6_avx5124_gen_syndrome, - raid6_avx5124_xor_syndrome, - raid6_have_avx512, - "avx512x4", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5124_gen_syndrome, + .xor_syndrome = raid6_avx5124_xor_syndrome, + .name = "avx512x4", }; #endif diff --git a/lib/raid6/mmx.c b/lib/raid/raid6/x86/mmx.c index 3a5bf53a297b..afa82536142d 100644 --- a/lib/raid6/mmx.c +++ b/lib/raid/raid6/x86/mmx.c @@ -1,20 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/mmx.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * MMX implementation of RAID-6 syndrome functions + * MMX implementation of RAID-6 syndrome functions. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Shared with raid6/sse1.c */ const struct raid6_mmx_constants { @@ -23,12 +16,6 @@ const struct raid6_mmx_constants { 0x1d1d1d1d1d1d1d1dULL, }; -static int raid6_have_mmx(void) -{ - /* Not really "boot_cpu" but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX); -} - /* * Plain MMX implementation */ @@ -70,11 +57,8 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx1 = { - raid6_mmx1_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx1", - 0 + .gen_syndrome = raid6_mmx1_gen_syndrome, + .name = "mmxx1", }; /* @@ -129,11 +113,6 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx2 = { - raid6_mmx2_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx2", - 0 + .gen_syndrome = raid6_mmx2_gen_syndrome, + .name = "mmxx2", }; - -#endif diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h new file mode 100644 index 000000000000..02f8843b0537 --- /dev/null +++ b/lib/raid/raid6/x86/pq_arch.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpufeature.h> + +extern const struct raid6_calls raid6_mmxx1; +extern const struct raid6_calls raid6_mmxx2; +extern const struct raid6_calls raid6_sse1x1; +extern const struct raid6_calls raid6_sse1x2; +extern const struct raid6_calls raid6_sse2x1; +extern const struct raid6_calls raid6_sse2x2; +extern const struct raid6_calls raid6_sse2x4; +extern const struct raid6_calls raid6_avx2x1; +extern const struct raid6_calls raid6_avx2x2; +extern const struct raid6_calls raid6_avx2x4; +extern const struct raid6_calls raid6_avx512x1; +extern const struct raid6_calls raid6_avx512x2; +extern const struct raid6_calls raid6_avx512x4; + +extern const struct raid6_recov_calls raid6_recov_ssse3; +extern const struct raid6_recov_calls raid6_recov_avx2; +extern const struct raid6_recov_calls raid6_recov_avx512; + +static inline int raid6_has_avx512(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512BW) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + boot_cpu_has(X86_FEATURE_AVX512DQ); +} + +static inline bool raid6_has_avx2(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); +} + +static inline bool raid6_has_ssse3(void) +{ + return boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2) && + boot_cpu_has(X86_FEATURE_SSSE3); +} + +static inline bool raid6_has_sse2(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + boot_cpu_has(X86_FEATURE_FXSR) && + boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2); +} + +static inline bool raid6_has_sse1_or_mmxext(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + (boot_cpu_has(X86_FEATURE_XMM) || + boot_cpu_has(X86_FEATURE_MMXEXT)); +} + +static __always_inline void __init arch_raid6_init(void) +{ + if (raid6_has_avx2()) { + raid6_algo_add(&raid6_avx2x1); + raid6_algo_add(&raid6_avx2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx2x4); + if (raid6_has_avx512()) { + raid6_algo_add(&raid6_avx512x1); + raid6_algo_add(&raid6_avx512x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx512x4); + } + } else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) { + /* x86_64 can assume SSE2 as baseline */ + raid6_algo_add(&raid6_sse2x1); + raid6_algo_add(&raid6_sse2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_sse2x4); + } else { + raid6_algo_add_default(); + if (raid6_has_sse1_or_mmxext()) { + raid6_algo_add(&raid6_sse1x1); + raid6_algo_add(&raid6_sse1x2); + } else if (boot_cpu_has(X86_FEATURE_MMX)) { + raid6_algo_add(&raid6_mmxx1); + raid6_algo_add(&raid6_mmxx2); + } + } + + if (raid6_has_avx512()) + raid6_recov_algo_add(&raid6_recov_avx512); + else if (raid6_has_avx2()) + raid6_recov_algo_add(&raid6_recov_avx2); + else if (raid6_has_ssse3()) + raid6_recov_algo_add(&raid6_recov_ssse3); +} diff --git a/lib/raid6/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c index 97d598d2535c..a714a780a2d8 100644 --- a/lib/raid6/recov_avx2.c +++ b/lib/raid/raid6/x86/recov_avx2.c @@ -4,14 +4,10 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -28,13 +24,13 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +192,10 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -303,11 +299,9 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx2 = { .data2 = raid6_2data_recov_avx2, .datap = raid6_datap_recov_avx2, - .valid = raid6_has_avx2, #ifdef CONFIG_X86_64 .name = "avx2x2", #else .name = "avx2x1", #endif - .priority = 2, }; diff --git a/lib/raid6/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c index 7986120ca444..ec72d5a30c01 100644 --- a/lib/raid6/recov_avx512.c +++ b/lib/raid/raid6/x86/recov_avx512.c @@ -6,18 +6,10 @@ * Author: Megha Dey <megha.dey@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -37,13 +29,13 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -238,10 +230,10 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -367,11 +359,9 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx512 = { .data2 = raid6_2data_recov_avx512, .datap = raid6_datap_recov_avx512, - .valid = raid6_has_avx512, #ifdef CONFIG_X86_64 .name = "avx512x2", #else .name = "avx512x1", #endif - .priority = 3, }; diff --git a/lib/raid6/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c index 2e849185c32b..700bd2c865ec 100644 --- a/lib/raid6/recov_ssse3.c +++ b/lib/raid/raid6/x86/recov_ssse3.c @@ -3,15 +3,10 @@ * Copyright (C) 2012 Intel Corporation */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_ssse3(void) -{ - return boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2) && - boot_cpu_has(X86_FEATURE_SSSE3); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -30,13 +25,13 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -203,10 +198,10 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -318,11 +313,9 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_ssse3 = { .data2 = raid6_2data_recov_ssse3, .datap = raid6_datap_recov_ssse3, - .valid = raid6_has_ssse3, #ifdef CONFIG_X86_64 .name = "ssse3x2", #else .name = "ssse3x1", #endif - .priority = 1, }; diff --git a/lib/raid6/sse1.c b/lib/raid/raid6/x86/sse1.c index 692fa3a93bf0..f4b260df522a 100644 --- a/lib/raid6/sse1.c +++ b/lib/raid/raid6/x86/sse1.c @@ -1,39 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse1.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * SSE-1/MMXEXT implementation of RAID-6 syndrome functions + * SSE-1/MMXEXT implementation of RAID-6 syndrome functions. * - * This is really an MMX implementation, but it requires SSE-1 or - * AMD MMXEXT for prefetch support and a few other features. The - * support for nontemporal memory accesses is enough to make this - * worthwhile as a separate implementation. + * This is really an MMX implementation, but it requires SSE-1 or AMD MMXEXT for + * prefetch support and a few other features. The support for nontemporal + * memory accesses is enough to make this worthwhile as a separate + * implementation. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Defined in raid6/mmx.c */ extern const struct raid6_mmx_constants { u64 x1d; } raid6_mmx_constants; -static int raid6_have_sse1_or_mmxext(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - (boot_cpu_has(X86_FEATURE_XMM) || - boot_cpu_has(X86_FEATURE_MMXEXT)); -} - /* * Plain SSE1 implementation */ @@ -86,11 +71,8 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x1 = { - raid6_sse11_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse11_gen_syndrome, + .name = "sse1x1", }; /* @@ -149,11 +131,6 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x2 = { - raid6_sse12_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse12_gen_syndrome, + .name = "sse1x2", }; - -#endif diff --git a/lib/raid6/sse2.c b/lib/raid/raid6/x86/sse2.c index 2930220249c9..43b09ce58270 100644 --- a/lib/raid6/sse2.c +++ b/lib/raid/raid6/x86/sse2.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse2.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * * SSE-2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_sse_constants { u64 x1d[2]; @@ -21,15 +15,6 @@ static const struct raid6_sse_constants { { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, }; -static int raid6_have_sse2(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - boot_cpu_has(X86_FEATURE_FXSR) && - boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2); -} - /* * Plain SSE2 implementation */ @@ -133,11 +118,9 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x1 = { - raid6_sse21_gen_syndrome, - raid6_sse21_xor_syndrome, - raid6_have_sse2, - "sse2x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse21_gen_syndrome, + .xor_syndrome = raid6_sse21_xor_syndrome, + .name = "sse2x1", }; /* @@ -263,11 +246,9 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x2 = { - raid6_sse22_gen_syndrome, - raid6_sse22_xor_syndrome, - raid6_have_sse2, - "sse2x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse22_gen_syndrome, + .xor_syndrome = raid6_sse22_xor_syndrome, + .name = "sse2x2", }; #ifdef CONFIG_X86_64 @@ -470,11 +451,9 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop, const struct raid6_calls raid6_sse2x4 = { - raid6_sse24_gen_syndrome, - raid6_sse24_xor_syndrome, - raid6_have_sse2, - "sse2x4", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse24_gen_syndrome, + .xor_syndrome = raid6_sse24_xor_syndrome, + .name = "sse2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile index 4d633dfd5b90..e8ecec3c09f9 100644 --- a/lib/raid/xor/Makefile +++ b/lib/raid/xor/Makefile @@ -17,9 +17,10 @@ endif xor-$(CONFIG_ALPHA) += alpha/xor.o xor-$(CONFIG_ARM) += arm/xor.o ifeq ($(CONFIG_ARM),y) -xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o +xor-$(CONFIG_KERNEL_MODE_NEON) += xor-neon.o arm/xor-neon-glue.o endif -xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o +xor-$(CONFIG_ARM64) += xor-neon.o arm64/xor-eor3.o \ + arm64/xor-neon-glue.o xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o @@ -31,11 +32,11 @@ xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o obj-y += tests/ -CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_xor-neon.o += $(CC_FLAGS_FPU) -I$(src)/$(SRCARCH) +CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU) -CFLAGS_arm64/xor-neon.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_arm64/xor-neon.o += $(CC_FLAGS_NO_FPU) +CFLAGS_arm64/xor-eor3.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm64/xor-eor3.o += $(CC_FLAGS_NO_FPU) CFLAGS_powerpc/xor_vmx.o += -mhard-float -maltivec \ $(call cc-option,-mabi=altivec) \ diff --git a/lib/raid/xor/arm/xor-neon.c b/lib/raid/xor/arm/xor-neon.c deleted file mode 100644 index 23147e3a7904..000000000000 --- a/lib/raid/xor/arm/xor-neon.c +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#include "xor_impl.h" -#include "xor_arch.h" - -#ifndef __ARM_NEON__ -#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon' -#endif - -/* - * Pull in the reference implementations while instructing GCC (through - * -ftree-vectorize) to attempt to exploit implicit parallelism and emit - * NEON instructions. Clang does this by default at O2 so no pragma is - * needed. - */ -#ifdef CONFIG_CC_IS_GCC -#pragma GCC optimize "tree-vectorize" -#endif - -#define NO_TEMPLATE -#include "../xor-8regs.c" - -__DO_XOR_BLOCKS(neon_inner, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5); diff --git a/lib/raid/xor/arm/xor-neon.h b/lib/raid/xor/arm/xor-neon.h new file mode 100644 index 000000000000..406e0356f05b --- /dev/null +++ b/lib/raid/xor/arm/xor-neon.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +extern struct xor_block_template xor_block_arm4regs; +extern struct xor_block_template xor_block_neon; + +void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, + unsigned int bytes); diff --git a/lib/raid/xor/arm/xor_arch.h b/lib/raid/xor/arm/xor_arch.h index 775ff835df65..f1ddb64fe62a 100644 --- a/lib/raid/xor/arm/xor_arch.h +++ b/lib/raid/xor/arm/xor_arch.h @@ -3,12 +3,7 @@ * Copyright (C) 2001 Russell King */ #include <asm/neon.h> - -extern struct xor_block_template xor_block_arm4regs; -extern struct xor_block_template xor_block_neon; - -void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt, - unsigned int bytes); +#include "xor-neon.h" static __always_inline void __init arch_xor_init(void) { diff --git a/lib/raid/xor/arm64/xor-eor3.c b/lib/raid/xor/arm64/xor-eor3.c new file mode 100644 index 000000000000..e44016c363f1 --- /dev/null +++ b/lib/raid/xor/arm64/xor-eor3.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <linux/cache.h> +#include <asm/neon-intrinsics.h> +#include "xor_impl.h" +#include "xor_arch.h" +#include "xor-neon.h" + +extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2); + +static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) +{ + uint64x2_t res; + + asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" + "eor3 %0.16b, %1.16b, %2.16b, %3.16b" + : "=w"(res) : "w"(p), "w"(q), "w"(r)); + return res; +} + +static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 */ + v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); + v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); + v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); + v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + } while (--lines > 0); +} + +static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1, + const unsigned long * __restrict p2, + const unsigned long * __restrict p3, + const unsigned long * __restrict p4, + const unsigned long * __restrict p5) +{ + uint64_t *dp1 = (uint64_t *)p1; + uint64_t *dp2 = (uint64_t *)p2; + uint64_t *dp3 = (uint64_t *)p3; + uint64_t *dp4 = (uint64_t *)p4; + uint64_t *dp5 = (uint64_t *)p5; + + register uint64x2_t v0, v1, v2, v3; + long lines = bytes / (sizeof(uint64x2_t) * 4); + + do { + /* p1 ^= p2 ^ p3 */ + v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), + vld1q_u64(dp3 + 0)); + v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), + vld1q_u64(dp3 + 2)); + v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), + vld1q_u64(dp3 + 4)); + v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), + vld1q_u64(dp3 + 6)); + + /* p1 ^= p4 ^ p5 */ + v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); + v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); + v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); + v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); + + /* store */ + vst1q_u64(dp1 + 0, v0); + vst1q_u64(dp1 + 2, v1); + vst1q_u64(dp1 + 4, v2); + vst1q_u64(dp1 + 6, v3); + + dp1 += 8; + dp2 += 8; + dp3 += 8; + dp4 += 8; + dp5 += 8; + } while (--lines > 0); +} + +__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4, + __xor_eor3_5); diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c index 1edaed8acffe..46b3c8bdc27f 100644 --- a/lib/raid/xor/xor-8regs.c +++ b/lib/raid/xor/xor-8regs.c @@ -93,11 +93,9 @@ xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1, } while (--lines > 0); } -#ifndef NO_TEMPLATE DO_XOR_BLOCKS(8regs, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5); struct xor_block_template xor_block_8regs = { .name = "8regs", .xor_gen = xor_gen_8regs, }; -#endif /* NO_TEMPLATE */ diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c index bd4e6e434418..50931fbf0324 100644 --- a/lib/raid/xor/xor-core.c +++ b/lib/raid/xor/xor-core.c @@ -8,6 +8,7 @@ #include <linux/module.h> #include <linux/gfp.h> +#include <linux/slab.h> #include <linux/raid/xor.h> #include <linux/jiffies.h> #include <linux/preempt.h> @@ -114,7 +115,7 @@ static int __init calibrate_xor_blocks(void) if (forced_template) return 0; - b1 = (void *) __get_free_pages(GFP_KERNEL, 2); + b1 = kmalloc(PAGE_SIZE * 4, GFP_KERNEL); if (!b1) { pr_warn("xor: Yikes! No memory available.\n"); return -ENOMEM; @@ -132,7 +133,7 @@ static int __init calibrate_xor_blocks(void) pr_info("xor: using function: %s (%d MB/sec)\n", fastest->name, fastest->speed); - free_pages((unsigned long)b1, 2); + kfree(b1); return 0; } diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/xor-neon.c index 97ef3cb92496..c7c3cf634e23 100644 --- a/lib/raid/xor/arm64/xor-neon.c +++ b/lib/raid/xor/xor-neon.c @@ -4,12 +4,11 @@ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd. */ -#include <linux/cache.h> -#include <asm/neon-intrinsics.h> #include "xor_impl.h" -#include "xor_arch.h" #include "xor-neon.h" +#include <asm/neon-intrinsics.h> + static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1, const unsigned long * __restrict p2) { @@ -175,138 +174,6 @@ static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1, __DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4, __xor_neon_5); -static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r) -{ - uint64x2_t res; - - asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n" - "eor3 %0.16b, %1.16b, %2.16b, %3.16b" - : "=w"(res) : "w"(p), "w"(q), "w"(r)); - return res; -} - -static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - } while (--lines > 0); -} - -static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - uint64_t *dp4 = (uint64_t *)p4; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* p1 ^= p4 */ - v0 = veorq_u64(v0, vld1q_u64(dp4 + 0)); - v1 = veorq_u64(v1, vld1q_u64(dp4 + 2)); - v2 = veorq_u64(v2, vld1q_u64(dp4 + 4)); - v3 = veorq_u64(v3, vld1q_u64(dp4 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - dp4 += 8; - } while (--lines > 0); -} - -static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1, - const unsigned long * __restrict p2, - const unsigned long * __restrict p3, - const unsigned long * __restrict p4, - const unsigned long * __restrict p5) -{ - uint64_t *dp1 = (uint64_t *)p1; - uint64_t *dp2 = (uint64_t *)p2; - uint64_t *dp3 = (uint64_t *)p3; - uint64_t *dp4 = (uint64_t *)p4; - uint64_t *dp5 = (uint64_t *)p5; - - register uint64x2_t v0, v1, v2, v3; - long lines = bytes / (sizeof(uint64x2_t) * 4); - - do { - /* p1 ^= p2 ^ p3 */ - v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0), - vld1q_u64(dp3 + 0)); - v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2), - vld1q_u64(dp3 + 2)); - v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4), - vld1q_u64(dp3 + 4)); - v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6), - vld1q_u64(dp3 + 6)); - - /* p1 ^= p4 ^ p5 */ - v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0)); - v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2)); - v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4)); - v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6)); - - /* store */ - vst1q_u64(dp1 + 0, v0); - vst1q_u64(dp1 + 2, v1); - vst1q_u64(dp1 + 4, v2); - vst1q_u64(dp1 + 6, v3); - - dp1 += 8; - dp2 += 8; - dp3 += 8; - dp4 += 8; - dp5 += 8; - } while (--lines > 0); -} - -__DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4, - __xor_eor3_5); +#ifdef CONFIG_ARM64 +extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2); +#endif diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile deleted file mode 100644 index 5be0a4e60ab1..000000000000 --- a/lib/raid6/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_RAID6_PQ) += raid6_pq.o - -raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ - int8.o - -raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o -raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o -raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o -raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o - -hostprogs += mktables - -ifeq ($(CONFIG_ALTIVEC),y) -altivec_flags := -maltivec $(call cc-option,-mabi=altivec) -# Enable <altivec.h> -altivec_flags += -isystem $(shell $(CC) -print-file-name=include) - -ifdef CONFIG_CC_IS_CLANG -# clang ppc port does not yet support -maltivec when -msoft-float is -# enabled. A future release of clang will resolve this -# https://llvm.org/pr31177 -CFLAGS_REMOVE_altivec1.o += -msoft-float -CFLAGS_REMOVE_altivec2.o += -msoft-float -CFLAGS_REMOVE_altivec4.o += -msoft-float -CFLAGS_REMOVE_altivec8.o += -msoft-float -CFLAGS_REMOVE_vpermxor1.o += -msoft-float -CFLAGS_REMOVE_vpermxor2.o += -msoft-float -CFLAGS_REMOVE_vpermxor4.o += -msoft-float -CFLAGS_REMOVE_vpermxor8.o += -msoft-float -endif -endif - -quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ - -targets += int1.c int2.c int4.c int8.c -$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_altivec1.o += $(altivec_flags) -CFLAGS_altivec2.o += $(altivec_flags) -CFLAGS_altivec4.o += $(altivec_flags) -CFLAGS_altivec8.o += $(altivec_flags) -targets += altivec1.c altivec2.c altivec4.c altivec8.c -$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_vpermxor1.o += $(altivec_flags) -CFLAGS_vpermxor2.o += $(altivec_flags) -CFLAGS_vpermxor4.o += $(altivec_flags) -CFLAGS_vpermxor8.o += $(altivec_flags) -targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c -$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_neon1.o += $(CC_FLAGS_FPU) -CFLAGS_neon2.o += $(CC_FLAGS_FPU) -CFLAGS_neon4.o += $(CC_FLAGS_FPU) -CFLAGS_neon8.o += $(CC_FLAGS_FPU) -CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU) -targets += neon1.c neon2.c neon4.c neon8.c -$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += s390vx8.c -$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -quiet_cmd_mktable = TABLE $@ - cmd_mktable = $(obj)/mktables > $@ - -targets += tables.c -$(obj)/tables.c: $(obj)/mktables FORCE - $(call if_changed,mktable) diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c deleted file mode 100644 index 799e0e5eac26..000000000000 --- a/lib/raid6/algos.c +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/algos.c - * - * Algorithm list and algorithm selection for RAID-6 - */ - -#include <linux/raid/pq.h> -#ifndef __KERNEL__ -#include <sys/mman.h> -#include <stdio.h> -#else -#include <linux/module.h> -#include <linux/gfp.h> -#endif - -struct raid6_calls raid6_call; -EXPORT_SYMBOL_GPL(raid6_call); - -const struct raid6_calls * const raid6_algos[] = { -#if defined(__i386__) && !defined(__arch_um__) - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x2, - &raid6_sse2x1, - &raid6_sse1x2, - &raid6_sse1x1, - &raid6_mmxx2, - &raid6_mmxx1, -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_avx512x4, - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x4, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x4, - &raid6_sse2x2, - &raid6_sse2x1, -#endif -#ifdef CONFIG_ALTIVEC - &raid6_vpermxor8, - &raid6_vpermxor4, - &raid6_vpermxor2, - &raid6_vpermxor1, - &raid6_altivec8, - &raid6_altivec4, - &raid6_altivec2, - &raid6_altivec1, -#endif -#if defined(CONFIG_S390) - &raid6_s390vx8, -#endif -#ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx8, - &raid6_neonx4, - &raid6_neonx2, - &raid6_neonx1, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_rvvx1, - &raid6_rvvx2, - &raid6_rvvx4, - &raid6_rvvx8, -#endif - &raid6_intx8, - &raid6_intx4, - &raid6_intx2, - &raid6_intx1, - NULL -}; - -void (*raid6_2data_recov)(int, size_t, int, int, void **); -EXPORT_SYMBOL_GPL(raid6_2data_recov); - -void (*raid6_datap_recov)(int, size_t, int, void **); -EXPORT_SYMBOL_GPL(raid6_datap_recov); - -const struct raid6_recov_calls *const raid6_recov_algos[] = { -#ifdef CONFIG_X86 - &raid6_recov_avx512, - &raid6_recov_avx2, - &raid6_recov_ssse3, -#endif -#ifdef CONFIG_S390 - &raid6_recov_s390xc, -#endif -#if defined(CONFIG_KERNEL_MODE_NEON) - &raid6_recov_neon, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_recov_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_recov_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_recov_rvv, -#endif - &raid6_recov_intx1, - NULL -}; - -#ifdef __KERNEL__ -#define RAID6_TIME_JIFFIES_LG2 4 -#else -/* Need more time to be stable in userspace */ -#define RAID6_TIME_JIFFIES_LG2 9 -#define time_before(x, y) ((x) < (y)) -#endif - -#define RAID6_TEST_DISKS 8 -#define RAID6_TEST_DISKS_ORDER 3 - -static inline const struct raid6_recov_calls *raid6_choose_recov(void) -{ - const struct raid6_recov_calls *const *algo; - const struct raid6_recov_calls *best; - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) - if (!(*algo)->valid || (*algo)->valid()) - best = *algo; - - if (best) { - raid6_2data_recov = best->data2; - raid6_datap_recov = best->datap; - - pr_info("raid6: using %s recovery algorithm\n", best->name); - } else - pr_err("raid6: Yikes! No recovery algorithm found!\n"); - - return best; -} - -static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) -{ - unsigned long perf, bestgenperf, j0, j1; - int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ - const struct raid6_calls *const *algo; - const struct raid6_calls *best; - - for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { - if (!best || (*algo)->priority >= best->priority) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - best = *algo; - break; - } - - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { - (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - if (perf > bestgenperf) { - bestgenperf = perf; - best = *algo; - } - pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf * HZ * (disks-2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - } - } - - if (!best) { - pr_err("raid6: Yikes! No algorithm found!\n"); - goto out; - } - - raid6_call = *best; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - pr_info("raid6: skipped pq benchmark and selected %s\n", - best->name); - goto out; - } - - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - - if (best->xor_syndrome) { - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { - best->xor_syndrome(disks, start, stop, - PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (perf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); - } - -out: - return best; -} - - -/* Try to pick the best algorithm */ -/* This code uses the gfmul table as convenient data set to abuse */ - -int __init raid6_select_algo(void) -{ - const int disks = RAID6_TEST_DISKS; - - const struct raid6_calls *gen_best; - const struct raid6_recov_calls *rec_best; - char *disk_ptr, *p; - void *dptrs[RAID6_TEST_DISKS]; - int i, cycle; - - /* prepare the buffer and fill it circularly with gfmul table */ - disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); - if (!disk_ptr) { - pr_err("raid6: Yikes! No memory available.\n"); - return -ENOMEM; - } - - p = disk_ptr; - for (i = 0; i < disks; i++) - dptrs[i] = p + PAGE_SIZE * i; - - cycle = ((disks - 2) * PAGE_SIZE) / 65536; - for (i = 0; i < cycle; i++) { - memcpy(p, raid6_gfmul, 65536); - p += 65536; - } - - if ((disks - 2) * PAGE_SIZE % 65536) - memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); - - /* select raid gen_syndrome function */ - gen_best = raid6_choose_gen(&dptrs, disks); - - /* select raid recover functions */ - rec_best = raid6_choose_recov(); - - free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); - - return gen_best && rec_best ? 0 : -EINVAL; -} - -static void raid6_exit(void) -{ - do { } while (0); -} - -subsys_initcall(raid6_select_algo); -module_exit(raid6_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h deleted file mode 100644 index acfc33ce7056..000000000000 --- a/lib/raid6/loongarch.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> - * - * raid6/loongarch.h - * - * Definitions common to LoongArch RAID-6 code only - */ - -#ifndef _LIB_RAID6_LOONGARCH_H -#define _LIB_RAID6_LOONGARCH_H - -#ifdef __KERNEL__ - -#include <asm/cpu-features.h> -#include <asm/fpu.h> - -#else /* for user-space testing */ - -#include <sys/auxv.h> - -/* have to supply these defines for glibc 2.37- and musl */ -#ifndef HWCAP_LOONGARCH_LSX -#define HWCAP_LOONGARCH_LSX (1 << 4) -#endif -#ifndef HWCAP_LOONGARCH_LASX -#define HWCAP_LOONGARCH_LASX (1 << 5) -#endif - -#define kernel_fpu_begin() -#define kernel_fpu_end() - -#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) -#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) - -#endif /* __KERNEL__ */ - -#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore deleted file mode 100644 index 1b68a77f348f..000000000000 --- a/lib/raid6/test/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/int.uc -/neon.uc -/raid6test diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile deleted file mode 100644 index 09bbe2b14cce..000000000000 --- a/lib/raid6/test/Makefile +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# This is a simple Makefile to test some of the RAID-6 code -# from userspace. -# - -pound := \# - -# Adjust as desired -CC = gcc -OPTFLAGS = -O2 -CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) -LD = ld -AWK = awk -f -AR = ar -RANLIB = ranlib -OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o - -ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/) -ifeq ($(ARCH),i386) - CFLAGS += -DCONFIG_X86_32 - IS_X86 = yes -endif -ifeq ($(ARCH),x86_64) - CFLAGS += -DCONFIG_X86_64 - IS_X86 = yes -endif - -ifeq ($(ARCH),arm) - CFLAGS += -I../../../arch/arm/include -mfpu=neon - HAS_NEON = yes -endif -ifeq ($(ARCH),aarch64) - CFLAGS += -I../../../arch/arm64/include - HAS_NEON = yes -endif - -ifeq ($(findstring riscv,$(ARCH)),riscv) - CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1 - HAS_RVV = yes -endif - -ifeq ($(findstring ppc,$(ARCH)),ppc) - CFLAGS += -I../../../arch/powerpc/include - HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) -endif - -ifeq ($(ARCH),loongarch64) - CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 - CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) - CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) -endif - -ifeq ($(IS_X86),yes) - OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o - CFLAGS += -DCONFIG_X86 -else ifeq ($(HAS_NEON),yes) - OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o - CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -else ifeq ($(ARCH),loongarch64) - OBJS += loongarch_simd.o recov_loongarch_simd.o -else ifeq ($(HAS_RVV),yes) - OBJS += rvv.o recov_rvv.o - CFLAGS += -DCONFIG_RISCV_ISA_V=1 -endif - -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< - -%.c: ../%.c - cp -f $< $@ - -%.uc: ../%.uc - cp -f $< $@ - -all: raid6.a raid6test - -raid6.a: $(OBJS) - rm -f $@ - $(AR) cq $@ $^ - $(RANLIB) $@ - -raid6test: test.c raid6.a - $(CC) $(CFLAGS) -o raid6test $^ - -neon1.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < neon.uc > $@ - -neon2.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < neon.uc > $@ - -neon4.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < neon.uc > $@ - -neon8.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < neon.uc > $@ - -altivec1.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@ - -altivec2.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < altivec.uc > $@ - -altivec4.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < altivec.uc > $@ - -altivec8.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < altivec.uc > $@ - -vpermxor1.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@ - -vpermxor2.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@ - -vpermxor4.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@ - -vpermxor8.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@ - -int1.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < int.uc > $@ - -int2.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < int.uc > $@ - -int4.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < int.uc > $@ - -int8.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < int.uc > $@ - -int16.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=16 < int.uc > $@ - -int32.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=32 < int.uc > $@ - -tables.c: mktables - ./mktables > tables.c - -clean: - rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test - -spotless: clean - rm -f *~ diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c deleted file mode 100644 index 841a55242aba..000000000000 --- a/lib/raid6/test/test.c +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6test.c - * - * Test RAID-6 recovery with various algorithms - */ - -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <linux/raid/pq.h> - -#define NDISKS 16 /* Including P and Q */ - -const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -char *dataptrs[NDISKS]; -char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -static void makedata(int start, int stop) -{ - int i, j; - - for (i = start; i <= stop; i++) { - for (j = 0; j < PAGE_SIZE; j++) - data[i][j] = rand(); - - dataptrs[i] = data[i]; - } -} - -static char disk_type(int d) -{ - switch (d) { - case NDISKS-2: - return 'P'; - case NDISKS-1: - return 'Q'; - default: - return 'D'; - } -} - -static int test_disks(int i, int j) -{ - int erra, errb; - - memset(recovi, 0xf0, PAGE_SIZE); - memset(recovj, 0xba, PAGE_SIZE); - - dataptrs[i] = recovi; - dataptrs[j] = recovj; - - raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs); - - erra = memcmp(data[i], recovi, PAGE_SIZE); - errb = memcmp(data[j], recovj, PAGE_SIZE); - - if (i < NDISKS-2 && j == NDISKS-1) { - /* We don't implement the DQ failure scenario, since it's - equivalent to a RAID-5 failure (XOR, then recompute Q) */ - erra = errb = 0; - } else { - printf("algo=%-8s faila=%3d(%c) failb=%3d(%c) %s\n", - raid6_call.name, - i, disk_type(i), - j, disk_type(j), - (!erra && !errb) ? "OK" : - !erra ? "ERRB" : - !errb ? "ERRA" : "ERRAB"); - } - - dataptrs[i] = data[i]; - dataptrs[j] = data[j]; - - return erra || errb; -} - -int main(int argc, char *argv[]) -{ - const struct raid6_calls *const *algo; - const struct raid6_recov_calls *const *ra; - int i, j, p1, p2; - int err = 0; - - makedata(0, NDISKS-1); - - for (ra = raid6_recov_algos; *ra; ra++) { - if ((*ra)->valid && !(*ra)->valid()) - continue; - - raid6_2data_recov = (*ra)->data2; - raid6_datap_recov = (*ra)->datap; - - printf("using recovery %s\n", (*ra)->name); - - for (algo = raid6_algos; *algo; algo++) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - raid6_call = **algo; - - /* Nuke syndromes */ - memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE); - - /* Generate assumed good syndrome */ - raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - - if (!raid6_call.xor_syndrome) - continue; - - for (p1 = 0; p1 < NDISKS-2; p1++) - for (p2 = p1; p2 < NDISKS-2; p2++) { - - /* Simulate rmw run */ - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - makedata(p1, p2); - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - } - - } - printf("\n"); - } - - printf("\n"); - /* Pick the best algorithm test */ - raid6_select_algo(); - - if (err) - printf("\n*** ERRORS FOUND ***\n"); - - return err; -} diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h deleted file mode 100644 index 9a6ff37115e7..000000000000 --- a/lib/raid6/x86.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* ----------------------------------------------------------------------- * - * - * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/x86.h - * - * Definitions common to x86 and x86-64 RAID-6 code only - */ - -#ifndef LINUX_RAID_RAID6X86_H -#define LINUX_RAID_RAID6X86_H - -#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__) - -#ifdef __KERNEL__ /* Real code */ - -#include <asm/fpu/api.h> - -#else /* Dummy code for user space testing */ - -static inline void kernel_fpu_begin(void) -{ -} - -static inline void kernel_fpu_end(void) -{ -} - -#define __aligned(x) __attribute__((aligned(x))) - -#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ -#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions - * (fast save and restore) */ -#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ -#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ -#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ -#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ -#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ -#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ -#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ -#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular) - * Instructions - */ -#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular) - * Instructions - */ -#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length) - * Extensions - */ -#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ - -/* Should work well enough on modern CPUs for testing */ -static inline int boot_cpu_has(int flag) -{ - u32 eax, ebx, ecx, edx; - - eax = (flag & 0x100) ? 7 : - (flag & 0x20) ? 0x80000001 : 1; - ecx = 0; - - asm volatile("cpuid" - : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx)); - - return ((flag & 0x100 ? ebx : - (flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1; -} - -#endif /* ndef __KERNEL__ */ - -#endif -#endif diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 04b3a808fca9..40cfb38ac919 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -687,6 +687,108 @@ void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, } EXPORT_SYMBOL_GPL(rhashtable_insert_slow); +/* Scan one element forward from prev_key's position in @tbl. + * Returns first rhash_head whose bucket > prev_key's bucket, or the + * element immediately after prev_key inside prev_key's bucket. + * Returns the first element if prev_key is NULL, NULL when @tbl is + * exhausted, or ERR_PTR(-ENOENT) if prev_key is not found in @tbl. + */ +static struct rhash_head *__rhashtable_next_in_table( + struct rhashtable *ht, struct bucket_table *tbl, + const void *prev_key) +{ + struct rhashtable_compare_arg arg = { .ht = ht, .key = prev_key }; + const struct rhashtable_params params = ht->p; + struct rhash_head *he; + unsigned int b = 0; + bool found = false; + + if (prev_key) { + b = rht_key_hashfn(ht, tbl, prev_key, params); + rht_for_each_rcu(he, tbl, b) { + bool match = params.obj_cmpfn + ? !params.obj_cmpfn(&arg, rht_obj(ht, he)) + : !rhashtable_compare(&arg, rht_obj(ht, he)); + if (found) { + if (match) + continue; + return he; + } + if (match) + found = true; + } + if (!found) + return ERR_PTR(-ENOENT); + b++; + } + + for (; b < tbl->size; b++) + rht_for_each_rcu(he, tbl, b) + return he; + return NULL; +} + +/** + * rhashtable_next_key - return next element after a given key + * @ht: hash table + * @prev_key: pointer to previous key, or NULL for the first element + * + * WARNING: this walk is highly unstable. Unlike rhashtable_walk_*(), + * it cannot detect a concurrent resize or rehash, so a full iteration + * is NOT guaranteed to terminate under adversarial or sustained + * rehashing. Callers MUST tolerate skipped and duplicated elements and + * SHOULD bound their loop externally. + * + * Returns the next element in best-effort iteration order, walking the + * @tbl chain (including any future_tbl in flight). Caller must hold RCU. + * + * Pass @prev_key == NULL to obtain the first element. To iterate, set + * @prev_key to the key of the previously returned element on each call, + * and stop when NULL is returned. + * + * Best-effort semantics: + * - Across the tbl->future_tbl chain, an element being migrated may + * transiently appear in both tables and be observed twice. + * - Concurrent inserts may or may not be observed. + * - Termination of a full iteration loop is NOT guaranteed under + * adversarial continuous rehash; callers MUST tolerate skips and + * repeats and SHOULD bound their loop externally. + * - Behavior on tables that contain duplicate keys is undefined: + * duplicates may be skipped, repeated, or trap the walk in a + * cycle. Callers requiring duplicate-key iteration must use + * rhashtable_walk_*() instead. + * - rhltable instances are not supported and return + * ERR_PTR(-EOPNOTSUPP). + * - If prev_key was concurrently deleted and is not present in any + * in-flight table, returns ERR_PTR(-ENOENT). + * + * Returns entry of the next element, or NULL when iteration is exhausted, + * or ERR_PTR(-ENOENT) if prev_key is not found, or + * ERR_PTR(-EOPNOTSUPP) if @ht is an rhltable. + */ +void *rhashtable_next_key(struct rhashtable *ht, const void *prev_key) +{ + struct bucket_table *tbl; + struct rhash_head *he; + + if (unlikely(ht->rhlist)) + return ERR_PTR(-EOPNOTSUPP); + + tbl = rht_dereference_rcu(ht->tbl, ht); + do { + he = __rhashtable_next_in_table(ht, tbl, prev_key); + if (!IS_ERR_OR_NULL(he)) + return rht_obj(ht, he); + if (!he) + prev_key = NULL; + /* See any new future_tbl attached during a rehash. */ + smp_rmb(); + tbl = rht_dereference_rcu(tbl->future_tbl, ht); + } while (tbl); + return he; /* NULL or -ENOENT */ +} +EXPORT_SYMBOL_GPL(rhashtable_next_key); + /** * rhashtable_walk_enter - Initialise an iterator * @ht: Table to walk over @@ -1057,8 +1159,9 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed) * .obj_hashfn = my_hash_fn, * }; */ -int rhashtable_init_noprof(struct rhashtable *ht, - const struct rhashtable_params *params) +int __rhashtable_init_noprof(struct rhashtable *ht, + const struct rhashtable_params *params, + struct lock_class_key *key) { struct bucket_table *tbl; size_t size; @@ -1068,7 +1171,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return -EINVAL; memset(ht, 0, sizeof(*ht)); - mutex_init(&ht->mutex); + mutex_init_with_key(&ht->mutex, key); spin_lock_init(&ht->lock); memcpy(&ht->p, params, sizeof(*params)); @@ -1120,7 +1223,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, return 0; } -EXPORT_SYMBOL_GPL(rhashtable_init_noprof); +EXPORT_SYMBOL_GPL(__rhashtable_init_noprof); /** * rhltable_init - initialize a new hash list table @@ -1131,15 +1234,17 @@ EXPORT_SYMBOL_GPL(rhashtable_init_noprof); * * See documentation for rhashtable_init. */ -int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params) +int __rhltable_init_noprof(struct rhltable *hlt, + const struct rhashtable_params *params, + struct lock_class_key *key) { int err; - err = rhashtable_init_noprof(&hlt->ht, params); + err = __rhashtable_init_noprof(&hlt->ht, params, key); hlt->ht.rhlist = true; return err; } -EXPORT_SYMBOL_GPL(rhltable_init_noprof); +EXPORT_SYMBOL_GPL(__rhltable_init_noprof); static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj, void (*free_fn)(void *ptr, void *arg), diff --git a/lib/scatterlist.c b/lib/scatterlist.c index b7fe91ef35b8..6ea40d2e6247 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -1366,6 +1366,7 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter, sg_max--; maxsize -= len; + start += len; ret += len; if (maxsize <= 0 || sg_max == 0) break; diff --git a/lib/seq_buf.c b/lib/seq_buf.c index f3f3436d60a9..b59488fa8135 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -298,6 +298,7 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, } return 0; } +EXPORT_SYMBOL_GPL(seq_buf_putmem_hex); /** * seq_buf_path - copy a path into the sequence buffer diff --git a/lib/string.c b/lib/string.c index b632c71df1a5..a4e8ad23577d 100644 --- a/lib/string.c +++ b/lib/string.c @@ -21,6 +21,7 @@ #include <linux/errno.h> #include <linux/limits.h> #include <linux/linkage.h> +#include <linux/minmax.h> #include <linux/stddef.h> #include <linux/string.h> #include <linux/types.h> @@ -88,22 +89,6 @@ char *strcpy(char *dest, const char *src) EXPORT_SYMBOL(strcpy); #endif -#ifndef __HAVE_ARCH_STRNCPY -char *strncpy(char *dest, const char *src, size_t count) -{ - char *tmp = dest; - - while (count) { - if ((*tmp = *src) != 0) - src++; - tmp++; - count--; - } - return dest; -} -EXPORT_SYMBOL(strncpy); -#endif - #ifdef __BIG_ENDIAN # define ALLBUTLAST_BYTE_MASK (~255ul) #else @@ -125,11 +110,8 @@ ssize_t sized_strscpy(char *dest, const char *src, size_t count) * If src is unaligned, don't cross a page boundary, * since we don't know if the next page is mapped. */ - if ((long)src & (sizeof(long) - 1)) { - size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)); - if (limit < max) - max = limit; - } + if ((long)src & (sizeof(long) - 1)) + max = min(PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)), max); #else /* If src or dest is unaligned, don't do word-at-a-time. */ if (((long) dest | (long) src) & (sizeof(long) - 1)) diff --git a/lib/string_helpers.c b/lib/string_helpers.c index 169eaf583494..98d6ed0eaab7 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c @@ -752,7 +752,7 @@ EXPORT_SYMBOL_GPL(kstrdup_and_replace); * kasprintf_strarray - allocate and fill array of sequential strings * @gfp: flags for the slab allocator * @prefix: prefix to be used - * @n: amount of lines to be allocated and filled + * @n: number of strings to be allocated and filled * * Allocates and fills @n strings using pattern "%s-%zu", where prefix * is provided by caller. The caller is responsible to free them with @@ -765,7 +765,7 @@ char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n) char **names; size_t i; - names = kcalloc(n + 1, sizeof(char *), gfp); + names = kcalloc(n, sizeof(char *), gfp); if (!names) return NULL; @@ -805,7 +805,7 @@ void kfree_strarray(char **array, size_t n) EXPORT_SYMBOL_GPL(kfree_strarray); struct strarray { - char **array; + char **array __counted_by_ptr(n); size_t n; }; @@ -824,13 +824,13 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n) if (!ptr) return ERR_PTR(-ENOMEM); + ptr->n = n; ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n); if (!ptr->array) { devres_free(ptr); return ERR_PTR(-ENOMEM); } - ptr->n = n; devres_add(dev, ptr); return ptr->array; diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index ee87fef66cb5..811128d0df16 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c @@ -198,6 +198,7 @@ static void __init test_kstrtoull_fail(void) {"10000000000000000000000000000000000000000000000000000000000000000", 2}, {"2000000000000000000000", 8}, {"18446744073709551616", 10}, + {"569202370375329612767", 10}, {"10000000000000000", 16}, /* negative */ {"-0", 0}, @@ -275,9 +276,11 @@ static void __init test_kstrtoll_fail(void) {"9223372036854775809", 10}, {"18446744073709551614", 10}, {"18446744073709551615", 10}, + {"569202370375329612767", 10}, {"-9223372036854775809", 10}, {"-18446744073709551614", 10}, {"-18446744073709551615", 10}, + {"-569202370375329612767", 10}, /* sign is first character if any */ {"-+1", 0}, {"-+1", 8}, @@ -334,6 +337,7 @@ static void __init test_kstrtou64_fail(void) {"-1", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, }; TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail); } @@ -386,6 +390,8 @@ static void __init test_kstrtos64_fail(void) {"18446744073709551615", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, + {"-569202370375329612767", 10}, }; TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail); } diff --git a/lib/test_bpf.c b/lib/test_bpf.c index 5892c0f17ddc..af6f3340c034 100644 --- a/lib/test_bpf.c +++ b/lib/test_bpf.c @@ -560,8 +560,23 @@ static int bpf_fill_max_jmp_never_taken(struct bpf_test *self) } /* ALU result computation used in tests */ -static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) +enum { F_ALU32 = 1, F_SIGNED = 2 }; + +static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags) { + bool is_signed = flags & F_SIGNED; + + /* Narrow operands for ALU32 */ + if (flags & F_ALU32) { + if (is_signed) { + v1 = (u64)(s32)v1; + v2 = (u64)(s32)v2; + } else { + v1 = (u32)v1; + v2 = (u32)v2; + } + } + *res = 0; switch (op) { case BPF_MOV: @@ -599,12 +614,28 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) case BPF_DIV: if (v2 == 0) return false; - *res = div64_u64(v1, v2); + if (!is_signed) { + *res = div64_u64(v1, v2); + } else { + if ((s64)v2 == -1) /* Handled by verifier */ + return false; + *res = (u64)div64_s64(v1, v2); + } break; case BPF_MOD: if (v2 == 0) return false; - div64_u64_rem(v1, v2, res); + if (!is_signed) { + div64_u64_rem(v1, v2, res); + } else { + if ((s64)v2 == -1) + return false; + /* + * Avoid s64 % s64 which generates __moddi3 on + * 32-bit architectures. Use div64_s64 instead. + */ + *res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2); + } break; } return true; @@ -612,7 +643,7 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op) /* Test an ALU shift operation for all valid shift values */ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, - u8 mode, bool alu32) + u8 mode, u32 flags) { static const s64 regs[] = { 0x0123456789abcdefLL, /* dword > 0, word < 0 */ @@ -620,7 +651,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, 0xfedcba0198765432LL, /* dword < 0, word < 0 */ 0x0123458967abcdefLL, /* dword > 0, word > 0 */ }; - int bits = alu32 ? 32 : 64; + int bits = (flags & F_ALU32) ? 32 : 64; int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3; struct bpf_insn *insn; int imm, k; @@ -643,7 +674,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, /* Perform operation */ insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3); insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm); - if (alu32) { + if (flags & F_ALU32) { if (mode == BPF_K) insn[i++] = BPF_ALU32_IMM(op, R1, imm); else @@ -653,14 +684,14 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, reg = (s32)reg; else reg = (u32)reg; - __bpf_alu_result(&val, reg, imm, op); + __bpf_alu_result(&val, reg, imm, op, 0); val = (u32)val; } else { if (mode == BPF_K) insn[i++] = BPF_ALU64_IMM(op, R1, imm); else insn[i++] = BPF_ALU64_REG(op, R1, R2); - __bpf_alu_result(&val, reg, imm, op); + __bpf_alu_result(&val, reg, imm, op, 0); } /* @@ -688,62 +719,62 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op, static int bpf_fill_alu64_lsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0); } static int bpf_fill_alu64_rsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0); } static int bpf_fill_alu64_arsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0); } static int bpf_fill_alu64_lsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0); } static int bpf_fill_alu64_rsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0); } static int bpf_fill_alu64_arsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0); } static int bpf_fill_alu32_lsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_rsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_arsh_imm(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32); } static int bpf_fill_alu32_lsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32); } static int bpf_fill_alu32_rsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32); } static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true); + return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32); } /* @@ -751,9 +782,9 @@ static int bpf_fill_alu32_arsh_reg(struct bpf_test *self) * for the case when the source and destination are the same. */ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, - bool alu32) + u32 flags) { - int bits = alu32 ? 32 : 64; + int bits = (flags & F_ALU32) ? 32 : 64; int len = 3 + 6 * bits; struct bpf_insn *insn; int i = 0; @@ -770,14 +801,14 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, /* Perform operation */ insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val); - if (alu32) + if (flags & F_ALU32) insn[i++] = BPF_ALU32_REG(op, R1, R1); else insn[i++] = BPF_ALU64_REG(op, R1, R1); /* Compute the reference result */ - __bpf_alu_result(&res, val, val, op); - if (alu32) + __bpf_alu_result(&res, val, val, op, 0); + if (flags & F_ALU32) res = (u32)res; i += __bpf_ld_imm64(&insn[i], R2, res); @@ -798,32 +829,32 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op, static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0); } static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0); } static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false); + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0); } static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32); } static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32); } static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self) { - return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true); + return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32); } /* @@ -936,17 +967,20 @@ static int __bpf_fill_pattern(struct bpf_test *self, void *arg, static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, struct bpf_insn *insns, s64 dst, s64 imm) { - int op = *(int *)arg; + int *a = arg; + int op = a[0]; + u32 flags = a[1]; + s16 off = (flags & F_SIGNED) ? 1 : 0; int i = 0; u64 res; if (!insns) return 7; - if (__bpf_alu_result(&res, dst, (s32)imm, op)) { + if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R3, res); - insns[i++] = BPF_ALU64_IMM(op, R1, imm); + insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off); insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); insns[i++] = BPF_EXIT_INSN(); } @@ -957,17 +991,20 @@ static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg, static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg, struct bpf_insn *insns, s64 dst, s64 imm) { - int op = *(int *)arg; + int *a = arg; + int op = a[0]; + u32 flags = a[1]; + s16 off = (flags & F_SIGNED) ? 1 : 0; int i = 0; u64 res; if (!insns) return 7; - if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) { + if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R3, (u32)res); - insns[i++] = BPF_ALU32_IMM(op, R1, imm); + insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off); insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1); insns[i++] = BPF_EXIT_INSN(); } @@ -985,7 +1022,7 @@ static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg, if (!insns) return 9; - if (__bpf_alu_result(&res, dst, src, op)) { + if (__bpf_alu_result(&res, dst, src, op, 0)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R2, src); i += __bpf_ld_imm64(&insns[i], R3, res); @@ -1007,7 +1044,7 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, if (!insns) return 9; - if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) { + if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) { i += __bpf_ld_imm64(&insns[i], R1, dst); i += __bpf_ld_imm64(&insns[i], R2, src); i += __bpf_ld_imm64(&insns[i], R3, (u32)res); @@ -1019,16 +1056,20 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg, return i; } -static int __bpf_fill_alu64_imm(struct bpf_test *self, int op) +static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags) { - return __bpf_fill_pattern(self, &op, 64, 32, + int arg[2] = {op, flags}; + + return __bpf_fill_pattern(self, &arg, 64, 32, PATTERN_BLOCK1, PATTERN_BLOCK2, &__bpf_emit_alu64_imm); } -static int __bpf_fill_alu32_imm(struct bpf_test *self, int op) +static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags) { - return __bpf_fill_pattern(self, &op, 64, 32, + int arg[2] = {op, flags}; + + return __bpf_fill_pattern(self, &arg, 64, 32, PATTERN_BLOCK1, PATTERN_BLOCK2, &__bpf_emit_alu32_imm); } @@ -1050,93 +1091,115 @@ static int __bpf_fill_alu32_reg(struct bpf_test *self, int op) /* ALU64 immediate operations */ static int bpf_fill_alu64_mov_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MOV); + return __bpf_fill_alu64_imm(self, BPF_MOV, 0); } static int bpf_fill_alu64_and_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_AND); + return __bpf_fill_alu64_imm(self, BPF_AND, 0); } static int bpf_fill_alu64_or_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_OR); + return __bpf_fill_alu64_imm(self, BPF_OR, 0); } static int bpf_fill_alu64_xor_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_XOR); + return __bpf_fill_alu64_imm(self, BPF_XOR, 0); } static int bpf_fill_alu64_add_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_ADD); + return __bpf_fill_alu64_imm(self, BPF_ADD, 0); } static int bpf_fill_alu64_sub_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_SUB); + return __bpf_fill_alu64_imm(self, BPF_SUB, 0); } static int bpf_fill_alu64_mul_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MUL); + return __bpf_fill_alu64_imm(self, BPF_MUL, 0); } static int bpf_fill_alu64_div_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_DIV); + return __bpf_fill_alu64_imm(self, BPF_DIV, 0); } static int bpf_fill_alu64_mod_imm(struct bpf_test *self) { - return __bpf_fill_alu64_imm(self, BPF_MOD); + return __bpf_fill_alu64_imm(self, BPF_MOD, 0); +} + +/* Signed ALU64 immediate operations */ +static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu64_smod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED); +} + +/* Signed ALU32 immediate operations */ +static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu32_smod_imm(struct bpf_test *self) +{ + return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED); } /* ALU32 immediate operations */ static int bpf_fill_alu32_mov_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MOV); + return __bpf_fill_alu32_imm(self, BPF_MOV, 0); } static int bpf_fill_alu32_and_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_AND); + return __bpf_fill_alu32_imm(self, BPF_AND, 0); } static int bpf_fill_alu32_or_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_OR); + return __bpf_fill_alu32_imm(self, BPF_OR, 0); } static int bpf_fill_alu32_xor_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_XOR); + return __bpf_fill_alu32_imm(self, BPF_XOR, 0); } static int bpf_fill_alu32_add_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_ADD); + return __bpf_fill_alu32_imm(self, BPF_ADD, 0); } static int bpf_fill_alu32_sub_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_SUB); + return __bpf_fill_alu32_imm(self, BPF_SUB, 0); } static int bpf_fill_alu32_mul_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MUL); + return __bpf_fill_alu32_imm(self, BPF_MUL, 0); } static int bpf_fill_alu32_div_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_DIV); + return __bpf_fill_alu32_imm(self, BPF_DIV, 0); } static int bpf_fill_alu32_mod_imm(struct bpf_test *self) { - return __bpf_fill_alu32_imm(self, BPF_MOD); + return __bpf_fill_alu32_imm(self, BPF_MOD, 0); } /* ALU64 register operations */ @@ -1235,7 +1298,8 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self) * Test JITs that implement complex ALU operations as function * calls, and must re-arrange operands for argument passing. */ -static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) +static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, + u32 flags) { int len = 2 + 10 * 10; struct bpf_insn *insns; @@ -1249,28 +1313,37 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) return -ENOMEM; /* Operand and result values according to operation */ - if (alu32) - dst = 0x76543210U; - else - dst = 0x7edcba9876543210ULL; + if (flags & F_SIGNED) { + if (flags & F_ALU32) + dst = -76543210; + else + dst = -7654321076543210LL; + } else { + if (flags & F_ALU32) + dst = 0x76543210U; + else + dst = 0x7edcba9876543210ULL; + } imm = 0x01234567U; if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) imm &= 31; - __bpf_alu_result(&res, dst, imm, op); + __bpf_alu_result(&res, dst, imm, op, flags); - if (alu32) + if (flags & F_ALU32) res = (u32)res; /* Check all operand registers */ for (rd = R0; rd <= R9; rd++) { i += __bpf_ld_imm64(&insns[i], rd, dst); - if (alu32) - insns[i++] = BPF_ALU32_IMM(op, rd, imm); + s16 off = (flags & F_SIGNED) ? 1 : 0; + + if (flags & F_ALU32) + insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off); else - insns[i++] = BPF_ALU64_IMM(op, rd, imm); + insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off); insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2); insns[i++] = BPF_MOV64_IMM(R0, __LINE__); @@ -1295,123 +1368,145 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32) /* ALU64 K registers */ static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOV, false); + return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0); } static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_AND, false); + return __bpf_fill_alu_imm_regs(self, BPF_AND, 0); } static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_OR, false); + return __bpf_fill_alu_imm_regs(self, BPF_OR, 0); } static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_XOR, false); + return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0); } static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_LSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0); } static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_RSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0); } static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false); + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0); } static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ADD, false); + return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0); } static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_SUB, false); + return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0); } static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MUL, false); + return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0); } static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_DIV, false); + return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0); } static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOD, false); + return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0); +} + +/* Signed ALU64 K registers */ +static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED); +} + +static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED); } /* ALU32 K registers */ static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOV, true); + return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32); } static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_AND, true); + return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32); } static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_OR, true); + return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32); } static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_XOR, true); + return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32); } static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_LSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32); } static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_RSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32); } static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true); + return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32); } static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_ADD, true); + return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32); } static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_SUB, true); + return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32); } static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MUL, true); + return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32); } static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_DIV, true); + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32); } static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self) { - return __bpf_fill_alu_imm_regs(self, BPF_MOD, true); + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32); +} + +/* Signed ALU32 K registers */ +static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED); +} + +static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self) +{ + return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED); } /* @@ -1442,8 +1537,8 @@ static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32) if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH) src &= 31; - __bpf_alu_result(&res, dst, src, op); - __bpf_alu_result(&same, src, src, op); + __bpf_alu_result(&res, dst, src, op, 0); + __bpf_alu_result(&same, src, src, op, 0); if (alu32) { res = (u32)res; @@ -1626,7 +1721,7 @@ static int __bpf_emit_atomic64(struct bpf_test *self, void *arg, res = src; break; default: - __bpf_alu_result(&res, dst, src, BPF_OP(op)); + __bpf_alu_result(&res, dst, src, BPF_OP(op), 0); } keep = 0x0123456789abcdefULL; @@ -1673,7 +1768,7 @@ static int __bpf_emit_atomic32(struct bpf_test *self, void *arg, res = src; break; default: - __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op)); + __bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0); } keep = 0x0123456789abcdefULL; @@ -1939,7 +2034,7 @@ static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op) res = mem; break; default: - __bpf_alu_result(&res, mem, upd, BPF_OP(op)); + __bpf_alu_result(&res, mem, upd, BPF_OP(op), 0); } /* Test all operand registers */ @@ -12354,6 +12449,22 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu64_mod_imm_regs, }, + { + "ALU64_SDIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sdiv_imm_regs, + }, + { + "ALU64_SMOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_smod_imm_regs, + }, /* ALU32 K registers */ { "ALU32_MOV_K: registers", @@ -12451,6 +12562,22 @@ static struct bpf_test tests[] = { { { 0, 1 } }, .fill_helper = bpf_fill_alu32_mod_imm_regs, }, + { + "ALU32_SDIV_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sdiv_imm_regs, + }, + { + "ALU32_SMOD_K: registers", + { }, + INTERNAL, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_smod_imm_regs, + }, /* ALU64 X register combinations */ { "ALU64_MOV_X: register combinations", @@ -12881,6 +13008,24 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu64_mod_imm, .nr_testruns = NR_PATTERN_RUNS, }, + { + "ALU64_SDIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_sdiv_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU64_SMOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu64_smod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, /* ALU32 immediate magnitudes */ { "ALU32_MOV_K: all immediate value magnitudes", @@ -12963,6 +13108,24 @@ static struct bpf_test tests[] = { .fill_helper = bpf_fill_alu32_mod_imm, .nr_testruns = NR_PATTERN_RUNS, }, + { + "ALU32_SDIV_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_sdiv_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, + { + "ALU32_SMOD_K: all immediate value magnitudes", + { }, + INTERNAL | FLAG_NO_DATA, + { }, + { { 0, 1 } }, + .fill_helper = bpf_fill_alu32_smod_imm, + .nr_testruns = NR_PATTERN_RUNS, + }, /* ALU64 register magnitudes */ { "ALU64_MOV_X: all register value magnitudes", diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 06b4a6a028e0..316f4dfcda65 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock, struct test_mutex_data { struct mutex mtx; int counter __guarded_by(&mtx); + + struct mutex mtx2; + int anyread __guarded_by(&mtx, &mtx2); + int *anyptr __pt_guarded_by(&mtx, &mtx2); }; static void __used test_mutex_init(struct test_mutex_data *d) @@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d) } } +static void __used test_mutex_multiguard(struct test_mutex_data *d) +{ + mutex_lock(&d->mtx); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx); + + mutex_lock(&d->mtx2); + (void)d->anyread; + (void)*d->anyptr; + mutex_unlock(&d->mtx2); + + mutex_lock(&d->mtx); + mutex_lock(&d->mtx2); + d->anyread++; + (*d->anyptr)++; + mutex_unlock(&d->mtx2); + mutex_unlock(&d->mtx); +} + struct test_seqlock_data { seqlock_t sl; int counter __guarded_by(&sl); diff --git a/lib/test_firmware.c b/lib/test_firmware.c index b471d720879a..7459bba65444 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -867,7 +867,7 @@ static int test_fw_run_batch_request(void *data) if (test_fw_config->into_buf) { void *test_buf; - test_buf = kzalloc(TEST_FIRMWARE_BUF_SIZE, GFP_KERNEL); + test_buf = kzalloc(test_fw_config->buf_size, GFP_KERNEL); if (!test_buf) return -ENOMEM; diff --git a/lib/test_fortify/write_overflow-strncpy-src.c b/lib/test_fortify/write_overflow-strncpy-src.c deleted file mode 100644 index 8dcfb8c788dd..000000000000 --- a/lib/test_fortify/write_overflow-strncpy-src.c +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#define TEST \ - strncpy(small, large_src, sizeof(small) + 1) - -#include "test_fortify.h" diff --git a/lib/test_fortify/write_overflow-strncpy.c b/lib/test_fortify/write_overflow-strncpy.c deleted file mode 100644 index b85f079c815d..000000000000 --- a/lib/test_fortify/write_overflow-strncpy.c +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#define TEST \ - strncpy(instance.buf, large_src, sizeof(instance.buf) + 1) - -#include "test_fortify.h" diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 213504915737..9c59d1ceb5b5 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -1063,6 +1063,25 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args, /* Try with smaller pages if large allocation fails */ if (!dpage && order) { dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr); + if (!dpage) { + /* Unlock and free pages already allocated. */ + while (i > 0) { + struct page *fpage; + + fpage = migrate_pfn_to_page(dst[--i]); + unlock_page(fpage); + __free_page(fpage); + } + /* Clear remaining dst entries to avoid + * migrate_vma_pages/finalize() using + * uninitialized values. + */ + while (i < (1 << order)) { + dst[i] = 0; + i++; + } + return VM_FAULT_OOM; + } lock_page(dpage); dst[i] = migrate_pfn(page_to_pfn(dpage)); dst_page = pfn_to_page(page_to_pfn(dpage)); @@ -1111,9 +1130,6 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, unsigned long *src_pfns; unsigned long *dst_pfns; - src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); - dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); - start = cmd->addr; end = start + size; if (end < start) @@ -1123,6 +1139,9 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, if (!mmget_not_zero(mm)) return -EINVAL; + src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL); + dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL); + cmd->cpages = 0; mmap_read_lock(mm); for (addr = start; addr < end; addr = next) { @@ -1148,7 +1167,11 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror, goto out; pr_debug("Migrating from device mem to sys mem\n"); - dmirror_devmem_fault_alloc_and_copy(&args, dmirror); + if (dmirror_devmem_fault_alloc_and_copy(&args, dmirror)) { + migrate_vma_finalize(&args); + ret = -ENOMEM; + goto out; + } migrate_vma_pages(&args); cmd->cpages += dmirror_successful_migrated_pages(&args); @@ -1253,8 +1276,8 @@ out: mmap_read_unlock(mm); mmput(mm); free_mem: - kfree(src_pfns); - kfree(dst_pfns); + kvfree(src_pfns); + kvfree(dst_pfns); return ret; } @@ -1679,12 +1702,20 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf) if (order) args.flags |= MIGRATE_VMA_SELECT_COMPOUND; - if (migrate_vma_setup(&args)) - return VM_FAULT_SIGBUS; + /* + * In practice migrate_vma_setup() should never fail unless the + * test is wrong as it just tests some static VMA properties. + */ + if (migrate_vma_setup(&args)) { + ret = VM_FAULT_SIGBUS; + goto err; + } ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror); - if (ret) + if (ret) { + migrate_vma_finalize(&args); goto err; + } migrate_vma_pages(&args); /* * No device finalize step is needed since diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 6298f66c964b..68c3b9da090e 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor, for (iter = 0; iter < 10; iter++) { /* Do a test of bulk allocations */ if (!want_rcu && !want_ctor) { - int ret; - - ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array); - if (!ret) { + if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, + bulk_array)) { fail = true; } else { int i; - for (i = 0; i < ret; i++) + for (i = 0; i < BULK_SIZE; i++) fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); - kmem_cache_free_bulk(c, ret, bulk_array); + kmem_cache_free_bulk(c, BULK_SIZE, bulk_array); } } @@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures) { struct kmem_cache *c; int i, iter, maxiter = 1024; - int num, bytes; + int bytes; bool fail = false; void *objects[10]; c = kmem_cache_create("test_cache", size, size, 0, NULL); for (iter = 0; (iter < maxiter) && !fail; iter++) { - num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), - objects); - for (i = 0; i < num; i++) { + if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), + objects)) + continue; + + for (i = 0; i < ARRAY_SIZE(objects); i++) { bytes = count_nonzero_bytes(objects[i], size); if (bytes) fail = true; fill_with_garbage(objects[i], size); } - if (num) - kmem_cache_free_bulk(c, num, objects); + kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects); } kmem_cache_destroy(c); *total_failures += fail; @@ -387,7 +386,7 @@ static int __init test_kmemcache(int *total_failures) ctor = flags & 1; rcu = flags & 2; zero = flags & 4; - if (ctor & zero) + if (ctor && zero) continue; num_tests += do_kmem_cache_size(size, ctor, rcu, zero, &failures); diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index 0b33559a910b..b767a38a74f9 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -679,6 +679,78 @@ out: return err; } +static int __init test_rhashtable_next_key(void) +{ + struct rhashtable_params params = test_rht_params; + struct test_obj_val key_missing = { .id = 99999, .tid = 0 }; + struct test_obj_val *prev_key = NULL; + struct rhashtable ht; + struct test_obj *objs, *cur; + int i, count = 0, err; + int visited_keys[8] = { 0 }; + const int n = ARRAY_SIZE(visited_keys); + + params.nelem_hint = n; + + err = rhashtable_init(&ht, ¶ms); + if (err) + return err; + + objs = kcalloc(n, sizeof(*objs), GFP_KERNEL); + if (!objs) { + rhashtable_destroy(&ht); + return -ENOMEM; + } + + for (i = 0; i < n; i++) { + objs[i].value.id = i; + err = rhashtable_insert_fast(&ht, &objs[i].node, params); + if (err) + goto out; + } + + rcu_read_lock(); + + /* NULL prev_key: walk from the beginning, expect all n elements. */ + while ((cur = rhashtable_next_key(&ht, prev_key))) { + if (IS_ERR(cur)) { + err = -EINVAL; + goto unlock; + } + count++; + prev_key = &cur->value; + visited_keys[cur->value.id] = 1; + if (count > n) + break; + } + + if (count != n) { + err = -EINVAL; + goto unlock; + } + + for (i = 0; i < n; i++) { + if (!visited_keys[i]) { + err = -EINVAL; + goto unlock; + } + } + + /* Non-existing prev_key: must return ERR_PTR(-ENOENT). */ + cur = rhashtable_next_key(&ht, &key_missing); + if (!IS_ERR(cur) || PTR_ERR(cur) != -ENOENT) + err = -EINVAL; + +unlock: + rcu_read_unlock(); +out: + for (i = 0; i < n; i++) + rhashtable_remove_fast(&ht, &objs[i].node, params); + kfree(objs); + rhashtable_destroy(&ht); + return err; +} + static int __init test_rht_init(void) { unsigned int entries; @@ -738,6 +810,9 @@ static int __init test_rht_init(void) test_insert_duplicates_run(); + pr_info("Testing rhashtable_next_key: %s\n", + test_rhashtable_next_key() == 0 ? "pass" : "FAIL"); + if (!tcount) return 0; diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c index 876c72c18a0c..b23f85e8f8ca 100644 --- a/lib/test_vmalloc.c +++ b/lib/test_vmalloc.c @@ -55,6 +55,7 @@ __param(int, run_test_mask, 7, "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n" "\t\tid: 1024, name: vm_map_ram_test\n" "\t\tid: 2048, name: no_block_alloc_test\n" + "\t\tid: 4096, name: vrealloc_test\n" /* Add a new test case description here. */ ); @@ -421,6 +422,66 @@ cleanup: return nr_allocated != map_nr_pages; } +static int vrealloc_test(void) +{ + void *ptr, *tmp; + int i; + + for (i = 0; i < test_loop_count; i++) { + int err = -1; + + ptr = vrealloc(NULL, PAGE_SIZE, GFP_KERNEL); + if (!ptr) + return -1; + + *((__u8 *)ptr) = 'a'; + + /* Grow: beyond allocated pages, triggers full realloc. */ + tmp = vrealloc(ptr, 4 * PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Shrink: crosses page boundary, frees tail pages. */ + tmp = vrealloc(ptr, PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Shrink: within same page, no page freeing. */ + tmp = vrealloc(ptr, PAGE_SIZE / 2, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + /* Grow: within allocated page, in-place, no realloc. */ + tmp = vrealloc(ptr, PAGE_SIZE, GFP_KERNEL); + if (!tmp) + goto error; + ptr = tmp; + + if (*((__u8 *)ptr) != 'a') + goto error; + + err = 0; +error: + vfree(ptr); + if (err) + return err; + } + + return 0; +} + struct test_case_desc { const char *test_name; int (*test_func)(void); @@ -440,6 +501,7 @@ static struct test_case_desc test_case_array[] = { { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test, }, { "vm_map_ram_test", vm_map_ram_test, }, { "no_block_alloc_test", no_block_alloc_test, true }, + { "vrealloc_test", vrealloc_test, }, /* Add a new test case here. */ }; diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 7e9c2fa52e35..4ead57602eac 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_BITS_TEST) += test_bits.o +obj-$(CONFIG_SHDI3_KUNIT_TEST) += shdi3_kunit.o obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o diff --git a/lib/tests/cmdline_kunit.c b/lib/tests/cmdline_kunit.c index c1602f797637..3f61ff8d3178 100644 --- a/lib/tests/cmdline_kunit.c +++ b/lib/tests/cmdline_kunit.c @@ -6,6 +6,7 @@ #include <kunit/test.h> #include <linux/kernel.h> #include <linux/random.h> +#include <linux/sizes.h> #include <linux/string.h> static const char *cmdline_test_strings[] = { @@ -139,11 +140,128 @@ static void cmdline_test_range(struct kunit *test) } while (++i < ARRAY_SIZE(cmdline_test_range_strings)); } +static void cmdline_test_next_arg_quoted_value(struct kunit *test) +{ + char in[] = "foo=\"bar baz\" qux=1"; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar baz"); + KUNIT_EXPECT_STREQ(test, next, "qux=1"); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "qux"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "1"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_bare_quote_regression(struct kunit *test) +{ + char in[] = "foo=bar \""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar"); + KUNIT_EXPECT_STREQ(test, next, "\""); + + /* This hits the i == 0 quoted-token case fixed by 9847f21225c4. */ + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, ""); + KUNIT_EXPECT_PTR_EQ(test, val, NULL); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_mixed_tokens(struct kunit *test) +{ + char in[] = "bbb= jjj kkk=\"a=b\""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "bbb"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, ""); + KUNIT_EXPECT_STREQ(test, next, "jjj kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "jjj"); + KUNIT_EXPECT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, next, "kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "kkk"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "a=b"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +struct cmdline_test_memparse_entry { + const char *input; + const char *unrecognized; + unsigned long long result; +}; + +static const struct cmdline_test_memparse_entry testdata[] = { + { "0", "", 0ULL }, + { "1", "", 1ULL }, + { "a", "a", 0ULL }, + { "k", "k", 0ULL }, + { "E", "E", 0ULL }, + { "0xb", "", 11ULL }, + { "0xz", "x", 0ULL }, + { "1234", "", 1234ULL }, + { "04567", "", 2423ULL }, + { "0x9876", "", 39030LL }, + { "05678", "8", 375ULL }, + { "0xabcdefz", "z", 11259375ULL }, + { "0cdba", "c", 0ULL }, + { "4K", "", SZ_4K }, + { "0x10k@0xaaaabbbb", "@", SZ_16K }, + { "32M", "", SZ_32M }, + { "067m:foo", ":", 55 * SZ_1M }, + { "2G;bar=baz", ";", SZ_2G }, + { "07gz", "z", 7ULL * SZ_1G }, + { "3T+data", "+", 3 * SZ_1T }, + { "04t,ro", ",", SZ_4T }, + { "012p", "", 11258999068426240ULL }, + { "7P,sync", ",", 7881299347898368ULL }, + { "0x2e", "", 46ULL }, + { "2E and more", " ", 2305843009213693952ULL }, + { "18446744073709551615", "", ULLONG_MAX }, + { "0xffffffffffffffff0", "", ULLONG_MAX }, + { "1111111111111111111T", "", ULLONG_MAX }, + { "222222222222222222222G", "", ULLONG_MAX }, + { "3333333333333333333333M", "", ULLONG_MAX }, +}; + +static void cmdline_test_memparse(struct kunit *test) +{ + const struct cmdline_test_memparse_entry *e; + unsigned long long ret; + char *retptr; + + for (e = testdata; e < testdata + ARRAY_SIZE(testdata); e++) { + ret = memparse(e->input, &retptr); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when parsing '%s'", e->input); + KUNIT_EXPECT_EQ_MSG(test, *retptr, *e->unrecognized, + " when parsing '%s'", e->input); + } +} + static struct kunit_case cmdline_test_cases[] = { KUNIT_CASE(cmdline_test_noint), KUNIT_CASE(cmdline_test_lead_int), KUNIT_CASE(cmdline_test_tail_int), KUNIT_CASE(cmdline_test_range), + KUNIT_CASE(cmdline_test_next_arg_quoted_value), + KUNIT_CASE(cmdline_test_next_arg_bare_quote_regression), + KUNIT_CASE(cmdline_test_next_arg_mixed_tokens), + KUNIT_CASE(cmdline_test_memparse), {} }; diff --git a/lib/tests/fortify_kunit.c b/lib/tests/fortify_kunit.c index fc9c76f026d6..413cdbf3dc0d 100644 --- a/lib/tests/fortify_kunit.c +++ b/lib/tests/fortify_kunit.c @@ -458,7 +458,7 @@ static void fortify_test_strnlen(struct kunit *test) /* Make string unterminated, and recount. */ pad.buf[end] = 'A'; end = sizeof(pad.buf); - /* Reading beyond with strncpy() will fail. */ + /* Reading beyond will fail. */ KUNIT_EXPECT_EQ(test, strnlen(pad.buf, end + 1), end); KUNIT_EXPECT_EQ(test, fortify_read_overflows, 1); KUNIT_EXPECT_EQ(test, strnlen(pad.buf, end + 2), end); @@ -531,64 +531,6 @@ static void fortify_test_strcpy(struct kunit *test) KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); } -static void fortify_test_strncpy(struct kunit *test) -{ - struct fortify_padding pad = { }; - char src[] = "Copy me fully into a small buffer and I will overflow!"; - size_t sizeof_buf = sizeof(pad.buf); - - OPTIMIZER_HIDE_VAR(sizeof_buf); - - /* Destination is %NUL-filled to start with. */ - KUNIT_EXPECT_EQ(test, pad.bytes_before, 0); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 3], '\0'); - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* Legitimate strncpy() 1 less than of max size. */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf - 1) - == pad.buf); - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 0); - /* Only last byte should be %NUL */ - KUNIT_EXPECT_EQ(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 3], '\0'); - - /* Legitimate (though unterminated) max-size strncpy. */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf) - == pad.buf); - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 0); - /* No trailing %NUL -- thanks strncpy API. */ - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* But we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* Now verify that FORTIFY is working... */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf + 1) - == pad.buf); - /* Should catch the overflow. */ - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 1); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* And we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); - - /* And further... */ - KUNIT_ASSERT_TRUE(test, strncpy(pad.buf, src, sizeof_buf + 2) - == pad.buf); - /* Should catch the overflow. */ - KUNIT_EXPECT_EQ(test, fortify_write_overflows, 2); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 1], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - KUNIT_EXPECT_NE(test, pad.buf[sizeof_buf - 2], '\0'); - /* And we will not have gone beyond. */ - KUNIT_EXPECT_EQ(test, pad.bytes_after, 0); -} - static void fortify_test_strscpy(struct kunit *test) { struct fortify_padding pad = { }; @@ -1100,7 +1042,6 @@ static struct kunit_case fortify_test_cases[] = { KUNIT_CASE(fortify_test_strlen), KUNIT_CASE(fortify_test_strnlen), KUNIT_CASE(fortify_test_strcpy), - KUNIT_CASE(fortify_test_strncpy), KUNIT_CASE(fortify_test_strscpy), KUNIT_CASE(fortify_test_strcat), KUNIT_CASE(fortify_test_strncat), diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c index f02f7b7aa796..d9690ba1db88 100644 --- a/lib/tests/kunit_iov_iter.c +++ b/lib/tests/kunit_iov_iter.c @@ -53,7 +53,7 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, size_t npages) { struct page **pages; - unsigned long got; + unsigned long got, last; void *buffer; unsigned int i; @@ -61,7 +61,15 @@ static void *__init iov_kunit_create_buffer(struct kunit *test, KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); *ppages = pages; - got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + got = 0; + while (true) { + last = got; + got = alloc_pages_bulk(GFP_KERNEL, npages, pages); + + if (last == got || got == npages) + break; + } + if (got != npages) { release_pages(pages, got); kvfree(pages); @@ -275,7 +283,7 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) struct page **spages, **bpages; u8 *scratch, *buffer; size_t bufsize, npages, size, copied; - int i, b, patt; + int i, patt; bufsize = 0x100000; npages = bufsize / PAGE_SIZE; @@ -298,10 +306,9 @@ static void __init iov_kunit_copy_to_bvec(struct kunit *test) KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); /* Build the expected image in the scratch buffer. */ - b = 0; patt = 0; memset(scratch, 0, bufsize); - for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { + for (pr = bvec_test_ranges; pr->from >= 0; pr++) { u8 *p = scratch + pr->page * PAGE_SIZE; for (i = pr->from; i < pr->to; i++) diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c index e4b0ecbee32f..4c08a7c6fb78 100644 --- a/lib/tests/liveupdate.c +++ b/lib/tests/liveupdate.c @@ -105,6 +105,9 @@ static void liveupdate_test_init(void) pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n", flb->compatible, ERR_PTR(err)); } + + if (!err) + liveupdate_flb_put_incoming(flb); } initialized = true; } diff --git a/lib/tests/printf_kunit.c b/lib/tests/printf_kunit.c index bb70b9cddadd..eccf041ebd56 100644 --- a/lib/tests/printf_kunit.c +++ b/lib/tests/printf_kunit.c @@ -319,7 +319,27 @@ symbol_ptr(struct kunit *kunittest) static void kernel_ptr(struct kunit *kunittest) { - /* We can't test this without access to kptr_restrict. */ + switch (kptr_restrict) { + case 0: + if (no_hash_pointers) { + test(PTR_STR, "%pK", PTR); + } else { + char buf[PLAIN_BUF_SIZE]; + + plain_hash_to_buffer(kunittest, PTR, buf, PLAIN_BUF_SIZE); + /* %pK behaves the same as hashing */ + test(buf, "%pK", PTR); + } + break; + case 1: + /* The KUnit kthread has all capabilities, including CAP_SYSLOG */ + test(PTR_STR, "%pK", PTR); + break; + case 2: + default: + test(ZEROS "00000000", "%pK", PTR); + break; + } } static void @@ -415,8 +435,10 @@ mac(struct kunit *kunittest) test("2d:48:d6:fc:7a:05", "%pM", addr); test("05:7a:fc:d6:48:2d", "%pMR", addr); + test("05:7A:FC:D6:48:2D", "%pMRU", addr); test("2d-48-d6-fc-7a-05", "%pMF", addr); test("2d48d6fc7a05", "%pm", addr); + test("2D48D6FC7A05", "%pmU", addr); test("057afcd6482d", "%pmR", addr); } diff --git a/lib/tests/seq_buf_kunit.c b/lib/tests/seq_buf_kunit.c index 8a01579a978e..eb466386bbef 100644 --- a/lib/tests/seq_buf_kunit.c +++ b/lib/tests/seq_buf_kunit.c @@ -184,6 +184,38 @@ static void seq_buf_get_buf_commit_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); } +static void seq_buf_putmem_hex_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 24); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 0809 "; +#else + const char *expected = "0706050403020100 0908 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), 0); + KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), strlen(expected)); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + +static void seq_buf_putmem_hex_overflow_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 20); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 "; +#else + const char *expected = "0706050403020100 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), -1); + KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), 20); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_init_test), KUNIT_CASE(seq_buf_declare_test), @@ -194,6 +226,8 @@ static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_printf_test), KUNIT_CASE(seq_buf_printf_overflow_test), KUNIT_CASE(seq_buf_get_buf_commit_test), + KUNIT_CASE(seq_buf_putmem_hex_test), + KUNIT_CASE(seq_buf_putmem_hex_overflow_test), {} }; diff --git a/lib/tests/shdi3_kunit.c b/lib/tests/shdi3_kunit.c new file mode 100644 index 000000000000..44f65e66512b --- /dev/null +++ b/lib/tests/shdi3_kunit.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR Apache-2.0 +/* + * Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3(). + */ + +#include <linux/array_size.h> +#include <linux/module.h> +#include <linux/libgcc.h> +#include <kunit/test.h> + +struct shdi3_test_entry { + long long input; + int shift; + long long result; +}; + +static const struct shdi3_test_entry ashldi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashldi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x2468ACF13579BDELL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BCLL }, + { 0x0123456789ABCDEFLL, 3, 0x91A2B3C4D5E6F78LL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDEF0LL }, + { 0x0123456789ABCDEFLL, 28, 0x789ABCDEF0000000LL }, + { 0x0123456789ABCDEFLL, 29, 0xF13579BDE0000000LL }, + { 0x0123456789ABCDEFLL, 30, 0xE26AF37BC0000000LL }, + { 0x0123456789ABCDEFLL, 31, 0xC4D5E6F780000000LL }, + { 0x0123456789ABCDEFLL, 32, 0x89ABCDEF00000000LL }, + { 0x0123456789ABCDEFLL, 33, 0x13579BDE00000000LL }, + { 0x0123456789ABCDEFLL, 34, 0x26AF37BC00000000LL }, + { 0x0123456789ABCDEFLL, 35, 0x4D5E6F7800000000LL }, + { 0x0123456789ABCDEFLL, 36, 0x9ABCDEF000000000LL }, + { 0x0123456789ABCDEFLL, 60, 0xF000000000000000LL }, + { 0x0123456789ABCDEFLL, 61, 0xE000000000000000LL }, + { 0x0123456789ABCDEFLL, 62, 0xC000000000000000LL }, + { 0x0123456789ABCDEFLL, 63, 0x8000000000000000LL }, +}; + +static void shdi3_test_ashldi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashldi3_testdata; + e < ashldi3_testdata + ARRAY_SIZE(ashldi3_testdata); e++) { + ret = __ashldi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashldi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry ashrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0xFF6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0xFFB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0xFFDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFFFFFFFFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0xFFFFFFFFF6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0xFFFFFFFFFB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0xFFFFFFFFFDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFFFFFFFFFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0xFFFFFFFFFF6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0xFFFFFFFFFFB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0xFFFFFFFFFFDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFFFFFFFFFFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xFFFFFFFFFFFFFFFALL }, + { 0xAEDCBA9876543210LL, 61, 0xFFFFFFFFFFFFFFFDLL }, + { 0xAEDCBA9876543210LL, 62, 0xFFFFFFFFFFFFFFFELL }, + { 0xAEDCBA9876543210LL, 63, 0xFFFFFFFFFFFFFFFFLL }, +}; + +static void shdi3_test_ashrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashrdi3_testdata; + e < ashrdi3_testdata + ARRAY_SIZE(ashrdi3_testdata); e++) { + ret = __ashrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry lshrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/lshrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0x7F6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0x3FB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0x1FDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0x7F6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0x3FB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0x1FDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0x7F6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0x3FB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0x1FDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xALL }, + { 0xAEDCBA9876543210LL, 61, 0x5LL }, + { 0xAEDCBA9876543210LL, 62, 0x2LL }, + { 0xAEDCBA9876543210LL, 63, 0x1LL }, +}; + +static void shdi3_test_lshrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = lshrdi3_testdata; + e < lshrdi3_testdata + ARRAY_SIZE(lshrdi3_testdata); e++) { + ret = __lshrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __lshrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static struct kunit_case shdi3_test_cases[] = { + KUNIT_CASE(shdi3_test_ashldi3), + KUNIT_CASE(shdi3_test_ashrdi3), + KUNIT_CASE(shdi3_test_lshrdi3), + {} +}; + +static struct kunit_suite shdi3_test_suite = { + .name = "shdi3", + .test_cases = shdi3_test_cases, +}; +kunit_test_suite(shdi3_test_suite); + +MODULE_DESCRIPTION("Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3()"); +MODULE_LICENSE("GPL"); diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c index c853046183d2..9fbe91079c7e 100644 --- a/lib/tests/string_helpers_kunit.c +++ b/lib/tests/string_helpers_kunit.c @@ -5,11 +5,16 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <kunit/test.h> + #include <linux/array_size.h> -#include <linux/kernel.h> +#include <linux/bug.h> +#include <linux/limits.h> +#include <linux/module.h> #include <linux/random.h> -#include <linux/string.h> +#include <linux/slab.h> +#include <linux/sprintf.h> #include <linux/string_helpers.h> +#include <linux/types.h> static void test_string_check_buf(struct kunit *test, const char *name, unsigned int flags, @@ -601,6 +606,11 @@ static void test_unescape(struct kunit *test) test_string_unescape(test, "unescape", i, false); test_string_unescape(test, "unescape inplace", get_random_u32_below(UNESCAPE_ALL_MASK + 1), true); +} + +static void test_escape(struct kunit *test) +{ + unsigned int i; /* Without dictionary */ for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) @@ -615,6 +625,7 @@ static struct kunit_case string_helpers_test_cases[] = { KUNIT_CASE(test_get_size), KUNIT_CASE(test_upper_lower), KUNIT_CASE(test_unescape), + KUNIT_CASE(test_escape), {} }; diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c index de71b2649dac..2ef64fbe67d6 100644 --- a/lib/tests/uuid_kunit.c +++ b/lib/tests/uuid_kunit.c @@ -86,11 +86,67 @@ static void uuid_test_uuid_invalid(struct kunit *test) } } +/* + * RFC 4122 section 4.4 says random UUIDs/GUIDs (version 4) must have: + * - version 4 in the high nibble of the version byte, + * - variant DCE 1.1 (binary 10x) in the high bits of byte 8. + * + * The version byte is byte 6 in the "wire" uuid_t layout and byte 7 in + * the byte-swapped guid_t layout. + */ +static void uuid_test_uuid_gen(struct kunit *test) +{ + uuid_t u; + + for (unsigned int i = 0; i < 8; i++) { + uuid_gen(&u); + KUNIT_EXPECT_EQ(test, u.b[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, u.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_guid_gen(struct kunit *test) +{ + guid_t g; + + for (unsigned int i = 0; i < 8; i++) { + guid_gen(&g); + KUNIT_EXPECT_EQ(test, g.b[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, g.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_uuid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_uuid(buf); + KUNIT_EXPECT_EQ(test, buf[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_guid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_guid(buf); + KUNIT_EXPECT_EQ(test, buf[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + static struct kunit_case uuid_test_cases[] = { KUNIT_CASE(uuid_test_guid_valid), KUNIT_CASE(uuid_test_uuid_valid), KUNIT_CASE(uuid_test_guid_invalid), KUNIT_CASE(uuid_test_uuid_invalid), + KUNIT_CASE(uuid_test_uuid_gen), + KUNIT_CASE(uuid_test_guid_gen), + KUNIT_CASE(uuid_test_generate_random_uuid), + KUNIT_CASE(uuid_test_generate_random_guid), {}, }; diff --git a/lib/usercopy.c b/lib/usercopy.c index b00a3a957de6..e2f0bf104a59 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,15 +12,13 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) +#if !defined(INLINE_COPY_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); } EXPORT_SYMBOL(_copy_from_user); -#endif -#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index da224011fafd..e0f289d3d110 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -126,7 +126,7 @@ bool vdso_get_timestamp(const struct vdso_time_data *vd, const struct vdso_clock } static __always_inline -const struct vdso_time_data *__arch_get_vdso_u_timens_data(const struct vdso_time_data *vd) +const struct vdso_time_data *vdso_timens_data(const struct vdso_time_data *vd) { return (void *)vd + PAGE_SIZE; } @@ -135,7 +135,7 @@ static __always_inline bool do_hres_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); + const struct vdso_time_data *vd = vdso_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; const struct vdso_clock *vc = vd->clock_data; u32 seq; @@ -191,7 +191,7 @@ static __always_inline bool do_coarse_timens(const struct vdso_time_data *vdns, const struct vdso_clock *vcns, clockid_t clk, struct __kernel_timespec *ts) { - const struct vdso_time_data *vd = __arch_get_vdso_u_timens_data(vdns); + const struct vdso_time_data *vd = vdso_timens_data(vdns); const struct timens_offset *offs = &vcns->offset[clk]; const struct vdso_clock *vc = vd->clock_data; const struct vdso_timestamp *vdso_ts; @@ -250,7 +250,7 @@ bool do_aux(const struct vdso_time_data *vd, clockid_t clock, struct __kernel_ti do { while (vdso_read_begin_timens(vc, &seq)) { /* Re-read from the real time data page, reload seq by looping */ - vd = __arch_get_vdso_u_timens_data(vd); + vd = vdso_timens_data(vd); vc = &vd->aux_clock_data[idx]; } @@ -360,7 +360,7 @@ __cvdso_gettimeofday_data(const struct vdso_time_data *vd, if (unlikely(tz != NULL)) { if (vdso_is_timens_clock(vc)) - vd = __arch_get_vdso_u_timens_data(vd); + vd = vdso_timens_data(vd); tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest; tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime; @@ -383,7 +383,7 @@ __cvdso_time_data(const struct vdso_time_data *vd, __kernel_old_time_t *time) __kernel_old_time_t t; if (vdso_is_timens_clock(vc)) { - vd = __arch_get_vdso_u_timens_data(vd); + vd = vdso_timens_data(vd); vc = vd->clock_data; } @@ -414,7 +414,7 @@ bool __cvdso_clock_getres_common(const struct vdso_time_data *vd, clockid_t cloc return false; if (vdso_is_timens_clock(vc)) - vd = __arch_get_vdso_u_timens_data(vd); + vd = vdso_timens_data(vd); /* * Convert the clockid to a bitmask and use it to check which diff --git a/lib/vsprintf.c b/lib/vsprintf.c index 9f359b31c8d1..2bc6ef483576 100644 --- a/lib/vsprintf.c +++ b/lib/vsprintf.c @@ -59,7 +59,7 @@ /* Disable pointer hashing if requested */ bool no_hash_pointers __ro_after_init; -EXPORT_SYMBOL_GPL(no_hash_pointers); +EXPORT_SYMBOL_FOR_MODULES(no_hash_pointers, "printf_kunit"); /* * Hashed pointers policy selected by "hash_pointers=..." boot param @@ -129,13 +129,6 @@ unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base) } EXPORT_SYMBOL(simple_strtoul); -unsigned long simple_strntoul(const char *cp, char **endp, unsigned int base, - size_t max_chars) -{ - return simple_strntoull(cp, endp, base, max_chars); -} -EXPORT_SYMBOL(simple_strntoul); - /** * simple_strtol - convert a string to a signed long * @cp: The start of the string @@ -857,6 +850,7 @@ static char *default_pointer(char *buf, char *end, const void *ptr, } int kptr_restrict __read_mostly; +EXPORT_SYMBOL_FOR_MODULES(kptr_restrict, "printf_kunit"); static noinline_for_stack char *restricted_pointer(char *buf, char *end, const void *ptr, @@ -1208,7 +1202,7 @@ char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec, } if (spec.field_width > 0) - len = min_t(int, spec.field_width, 64); + len = min(spec.field_width, 64); for (i = 0; i < len; ++i) { if (buf < end) @@ -1233,7 +1227,7 @@ char *bitmap_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { const int CHUNKSZ = 32; - int nr_bits = max_t(int, spec.field_width, 0); + int nr_bits = max(spec.field_width, 0); int i, chunksz; bool first = true; @@ -1276,7 +1270,7 @@ static noinline_for_stack char *bitmap_list_string(char *buf, char *end, const unsigned long *bitmap, struct printf_spec spec, const char *fmt) { - int nr_bits = max_t(int, spec.field_width, 0); + int nr_bits = max(spec.field_width, 0); bool first = true; int rbot, rtop; @@ -1309,31 +1303,39 @@ char *mac_address_string(char *buf, char *end, u8 *addr, char mac_addr[sizeof("xx:xx:xx:xx:xx:xx")]; char *p = mac_addr; int i; - char separator; + char separator = ':'; bool reversed = false; + bool uc = false; if (check_pointer(&buf, end, addr, spec)) return buf; switch (fmt[1]) { case 'F': + uc = fmt[2] == 'U'; separator = '-'; break; case 'R': + uc = fmt[2] == 'U'; reversed = true; - fallthrough; + break; + + case 'U': + uc = true; + break; default: - separator = ':'; break; } for (i = 0; i < 6; i++) { - if (reversed) - p = hex_byte_pack(p, addr[5 - i]); + u8 byte = reversed ? addr[5 - i] : addr[i]; + + if (uc) + p = hex_byte_pack_upper(p, byte); else - p = hex_byte_pack(p, addr[i]); + p = hex_byte_pack(p, byte); if (fmt[0] == 'M' && i != 5) *p++ = separator; @@ -2361,13 +2363,13 @@ static int __init hash_pointers_mode_parse(char *str) if (!str) { pr_warn("Hash pointers mode empty; falling back to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; - } else if (strncmp(str, "auto", 4) == 0) { + } else if (strcmp(str, "auto") == 0) { pr_info("Hash pointers mode set to auto.\n"); hash_pointers_mode = HASH_PTR_AUTO; - } else if (strncmp(str, "never", 5) == 0) { + } else if (strcmp(str, "never") == 0) { pr_info("Hash pointers mode set to never.\n"); hash_pointers_mode = HASH_PTR_NEVER; - } else if (strncmp(str, "always", 6) == 0) { + } else if (strcmp(str, "always") == 0) { pr_info("Hash pointers mode set to always.\n"); hash_pointers_mode = HASH_PTR_ALWAYS; } else { @@ -2416,6 +2418,7 @@ early_param("no_hash_pointers", no_hash_pointers_enable); * - 'MF' For a 6-byte MAC FDDI address, it prints the address * with a dash-separated hex notation * - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth) + * - '[mM][FR][U]' One of the above in the upper case * - 'I' [46] for IPv4/IPv6 addresses printed in the usual way * IPv4 uses dot-separated decimal without leading 0's (1.2.3.4) * IPv6 uses colon separated network-order 16 bit hex with leading 0's @@ -2550,6 +2553,7 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr, case 'm': /* Contiguous: 000102030405 */ /* [mM]F (FDDI) */ /* [mM]R (Reverse order; Bluetooth) */ + /* [mM][FR][U] (One of the above in the upper case) */ return mac_address_string(buf, end, ptr, spec, fmt); case 'I': /* Formatted IP supported * 4: 1.2.3.4 @@ -2640,6 +2644,18 @@ static unsigned char spec_flag(unsigned char c) return (c < sizeof(spec_flag_array)) ? spec_flag_array[c] : 0; } +static void set_field_width(struct printf_spec *spec, int width) +{ + spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); + WARN_ONCE(spec->field_width != width, "field width %d out of range", width); +} + +static void set_precision(struct printf_spec *spec, int prec) +{ + spec->precision = clamp(prec, 0, PRECISION_MAX); + WARN_ONCE(spec->precision < prec, "precision %d too large", prec); +} + /* * Helper function to decode printf style format. * Each call decode a token from the format and return the @@ -2710,7 +2726,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec) spec->field_width = -1; if (isdigit(*fmt.str)) - spec->field_width = skip_atoi(&fmt.str); + set_field_width(spec, skip_atoi(&fmt.str)); else if (unlikely(*fmt.str == '*')) { /* it's the next argument */ fmt.state = FORMAT_STATE_WIDTH; @@ -2724,9 +2740,7 @@ precision: if (unlikely(*fmt.str == '.')) { fmt.str++; if (isdigit(*fmt.str)) { - spec->precision = skip_atoi(&fmt.str); - if (spec->precision < 0) - spec->precision = 0; + set_precision(spec, skip_atoi(&fmt.str)); } else if (*fmt.str == '*') { /* it's the next argument */ fmt.state = FORMAT_STATE_PRECISION; @@ -2799,24 +2813,6 @@ qualifier: return fmt; } -static void -set_field_width(struct printf_spec *spec, int width) -{ - spec->field_width = width; - if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) { - spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX); - } -} - -static void -set_precision(struct printf_spec *spec, int prec) -{ - spec->precision = prec; - if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) { - spec->precision = clamp(prec, 0, PRECISION_MAX); - } -} - /* * Turn a 1/2/4-byte value into a 64-bit one for printing: truncate * as necessary and deal with signedness. @@ -2864,6 +2860,7 @@ static unsigned long long convert_num_spec(unsigned int val, int size, struct pr int vsnprintf(char *buf, size_t size, const char *fmt_str, va_list args) { char *str, *end; + size_t ret_size; struct printf_spec spec = {0}; struct fmt fmt = { .str = fmt_str, @@ -2983,8 +2980,12 @@ out: } /* the trailing null byte doesn't count towards the total */ - return str-buf; + ret_size = str - buf; + /* Make sure the return value is within the positive integer range */ + if (WARN_ON_ONCE(ret_size > INT_MAX)) + ret_size = INT_MAX; + return ret_size; } EXPORT_SYMBOL(vsnprintf); @@ -3288,6 +3289,7 @@ int bstr_printf(char *buf, size_t size, const char *fmt_str, const u32 *bin_buf) struct printf_spec spec = {0}; char *str, *end; const char *args = (const char *)bin_buf; + size_t ret_size; if (WARN_ON_ONCE(size > INT_MAX)) return 0; @@ -3436,7 +3438,12 @@ out: #undef get_arg /* the trailing null byte doesn't count towards the total */ - return str - buf; + ret_size = str - buf; + + /* Make sure the return value is within the positive integer range */ + if (WARN_ON_ONCE(ret_size > INT_MAX)) + ret_size = INT_MAX; + return ret_size; } EXPORT_SYMBOL_GPL(bstr_printf); |