summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-04-16 20:11:56 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-04-16 20:11:56 -0700
commit440d6635b20037bc9ad46b20817d7b61cef0fc1b (patch)
tree1a5e8962ae974aff248dbf594ae39f237b6c637f /arch
parent0b2f2b1fc0c61e602a6babf580b91f895b0ea80a (diff)
parent70b672833f4025341c11b22c7f83778a5cd611bc (diff)
downloadlinux-next-440d6635b20037bc9ad46b20817d7b61cef0fc1b.tar.gz
linux-next-440d6635b20037bc9ad46b20817d7b61cef0fc1b.zip
Merge tag 'mm-nonmm-stable-2026-04-15-04-20' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull non-MM updates from Andrew Morton: - "pid: make sub-init creation retryable" (Oleg Nesterov) Make creation of init in a new namespace more robust by clearing away some historical cruft which is no longer needed. Also some documentation fixups - "selftests/fchmodat2: Error handling and general" (Mark Brown) Fix and a cleanup for the fchmodat2() syscall selftest - "lib: polynomial: Move to math/ and clean up" (Andy Shevchenko) - "hung_task: Provide runtime reset interface for hung task detector" (Aaron Tomlin) Give administrators the ability to zero out /proc/sys/kernel/hung_task_detect_count - "tools/getdelays: use the static UAPI headers from tools/include/uapi" (Thomas Weißschuh) Teach getdelays to use the in-kernel UAPI headers rather than the system-provided ones - "watchdog/hardlockup: Improvements to hardlockup" (Mayank Rungta) Several cleanups and fixups to the hardlockup detector code and its documentation - "lib/bch: fix undefined behavior from signed left-shifts" (Josh Law) A couple of small/theoretical fixes in the bch code - "ocfs2/dlm: fix two bugs in dlm_match_regions()" (Junrui Luo) - "cleanup the RAID5 XOR library" (Christoph Hellwig) A quite far-reaching cleanup to this code. I can't do better than to quote Christoph: "The XOR library used for the RAID5 parity is a bit of a mess right now. The main file sits in crypto/ despite not being cryptography and not using the crypto API, with the generic implementations sitting in include/asm-generic and the arch implementations sitting in an asm/ header in theory. The latter doesn't work for many cases, so architectures often build the code directly into the core kernel, or create another module for the architecture code. Change this to a single module in lib/ that also contains the architecture optimizations, similar to the library work Eric Biggers has done for the CRC and crypto libraries later. After that it changes to better calling conventions that allow for smarter architecture implementations (although none is contained here yet), and uses static_call to avoid indirection function call overhead" - "lib/list_sort: Clean up list_sort() scheduling workarounds" (Kuan-Wei Chiu) Clean up this library code by removing a hacky thing which was added for UBIFS, which UBIFS doesn't actually need - "Fix bugs in extract_iter_to_sg()" (Christian Ehrhardt) Fix a few bugs in the scatterlist code, add in-kernel tests for the now-fixed bugs and fix a leak in the test itself - "kdump: Enable LUKS-encrypted dump target support in ARM64 and PowerPC" (Coiby Xu) Enable support of the LUKS-encrypted device dump target on arm64 and powerpc - "ocfs2: consolidate extent list validation into block read callbacks" (Joseph Qi) Cleanup, simplify, and make more robust ocfs2's validation of extent list fields (Kernel test robot loves mounting corrupted fs images!) * tag 'mm-nonmm-stable-2026-04-15-04-20' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (127 commits) ocfs2: validate group add input before caching ocfs2: validate bg_bits during freefrag scan ocfs2: fix listxattr handling when the buffer is full doc: watchdog: fix typos etc update Sean's email address ocfs2: use get_random_u32() where appropriate ocfs2: split transactions in dio completion to avoid credit exhaustion ocfs2: remove redundant l_next_free_rec check in __ocfs2_find_path() ocfs2: validate extent block list fields during block read ocfs2: remove empty extent list check in ocfs2_dx_dir_lookup_rec() ocfs2: validate dx_root extent list fields during block read ocfs2: fix use-after-free in ocfs2_fault() when VM_FAULT_RETRY ocfs2: handle invalid dinode in ocfs2_group_extend .get_maintainer.ignore: add Askar ocfs2: validate bg_list extent bounds in discontig groups checkpatch: exclude forward declarations of const structs tools/accounting: handle truncated taskstats netlink messages taskstats: set version in TGID exit notifications ocfs2/heartbeat: fix slot mapping rollback leaks on error paths arm64,ppc64le/kdump: pass dm-crypt keys to kdump kernel ...
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/include/asm/xor.h866
-rw-r--r--arch/arm/include/asm/xor.h225
-rw-r--r--arch/arm/lib/Makefile5
-rw-r--r--arch/arm/lib/xor-neon.c38
-rw-r--r--arch/arm64/include/asm/xor.h73
-rw-r--r--arch/arm64/kernel/machine_kexec_file.c4
-rw-r--r--arch/arm64/lib/Makefile6
-rw-r--r--arch/arm64/lib/xor-neon.c338
-rw-r--r--arch/loongarch/include/asm/xor.h68
-rw-r--r--arch/loongarch/include/asm/xor_simd.h34
-rw-r--r--arch/loongarch/lib/Makefile2
-rw-r--r--arch/loongarch/lib/xor_simd.c93
-rw-r--r--arch/loongarch/lib/xor_simd.h38
-rw-r--r--arch/loongarch/lib/xor_simd_glue.c72
-rw-r--r--arch/loongarch/lib/xor_template.c110
-rw-r--r--arch/powerpc/include/asm/xor.h47
-rw-r--r--arch/powerpc/include/asm/xor_altivec.h22
-rw-r--r--arch/powerpc/kexec/elf_64.c4
-rw-r--r--arch/powerpc/lib/Makefile5
-rw-r--r--arch/powerpc/lib/xor_vmx.c156
-rw-r--r--arch/powerpc/lib/xor_vmx.h22
-rw-r--r--arch/powerpc/lib/xor_vmx_glue.c63
-rw-r--r--arch/riscv/include/asm/xor.h68
-rw-r--r--arch/riscv/lib/Makefile1
-rw-r--r--arch/riscv/lib/xor.S81
-rw-r--r--arch/s390/include/asm/xor.h21
-rw-r--r--arch/s390/lib/Makefile2
-rw-r--r--arch/s390/lib/xor.c136
-rw-r--r--arch/sparc/include/asm/asm-prototypes.h1
-rw-r--r--arch/sparc/include/asm/xor.h9
-rw-r--r--arch/sparc/include/asm/xor_32.h268
-rw-r--r--arch/sparc/include/asm/xor_64.h79
-rw-r--r--arch/sparc/lib/Makefile2
-rw-r--r--arch/sparc/lib/xor.S646
-rw-r--r--arch/um/include/asm/xor.h24
-rw-r--r--arch/x86/include/asm/xor.h502
-rw-r--r--arch/x86/include/asm/xor_32.h573
-rw-r--r--arch/x86/include/asm/xor_64.h28
-rw-r--r--arch/x86/include/asm/xor_avx.h178
-rw-r--r--arch/x86/kernel/kexec-bzimage64.c6
40 files changed, 11 insertions, 4905 deletions
diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h
deleted file mode 100644
index e0de0c233ab9..000000000000
--- a/arch/alpha/include/asm/xor.h
+++ /dev/null
@@ -1,866 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm-alpha/xor.h
- *
- * Optimized RAID-5 checksumming functions for alpha EV5 and EV6
- */
-
-extern void
-xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-extern void
-xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-extern void
-xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-extern void
-xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-extern void
-xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-extern void
-xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-extern void
-xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-extern void
-xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-asm(" \n\
- .text \n\
- .align 3 \n\
- .ent xor_alpha_2 \n\
-xor_alpha_2: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-2: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,8($17) \n\
- ldq $3,8($18) \n\
- \n\
- ldq $4,16($17) \n\
- ldq $5,16($18) \n\
- ldq $6,24($17) \n\
- ldq $7,24($18) \n\
- \n\
- ldq $19,32($17) \n\
- ldq $20,32($18) \n\
- ldq $21,40($17) \n\
- ldq $22,40($18) \n\
- \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- ldq $25,56($17) \n\
- xor $0,$1,$0 # 7 cycles from $1 load \n\
- \n\
- ldq $27,56($18) \n\
- xor $2,$3,$2 \n\
- stq $0,0($17) \n\
- xor $4,$5,$4 \n\
- \n\
- stq $2,8($17) \n\
- xor $6,$7,$6 \n\
- stq $4,16($17) \n\
- xor $19,$20,$19 \n\
- \n\
- stq $6,24($17) \n\
- xor $21,$22,$21 \n\
- stq $19,32($17) \n\
- xor $23,$24,$23 \n\
- \n\
- stq $21,40($17) \n\
- xor $25,$27,$25 \n\
- stq $23,48($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $25,56($17) \n\
- addq $17,64,$17 \n\
- addq $18,64,$18 \n\
- bgt $16,2b \n\
- \n\
- ret \n\
- .end xor_alpha_2 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_3 \n\
-xor_alpha_3: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-3: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,8($17) \n\
- \n\
- ldq $4,8($18) \n\
- ldq $6,16($17) \n\
- ldq $7,16($18) \n\
- ldq $21,24($17) \n\
- \n\
- ldq $22,24($18) \n\
- ldq $24,32($17) \n\
- ldq $25,32($18) \n\
- ldq $5,8($19) \n\
- \n\
- ldq $20,16($19) \n\
- ldq $23,24($19) \n\
- ldq $27,32($19) \n\
- nop \n\
- \n\
- xor $0,$1,$1 # 8 cycles from $0 load \n\
- xor $3,$4,$4 # 6 cycles from $4 load \n\
- xor $6,$7,$7 # 6 cycles from $7 load \n\
- xor $21,$22,$22 # 5 cycles from $22 load \n\
- \n\
- xor $1,$2,$2 # 9 cycles from $2 load \n\
- xor $24,$25,$25 # 5 cycles from $25 load \n\
- stq $2,0($17) \n\
- xor $4,$5,$5 # 6 cycles from $5 load \n\
- \n\
- stq $5,8($17) \n\
- xor $7,$20,$20 # 7 cycles from $20 load \n\
- stq $20,16($17) \n\
- xor $22,$23,$23 # 7 cycles from $23 load \n\
- \n\
- stq $23,24($17) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- stq $27,32($17) \n\
- nop \n\
- \n\
- ldq $0,40($17) \n\
- ldq $1,40($18) \n\
- ldq $3,48($17) \n\
- ldq $4,48($18) \n\
- \n\
- ldq $6,56($17) \n\
- ldq $7,56($18) \n\
- ldq $2,40($19) \n\
- ldq $5,48($19) \n\
- \n\
- ldq $20,56($19) \n\
- xor $0,$1,$1 # 4 cycles from $1 load \n\
- xor $3,$4,$4 # 5 cycles from $4 load \n\
- xor $6,$7,$7 # 5 cycles from $7 load \n\
- \n\
- xor $1,$2,$2 # 4 cycles from $2 load \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- stq $2,40($17) \n\
- xor $7,$20,$20 # 4 cycles from $20 load \n\
- \n\
- stq $5,48($17) \n\
- subq $16,1,$16 \n\
- stq $20,56($17) \n\
- addq $19,64,$19 \n\
- \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,3b \n\
- ret \n\
- .end xor_alpha_3 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_4 \n\
-xor_alpha_4: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-4: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,8($17) \n\
- ldq $5,8($18) \n\
- ldq $6,8($19) \n\
- ldq $7,8($20) \n\
- \n\
- ldq $21,16($17) \n\
- ldq $22,16($18) \n\
- ldq $23,16($19) \n\
- ldq $24,16($20) \n\
- \n\
- ldq $25,24($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $27,24($18) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,24($19) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $3,0($17) \n\
- xor $6,$7,$7 \n\
- xor $21,$22,$22 # 7 cycles from $22 load \n\
- xor $5,$7,$7 \n\
- \n\
- stq $7,8($17) \n\
- xor $23,$24,$24 # 7 cycles from $24 load \n\
- ldq $2,32($17) \n\
- xor $22,$24,$24 \n\
- \n\
- ldq $3,32($18) \n\
- ldq $4,32($19) \n\
- ldq $5,32($20) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- \n\
- ldq $6,40($17) \n\
- ldq $7,40($18) \n\
- ldq $21,40($19) \n\
- ldq $22,40($20) \n\
- \n\
- stq $24,16($17) \n\
- xor $0,$1,$1 # 9 cycles from $1 load \n\
- xor $2,$3,$3 # 5 cycles from $3 load \n\
- xor $27,$1,$1 \n\
- \n\
- stq $1,24($17) \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- \n\
- ldq $25,48($19) \n\
- xor $3,$5,$5 \n\
- ldq $27,48($20) \n\
- ldq $0,56($17) \n\
- \n\
- ldq $1,56($18) \n\
- ldq $2,56($19) \n\
- xor $6,$7,$7 # 8 cycles from $6 load \n\
- ldq $3,56($20) \n\
- \n\
- stq $5,32($17) \n\
- xor $21,$22,$22 # 8 cycles from $22 load \n\
- xor $7,$22,$22 \n\
- xor $23,$24,$24 # 5 cycles from $24 load \n\
- \n\
- stq $22,40($17) \n\
- xor $25,$27,$27 # 5 cycles from $27 load \n\
- xor $24,$27,$27 \n\
- xor $0,$1,$1 # 5 cycles from $1 load \n\
- \n\
- stq $27,48($17) \n\
- xor $2,$3,$3 # 4 cycles from $3 load \n\
- xor $1,$3,$3 \n\
- subq $16,1,$16 \n\
- \n\
- stq $3,56($17) \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,4b \n\
- ret \n\
- .end xor_alpha_4 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_5 \n\
-xor_alpha_5: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-5: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,0($21) \n\
- ldq $5,8($17) \n\
- ldq $6,8($18) \n\
- ldq $7,8($19) \n\
- \n\
- ldq $22,8($20) \n\
- ldq $23,8($21) \n\
- ldq $24,16($17) \n\
- ldq $25,16($18) \n\
- \n\
- ldq $27,16($19) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $28,16($20) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,16($21) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($17) \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- \n\
- stq $4,0($17) \n\
- xor $5,$6,$6 # 7 cycles from $6 load \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $6,$23,$23 # 7 cycles from $23 load \n\
- \n\
- ldq $2,24($18) \n\
- xor $22,$23,$23 \n\
- ldq $3,24($19) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $23,8($17) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- ldq $4,24($20) \n\
- xor $28,$0,$0 # 7 cycles from $0 load \n\
- \n\
- ldq $5,24($21) \n\
- xor $27,$0,$0 \n\
- ldq $6,32($17) \n\
- ldq $7,32($18) \n\
- \n\
- stq $0,16($17) \n\
- xor $1,$2,$2 # 6 cycles from $2 load \n\
- ldq $22,32($19) \n\
- xor $3,$4,$4 # 4 cycles from $4 load \n\
- \n\
- ldq $23,32($20) \n\
- xor $2,$4,$4 \n\
- ldq $24,32($21) \n\
- ldq $25,40($17) \n\
- \n\
- ldq $27,40($18) \n\
- ldq $28,40($19) \n\
- ldq $0,40($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $5,24($17) \n\
- xor $6,$7,$7 # 7 cycles from $7 load \n\
- ldq $1,40($21) \n\
- ldq $2,48($17) \n\
- \n\
- ldq $3,48($18) \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- ldq $4,48($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- ldq $5,48($20) \n\
- xor $22,$24,$24 \n\
- ldq $6,48($21) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- \n\
- stq $24,32($17) \n\
- xor $27,$28,$28 # 8 cycles from $28 load \n\
- ldq $7,56($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- \n\
- ldq $22,56($18) \n\
- ldq $23,56($19) \n\
- ldq $24,56($20) \n\
- ldq $25,56($21) \n\
- \n\
- xor $28,$1,$1 \n\
- xor $2,$3,$3 # 9 cycles from $3 load \n\
- xor $3,$4,$4 # 9 cycles from $4 load \n\
- xor $5,$6,$6 # 8 cycles from $6 load \n\
- \n\
- stq $1,40($17) \n\
- xor $4,$6,$6 \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- stq $6,48($17) \n\
- xor $22,$24,$24 \n\
- subq $16,1,$16 \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $25,56($17) \n\
- addq $21,64,$21 \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,5b \n\
- ret \n\
- .end xor_alpha_5 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_2 \n\
-xor_alpha_prefetch_2: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- .align 4 \n\
-2: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,8($17) \n\
- ldq $3,8($18) \n\
- \n\
- ldq $4,16($17) \n\
- ldq $5,16($18) \n\
- ldq $6,24($17) \n\
- ldq $7,24($18) \n\
- \n\
- ldq $19,32($17) \n\
- ldq $20,32($18) \n\
- ldq $21,40($17) \n\
- ldq $22,40($18) \n\
- \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- ldq $25,56($17) \n\
- ldq $27,56($18) \n\
- \n\
- ldq $31,256($17) \n\
- xor $0,$1,$0 # 8 cycles from $1 load \n\
- ldq $31,256($18) \n\
- xor $2,$3,$2 \n\
- \n\
- stq $0,0($17) \n\
- xor $4,$5,$4 \n\
- stq $2,8($17) \n\
- xor $6,$7,$6 \n\
- \n\
- stq $4,16($17) \n\
- xor $19,$20,$19 \n\
- stq $6,24($17) \n\
- xor $21,$22,$21 \n\
- \n\
- stq $19,32($17) \n\
- xor $23,$24,$23 \n\
- stq $21,40($17) \n\
- xor $25,$27,$25 \n\
- \n\
- stq $23,48($17) \n\
- subq $16,1,$16 \n\
- stq $25,56($17) \n\
- addq $17,64,$17 \n\
- \n\
- addq $18,64,$18 \n\
- bgt $16,2b \n\
- ret \n\
- .end xor_alpha_prefetch_2 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_3 \n\
-xor_alpha_prefetch_3: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- .align 4 \n\
-3: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,8($17) \n\
- \n\
- ldq $4,8($18) \n\
- ldq $6,16($17) \n\
- ldq $7,16($18) \n\
- ldq $21,24($17) \n\
- \n\
- ldq $22,24($18) \n\
- ldq $24,32($17) \n\
- ldq $25,32($18) \n\
- ldq $5,8($19) \n\
- \n\
- ldq $20,16($19) \n\
- ldq $23,24($19) \n\
- ldq $27,32($19) \n\
- nop \n\
- \n\
- xor $0,$1,$1 # 8 cycles from $0 load \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- xor $6,$7,$7 # 6 cycles from $7 load \n\
- xor $21,$22,$22 # 5 cycles from $22 load \n\
- \n\
- xor $1,$2,$2 # 9 cycles from $2 load \n\
- xor $24,$25,$25 # 5 cycles from $25 load \n\
- stq $2,0($17) \n\
- xor $4,$5,$5 # 6 cycles from $5 load \n\
- \n\
- stq $5,8($17) \n\
- xor $7,$20,$20 # 7 cycles from $20 load \n\
- stq $20,16($17) \n\
- xor $22,$23,$23 # 7 cycles from $23 load \n\
- \n\
- stq $23,24($17) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- stq $27,32($17) \n\
- nop \n\
- \n\
- ldq $0,40($17) \n\
- ldq $1,40($18) \n\
- ldq $3,48($17) \n\
- ldq $4,48($18) \n\
- \n\
- ldq $6,56($17) \n\
- ldq $7,56($18) \n\
- ldq $2,40($19) \n\
- ldq $5,48($19) \n\
- \n\
- ldq $20,56($19) \n\
- ldq $31,256($17) \n\
- ldq $31,256($18) \n\
- ldq $31,256($19) \n\
- \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- xor $3,$4,$4 # 5 cycles from $4 load \n\
- xor $6,$7,$7 # 5 cycles from $7 load \n\
- xor $1,$2,$2 # 4 cycles from $2 load \n\
- \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- xor $7,$20,$20 # 4 cycles from $20 load \n\
- stq $2,40($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $5,48($17) \n\
- addq $19,64,$19 \n\
- stq $20,56($17) \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,3b \n\
- ret \n\
- .end xor_alpha_prefetch_3 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_4 \n\
-xor_alpha_prefetch_4: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- ldq $31, 0($20) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- ldq $31, 64($20) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- ldq $31, 128($20) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- ldq $31, 192($20) \n\
- .align 4 \n\
-4: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,8($17) \n\
- ldq $5,8($18) \n\
- ldq $6,8($19) \n\
- ldq $7,8($20) \n\
- \n\
- ldq $21,16($17) \n\
- ldq $22,16($18) \n\
- ldq $23,16($19) \n\
- ldq $24,16($20) \n\
- \n\
- ldq $25,24($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $27,24($18) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,24($19) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $3,0($17) \n\
- xor $6,$7,$7 \n\
- xor $21,$22,$22 # 7 cycles from $22 load \n\
- xor $5,$7,$7 \n\
- \n\
- stq $7,8($17) \n\
- xor $23,$24,$24 # 7 cycles from $24 load \n\
- ldq $2,32($17) \n\
- xor $22,$24,$24 \n\
- \n\
- ldq $3,32($18) \n\
- ldq $4,32($19) \n\
- ldq $5,32($20) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- \n\
- ldq $6,40($17) \n\
- ldq $7,40($18) \n\
- ldq $21,40($19) \n\
- ldq $22,40($20) \n\
- \n\
- stq $24,16($17) \n\
- xor $0,$1,$1 # 9 cycles from $1 load \n\
- xor $2,$3,$3 # 5 cycles from $3 load \n\
- xor $27,$1,$1 \n\
- \n\
- stq $1,24($17) \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- ldq $23,48($17) \n\
- xor $3,$5,$5 \n\
- \n\
- ldq $24,48($18) \n\
- ldq $25,48($19) \n\
- ldq $27,48($20) \n\
- ldq $0,56($17) \n\
- \n\
- ldq $1,56($18) \n\
- ldq $2,56($19) \n\
- ldq $3,56($20) \n\
- xor $6,$7,$7 # 8 cycles from $6 load \n\
- \n\
- ldq $31,256($17) \n\
- xor $21,$22,$22 # 8 cycles from $22 load \n\
- ldq $31,256($18) \n\
- xor $7,$22,$22 \n\
- \n\
- ldq $31,256($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- ldq $31,256($20) \n\
- xor $25,$27,$27 # 6 cycles from $27 load \n\
- \n\
- stq $5,32($17) \n\
- xor $24,$27,$27 \n\
- xor $0,$1,$1 # 7 cycles from $1 load \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- stq $22,40($17) \n\
- xor $1,$3,$3 \n\
- stq $27,48($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $3,56($17) \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,4b \n\
- ret \n\
- .end xor_alpha_prefetch_4 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_5 \n\
-xor_alpha_prefetch_5: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- ldq $31, 0($20) \n\
- ldq $31, 0($21) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- ldq $31, 64($20) \n\
- ldq $31, 64($21) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- ldq $31, 128($20) \n\
- ldq $31, 128($21) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- ldq $31, 192($20) \n\
- ldq $31, 192($21) \n\
- .align 4 \n\
-5: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,0($21) \n\
- ldq $5,8($17) \n\
- ldq $6,8($18) \n\
- ldq $7,8($19) \n\
- \n\
- ldq $22,8($20) \n\
- ldq $23,8($21) \n\
- ldq $24,16($17) \n\
- ldq $25,16($18) \n\
- \n\
- ldq $27,16($19) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $28,16($20) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,16($21) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($17) \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- \n\
- stq $4,0($17) \n\
- xor $5,$6,$6 # 7 cycles from $6 load \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $6,$23,$23 # 7 cycles from $23 load \n\
- \n\
- ldq $2,24($18) \n\
- xor $22,$23,$23 \n\
- ldq $3,24($19) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $23,8($17) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- ldq $4,24($20) \n\
- xor $28,$0,$0 # 7 cycles from $0 load \n\
- \n\
- ldq $5,24($21) \n\
- xor $27,$0,$0 \n\
- ldq $6,32($17) \n\
- ldq $7,32($18) \n\
- \n\
- stq $0,16($17) \n\
- xor $1,$2,$2 # 6 cycles from $2 load \n\
- ldq $22,32($19) \n\
- xor $3,$4,$4 # 4 cycles from $4 load \n\
- \n\
- ldq $23,32($20) \n\
- xor $2,$4,$4 \n\
- ldq $24,32($21) \n\
- ldq $25,40($17) \n\
- \n\
- ldq $27,40($18) \n\
- ldq $28,40($19) \n\
- ldq $0,40($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $5,24($17) \n\
- xor $6,$7,$7 # 7 cycles from $7 load \n\
- ldq $1,40($21) \n\
- ldq $2,48($17) \n\
- \n\
- ldq $3,48($18) \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- ldq $4,48($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- ldq $5,48($20) \n\
- xor $22,$24,$24 \n\
- ldq $6,48($21) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- \n\
- stq $24,32($17) \n\
- xor $27,$28,$28 # 8 cycles from $28 load \n\
- ldq $7,56($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- \n\
- ldq $22,56($18) \n\
- ldq $23,56($19) \n\
- ldq $24,56($20) \n\
- ldq $25,56($21) \n\
- \n\
- ldq $31,256($17) \n\
- xor $28,$1,$1 \n\
- ldq $31,256($18) \n\
- xor $2,$3,$3 # 9 cycles from $3 load \n\
- \n\
- ldq $31,256($19) \n\
- xor $3,$4,$4 # 9 cycles from $4 load \n\
- ldq $31,256($20) \n\
- xor $5,$6,$6 # 8 cycles from $6 load \n\
- \n\
- stq $1,40($17) \n\
- xor $4,$6,$6 \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- stq $6,48($17) \n\
- xor $22,$24,$24 \n\
- ldq $31,256($21) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $25,56($17) \n\
- subq $16,1,$16 \n\
- addq $21,64,$21 \n\
- addq $20,64,$20 \n\
- \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,5b \n\
- \n\
- ret \n\
- .end xor_alpha_prefetch_5 \n\
-");
-
-static struct xor_block_template xor_block_alpha = {
- .name = "alpha",
- .do_2 = xor_alpha_2,
- .do_3 = xor_alpha_3,
- .do_4 = xor_alpha_4,
- .do_5 = xor_alpha_5,
-};
-
-static struct xor_block_template xor_block_alpha_prefetch = {
- .name = "alpha prefetch",
- .do_2 = xor_alpha_prefetch_2,
- .do_3 = xor_alpha_prefetch_3,
- .do_4 = xor_alpha_prefetch_4,
- .do_5 = xor_alpha_prefetch_5,
-};
-
-/* For grins, also test the generic routines. */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_alpha); \
- xor_speed(&xor_block_alpha_prefetch); \
- } while (0)
-
-/* Force the use of alpha_prefetch if EV6, as it is significantly
- faster in the cold cache case. */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- (implver() == IMPLVER_EV6 ? &xor_block_alpha_prefetch : FASTEST)
diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
deleted file mode 100644
index 934b549905f5..000000000000
--- a/arch/arm/include/asm/xor.h
+++ /dev/null
@@ -1,225 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * arch/arm/include/asm/xor.h
- *
- * Copyright (C) 2001 Russell King
- */
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
-#include <asm/neon.h>
-
-#define __XOR(a1, a2) a1 ^= a2
-
-#define GET_BLOCK_2(dst) \
- __asm__("ldmia %0, {%1, %2}" \
- : "=r" (dst), "=r" (a1), "=r" (a2) \
- : "0" (dst))
-
-#define GET_BLOCK_4(dst) \
- __asm__("ldmia %0, {%1, %2, %3, %4}" \
- : "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
- : "0" (dst))
-
-#define XOR_BLOCK_2(src) \
- __asm__("ldmia %0!, {%1, %2}" \
- : "=r" (src), "=r" (b1), "=r" (b2) \
- : "0" (src)); \
- __XOR(a1, b1); __XOR(a2, b2);
-
-#define XOR_BLOCK_4(src) \
- __asm__("ldmia %0!, {%1, %2, %3, %4}" \
- : "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
- : "0" (src)); \
- __XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
-
-#define PUT_BLOCK_2(dst) \
- __asm__ __volatile__("stmia %0!, {%2, %3}" \
- : "=r" (dst) \
- : "0" (dst), "r" (a1), "r" (a2))
-
-#define PUT_BLOCK_4(dst) \
- __asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
- : "=r" (dst) \
- : "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
-
-static void
-xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 4;
- register unsigned int a1 __asm__("r4");
- register unsigned int a2 __asm__("r5");
- register unsigned int a3 __asm__("r6");
- register unsigned int a4 __asm__("r10");
- register unsigned int b1 __asm__("r8");
- register unsigned int b2 __asm__("r9");
- register unsigned int b3 __asm__("ip");
- register unsigned int b4 __asm__("lr");
-
- do {
- GET_BLOCK_4(p1);
- XOR_BLOCK_4(p2);
- PUT_BLOCK_4(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 4;
- register unsigned int a1 __asm__("r4");
- register unsigned int a2 __asm__("r5");
- register unsigned int a3 __asm__("r6");
- register unsigned int a4 __asm__("r10");
- register unsigned int b1 __asm__("r8");
- register unsigned int b2 __asm__("r9");
- register unsigned int b3 __asm__("ip");
- register unsigned int b4 __asm__("lr");
-
- do {
- GET_BLOCK_4(p1);
- XOR_BLOCK_4(p2);
- XOR_BLOCK_4(p3);
- PUT_BLOCK_4(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 2;
- register unsigned int a1 __asm__("r8");
- register unsigned int a2 __asm__("r9");
- register unsigned int b1 __asm__("ip");
- register unsigned int b2 __asm__("lr");
-
- do {
- GET_BLOCK_2(p1);
- XOR_BLOCK_2(p2);
- XOR_BLOCK_2(p3);
- XOR_BLOCK_2(p4);
- PUT_BLOCK_2(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 2;
- register unsigned int a1 __asm__("r8");
- register unsigned int a2 __asm__("r9");
- register unsigned int b1 __asm__("ip");
- register unsigned int b2 __asm__("lr");
-
- do {
- GET_BLOCK_2(p1);
- XOR_BLOCK_2(p2);
- XOR_BLOCK_2(p3);
- XOR_BLOCK_2(p4);
- XOR_BLOCK_2(p5);
- PUT_BLOCK_2(p1);
- } while (--lines);
-}
-
-static struct xor_block_template xor_block_arm4regs = {
- .name = "arm4regs",
- .do_2 = xor_arm4regs_2,
- .do_3 = xor_arm4regs_3,
- .do_4 = xor_arm4regs_4,
- .do_5 = xor_arm4regs_5,
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_arm4regs); \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- NEON_TEMPLATES; \
- } while (0)
-
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_neon_inner;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- if (in_interrupt()) {
- xor_arm4regs_2(bytes, p1, p2);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_2(bytes, p1, p2);
- kernel_neon_end();
- }
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- if (in_interrupt()) {
- xor_arm4regs_3(bytes, p1, p2, p3);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_3(bytes, p1, p2, p3);
- kernel_neon_end();
- }
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- if (in_interrupt()) {
- xor_arm4regs_4(bytes, p1, p2, p3, p4);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
- kernel_neon_end();
- }
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- if (in_interrupt()) {
- xor_arm4regs_5(bytes, p1, p2, p3, p4, p5);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
- kernel_neon_end();
- }
-}
-
-static struct xor_block_template xor_block_neon = {
- .name = "neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
-};
-
-#define NEON_TEMPLATES \
- do { if (cpu_has_neon()) xor_speed(&xor_block_neon); } while (0)
-#else
-#define NEON_TEMPLATES
-#endif
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 0ca5aae1bcc3..9295055cdfc9 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -39,9 +39,4 @@ endif
$(obj)/csumpartialcopy.o: $(obj)/csumpartialcopygeneric.S
$(obj)/csumpartialcopyuser.o: $(obj)/csumpartialcopygeneric.S
-ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
- CFLAGS_xor-neon.o += $(CC_FLAGS_FPU)
- obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
-endif
-
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c
deleted file mode 100644
index cf57fca97908..000000000000
--- a/arch/arm/lib/xor-neon.c
+++ /dev/null
@@ -1,38 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * linux/arch/arm/lib/xor-neon.c
- *
- * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/raid/xor.h>
-#include <linux/module.h>
-
-MODULE_DESCRIPTION("NEON accelerated XOR implementation");
-MODULE_LICENSE("GPL");
-
-#ifndef __ARM_NEON__
-#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
-#endif
-
-/*
- * Pull in the reference implementations while instructing GCC (through
- * -ftree-vectorize) to attempt to exploit implicit parallelism and emit
- * NEON instructions. Clang does this by default at O2 so no pragma is
- * needed.
- */
-#ifdef CONFIG_CC_IS_GCC
-#pragma GCC optimize "tree-vectorize"
-#endif
-
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#include <asm-generic/xor.h>
-
-struct xor_block_template const xor_block_neon_inner = {
- .name = "__inner_neon__",
- .do_2 = xor_8regs_2,
- .do_3 = xor_8regs_3,
- .do_4 = xor_8regs_4,
- .do_5 = xor_8regs_5,
-};
-EXPORT_SYMBOL(xor_block_neon_inner);
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
deleted file mode 100644
index c38e3d017a79..000000000000
--- a/arch/arm64/include/asm/xor.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * arch/arm64/include/asm/xor.h
- *
- * Authors: Jackie Liu <liuyun01@kylinos.cn>
- * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
- */
-
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
-#include <asm/simd.h>
-
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_inner_neon;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_2(bytes, p1, p2);
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_3(bytes, p1, p2, p3);
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
-}
-
-static struct xor_block_template xor_block_arm64 = {
- .name = "arm64_neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
-};
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- if (cpu_has_neon()) { \
- xor_speed(&xor_block_arm64);\
- } \
- } while (0)
-
-#endif /* ! CONFIG_KERNEL_MODE_NEON */
diff --git a/arch/arm64/kernel/machine_kexec_file.c b/arch/arm64/kernel/machine_kexec_file.c
index fba260ad87a9..e31fabed378a 100644
--- a/arch/arm64/kernel/machine_kexec_file.c
+++ b/arch/arm64/kernel/machine_kexec_file.c
@@ -134,6 +134,10 @@ int load_other_segments(struct kimage *image,
kexec_dprintk("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
+
+ ret = crash_load_dm_crypt_keys(image);
+ if (ret)
+ goto out_err;
}
#endif
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 633e5223d944..448c917494f3 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -5,12 +5,6 @@ lib-y := clear_user.o delay.o copy_from_user.o \
memset.o memcmp.o strcmp.o strncmp.o strlen.o \
strnlen.o strchr.o strrchr.o tishift.o
-ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
-obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
-CFLAGS_xor-neon.o += $(CC_FLAGS_FPU)
-CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU)
-endif
-
lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c
deleted file mode 100644
index 8fffebfa17b2..000000000000
--- a/arch/arm64/lib/xor-neon.c
+++ /dev/null
@@ -1,338 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * arch/arm64/lib/xor-neon.c
- *
- * Authors: Jackie Liu <liuyun01@kylinos.cn>
- * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
- */
-
-#include <linux/raid/xor.h>
-#include <linux/module.h>
-#include <asm/neon-intrinsics.h>
-
-static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 */
- v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
- v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
- v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
- v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- } while (--lines > 0);
-}
-
-static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 */
- v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
- v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
- v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
- v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
-
- /* p1 ^= p3 */
- v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- } while (--lines > 0);
-}
-
-static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
- uint64_t *dp4 = (uint64_t *)p4;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 */
- v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
- v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
- v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
- v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
-
- /* p1 ^= p3 */
- v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
-
- /* p1 ^= p4 */
- v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- dp4 += 8;
- } while (--lines > 0);
-}
-
-static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
- uint64_t *dp4 = (uint64_t *)p4;
- uint64_t *dp5 = (uint64_t *)p5;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 */
- v0 = veorq_u64(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0));
- v1 = veorq_u64(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2));
- v2 = veorq_u64(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4));
- v3 = veorq_u64(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6));
-
- /* p1 ^= p3 */
- v0 = veorq_u64(v0, vld1q_u64(dp3 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp3 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp3 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp3 + 6));
-
- /* p1 ^= p4 */
- v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
-
- /* p1 ^= p5 */
- v0 = veorq_u64(v0, vld1q_u64(dp5 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp5 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp5 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp5 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- dp4 += 8;
- dp5 += 8;
- } while (--lines > 0);
-}
-
-struct xor_block_template xor_block_inner_neon __ro_after_init = {
- .name = "__inner_neon__",
- .do_2 = xor_arm64_neon_2,
- .do_3 = xor_arm64_neon_3,
- .do_4 = xor_arm64_neon_4,
- .do_5 = xor_arm64_neon_5,
-};
-EXPORT_SYMBOL(xor_block_inner_neon);
-
-static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
-{
- uint64x2_t res;
-
- asm(ARM64_ASM_PREAMBLE ".arch_extension sha3\n"
- "eor3 %0.16b, %1.16b, %2.16b, %3.16b"
- : "=w"(res) : "w"(p), "w"(q), "w"(r));
- return res;
-}
-
-static void xor_arm64_eor3_3(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 ^ p3 */
- v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
- vld1q_u64(dp3 + 0));
- v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
- vld1q_u64(dp3 + 2));
- v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
- vld1q_u64(dp3 + 4));
- v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
- vld1q_u64(dp3 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- } while (--lines > 0);
-}
-
-static void xor_arm64_eor3_4(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
- uint64_t *dp4 = (uint64_t *)p4;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 ^ p3 */
- v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
- vld1q_u64(dp3 + 0));
- v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
- vld1q_u64(dp3 + 2));
- v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
- vld1q_u64(dp3 + 4));
- v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
- vld1q_u64(dp3 + 6));
-
- /* p1 ^= p4 */
- v0 = veorq_u64(v0, vld1q_u64(dp4 + 0));
- v1 = veorq_u64(v1, vld1q_u64(dp4 + 2));
- v2 = veorq_u64(v2, vld1q_u64(dp4 + 4));
- v3 = veorq_u64(v3, vld1q_u64(dp4 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- dp4 += 8;
- } while (--lines > 0);
-}
-
-static void xor_arm64_eor3_5(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- uint64_t *dp1 = (uint64_t *)p1;
- uint64_t *dp2 = (uint64_t *)p2;
- uint64_t *dp3 = (uint64_t *)p3;
- uint64_t *dp4 = (uint64_t *)p4;
- uint64_t *dp5 = (uint64_t *)p5;
-
- register uint64x2_t v0, v1, v2, v3;
- long lines = bytes / (sizeof(uint64x2_t) * 4);
-
- do {
- /* p1 ^= p2 ^ p3 */
- v0 = eor3(vld1q_u64(dp1 + 0), vld1q_u64(dp2 + 0),
- vld1q_u64(dp3 + 0));
- v1 = eor3(vld1q_u64(dp1 + 2), vld1q_u64(dp2 + 2),
- vld1q_u64(dp3 + 2));
- v2 = eor3(vld1q_u64(dp1 + 4), vld1q_u64(dp2 + 4),
- vld1q_u64(dp3 + 4));
- v3 = eor3(vld1q_u64(dp1 + 6), vld1q_u64(dp2 + 6),
- vld1q_u64(dp3 + 6));
-
- /* p1 ^= p4 ^ p5 */
- v0 = eor3(v0, vld1q_u64(dp4 + 0), vld1q_u64(dp5 + 0));
- v1 = eor3(v1, vld1q_u64(dp4 + 2), vld1q_u64(dp5 + 2));
- v2 = eor3(v2, vld1q_u64(dp4 + 4), vld1q_u64(dp5 + 4));
- v3 = eor3(v3, vld1q_u64(dp4 + 6), vld1q_u64(dp5 + 6));
-
- /* store */
- vst1q_u64(dp1 + 0, v0);
- vst1q_u64(dp1 + 2, v1);
- vst1q_u64(dp1 + 4, v2);
- vst1q_u64(dp1 + 6, v3);
-
- dp1 += 8;
- dp2 += 8;
- dp3 += 8;
- dp4 += 8;
- dp5 += 8;
- } while (--lines > 0);
-}
-
-static int __init xor_neon_init(void)
-{
- if (cpu_have_named_feature(SHA3)) {
- xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
- xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
- xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
- }
- return 0;
-}
-module_init(xor_neon_init);
-
-static void __exit xor_neon_exit(void)
-{
-}
-module_exit(xor_neon_exit);
-
-MODULE_AUTHOR("Jackie Liu <liuyun01@kylinos.cn>");
-MODULE_DESCRIPTION("ARMv8 XOR Extensions");
-MODULE_LICENSE("GPL");
diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h
deleted file mode 100644
index 12467fffee46..000000000000
--- a/arch/loongarch/include/asm/xor.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_H
-#define _ASM_LOONGARCH_XOR_H
-
-#include <asm/cpu-features.h>
-#include <asm/xor_simd.h>
-
-#ifdef CONFIG_CPU_HAS_LSX
-static struct xor_block_template xor_block_lsx = {
- .name = "lsx",
- .do_2 = xor_lsx_2,
- .do_3 = xor_lsx_3,
- .do_4 = xor_lsx_4,
- .do_5 = xor_lsx_5,
-};
-
-#define XOR_SPEED_LSX() \
- do { \
- if (cpu_has_lsx) \
- xor_speed(&xor_block_lsx); \
- } while (0)
-#else /* CONFIG_CPU_HAS_LSX */
-#define XOR_SPEED_LSX()
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-static struct xor_block_template xor_block_lasx = {
- .name = "lasx",
- .do_2 = xor_lasx_2,
- .do_3 = xor_lasx_3,
- .do_4 = xor_lasx_4,
- .do_5 = xor_lasx_5,
-};
-
-#define XOR_SPEED_LASX() \
- do { \
- if (cpu_has_lasx) \
- xor_speed(&xor_block_lasx); \
- } while (0)
-#else /* CONFIG_CPU_HAS_LASX */
-#define XOR_SPEED_LASX()
-#endif /* CONFIG_CPU_HAS_LASX */
-
-/*
- * For grins, also test the generic routines.
- *
- * More importantly: it cannot be ruled out at this point of time, that some
- * future (maybe reduced) models could run the vector algorithms slower than
- * the scalar ones, maybe for errata or micro-op reasons. It may be
- * appropriate to revisit this after one or two more uarch generations.
- */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- XOR_SPEED_LSX(); \
- XOR_SPEED_LASX(); \
-} while (0)
-
-#endif /* _ASM_LOONGARCH_XOR_H */
diff --git a/arch/loongarch/include/asm/xor_simd.h b/arch/loongarch/include/asm/xor_simd.h
deleted file mode 100644
index 471b96332f38..000000000000
--- a/arch/loongarch/include/asm/xor_simd.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_SIMD_H
-#define _ASM_LOONGARCH_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* _ASM_LOONGARCH_XOR_SIMD_H */
diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile
index ccea3bbd4353..827a88529a42 100644
--- a/arch/loongarch/lib/Makefile
+++ b/arch/loongarch/lib/Makefile
@@ -8,6 +8,4 @@ lib-y += delay.o memset.o memcpy.o memmove.o \
obj-$(CONFIG_ARCH_SUPPORTS_INT128) += tishift.o
-obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o
-
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/loongarch/lib/xor_simd.c b/arch/loongarch/lib/xor_simd.c
deleted file mode 100644
index 84cd24b728c4..000000000000
--- a/arch/loongarch/lib/xor_simd.c
+++ /dev/null
@@ -1,93 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * LoongArch SIMD XOR operations
- *
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-
-#include "xor_simd.h"
-
-/*
- * Process one cache line (64 bytes) per loop. This is assuming all future
- * popular LoongArch cores are similar performance-characteristics-wise to the
- * current models.
- */
-#define LINE_WIDTH 64
-
-#ifdef CONFIG_CPU_HAS_LSX
-
-#define LD(reg, base, offset) \
- "vld $vr" #reg ", %[" #base "], " #offset "\n\t"
-#define ST(reg, base, offset) \
- "vst $vr" #reg ", %[" #base "], " #offset "\n\t"
-#define XOR(dj, k) "vxor.v $vr" #dj ", $vr" #dj ", $vr" #k "\n\t"
-
-#define LD_INOUT_LINE(base) \
- LD(0, base, 0) \
- LD(1, base, 16) \
- LD(2, base, 32) \
- LD(3, base, 48)
-
-#define LD_AND_XOR_LINE(base) \
- LD(4, base, 0) \
- LD(5, base, 16) \
- LD(6, base, 32) \
- LD(7, base, 48) \
- XOR(0, 4) \
- XOR(1, 5) \
- XOR(2, 6) \
- XOR(3, 7)
-
-#define ST_LINE(base) \
- ST(0, base, 0) \
- ST(1, base, 16) \
- ST(2, base, 32) \
- ST(3, base, 48)
-
-#define XOR_FUNC_NAME(nr) __xor_lsx_##nr
-#include "xor_template.c"
-
-#undef LD
-#undef ST
-#undef XOR
-#undef LD_INOUT_LINE
-#undef LD_AND_XOR_LINE
-#undef ST_LINE
-#undef XOR_FUNC_NAME
-
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-
-#define LD(reg, base, offset) \
- "xvld $xr" #reg ", %[" #base "], " #offset "\n\t"
-#define ST(reg, base, offset) \
- "xvst $xr" #reg ", %[" #base "], " #offset "\n\t"
-#define XOR(dj, k) "xvxor.v $xr" #dj ", $xr" #dj ", $xr" #k "\n\t"
-
-#define LD_INOUT_LINE(base) \
- LD(0, base, 0) \
- LD(1, base, 32)
-
-#define LD_AND_XOR_LINE(base) \
- LD(2, base, 0) \
- LD(3, base, 32) \
- XOR(0, 2) \
- XOR(1, 3)
-
-#define ST_LINE(base) \
- ST(0, base, 0) \
- ST(1, base, 32)
-
-#define XOR_FUNC_NAME(nr) __xor_lasx_##nr
-#include "xor_template.c"
-
-#undef LD
-#undef ST
-#undef XOR
-#undef LD_INOUT_LINE
-#undef LD_AND_XOR_LINE
-#undef ST_LINE
-#undef XOR_FUNC_NAME
-
-#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/arch/loongarch/lib/xor_simd.h b/arch/loongarch/lib/xor_simd.h
deleted file mode 100644
index f50f32514d80..000000000000
--- a/arch/loongarch/lib/xor_simd.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Simple interface to link xor_simd.c and xor_simd_glue.c
- *
- * Separating these files ensures that no SIMD instructions are run outside of
- * the kfpu critical section.
- */
-
-#ifndef __LOONGARCH_LIB_XOR_SIMD_H
-#define __LOONGARCH_LIB_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void __xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void __xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void __xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void __xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void __xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void __xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* __LOONGARCH_LIB_XOR_SIMD_H */
diff --git a/arch/loongarch/lib/xor_simd_glue.c b/arch/loongarch/lib/xor_simd_glue.c
deleted file mode 100644
index 393f689dbcf6..000000000000
--- a/arch/loongarch/lib/xor_simd_glue.c
+++ /dev/null
@@ -1,72 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * LoongArch SIMD XOR operations
- *
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-
-#include <linux/export.h>
-#include <linux/sched.h>
-#include <asm/fpu.h>
-#include <asm/xor_simd.h>
-#include "xor_simd.h"
-
-#define MAKE_XOR_GLUE_2(flavor) \
-void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_2(bytes, p1, p2); \
- kernel_fpu_end(); \
-} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_2)
-
-#define MAKE_XOR_GLUE_3(flavor) \
-void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_3(bytes, p1, p2, p3); \
- kernel_fpu_end(); \
-} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_3)
-
-#define MAKE_XOR_GLUE_4(flavor) \
-void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_4(bytes, p1, p2, p3, p4); \
- kernel_fpu_end(); \
-} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_4)
-
-#define MAKE_XOR_GLUE_5(flavor) \
-void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4, \
- const unsigned long * __restrict p5) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_5(bytes, p1, p2, p3, p4, p5); \
- kernel_fpu_end(); \
-} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_5)
-
-#define MAKE_XOR_GLUES(flavor) \
- MAKE_XOR_GLUE_2(flavor); \
- MAKE_XOR_GLUE_3(flavor); \
- MAKE_XOR_GLUE_4(flavor); \
- MAKE_XOR_GLUE_5(flavor)
-
-#ifdef CONFIG_CPU_HAS_LSX
-MAKE_XOR_GLUES(lsx);
-#endif
-
-#ifdef CONFIG_CPU_HAS_LASX
-MAKE_XOR_GLUES(lasx);
-#endif
diff --git a/arch/loongarch/lib/xor_template.c b/arch/loongarch/lib/xor_template.c
deleted file mode 100644
index 0358ced7fe33..000000000000
--- a/arch/loongarch/lib/xor_template.c
+++ /dev/null
@@ -1,110 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- *
- * Template for XOR operations, instantiated in xor_simd.c.
- *
- * Expected preprocessor definitions:
- *
- * - LINE_WIDTH
- * - XOR_FUNC_NAME(nr)
- * - LD_INOUT_LINE(buf)
- * - LD_AND_XOR_LINE(buf)
- * - ST_LINE(buf)
- */
-
-void XOR_FUNC_NAME(2)(unsigned long bytes,
- unsigned long * __restrict v1,
- const unsigned long * __restrict v2)
-{
- unsigned long lines = bytes / LINE_WIDTH;
-
- do {
- __asm__ __volatile__ (
- LD_INOUT_LINE(v1)
- LD_AND_XOR_LINE(v2)
- ST_LINE(v1)
- : : [v1] "r"(v1), [v2] "r"(v2) : "memory"
- );
-
- v1 += LINE_WIDTH / sizeof(unsigned long);
- v2 += LINE_WIDTH / sizeof(unsigned long);
- } while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(3)(unsigned long bytes,
- unsigned long * __restrict v1,
- const unsigned long * __restrict v2,
- const unsigned long * __restrict v3)
-{
- unsigned long lines = bytes / LINE_WIDTH;
-
- do {
- __asm__ __volatile__ (
- LD_INOUT_LINE(v1)
- LD_AND_XOR_LINE(v2)
- LD_AND_XOR_LINE(v3)
- ST_LINE(v1)
- : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3) : "memory"
- );
-
- v1 += LINE_WIDTH / sizeof(unsigned long);
- v2 += LINE_WIDTH / sizeof(unsigned long);
- v3 += LINE_WIDTH / sizeof(unsigned long);
- } while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(4)(unsigned long bytes,
- unsigned long * __restrict v1,
- const unsigned long * __restrict v2,
- const unsigned long * __restrict v3,
- const unsigned long * __restrict v4)
-{
- unsigned long lines = bytes / LINE_WIDTH;
-
- do {
- __asm__ __volatile__ (
- LD_INOUT_LINE(v1)
- LD_AND_XOR_LINE(v2)
- LD_AND_XOR_LINE(v3)
- LD_AND_XOR_LINE(v4)
- ST_LINE(v1)
- : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4)
- : "memory"
- );
-
- v1 += LINE_WIDTH / sizeof(unsigned long);
- v2 += LINE_WIDTH / sizeof(unsigned long);
- v3 += LINE_WIDTH / sizeof(unsigned long);
- v4 += LINE_WIDTH / sizeof(unsigned long);
- } while (--lines > 0);
-}
-
-void XOR_FUNC_NAME(5)(unsigned long bytes,
- unsigned long * __restrict v1,
- const unsigned long * __restrict v2,
- const unsigned long * __restrict v3,
- const unsigned long * __restrict v4,
- const unsigned long * __restrict v5)
-{
- unsigned long lines = bytes / LINE_WIDTH;
-
- do {
- __asm__ __volatile__ (
- LD_INOUT_LINE(v1)
- LD_AND_XOR_LINE(v2)
- LD_AND_XOR_LINE(v3)
- LD_AND_XOR_LINE(v4)
- LD_AND_XOR_LINE(v5)
- ST_LINE(v1)
- : : [v1] "r"(v1), [v2] "r"(v2), [v3] "r"(v3), [v4] "r"(v4),
- [v5] "r"(v5) : "memory"
- );
-
- v1 += LINE_WIDTH / sizeof(unsigned long);
- v2 += LINE_WIDTH / sizeof(unsigned long);
- v3 += LINE_WIDTH / sizeof(unsigned long);
- v4 += LINE_WIDTH / sizeof(unsigned long);
- v5 += LINE_WIDTH / sizeof(unsigned long);
- } while (--lines > 0);
-}
diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h
deleted file mode 100644
index 37d05c11d09c..000000000000
--- a/arch/powerpc/include/asm/xor.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- *
- * Copyright (C) IBM Corporation, 2012
- *
- * Author: Anton Blanchard <anton@au.ibm.com>
- */
-#ifndef _ASM_POWERPC_XOR_H
-#define _ASM_POWERPC_XOR_H
-
-#ifdef CONFIG_ALTIVEC
-
-#include <asm/cputable.h>
-#include <asm/cpu_has_feature.h>
-#include <asm/xor_altivec.h>
-
-static struct xor_block_template xor_block_altivec = {
- .name = "altivec",
- .do_2 = xor_altivec_2,
- .do_3 = xor_altivec_3,
- .do_4 = xor_altivec_4,
- .do_5 = xor_altivec_5,
-};
-
-#define XOR_SPEED_ALTIVEC() \
- do { \
- if (cpu_has_feature(CPU_FTR_ALTIVEC)) \
- xor_speed(&xor_block_altivec); \
- } while (0)
-#else
-#define XOR_SPEED_ALTIVEC()
-#endif
-
-/* Also try the generic routines. */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- XOR_SPEED_ALTIVEC(); \
-} while (0)
-
-#endif /* _ASM_POWERPC_XOR_H */
diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h
deleted file mode 100644
index 294620a25f80..000000000000
--- a/arch/powerpc/include/asm/xor_altivec.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_POWERPC_XOR_ALTIVEC_H
-#define _ASM_POWERPC_XOR_ALTIVEC_H
-
-#ifdef CONFIG_ALTIVEC
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-#endif
-#endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
diff --git a/arch/powerpc/kexec/elf_64.c b/arch/powerpc/kexec/elf_64.c
index 5d6d616404cf..ea50a072debf 100644
--- a/arch/powerpc/kexec/elf_64.c
+++ b/arch/powerpc/kexec/elf_64.c
@@ -79,6 +79,10 @@ static void *elf64_load(struct kimage *image, char *kernel_buf,
goto out;
}
+ ret = crash_load_dm_crypt_keys(image);
+ if (ret)
+ goto out;
+
/* Setup cmdline for kdump kernel case */
modified_cmdline = setup_kdump_cmdline(image, cmdline,
cmdline_len);
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index bcdf387f3998..b1b7a1419653 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -72,9 +72,4 @@ obj-$(CONFIG_PPC_LIB_RHEAP) += rheap.o
obj-$(CONFIG_FTR_FIXUP_SELFTEST) += feature-fixups-test.o
-obj-$(CONFIG_ALTIVEC) += xor_vmx.o xor_vmx_glue.o
-CFLAGS_xor_vmx.o += -mhard-float -maltivec $(call cc-option,-mabi=altivec)
-# Enable <altivec.h>
-CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include)
-
obj-$(CONFIG_PPC64) += $(obj64-y)
diff --git a/arch/powerpc/lib/xor_vmx.c b/arch/powerpc/lib/xor_vmx.c
deleted file mode 100644
index aab49d056d18..000000000000
--- a/arch/powerpc/lib/xor_vmx.c
+++ /dev/null
@@ -1,156 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *
- * Copyright (C) IBM Corporation, 2012
- *
- * Author: Anton Blanchard <anton@au.ibm.com>
- */
-
-/*
- * Sparse (as at v0.5.0) gets very, very confused by this file.
- * Make it a bit simpler for it.
- */
-#if !defined(__CHECKER__)
-#include <altivec.h>
-#else
-#define vec_xor(a, b) a ^ b
-#define vector __attribute__((vector_size(16)))
-#endif
-
-#include "xor_vmx.h"
-
-typedef vector signed char unative_t;
-
-#define DEFINE(V) \
- unative_t *V = (unative_t *)V##_in; \
- unative_t V##_0, V##_1, V##_2, V##_3
-
-#define LOAD(V) \
- do { \
- V##_0 = V[0]; \
- V##_1 = V[1]; \
- V##_2 = V[2]; \
- V##_3 = V[3]; \
- } while (0)
-
-#define STORE(V) \
- do { \
- V[0] = V##_0; \
- V[1] = V##_1; \
- V[2] = V##_2; \
- V[3] = V##_3; \
- } while (0)
-
-#define XOR(V1, V2) \
- do { \
- V1##_0 = vec_xor(V1##_0, V2##_0); \
- V1##_1 = vec_xor(V1##_1, V2##_1); \
- V1##_2 = vec_xor(V1##_2, V2##_2); \
- V1##_3 = vec_xor(V1##_3, V2##_3); \
- } while (0)
-
-void __xor_altivec_2(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in)
-{
- DEFINE(v1);
- DEFINE(v2);
- unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
- do {
- LOAD(v1);
- LOAD(v2);
- XOR(v1, v2);
- STORE(v1);
-
- v1 += 4;
- v2 += 4;
- } while (--lines > 0);
-}
-
-void __xor_altivec_3(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in)
-{
- DEFINE(v1);
- DEFINE(v2);
- DEFINE(v3);
- unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
- do {
- LOAD(v1);
- LOAD(v2);
- LOAD(v3);
- XOR(v1, v2);
- XOR(v1, v3);
- STORE(v1);
-
- v1 += 4;
- v2 += 4;
- v3 += 4;
- } while (--lines > 0);
-}
-
-void __xor_altivec_4(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in,
- const unsigned long * __restrict v4_in)
-{
- DEFINE(v1);
- DEFINE(v2);
- DEFINE(v3);
- DEFINE(v4);
- unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
- do {
- LOAD(v1);
- LOAD(v2);
- LOAD(v3);
- LOAD(v4);
- XOR(v1, v2);
- XOR(v3, v4);
- XOR(v1, v3);
- STORE(v1);
-
- v1 += 4;
- v2 += 4;
- v3 += 4;
- v4 += 4;
- } while (--lines > 0);
-}
-
-void __xor_altivec_5(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in,
- const unsigned long * __restrict v4_in,
- const unsigned long * __restrict v5_in)
-{
- DEFINE(v1);
- DEFINE(v2);
- DEFINE(v3);
- DEFINE(v4);
- DEFINE(v5);
- unsigned long lines = bytes / (sizeof(unative_t)) / 4;
-
- do {
- LOAD(v1);
- LOAD(v2);
- LOAD(v3);
- LOAD(v4);
- LOAD(v5);
- XOR(v1, v2);
- XOR(v3, v4);
- XOR(v1, v5);
- XOR(v1, v3);
- STORE(v1);
-
- v1 += 4;
- v2 += 4;
- v3 += 4;
- v4 += 4;
- v5 += 4;
- } while (--lines > 0);
-}
diff --git a/arch/powerpc/lib/xor_vmx.h b/arch/powerpc/lib/xor_vmx.h
deleted file mode 100644
index 573c41d90dac..000000000000
--- a/arch/powerpc/lib/xor_vmx.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Simple interface to link xor_vmx.c and xor_vmx_glue.c
- *
- * Separating these file ensures that no altivec instructions are run
- * outside of the enable/disable altivec block.
- */
-
-void __xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void __xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void __xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
diff --git a/arch/powerpc/lib/xor_vmx_glue.c b/arch/powerpc/lib/xor_vmx_glue.c
deleted file mode 100644
index 35d917ece4d1..000000000000
--- a/arch/powerpc/lib/xor_vmx_glue.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Altivec XOR operations
- *
- * Copyright 2017 IBM Corp.
- */
-
-#include <linux/preempt.h>
-#include <linux/export.h>
-#include <linux/sched.h>
-#include <asm/switch_to.h>
-#include <asm/xor_altivec.h>
-#include "xor_vmx.h"
-
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_2(bytes, p1, p2);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_2);
-
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_3(bytes, p1, p2, p3);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_3);
-
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_4(bytes, p1, p2, p3, p4);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_4);
-
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_5(bytes, p1, p2, p3, p4, p5);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_5);
diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h
deleted file mode 100644
index 96011861e46b..000000000000
--- a/arch/riscv/include/asm/xor.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2021 SiFive
- */
-
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#ifdef CONFIG_RISCV_ISA_V
-#include <asm/vector.h>
-#include <asm/switch_to.h>
-#include <asm/asm-prototypes.h>
-
-static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2)
-{
- kernel_vector_begin();
- xor_regs_2_(bytes, p1, p2);
- kernel_vector_end();
-}
-
-static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3)
-{
- kernel_vector_begin();
- xor_regs_3_(bytes, p1, p2, p3);
- kernel_vector_end();
-}
-
-static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4)
-{
- kernel_vector_begin();
- xor_regs_4_(bytes, p1, p2, p3, p4);
- kernel_vector_end();
-}
-
-static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4,
- const unsigned long *__restrict p5)
-{
- kernel_vector_begin();
- xor_regs_5_(bytes, p1, p2, p3, p4, p5);
- kernel_vector_end();
-}
-
-static struct xor_block_template xor_block_rvv = {
- .name = "rvv",
- .do_2 = xor_vector_2,
- .do_3 = xor_vector_3,
- .do_4 = xor_vector_4,
- .do_5 = xor_vector_5
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- if (has_vector()) { \
- xor_speed(&xor_block_rvv);\
- } \
- } while (0)
-#endif
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index bbc031124974..e220c35764eb 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -16,5 +16,4 @@ lib-$(CONFIG_MMU) += uaccess.o
lib-$(CONFIG_64BIT) += tishift.o
lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
-lib-$(CONFIG_RISCV_ISA_V) += xor.o
lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o
diff --git a/arch/riscv/lib/xor.S b/arch/riscv/lib/xor.S
deleted file mode 100644
index b28f2430e52f..000000000000
--- a/arch/riscv/lib/xor.S
+++ /dev/null
@@ -1,81 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2021 SiFive
- */
-#include <linux/linkage.h>
-#include <linux/export.h>
-#include <asm/asm.h>
-
-SYM_FUNC_START(xor_regs_2_)
- vsetvli a3, a0, e8, m8, ta, ma
- vle8.v v0, (a1)
- vle8.v v8, (a2)
- sub a0, a0, a3
- vxor.vv v16, v0, v8
- add a2, a2, a3
- vse8.v v16, (a1)
- add a1, a1, a3
- bnez a0, xor_regs_2_
- ret
-SYM_FUNC_END(xor_regs_2_)
-EXPORT_SYMBOL(xor_regs_2_)
-
-SYM_FUNC_START(xor_regs_3_)
- vsetvli a4, a0, e8, m8, ta, ma
- vle8.v v0, (a1)
- vle8.v v8, (a2)
- sub a0, a0, a4
- vxor.vv v0, v0, v8
- vle8.v v16, (a3)
- add a2, a2, a4
- vxor.vv v16, v0, v16
- add a3, a3, a4
- vse8.v v16, (a1)
- add a1, a1, a4
- bnez a0, xor_regs_3_
- ret
-SYM_FUNC_END(xor_regs_3_)
-EXPORT_SYMBOL(xor_regs_3_)
-
-SYM_FUNC_START(xor_regs_4_)
- vsetvli a5, a0, e8, m8, ta, ma
- vle8.v v0, (a1)
- vle8.v v8, (a2)
- sub a0, a0, a5
- vxor.vv v0, v0, v8
- vle8.v v16, (a3)
- add a2, a2, a5
- vxor.vv v0, v0, v16
- vle8.v v24, (a4)
- add a3, a3, a5
- vxor.vv v16, v0, v24
- add a4, a4, a5
- vse8.v v16, (a1)
- add a1, a1, a5
- bnez a0, xor_regs_4_
- ret
-SYM_FUNC_END(xor_regs_4_)
-EXPORT_SYMBOL(xor_regs_4_)
-
-SYM_FUNC_START(xor_regs_5_)
- vsetvli a6, a0, e8, m8, ta, ma
- vle8.v v0, (a1)
- vle8.v v8, (a2)
- sub a0, a0, a6
- vxor.vv v0, v0, v8
- vle8.v v16, (a3)
- add a2, a2, a6
- vxor.vv v0, v0, v16
- vle8.v v24, (a4)
- add a3, a3, a6
- vxor.vv v0, v0, v24
- vle8.v v8, (a5)
- add a4, a4, a6
- vxor.vv v16, v0, v8
- add a5, a5, a6
- vse8.v v16, (a1)
- add a1, a1, a6
- bnez a0, xor_regs_5_
- ret
-SYM_FUNC_END(xor_regs_5_)
-EXPORT_SYMBOL(xor_regs_5_)
diff --git a/arch/s390/include/asm/xor.h b/arch/s390/include/asm/xor.h
deleted file mode 100644
index 857d6759b67f..000000000000
--- a/arch/s390/include/asm/xor.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Optimited xor routines
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- */
-#ifndef _ASM_S390_XOR_H
-#define _ASM_S390_XOR_H
-
-extern struct xor_block_template xor_block_xc;
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_xc); \
-} while (0)
-
-#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_xc)
-
-#endif /* _ASM_S390_XOR_H */
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index f43f897d3fc0..2bf47204f6ab 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -5,7 +5,7 @@
lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o
lib-y += csum-partial.o
-obj-y += mem.o xor.o
+obj-y += mem.o
lib-$(CONFIG_KPROBES) += probes.o
lib-$(CONFIG_UPROBES) += probes.o
obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c
deleted file mode 100644
index 5363e4c2462d..000000000000
--- a/arch/s390/lib/xor.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Optimized xor_block operation for RAID4/5
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- */
-
-#include <linux/types.h>
-#include <linux/export.h>
-#include <linux/raid/xor.h>
-#include <asm/xor.h>
-
-static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- asm volatile(
- " aghi %0,-1\n"
- " jm 3f\n"
- " srlg 0,%0,8\n"
- " ltgr 0,0\n"
- " jz 1f\n"
- "0: xc 0(256,%1),0(%2)\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- " brctg 0,0b\n"
- "1: exrl %0,2f\n"
- " j 3f\n"
- "2: xc 0(1,%1),0(%2)\n"
- "3:"
- : "+a" (bytes), "+a" (p1), "+a" (p2)
- : : "0", "cc", "memory");
-}
-
-static void xor_xc_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- asm volatile(
- " aghi %0,-1\n"
- " jm 4f\n"
- " srlg 0,%0,8\n"
- " ltgr 0,0\n"
- " jz 1f\n"
- "0: xc 0(256,%1),0(%2)\n"
- " xc 0(256,%1),0(%3)\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- " la %3,256(%3)\n"
- " brctg 0,0b\n"
- "1: exrl %0,2f\n"
- " exrl %0,3f\n"
- " j 4f\n"
- "2: xc 0(1,%1),0(%2)\n"
- "3: xc 0(1,%1),0(%3)\n"
- "4:"
- : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3)
- : : "0", "cc", "memory");
-}
-
-static void xor_xc_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- asm volatile(
- " aghi %0,-1\n"
- " jm 5f\n"
- " srlg 0,%0,8\n"
- " ltgr 0,0\n"
- " jz 1f\n"
- "0: xc 0(256,%1),0(%2)\n"
- " xc 0(256,%1),0(%3)\n"
- " xc 0(256,%1),0(%4)\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- " la %3,256(%3)\n"
- " la %4,256(%4)\n"
- " brctg 0,0b\n"
- "1: exrl %0,2f\n"
- " exrl %0,3f\n"
- " exrl %0,4f\n"
- " j 5f\n"
- "2: xc 0(1,%1),0(%2)\n"
- "3: xc 0(1,%1),0(%3)\n"
- "4: xc 0(1,%1),0(%4)\n"
- "5:"
- : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4)
- : : "0", "cc", "memory");
-}
-
-static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- asm volatile(
- " aghi %0,-1\n"
- " jm 6f\n"
- " srlg 0,%0,8\n"
- " ltgr 0,0\n"
- " jz 1f\n"
- "0: xc 0(256,%1),0(%2)\n"
- " xc 0(256,%1),0(%3)\n"
- " xc 0(256,%1),0(%4)\n"
- " xc 0(256,%1),0(%5)\n"
- " la %1,256(%1)\n"
- " la %2,256(%2)\n"
- " la %3,256(%3)\n"
- " la %4,256(%4)\n"
- " la %5,256(%5)\n"
- " brctg 0,0b\n"
- "1: exrl %0,2f\n"
- " exrl %0,3f\n"
- " exrl %0,4f\n"
- " exrl %0,5f\n"
- " j 6f\n"
- "2: xc 0(1,%1),0(%2)\n"
- "3: xc 0(1,%1),0(%3)\n"
- "4: xc 0(1,%1),0(%4)\n"
- "5: xc 0(1,%1),0(%5)\n"
- "6:"
- : "+a" (bytes), "+a" (p1), "+a" (p2), "+a" (p3), "+a" (p4),
- "+a" (p5)
- : : "0", "cc", "memory");
-}
-
-struct xor_block_template xor_block_xc = {
- .name = "xc",
- .do_2 = xor_xc_2,
- .do_3 = xor_xc_3,
- .do_4 = xor_xc_4,
- .do_5 = xor_xc_5,
-};
-EXPORT_SYMBOL(xor_block_xc);
diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h
index 08810808ca6d..bbd1a8afaabf 100644
--- a/arch/sparc/include/asm/asm-prototypes.h
+++ b/arch/sparc/include/asm/asm-prototypes.h
@@ -14,7 +14,6 @@
#include <asm/oplib.h>
#include <asm/pgtable.h>
#include <asm/trap_block.h>
-#include <asm/xor.h>
void *__memscan_zero(void *, size_t);
void *__memscan_generic(void *, int, size_t);
diff --git a/arch/sparc/include/asm/xor.h b/arch/sparc/include/asm/xor.h
deleted file mode 100644
index f4c651e203c4..000000000000
--- a/arch/sparc/include/asm/xor.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef ___ASM_SPARC_XOR_H
-#define ___ASM_SPARC_XOR_H
-#if defined(__sparc__) && defined(__arch64__)
-#include <asm/xor_64.h>
-#else
-#include <asm/xor_32.h>
-#endif
-#endif
diff --git a/arch/sparc/include/asm/xor_32.h b/arch/sparc/include/asm/xor_32.h
deleted file mode 100644
index 0351813cf3af..000000000000
--- a/arch/sparc/include/asm/xor_32.h
+++ /dev/null
@@ -1,268 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm/xor.h
- *
- * Optimized RAID-5 checksumming functions for 32-bit Sparc.
- */
-
-/*
- * High speed xor_block operation for RAID4/5 utilizing the
- * ldd/std SPARC instructions.
- *
- * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
- */
-
-static void
-sparc_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- int lines = bytes / (sizeof (long)) / 8;
-
- do {
- __asm__ __volatile__(
- "ldd [%0 + 0x00], %%g2\n\t"
- "ldd [%0 + 0x08], %%g4\n\t"
- "ldd [%0 + 0x10], %%o0\n\t"
- "ldd [%0 + 0x18], %%o2\n\t"
- "ldd [%1 + 0x00], %%o4\n\t"
- "ldd [%1 + 0x08], %%l0\n\t"
- "ldd [%1 + 0x10], %%l2\n\t"
- "ldd [%1 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "std %%g2, [%0 + 0x00]\n\t"
- "std %%g4, [%0 + 0x08]\n\t"
- "std %%o0, [%0 + 0x10]\n\t"
- "std %%o2, [%0 + 0x18]\n"
- :
- : "r" (p1), "r" (p2)
- : "g2", "g3", "g4", "g5",
- "o0", "o1", "o2", "o3", "o4", "o5",
- "l0", "l1", "l2", "l3", "l4", "l5");
- p1 += 8;
- p2 += 8;
- } while (--lines > 0);
-}
-
-static void
-sparc_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- int lines = bytes / (sizeof (long)) / 8;
-
- do {
- __asm__ __volatile__(
- "ldd [%0 + 0x00], %%g2\n\t"
- "ldd [%0 + 0x08], %%g4\n\t"
- "ldd [%0 + 0x10], %%o0\n\t"
- "ldd [%0 + 0x18], %%o2\n\t"
- "ldd [%1 + 0x00], %%o4\n\t"
- "ldd [%1 + 0x08], %%l0\n\t"
- "ldd [%1 + 0x10], %%l2\n\t"
- "ldd [%1 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%2 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%2 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%2 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%2 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "std %%g2, [%0 + 0x00]\n\t"
- "std %%g4, [%0 + 0x08]\n\t"
- "std %%o0, [%0 + 0x10]\n\t"
- "std %%o2, [%0 + 0x18]\n"
- :
- : "r" (p1), "r" (p2), "r" (p3)
- : "g2", "g3", "g4", "g5",
- "o0", "o1", "o2", "o3", "o4", "o5",
- "l0", "l1", "l2", "l3", "l4", "l5");
- p1 += 8;
- p2 += 8;
- p3 += 8;
- } while (--lines > 0);
-}
-
-static void
-sparc_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- int lines = bytes / (sizeof (long)) / 8;
-
- do {
- __asm__ __volatile__(
- "ldd [%0 + 0x00], %%g2\n\t"
- "ldd [%0 + 0x08], %%g4\n\t"
- "ldd [%0 + 0x10], %%o0\n\t"
- "ldd [%0 + 0x18], %%o2\n\t"
- "ldd [%1 + 0x00], %%o4\n\t"
- "ldd [%1 + 0x08], %%l0\n\t"
- "ldd [%1 + 0x10], %%l2\n\t"
- "ldd [%1 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%2 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%2 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%2 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%2 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%3 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%3 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%3 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%3 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "std %%g2, [%0 + 0x00]\n\t"
- "std %%g4, [%0 + 0x08]\n\t"
- "std %%o0, [%0 + 0x10]\n\t"
- "std %%o2, [%0 + 0x18]\n"
- :
- : "r" (p1), "r" (p2), "r" (p3), "r" (p4)
- : "g2", "g3", "g4", "g5",
- "o0", "o1", "o2", "o3", "o4", "o5",
- "l0", "l1", "l2", "l3", "l4", "l5");
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- } while (--lines > 0);
-}
-
-static void
-sparc_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- int lines = bytes / (sizeof (long)) / 8;
-
- do {
- __asm__ __volatile__(
- "ldd [%0 + 0x00], %%g2\n\t"
- "ldd [%0 + 0x08], %%g4\n\t"
- "ldd [%0 + 0x10], %%o0\n\t"
- "ldd [%0 + 0x18], %%o2\n\t"
- "ldd [%1 + 0x00], %%o4\n\t"
- "ldd [%1 + 0x08], %%l0\n\t"
- "ldd [%1 + 0x10], %%l2\n\t"
- "ldd [%1 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%2 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%2 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%2 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%2 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%3 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%3 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%3 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%3 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "ldd [%4 + 0x00], %%o4\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "ldd [%4 + 0x08], %%l0\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "ldd [%4 + 0x10], %%l2\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "ldd [%4 + 0x18], %%l4\n\t"
- "xor %%g2, %%o4, %%g2\n\t"
- "xor %%g3, %%o5, %%g3\n\t"
- "xor %%g4, %%l0, %%g4\n\t"
- "xor %%g5, %%l1, %%g5\n\t"
- "xor %%o0, %%l2, %%o0\n\t"
- "xor %%o1, %%l3, %%o1\n\t"
- "xor %%o2, %%l4, %%o2\n\t"
- "xor %%o3, %%l5, %%o3\n\t"
- "std %%g2, [%0 + 0x00]\n\t"
- "std %%g4, [%0 + 0x08]\n\t"
- "std %%o0, [%0 + 0x10]\n\t"
- "std %%o2, [%0 + 0x18]\n"
- :
- : "r" (p1), "r" (p2), "r" (p3), "r" (p4), "r" (p5)
- : "g2", "g3", "g4", "g5",
- "o0", "o1", "o2", "o3", "o4", "o5",
- "l0", "l1", "l2", "l3", "l4", "l5");
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- p5 += 8;
- } while (--lines > 0);
-}
-
-static struct xor_block_template xor_block_SPARC = {
- .name = "SPARC",
- .do_2 = sparc_2,
- .do_3 = sparc_3,
- .do_4 = sparc_4,
- .do_5 = sparc_5,
-};
-
-/* For grins, also test the generic routines. */
-#include <asm-generic/xor.h>
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_SPARC); \
- } while (0)
diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h
deleted file mode 100644
index caaddea8ad79..000000000000
--- a/arch/sparc/include/asm/xor_64.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm/xor.h
- *
- * High speed xor_block operation for RAID4/5 utilizing the
- * UltraSparc Visual Instruction Set and Niagara block-init
- * twin-load instructions.
- *
- * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
- */
-
-#include <asm/spitfire.h>
-
-void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_vis_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void xor_vis_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-/* XXX Ugh, write cheetah versions... -DaveM */
-
-static struct xor_block_template xor_block_VIS = {
- .name = "VIS",
- .do_2 = xor_vis_2,
- .do_3 = xor_vis_3,
- .do_4 = xor_vis_4,
- .do_5 = xor_vis_5,
-};
-
-void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_niagara_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void xor_niagara_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-static struct xor_block_template xor_block_niagara = {
- .name = "Niagara",
- .do_2 = xor_niagara_2,
- .do_3 = xor_niagara_3,
- .do_4 = xor_niagara_4,
- .do_5 = xor_niagara_5,
-};
-
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_VIS); \
- xor_speed(&xor_block_niagara); \
- } while (0)
-
-/* For VIS for everything except Niagara. */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- ((tlb_type == hypervisor && \
- (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA5)) ? \
- &xor_block_niagara : \
- &xor_block_VIS)
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 783bdec0d7be..dd10cdd6f062 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -48,7 +48,7 @@ lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
-lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
+lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o hweight.o ffs.o
obj-$(CONFIG_SPARC64) += iomap.o
obj-$(CONFIG_SPARC32) += atomic32.o
diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S
deleted file mode 100644
index 35461e3b2a9b..000000000000
--- a/arch/sparc/lib/xor.S
+++ /dev/null
@@ -1,646 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * arch/sparc64/lib/xor.S
- *
- * High speed xor_block operation for RAID4/5 utilizing the
- * UltraSparc Visual Instruction Set and Niagara store-init/twin-load.
- *
- * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
- * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
- */
-
-#include <linux/export.h>
-#include <linux/linkage.h>
-#include <asm/visasm.h>
-#include <asm/asi.h>
-#include <asm/dcu.h>
-#include <asm/spitfire.h>
-
-/*
- * Requirements:
- * !(((long)dest | (long)sourceN) & (64 - 1)) &&
- * !(len & 127) && len >= 256
- */
- .text
-
- /* VIS versions. */
-ENTRY(xor_vis_2)
- rd %fprs, %o5
- andcc %o5, FPRS_FEF|FPRS_DU, %g0
- be,pt %icc, 0f
- sethi %hi(VISenter), %g1
- jmpl %g1 + %lo(VISenter), %g7
- add %g7, 8, %g7
-0: wr %g0, FPRS_FEF, %fprs
- rd %asi, %g1
- wr %g0, ASI_BLK_P, %asi
- membar #LoadStore|#StoreLoad|#StoreStore
- sub %o0, 128, %o0
- ldda [%o1] %asi, %f0
- ldda [%o2] %asi, %f16
-
-2: ldda [%o1 + 64] %asi, %f32
- fxor %f0, %f16, %f16
- fxor %f2, %f18, %f18
- fxor %f4, %f20, %f20
- fxor %f6, %f22, %f22
- fxor %f8, %f24, %f24
- fxor %f10, %f26, %f26
- fxor %f12, %f28, %f28
- fxor %f14, %f30, %f30
- stda %f16, [%o1] %asi
- ldda [%o2 + 64] %asi, %f48
- ldda [%o1 + 128] %asi, %f0
- fxor %f32, %f48, %f48
- fxor %f34, %f50, %f50
- add %o1, 128, %o1
- fxor %f36, %f52, %f52
- add %o2, 128, %o2
- fxor %f38, %f54, %f54
- subcc %o0, 128, %o0
- fxor %f40, %f56, %f56
- fxor %f42, %f58, %f58
- fxor %f44, %f60, %f60
- fxor %f46, %f62, %f62
- stda %f48, [%o1 - 64] %asi
- bne,pt %xcc, 2b
- ldda [%o2] %asi, %f16
-
- ldda [%o1 + 64] %asi, %f32
- fxor %f0, %f16, %f16
- fxor %f2, %f18, %f18
- fxor %f4, %f20, %f20
- fxor %f6, %f22, %f22
- fxor %f8, %f24, %f24
- fxor %f10, %f26, %f26
- fxor %f12, %f28, %f28
- fxor %f14, %f30, %f30
- stda %f16, [%o1] %asi
- ldda [%o2 + 64] %asi, %f48
- membar #Sync
- fxor %f32, %f48, %f48
- fxor %f34, %f50, %f50
- fxor %f36, %f52, %f52
- fxor %f38, %f54, %f54
- fxor %f40, %f56, %f56
- fxor %f42, %f58, %f58
- fxor %f44, %f60, %f60
- fxor %f46, %f62, %f62
- stda %f48, [%o1 + 64] %asi
- membar #Sync|#StoreStore|#StoreLoad
- wr %g1, %g0, %asi
- retl
- wr %g0, 0, %fprs
-ENDPROC(xor_vis_2)
-EXPORT_SYMBOL(xor_vis_2)
-
-ENTRY(xor_vis_3)
- rd %fprs, %o5
- andcc %o5, FPRS_FEF|FPRS_DU, %g0
- be,pt %icc, 0f
- sethi %hi(VISenter), %g1
- jmpl %g1 + %lo(VISenter), %g7
- add %g7, 8, %g7
-0: wr %g0, FPRS_FEF, %fprs
- rd %asi, %g1
- wr %g0, ASI_BLK_P, %asi
- membar #LoadStore|#StoreLoad|#StoreStore
- sub %o0, 64, %o0
- ldda [%o1] %asi, %f0
- ldda [%o2] %asi, %f16
-
-3: ldda [%o3] %asi, %f32
- fxor %f0, %f16, %f48
- fxor %f2, %f18, %f50
- add %o1, 64, %o1
- fxor %f4, %f20, %f52
- fxor %f6, %f22, %f54
- add %o2, 64, %o2
- fxor %f8, %f24, %f56
- fxor %f10, %f26, %f58
- fxor %f12, %f28, %f60
- fxor %f14, %f30, %f62
- ldda [%o1] %asi, %f0
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- add %o3, 64, %o3
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- subcc %o0, 64, %o0
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- stda %f48, [%o1 - 64] %asi
- bne,pt %xcc, 3b
- ldda [%o2] %asi, %f16
-
- ldda [%o3] %asi, %f32
- fxor %f0, %f16, %f48
- fxor %f2, %f18, %f50
- fxor %f4, %f20, %f52
- fxor %f6, %f22, %f54
- fxor %f8, %f24, %f56
- fxor %f10, %f26, %f58
- fxor %f12, %f28, %f60
- fxor %f14, %f30, %f62
- membar #Sync
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- stda %f48, [%o1] %asi
- membar #Sync|#StoreStore|#StoreLoad
- wr %g1, %g0, %asi
- retl
- wr %g0, 0, %fprs
-ENDPROC(xor_vis_3)
-EXPORT_SYMBOL(xor_vis_3)
-
-ENTRY(xor_vis_4)
- rd %fprs, %o5
- andcc %o5, FPRS_FEF|FPRS_DU, %g0
- be,pt %icc, 0f
- sethi %hi(VISenter), %g1
- jmpl %g1 + %lo(VISenter), %g7
- add %g7, 8, %g7
-0: wr %g0, FPRS_FEF, %fprs
- rd %asi, %g1
- wr %g0, ASI_BLK_P, %asi
- membar #LoadStore|#StoreLoad|#StoreStore
- sub %o0, 64, %o0
- ldda [%o1] %asi, %f0
- ldda [%o2] %asi, %f16
-
-4: ldda [%o3] %asi, %f32
- fxor %f0, %f16, %f16
- fxor %f2, %f18, %f18
- add %o1, 64, %o1
- fxor %f4, %f20, %f20
- fxor %f6, %f22, %f22
- add %o2, 64, %o2
- fxor %f8, %f24, %f24
- fxor %f10, %f26, %f26
- fxor %f12, %f28, %f28
- fxor %f14, %f30, %f30
- ldda [%o4] %asi, %f48
- fxor %f16, %f32, %f32
- fxor %f18, %f34, %f34
- fxor %f20, %f36, %f36
- fxor %f22, %f38, %f38
- add %o3, 64, %o3
- fxor %f24, %f40, %f40
- fxor %f26, %f42, %f42
- fxor %f28, %f44, %f44
- fxor %f30, %f46, %f46
- ldda [%o1] %asi, %f0
- fxor %f32, %f48, %f48
- fxor %f34, %f50, %f50
- fxor %f36, %f52, %f52
- add %o4, 64, %o4
- fxor %f38, %f54, %f54
- fxor %f40, %f56, %f56
- fxor %f42, %f58, %f58
- subcc %o0, 64, %o0
- fxor %f44, %f60, %f60
- fxor %f46, %f62, %f62
- stda %f48, [%o1 - 64] %asi
- bne,pt %xcc, 4b
- ldda [%o2] %asi, %f16
-
- ldda [%o3] %asi, %f32
- fxor %f0, %f16, %f16
- fxor %f2, %f18, %f18
- fxor %f4, %f20, %f20
- fxor %f6, %f22, %f22
- fxor %f8, %f24, %f24
- fxor %f10, %f26, %f26
- fxor %f12, %f28, %f28
- fxor %f14, %f30, %f30
- ldda [%o4] %asi, %f48
- fxor %f16, %f32, %f32
- fxor %f18, %f34, %f34
- fxor %f20, %f36, %f36
- fxor %f22, %f38, %f38
- fxor %f24, %f40, %f40
- fxor %f26, %f42, %f42
- fxor %f28, %f44, %f44
- fxor %f30, %f46, %f46
- membar #Sync
- fxor %f32, %f48, %f48
- fxor %f34, %f50, %f50
- fxor %f36, %f52, %f52
- fxor %f38, %f54, %f54
- fxor %f40, %f56, %f56
- fxor %f42, %f58, %f58
- fxor %f44, %f60, %f60
- fxor %f46, %f62, %f62
- stda %f48, [%o1] %asi
- membar #Sync|#StoreStore|#StoreLoad
- wr %g1, %g0, %asi
- retl
- wr %g0, 0, %fprs
-ENDPROC(xor_vis_4)
-EXPORT_SYMBOL(xor_vis_4)
-
-ENTRY(xor_vis_5)
- save %sp, -192, %sp
- rd %fprs, %o5
- andcc %o5, FPRS_FEF|FPRS_DU, %g0
- be,pt %icc, 0f
- sethi %hi(VISenter), %g1
- jmpl %g1 + %lo(VISenter), %g7
- add %g7, 8, %g7
-0: wr %g0, FPRS_FEF, %fprs
- rd %asi, %g1
- wr %g0, ASI_BLK_P, %asi
- membar #LoadStore|#StoreLoad|#StoreStore
- sub %i0, 64, %i0
- ldda [%i1] %asi, %f0
- ldda [%i2] %asi, %f16
-
-5: ldda [%i3] %asi, %f32
- fxor %f0, %f16, %f48
- fxor %f2, %f18, %f50
- add %i1, 64, %i1
- fxor %f4, %f20, %f52
- fxor %f6, %f22, %f54
- add %i2, 64, %i2
- fxor %f8, %f24, %f56
- fxor %f10, %f26, %f58
- fxor %f12, %f28, %f60
- fxor %f14, %f30, %f62
- ldda [%i4] %asi, %f16
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- add %i3, 64, %i3
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- ldda [%i5] %asi, %f32
- fxor %f48, %f16, %f48
- fxor %f50, %f18, %f50
- add %i4, 64, %i4
- fxor %f52, %f20, %f52
- fxor %f54, %f22, %f54
- add %i5, 64, %i5
- fxor %f56, %f24, %f56
- fxor %f58, %f26, %f58
- fxor %f60, %f28, %f60
- fxor %f62, %f30, %f62
- ldda [%i1] %asi, %f0
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- subcc %i0, 64, %i0
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- stda %f48, [%i1 - 64] %asi
- bne,pt %xcc, 5b
- ldda [%i2] %asi, %f16
-
- ldda [%i3] %asi, %f32
- fxor %f0, %f16, %f48
- fxor %f2, %f18, %f50
- fxor %f4, %f20, %f52
- fxor %f6, %f22, %f54
- fxor %f8, %f24, %f56
- fxor %f10, %f26, %f58
- fxor %f12, %f28, %f60
- fxor %f14, %f30, %f62
- ldda [%i4] %asi, %f16
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- ldda [%i5] %asi, %f32
- fxor %f48, %f16, %f48
- fxor %f50, %f18, %f50
- fxor %f52, %f20, %f52
- fxor %f54, %f22, %f54
- fxor %f56, %f24, %f56
- fxor %f58, %f26, %f58
- fxor %f60, %f28, %f60
- fxor %f62, %f30, %f62
- membar #Sync
- fxor %f48, %f32, %f48
- fxor %f50, %f34, %f50
- fxor %f52, %f36, %f52
- fxor %f54, %f38, %f54
- fxor %f56, %f40, %f56
- fxor %f58, %f42, %f58
- fxor %f60, %f44, %f60
- fxor %f62, %f46, %f62
- stda %f48, [%i1] %asi
- membar #Sync|#StoreStore|#StoreLoad
- wr %g1, %g0, %asi
- wr %g0, 0, %fprs
- ret
- restore
-ENDPROC(xor_vis_5)
-EXPORT_SYMBOL(xor_vis_5)
-
- /* Niagara versions. */
-ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
- save %sp, -192, %sp
- prefetch [%i1], #n_writes
- prefetch [%i2], #one_read
- rd %asi, %g7
- wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
- srlx %i0, 6, %g1
- mov %i1, %i0
- mov %i2, %i1
-1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src + 0x00 */
- ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src + 0x10 */
- ldda [%i1 + 0x20] %asi, %g2 /* %g2/%g3 = src + 0x20 */
- ldda [%i1 + 0x30] %asi, %l0 /* %l0/%l1 = src + 0x30 */
- prefetch [%i1 + 0x40], #one_read
- ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */
- ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */
- ldda [%i0 + 0x20] %asi, %o4 /* %o4/%o5 = dest + 0x20 */
- ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */
- prefetch [%i0 + 0x40], #n_writes
- xor %o0, %i2, %o0
- xor %o1, %i3, %o1
- stxa %o0, [%i0 + 0x00] %asi
- stxa %o1, [%i0 + 0x08] %asi
- xor %o2, %i4, %o2
- xor %o3, %i5, %o3
- stxa %o2, [%i0 + 0x10] %asi
- stxa %o3, [%i0 + 0x18] %asi
- xor %o4, %g2, %o4
- xor %o5, %g3, %o5
- stxa %o4, [%i0 + 0x20] %asi
- stxa %o5, [%i0 + 0x28] %asi
- xor %l2, %l0, %l2
- xor %l3, %l1, %l3
- stxa %l2, [%i0 + 0x30] %asi
- stxa %l3, [%i0 + 0x38] %asi
- add %i0, 0x40, %i0
- subcc %g1, 1, %g1
- bne,pt %xcc, 1b
- add %i1, 0x40, %i1
- membar #Sync
- wr %g7, 0x0, %asi
- ret
- restore
-ENDPROC(xor_niagara_2)
-EXPORT_SYMBOL(xor_niagara_2)
-
-ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
- save %sp, -192, %sp
- prefetch [%i1], #n_writes
- prefetch [%i2], #one_read
- prefetch [%i3], #one_read
- rd %asi, %g7
- wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
- srlx %i0, 6, %g1
- mov %i1, %i0
- mov %i2, %i1
- mov %i3, %l7
-1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
- ldda [%i1 + 0x10] %asi, %i4 /* %i4/%i5 = src1 + 0x10 */
- ldda [%l7 + 0x00] %asi, %g2 /* %g2/%g3 = src2 + 0x00 */
- ldda [%l7 + 0x10] %asi, %l0 /* %l0/%l1 = src2 + 0x10 */
- ldda [%i0 + 0x00] %asi, %o0 /* %o0/%o1 = dest + 0x00 */
- ldda [%i0 + 0x10] %asi, %o2 /* %o2/%o3 = dest + 0x10 */
- xor %g2, %i2, %g2
- xor %g3, %i3, %g3
- xor %o0, %g2, %o0
- xor %o1, %g3, %o1
- stxa %o0, [%i0 + 0x00] %asi
- stxa %o1, [%i0 + 0x08] %asi
- ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
- ldda [%l7 + 0x20] %asi, %g2 /* %g2/%g3 = src2 + 0x20 */
- ldda [%i0 + 0x20] %asi, %o0 /* %o0/%o1 = dest + 0x20 */
- xor %l0, %i4, %l0
- xor %l1, %i5, %l1
- xor %o2, %l0, %o2
- xor %o3, %l1, %o3
- stxa %o2, [%i0 + 0x10] %asi
- stxa %o3, [%i0 + 0x18] %asi
- ldda [%i1 + 0x30] %asi, %i4 /* %i4/%i5 = src1 + 0x30 */
- ldda [%l7 + 0x30] %asi, %l0 /* %l0/%l1 = src2 + 0x30 */
- ldda [%i0 + 0x30] %asi, %o2 /* %o2/%o3 = dest + 0x30 */
- prefetch [%i1 + 0x40], #one_read
- prefetch [%l7 + 0x40], #one_read
- prefetch [%i0 + 0x40], #n_writes
- xor %g2, %i2, %g2
- xor %g3, %i3, %g3
- xor %o0, %g2, %o0
- xor %o1, %g3, %o1
- stxa %o0, [%i0 + 0x20] %asi
- stxa %o1, [%i0 + 0x28] %asi
- xor %l0, %i4, %l0
- xor %l1, %i5, %l1
- xor %o2, %l0, %o2
- xor %o3, %l1, %o3
- stxa %o2, [%i0 + 0x30] %asi
- stxa %o3, [%i0 + 0x38] %asi
- add %i0, 0x40, %i0
- add %i1, 0x40, %i1
- subcc %g1, 1, %g1
- bne,pt %xcc, 1b
- add %l7, 0x40, %l7
- membar #Sync
- wr %g7, 0x0, %asi
- ret
- restore
-ENDPROC(xor_niagara_3)
-EXPORT_SYMBOL(xor_niagara_3)
-
-ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
- save %sp, -192, %sp
- prefetch [%i1], #n_writes
- prefetch [%i2], #one_read
- prefetch [%i3], #one_read
- prefetch [%i4], #one_read
- rd %asi, %g7
- wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
- srlx %i0, 6, %g1
- mov %i1, %i0
- mov %i2, %i1
- mov %i3, %l7
- mov %i4, %l6
-1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
- ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */
- ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */
- ldda [%i0 + 0x00] %asi, %l0 /* %l0/%l1 = dest + 0x00 */
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- stxa %l0, [%i0 + 0x00] %asi
- stxa %l1, [%i0 + 0x08] %asi
- ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */
- ldda [%i0 + 0x10] %asi, %l0 /* %l0/%l1 = dest + 0x10 */
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- stxa %l0, [%i0 + 0x10] %asi
- stxa %l1, [%i0 + 0x18] %asi
- ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */
- ldda [%i0 + 0x20] %asi, %l0 /* %l0/%l1 = dest + 0x20 */
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- stxa %l0, [%i0 + 0x20] %asi
- stxa %l1, [%i0 + 0x28] %asi
- ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */
- ldda [%i0 + 0x30] %asi, %l0 /* %l0/%l1 = dest + 0x30 */
-
- prefetch [%i1 + 0x40], #one_read
- prefetch [%l7 + 0x40], #one_read
- prefetch [%l6 + 0x40], #one_read
- prefetch [%i0 + 0x40], #n_writes
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- stxa %l0, [%i0 + 0x30] %asi
- stxa %l1, [%i0 + 0x38] %asi
-
- add %i0, 0x40, %i0
- add %i1, 0x40, %i1
- add %l7, 0x40, %l7
- subcc %g1, 1, %g1
- bne,pt %xcc, 1b
- add %l6, 0x40, %l6
- membar #Sync
- wr %g7, 0x0, %asi
- ret
- restore
-ENDPROC(xor_niagara_4)
-EXPORT_SYMBOL(xor_niagara_4)
-
-ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */
- save %sp, -192, %sp
- prefetch [%i1], #n_writes
- prefetch [%i2], #one_read
- prefetch [%i3], #one_read
- prefetch [%i4], #one_read
- prefetch [%i5], #one_read
- rd %asi, %g7
- wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi
- srlx %i0, 6, %g1
- mov %i1, %i0
- mov %i2, %i1
- mov %i3, %l7
- mov %i4, %l6
- mov %i5, %l5
-1: ldda [%i1 + 0x00] %asi, %i2 /* %i2/%i3 = src1 + 0x00 */
- ldda [%l7 + 0x00] %asi, %i4 /* %i4/%i5 = src2 + 0x00 */
- ldda [%l6 + 0x00] %asi, %g2 /* %g2/%g3 = src3 + 0x00 */
- ldda [%l5 + 0x00] %asi, %l0 /* %l0/%l1 = src4 + 0x00 */
- ldda [%i0 + 0x00] %asi, %l2 /* %l2/%l3 = dest + 0x00 */
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x10] %asi, %i2 /* %i2/%i3 = src1 + 0x10 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x10] %asi, %i4 /* %i4/%i5 = src2 + 0x10 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- ldda [%l6 + 0x10] %asi, %g2 /* %g2/%g3 = src3 + 0x10 */
- xor %l2, %l0, %l2
- xor %l3, %l1, %l3
- stxa %l2, [%i0 + 0x00] %asi
- stxa %l3, [%i0 + 0x08] %asi
- ldda [%l5 + 0x10] %asi, %l0 /* %l0/%l1 = src4 + 0x10 */
- ldda [%i0 + 0x10] %asi, %l2 /* %l2/%l3 = dest + 0x10 */
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x20] %asi, %i2 /* %i2/%i3 = src1 + 0x20 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x20] %asi, %i4 /* %i4/%i5 = src2 + 0x20 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- ldda [%l6 + 0x20] %asi, %g2 /* %g2/%g3 = src3 + 0x20 */
- xor %l2, %l0, %l2
- xor %l3, %l1, %l3
- stxa %l2, [%i0 + 0x10] %asi
- stxa %l3, [%i0 + 0x18] %asi
- ldda [%l5 + 0x20] %asi, %l0 /* %l0/%l1 = src4 + 0x20 */
- ldda [%i0 + 0x20] %asi, %l2 /* %l2/%l3 = dest + 0x20 */
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- ldda [%i1 + 0x30] %asi, %i2 /* %i2/%i3 = src1 + 0x30 */
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- ldda [%l7 + 0x30] %asi, %i4 /* %i4/%i5 = src2 + 0x30 */
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- ldda [%l6 + 0x30] %asi, %g2 /* %g2/%g3 = src3 + 0x30 */
- xor %l2, %l0, %l2
- xor %l3, %l1, %l3
- stxa %l2, [%i0 + 0x20] %asi
- stxa %l3, [%i0 + 0x28] %asi
- ldda [%l5 + 0x30] %asi, %l0 /* %l0/%l1 = src4 + 0x30 */
- ldda [%i0 + 0x30] %asi, %l2 /* %l2/%l3 = dest + 0x30 */
-
- prefetch [%i1 + 0x40], #one_read
- prefetch [%l7 + 0x40], #one_read
- prefetch [%l6 + 0x40], #one_read
- prefetch [%l5 + 0x40], #one_read
- prefetch [%i0 + 0x40], #n_writes
-
- xor %i4, %i2, %i4
- xor %i5, %i3, %i5
- xor %g2, %i4, %g2
- xor %g3, %i5, %g3
- xor %l0, %g2, %l0
- xor %l1, %g3, %l1
- xor %l2, %l0, %l2
- xor %l3, %l1, %l3
- stxa %l2, [%i0 + 0x30] %asi
- stxa %l3, [%i0 + 0x38] %asi
-
- add %i0, 0x40, %i0
- add %i1, 0x40, %i1
- add %l7, 0x40, %l7
- add %l6, 0x40, %l6
- subcc %g1, 1, %g1
- bne,pt %xcc, 1b
- add %l5, 0x40, %l5
- membar #Sync
- wr %g7, 0x0, %asi
- ret
- restore
-ENDPROC(xor_niagara_5)
-EXPORT_SYMBOL(xor_niagara_5)
diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h
deleted file mode 100644
index 647fae200c5d..000000000000
--- a/arch/um/include/asm/xor.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_UM_XOR_H
-#define _ASM_UM_XOR_H
-
-#ifdef CONFIG_64BIT
-#undef CONFIG_X86_32
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_sse_pf64))
-#else
-#define CONFIG_X86_32 1
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_8regs))
-#endif
-
-#include <asm/cpufeature.h>
-#include <../../x86/include/asm/xor.h>
-#include <linux/time-internal.h>
-
-#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
-#undef XOR_SELECT_TEMPLATE
-/* pick an arbitrary one - measuring isn't possible with inf-cpu */
-#define XOR_SELECT_TEMPLATE(x) \
- (time_travel_mode == TT_MODE_INFCPU ? TT_CPU_INF_XOR_DEFAULT : x)
-#endif
-
-#endif
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
deleted file mode 100644
index 7b0307acc410..000000000000
--- a/arch/x86/include/asm/xor.h
+++ /dev/null
@@ -1,502 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_H
-#define _ASM_X86_XOR_H
-
-/*
- * Optimized RAID-5 checksumming functions for SSE.
- */
-
-/*
- * Cache avoiding checksumming functions utilizing KNI instructions
- * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
- */
-
-/*
- * Based on
- * High-speed RAID5 checksumming functions utilizing SSE instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
-
-/*
- * x86-64 changes / gcc fixes from Andi Kleen.
- * Copyright 2002 Andi Kleen, SuSE Labs.
- *
- * This hasn't been optimized for the hammer yet, but there are likely
- * no advantages to be gotten from x86-64 here anyways.
- */
-
-#include <asm/fpu/api.h>
-
-#ifdef CONFIG_X86_32
-/* reduce register pressure */
-# define XOR_CONSTANT_CONSTRAINT "i"
-#else
-# define XOR_CONSTANT_CONSTRAINT "re"
-#endif
-
-#define OFFS(x) "16*("#x")"
-#define PF_OFFS(x) "256+16*("#x")"
-#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
-#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
-#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
-#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
-#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
-#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
-#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
-#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
-#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
-#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
-#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
-#define NOP(x)
-
-#define BLK64(pf, op, i) \
- pf(i) \
- op(i, 0) \
- op(i + 1, 1) \
- op(i + 2, 2) \
- op(i + 3, 3)
-
-static void
-xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- PF1(i) \
- PF1(i + 2) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- PF3(i) \
- PF3(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- XO3(i, 0) \
- XO3(i + 1, 1) \
- XO3(i + 2, 2) \
- XO3(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1),
- [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(PF3, XO3, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1),
- [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- PF3(i) \
- PF3(i + 2) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- PF4(i) \
- PF4(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO3(i, 0) \
- XO3(i + 1, 1) \
- XO3(i + 2, 2) \
- XO3(i + 3, 3) \
- XO4(i, 0) \
- XO4(i + 1, 1) \
- XO4(i + 2, 2) \
- XO4(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " add %[inc], %[p5] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
- [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(PF3, XO3, i) \
- BLK64(PF4, XO4, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " add %[inc], %[p5] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
- [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_sse_pf64 = {
- .name = "prefetch64-sse",
- .do_2 = xor_sse_2_pf64,
- .do_3 = xor_sse_3_pf64,
- .do_4 = xor_sse_4_pf64,
- .do_5 = xor_sse_5_pf64,
-};
-
-#undef LD
-#undef XO1
-#undef XO2
-#undef XO3
-#undef XO4
-#undef ST
-#undef NOP
-#undef BLK64
-#undef BLOCK
-
-#undef XOR_CONSTANT_CONSTRAINT
-
-#ifdef CONFIG_X86_32
-# include <asm/xor_32.h>
-#else
-# include <asm/xor_64.h>
-#endif
-
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- AVX_SELECT(FASTEST)
-
-#endif /* _ASM_X86_XOR_H */
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
deleted file mode 100644
index 7a6b9474591e..000000000000
--- a/arch/x86/include/asm/xor_32.h
+++ /dev/null
@@ -1,573 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_32_H
-#define _ASM_X86_XOR_32_H
-
-/*
- * Optimized RAID-5 checksumming functions for MMX.
- */
-
-/*
- * High-speed RAID5 checksumming functions utilizing MMX instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
-
-#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
-#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
-#define XO1(x, y) " pxor 8*("#x")(%2), %%mm"#y" ;\n"
-#define XO2(x, y) " pxor 8*("#x")(%3), %%mm"#y" ;\n"
-#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
-#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
-
-#include <asm/fpu/api.h>
-
-static void
-xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 7;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- XO1(i, 0) \
- ST(i, 0) \
- XO1(i+1, 1) \
- ST(i+1, 1) \
- XO1(i + 2, 2) \
- ST(i + 2, 2) \
- XO1(i + 3, 3) \
- ST(i + 3, 3)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " addl $128, %1 ;\n"
- " addl $128, %2 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2)
- :
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 7;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- XO2(i, 0) \
- ST(i, 0) \
- XO2(i + 1, 1) \
- ST(i + 1, 1) \
- XO2(i + 2, 2) \
- ST(i + 2, 2) \
- XO2(i + 3, 3) \
- ST(i + 3, 3)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " addl $128, %1 ;\n"
- " addl $128, %2 ;\n"
- " addl $128, %3 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3)
- :
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 7;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- XO3(i, 0) \
- ST(i, 0) \
- XO3(i + 1, 1) \
- ST(i + 1, 1) \
- XO3(i + 2, 2) \
- ST(i + 2, 2) \
- XO3(i + 3, 3) \
- ST(i + 3, 3)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " addl $128, %1 ;\n"
- " addl $128, %2 ;\n"
- " addl $128, %3 ;\n"
- " addl $128, %4 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
- :
- : "memory");
-
- kernel_fpu_end();
-}
-
-
-static void
-xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 7;
-
- kernel_fpu_begin();
-
- /* Make sure GCC forgets anything it knows about p4 or p5,
- such that it won't pass to the asm volatile below a
- register that is shared with any other variable. That's
- because we modify p4 and p5 there, but we can't mark them
- as read/write, otherwise we'd overflow the 10-asm-operands
- limit of GCC < 3.1. */
- asm("" : "+r" (p4), "+r" (p5));
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- XO3(i, 0) \
- XO3(i + 1, 1) \
- XO3(i + 2, 2) \
- XO3(i + 3, 3) \
- XO4(i, 0) \
- ST(i, 0) \
- XO4(i + 1, 1) \
- ST(i + 1, 1) \
- XO4(i + 2, 2) \
- ST(i + 2, 2) \
- XO4(i + 3, 3) \
- ST(i + 3, 3)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " addl $128, %1 ;\n"
- " addl $128, %2 ;\n"
- " addl $128, %3 ;\n"
- " addl $128, %4 ;\n"
- " addl $128, %5 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3)
- : "r" (p4), "r" (p5)
- : "memory");
-
- /* p4 and p5 were modified, and now the variables are dead.
- Clobber them just to be sure nobody does something stupid
- like assuming they have some legal value. */
- asm("" : "=r" (p4), "=r" (p5));
-
- kernel_fpu_end();
-}
-
-#undef LD
-#undef XO1
-#undef XO2
-#undef XO3
-#undef XO4
-#undef ST
-#undef BLOCK
-
-static void
-xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 6;
-
- kernel_fpu_begin();
-
- asm volatile(
- " .align 32 ;\n"
- " 1: ;\n"
- " movq (%1), %%mm0 ;\n"
- " movq 8(%1), %%mm1 ;\n"
- " pxor (%2), %%mm0 ;\n"
- " movq 16(%1), %%mm2 ;\n"
- " movq %%mm0, (%1) ;\n"
- " pxor 8(%2), %%mm1 ;\n"
- " movq 24(%1), %%mm3 ;\n"
- " movq %%mm1, 8(%1) ;\n"
- " pxor 16(%2), %%mm2 ;\n"
- " movq 32(%1), %%mm4 ;\n"
- " movq %%mm2, 16(%1) ;\n"
- " pxor 24(%2), %%mm3 ;\n"
- " movq 40(%1), %%mm5 ;\n"
- " movq %%mm3, 24(%1) ;\n"
- " pxor 32(%2), %%mm4 ;\n"
- " movq 48(%1), %%mm6 ;\n"
- " movq %%mm4, 32(%1) ;\n"
- " pxor 40(%2), %%mm5 ;\n"
- " movq 56(%1), %%mm7 ;\n"
- " movq %%mm5, 40(%1) ;\n"
- " pxor 48(%2), %%mm6 ;\n"
- " pxor 56(%2), %%mm7 ;\n"
- " movq %%mm6, 48(%1) ;\n"
- " movq %%mm7, 56(%1) ;\n"
-
- " addl $64, %1 ;\n"
- " addl $64, %2 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2)
- :
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 6;
-
- kernel_fpu_begin();
-
- asm volatile(
- " .align 32,0x90 ;\n"
- " 1: ;\n"
- " movq (%1), %%mm0 ;\n"
- " movq 8(%1), %%mm1 ;\n"
- " pxor (%2), %%mm0 ;\n"
- " movq 16(%1), %%mm2 ;\n"
- " pxor 8(%2), %%mm1 ;\n"
- " pxor (%3), %%mm0 ;\n"
- " pxor 16(%2), %%mm2 ;\n"
- " movq %%mm0, (%1) ;\n"
- " pxor 8(%3), %%mm1 ;\n"
- " pxor 16(%3), %%mm2 ;\n"
- " movq 24(%1), %%mm3 ;\n"
- " movq %%mm1, 8(%1) ;\n"
- " movq 32(%1), %%mm4 ;\n"
- " movq 40(%1), %%mm5 ;\n"
- " pxor 24(%2), %%mm3 ;\n"
- " movq %%mm2, 16(%1) ;\n"
- " pxor 32(%2), %%mm4 ;\n"
- " pxor 24(%3), %%mm3 ;\n"
- " pxor 40(%2), %%mm5 ;\n"
- " movq %%mm3, 24(%1) ;\n"
- " pxor 32(%3), %%mm4 ;\n"
- " pxor 40(%3), %%mm5 ;\n"
- " movq 48(%1), %%mm6 ;\n"
- " movq %%mm4, 32(%1) ;\n"
- " movq 56(%1), %%mm7 ;\n"
- " pxor 48(%2), %%mm6 ;\n"
- " movq %%mm5, 40(%1) ;\n"
- " pxor 56(%2), %%mm7 ;\n"
- " pxor 48(%3), %%mm6 ;\n"
- " pxor 56(%3), %%mm7 ;\n"
- " movq %%mm6, 48(%1) ;\n"
- " movq %%mm7, 56(%1) ;\n"
-
- " addl $64, %1 ;\n"
- " addl $64, %2 ;\n"
- " addl $64, %3 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3)
- :
- : "memory" );
-
- kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 6;
-
- kernel_fpu_begin();
-
- asm volatile(
- " .align 32,0x90 ;\n"
- " 1: ;\n"
- " movq (%1), %%mm0 ;\n"
- " movq 8(%1), %%mm1 ;\n"
- " pxor (%2), %%mm0 ;\n"
- " movq 16(%1), %%mm2 ;\n"
- " pxor 8(%2), %%mm1 ;\n"
- " pxor (%3), %%mm0 ;\n"
- " pxor 16(%2), %%mm2 ;\n"
- " pxor 8(%3), %%mm1 ;\n"
- " pxor (%4), %%mm0 ;\n"
- " movq 24(%1), %%mm3 ;\n"
- " pxor 16(%3), %%mm2 ;\n"
- " pxor 8(%4), %%mm1 ;\n"
- " movq %%mm0, (%1) ;\n"
- " movq 32(%1), %%mm4 ;\n"
- " pxor 24(%2), %%mm3 ;\n"
- " pxor 16(%4), %%mm2 ;\n"
- " movq %%mm1, 8(%1) ;\n"
- " movq 40(%1), %%mm5 ;\n"
- " pxor 32(%2), %%mm4 ;\n"
- " pxor 24(%3), %%mm3 ;\n"
- " movq %%mm2, 16(%1) ;\n"
- " pxor 40(%2), %%mm5 ;\n"
- " pxor 32(%3), %%mm4 ;\n"
- " pxor 24(%4), %%mm3 ;\n"
- " movq %%mm3, 24(%1) ;\n"
- " movq 56(%1), %%mm7 ;\n"
- " movq 48(%1), %%mm6 ;\n"
- " pxor 40(%3), %%mm5 ;\n"
- " pxor 32(%4), %%mm4 ;\n"
- " pxor 48(%2), %%mm6 ;\n"
- " movq %%mm4, 32(%1) ;\n"
- " pxor 56(%2), %%mm7 ;\n"
- " pxor 40(%4), %%mm5 ;\n"
- " pxor 48(%3), %%mm6 ;\n"
- " pxor 56(%3), %%mm7 ;\n"
- " movq %%mm5, 40(%1) ;\n"
- " pxor 48(%4), %%mm6 ;\n"
- " pxor 56(%4), %%mm7 ;\n"
- " movq %%mm6, 48(%1) ;\n"
- " movq %%mm7, 56(%1) ;\n"
-
- " addl $64, %1 ;\n"
- " addl $64, %2 ;\n"
- " addl $64, %3 ;\n"
- " addl $64, %4 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
- :
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 6;
-
- kernel_fpu_begin();
-
- /* Make sure GCC forgets anything it knows about p4 or p5,
- such that it won't pass to the asm volatile below a
- register that is shared with any other variable. That's
- because we modify p4 and p5 there, but we can't mark them
- as read/write, otherwise we'd overflow the 10-asm-operands
- limit of GCC < 3.1. */
- asm("" : "+r" (p4), "+r" (p5));
-
- asm volatile(
- " .align 32,0x90 ;\n"
- " 1: ;\n"
- " movq (%1), %%mm0 ;\n"
- " movq 8(%1), %%mm1 ;\n"
- " pxor (%2), %%mm0 ;\n"
- " pxor 8(%2), %%mm1 ;\n"
- " movq 16(%1), %%mm2 ;\n"
- " pxor (%3), %%mm0 ;\n"
- " pxor 8(%3), %%mm1 ;\n"
- " pxor 16(%2), %%mm2 ;\n"
- " pxor (%4), %%mm0 ;\n"
- " pxor 8(%4), %%mm1 ;\n"
- " pxor 16(%3), %%mm2 ;\n"
- " movq 24(%1), %%mm3 ;\n"
- " pxor (%5), %%mm0 ;\n"
- " pxor 8(%5), %%mm1 ;\n"
- " movq %%mm0, (%1) ;\n"
- " pxor 16(%4), %%mm2 ;\n"
- " pxor 24(%2), %%mm3 ;\n"
- " movq %%mm1, 8(%1) ;\n"
- " pxor 16(%5), %%mm2 ;\n"
- " pxor 24(%3), %%mm3 ;\n"
- " movq 32(%1), %%mm4 ;\n"
- " movq %%mm2, 16(%1) ;\n"
- " pxor 24(%4), %%mm3 ;\n"
- " pxor 32(%2), %%mm4 ;\n"
- " movq 40(%1), %%mm5 ;\n"
- " pxor 24(%5), %%mm3 ;\n"
- " pxor 32(%3), %%mm4 ;\n"
- " pxor 40(%2), %%mm5 ;\n"
- " movq %%mm3, 24(%1) ;\n"
- " pxor 32(%4), %%mm4 ;\n"
- " pxor 40(%3), %%mm5 ;\n"
- " movq 48(%1), %%mm6 ;\n"
- " movq 56(%1), %%mm7 ;\n"
- " pxor 32(%5), %%mm4 ;\n"
- " pxor 40(%4), %%mm5 ;\n"
- " pxor 48(%2), %%mm6 ;\n"
- " pxor 56(%2), %%mm7 ;\n"
- " movq %%mm4, 32(%1) ;\n"
- " pxor 48(%3), %%mm6 ;\n"
- " pxor 56(%3), %%mm7 ;\n"
- " pxor 40(%5), %%mm5 ;\n"
- " pxor 48(%4), %%mm6 ;\n"
- " pxor 56(%4), %%mm7 ;\n"
- " movq %%mm5, 40(%1) ;\n"
- " pxor 48(%5), %%mm6 ;\n"
- " pxor 56(%5), %%mm7 ;\n"
- " movq %%mm6, 48(%1) ;\n"
- " movq %%mm7, 56(%1) ;\n"
-
- " addl $64, %1 ;\n"
- " addl $64, %2 ;\n"
- " addl $64, %3 ;\n"
- " addl $64, %4 ;\n"
- " addl $64, %5 ;\n"
- " decl %0 ;\n"
- " jnz 1b ;\n"
- : "+r" (lines),
- "+r" (p1), "+r" (p2), "+r" (p3)
- : "r" (p4), "r" (p5)
- : "memory");
-
- /* p4 and p5 were modified, and now the variables are dead.
- Clobber them just to be sure nobody does something stupid
- like assuming they have some legal value. */
- asm("" : "=r" (p4), "=r" (p5));
-
- kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_pII_mmx = {
- .name = "pII_mmx",
- .do_2 = xor_pII_mmx_2,
- .do_3 = xor_pII_mmx_3,
- .do_4 = xor_pII_mmx_4,
- .do_5 = xor_pII_mmx_5,
-};
-
-static struct xor_block_template xor_block_p5_mmx = {
- .name = "p5_mmx",
- .do_2 = xor_p5_mmx_2,
- .do_3 = xor_p5_mmx_3,
- .do_4 = xor_p5_mmx_4,
- .do_5 = xor_p5_mmx_5,
-};
-
-static struct xor_block_template xor_block_pIII_sse = {
- .name = "pIII_sse",
- .do_2 = xor_sse_2,
- .do_3 = xor_sse_3,
- .do_4 = xor_sse_4,
- .do_5 = xor_sse_5,
-};
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* Also try the generic routines. */
-#include <asm-generic/xor.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
- We may also be able to load into the L1 only depending on how the cpu
- deals with a load to a line that is being prefetched. */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- AVX_XOR_SPEED; \
- if (boot_cpu_has(X86_FEATURE_XMM)) { \
- xor_speed(&xor_block_pIII_sse); \
- xor_speed(&xor_block_sse_pf64); \
- } else if (boot_cpu_has(X86_FEATURE_MMX)) { \
- xor_speed(&xor_block_pII_mmx); \
- xor_speed(&xor_block_p5_mmx); \
- } else { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- } \
-} while (0)
-
-#endif /* _ASM_X86_XOR_32_H */
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
deleted file mode 100644
index 0307e4ec5044..000000000000
--- a/arch/x86/include/asm/xor_64.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_XOR_64_H
-#define _ASM_X86_XOR_64_H
-
-static struct xor_block_template xor_block_sse = {
- .name = "generic_sse",
- .do_2 = xor_sse_2,
- .do_3 = xor_sse_3,
- .do_4 = xor_sse_4,
- .do_5 = xor_sse_5,
-};
-
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
- We may also be able to load into the L1 only depending on how the cpu
- deals with a load to a line that is being prefetched. */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- AVX_XOR_SPEED; \
- xor_speed(&xor_block_sse_pf64); \
- xor_speed(&xor_block_sse); \
-} while (0)
-
-#endif /* _ASM_X86_XOR_64_H */
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
deleted file mode 100644
index 7f81dd5897f4..000000000000
--- a/arch/x86/include/asm/xor_avx.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef _ASM_X86_XOR_AVX_H
-#define _ASM_X86_XOR_AVX_H
-
-/*
- * Optimized RAID-5 checksumming functions for AVX
- *
- * Copyright (C) 2012 Intel Corporation
- * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
- *
- * Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
- */
-
-#include <linux/compiler.h>
-#include <asm/fpu/api.h>
-
-#define BLOCK4(i) \
- BLOCK(32 * i, 0) \
- BLOCK(32 * (i + 1), 1) \
- BLOCK(32 * (i + 2), 2) \
- BLOCK(32 * (i + 3), 3)
-
-#define BLOCK16() \
- BLOCK4(0) \
- BLOCK4(4) \
- BLOCK4(8) \
- BLOCK4(12)
-
-static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0,
- const unsigned long * __restrict p1)
-{
- unsigned long lines = bytes >> 9;
-
- kernel_fpu_begin();
-
- while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
- asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p1[i / sizeof(*p1)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p0[i / sizeof(*p0)])); \
- asm volatile("vmovdqa %%ymm" #reg ", %0" : \
- "=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
- BLOCK16()
-
- p0 = (unsigned long *)((uintptr_t)p0 + 512);
- p1 = (unsigned long *)((uintptr_t)p1 + 512);
- }
-
- kernel_fpu_end();
-}
-
-static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
- const unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 9;
-
- kernel_fpu_begin();
-
- while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
- asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p2[i / sizeof(*p2)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p1[i / sizeof(*p1)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p0[i / sizeof(*p0)])); \
- asm volatile("vmovdqa %%ymm" #reg ", %0" : \
- "=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
- BLOCK16()
-
- p0 = (unsigned long *)((uintptr_t)p0 + 512);
- p1 = (unsigned long *)((uintptr_t)p1 + 512);
- p2 = (unsigned long *)((uintptr_t)p2 + 512);
- }
-
- kernel_fpu_end();
-}
-
-static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
- const unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 9;
-
- kernel_fpu_begin();
-
- while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
- asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p3[i / sizeof(*p3)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p2[i / sizeof(*p2)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p1[i / sizeof(*p1)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p0[i / sizeof(*p0)])); \
- asm volatile("vmovdqa %%ymm" #reg ", %0" : \
- "=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
- BLOCK16();
-
- p0 = (unsigned long *)((uintptr_t)p0 + 512);
- p1 = (unsigned long *)((uintptr_t)p1 + 512);
- p2 = (unsigned long *)((uintptr_t)p2 + 512);
- p3 = (unsigned long *)((uintptr_t)p3 + 512);
- }
-
- kernel_fpu_end();
-}
-
-static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
- const unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 9;
-
- kernel_fpu_begin();
-
- while (lines--) {
-#undef BLOCK
-#define BLOCK(i, reg) \
-do { \
- asm volatile("vmovdqa %0, %%ymm" #reg : : "m" (p4[i / sizeof(*p4)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p3[i / sizeof(*p3)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p2[i / sizeof(*p2)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p1[i / sizeof(*p1)])); \
- asm volatile("vxorps %0, %%ymm" #reg ", %%ymm" #reg : : \
- "m" (p0[i / sizeof(*p0)])); \
- asm volatile("vmovdqa %%ymm" #reg ", %0" : \
- "=m" (p0[i / sizeof(*p0)])); \
-} while (0);
-
- BLOCK16()
-
- p0 = (unsigned long *)((uintptr_t)p0 + 512);
- p1 = (unsigned long *)((uintptr_t)p1 + 512);
- p2 = (unsigned long *)((uintptr_t)p2 + 512);
- p3 = (unsigned long *)((uintptr_t)p3 + 512);
- p4 = (unsigned long *)((uintptr_t)p4 + 512);
- }
-
- kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_avx = {
- .name = "avx",
- .do_2 = xor_avx_2,
- .do_3 = xor_avx_3,
- .do_4 = xor_avx_4,
- .do_5 = xor_avx_5,
-};
-
-#define AVX_XOR_SPEED \
-do { \
- if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE)) \
- xor_speed(&xor_block_avx); \
-} while (0)
-
-#define AVX_SELECT(FASTEST) \
- (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE) ? &xor_block_avx : FASTEST)
-
-#endif
diff --git a/arch/x86/kernel/kexec-bzimage64.c b/arch/x86/kernel/kexec-bzimage64.c
index 5630c7dca1f3..7e980ea49d8d 100644
--- a/arch/x86/kernel/kexec-bzimage64.c
+++ b/arch/x86/kernel/kexec-bzimage64.c
@@ -525,12 +525,8 @@ static void *bzImage64_load(struct kimage *image, char *kernel,
if (ret)
return ERR_PTR(ret);
ret = crash_load_dm_crypt_keys(image);
- if (ret == -ENOENT) {
- kexec_dprintk("No dm crypt key to load\n");
- } else if (ret) {
- pr_err("Failed to load dm crypt keys\n");
+ if (ret)
return ERR_PTR(ret);
- }
if (image->dm_crypt_keys_addr &&
cmdline_len + MAX_ELFCOREHDR_STR_LEN + MAX_DMCRYPTKEYS_STR_LEN >
header->cmdline_size) {