lib/crypto: x86/aes-ctr: Migrate AVX-optimized code into library

Migrate aes-ctr-avx-x86_64.S into lib/crypto/, wiring it up to the CTR
and XCTR library functions instead of the crypto_skcipher API.  It still
remains available through crypto_skcipher via crypto/aes.c.

Some slight adjustments to the assembly code were needed:

- Take 'struct aes_enckey' instead of 'struct crypto_aes_ctx'.

- Upgrade the length argument from 32-bit to 64-bit so that it's
  compatible with the library's use of size_t (at least assuming no
  lengths over S64_MAX, which seems quite safe to assume...)

- Remove the CFI stubs, as the functions are now called directly.

- Adjust the argument order to match the caller.  Not strictly required,
  but it's easiest to handle this now when changing the function
  prototypes anyway and adding the new glue code.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://patch.msgid.link/20260927224418.109759-14-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

4 files changed, 2 insertions, 727 deletions

diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 79e610b5e5bc..778ab164c571 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -3,7 +3,7 @@
 menu "Accelerated Cryptographic Algorithms for CPU (x86)"
 
 config CRYPTO_AES_NI_INTEL
-	tristate "Ciphers: AES, modes: CTR, XCTR, XTS, GCM (AES-NI/VAES)"
+	tristate "Ciphers: AES, modes: XTS, GCM (AES-NI/VAES)"
 	depends on 64BIT
 	select CRYPTO_AEAD
 	select CRYPTO_LIB_AES
@@ -11,7 +11,7 @@ config CRYPTO_AES_NI_INTEL
 	select CRYPTO_SKCIPHER
 	help
 	  AEAD cipher: AES with GCM
-	  Length-preserving ciphers: AES with CTR, XCTR, XTS
+	  Length-preserving ciphers: AES with XTS
 
 	  Architecture: x86_64 using:
 	  - AES-NI (AES new instructions)
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 1c3feb6d72b7..0c016ba87373 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -42,7 +42,6 @@ aegis128-aesni-y := aegis128-aesni-asm.o aegis128-aesni-glue.o
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
 aesni-intel-y := aesni-intel_asm.o \
 		 aesni-intel_glue.o \
-		 aes-ctr-avx-x86_64.o \
 		 aes-gcm-aesni-x86_64.o \
 		 aes-gcm-vaes-avx2.o \
 		 aes-gcm-vaes-avx512.o \
diff --git a/arch/x86/crypto/aes-ctr-avx-x86_64.S b/arch/x86/crypto/aes-ctr-avx-x86_64.S
deleted file mode 100644
index 2745918f68ee..000000000000
--- a/arch/x86/crypto/aes-ctr-avx-x86_64.S
+++ /dev/null
@@ -1,571 +0,0 @@
-/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */
-//
-// Copyright 2025 Google LLC
-//
-// Author: Eric Biggers <ebiggers@google.com>
-//
-// This file is dual-licensed, meaning that you can use it under your choice of
-// either of the following two licenses:
-//
-// Licensed under the Apache License 2.0 (the "License").  You may obtain a copy
-// of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-// or
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice,
-//    this list of conditions and the following disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above copyright
-//    notice, this list of conditions and the following disclaimer in the
-//    documentation and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-// POSSIBILITY OF SUCH DAMAGE.
-//
-//------------------------------------------------------------------------------
-//
-// This file contains x86_64 assembly implementations of AES-CTR and AES-XCTR
-// using the following sets of CPU features:
-//	- AES-NI && AVX
-//	- VAES && AVX2
-//	- VAES && AVX512BW && AVX512VL && BMI2
-//
-// See the function definitions at the bottom of the file for more information.
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-.section .rodata
-.p2align 4
-
-.Lbswap_mask:
-	.octa	0x000102030405060708090a0b0c0d0e0f
-
-.Lctr_pattern:
-	.quad	0, 0
-.Lone:
-	.quad	1, 0
-.Ltwo:
-	.quad	2, 0
-	.quad	3, 0
-
-.Lfour:
-	.quad	4, 0
-
-.text
-
-// Move a vector between memory and a register.
-.macro	_vmovdqu	src, dst
-.if VL < 64
-	vmovdqu		\src, \dst
-.else
-	vmovdqu8	\src, \dst
-.endif
-.endm
-
-// Move a vector between registers.
-.macro	_vmovdqa	src, dst
-.if VL < 64
-	vmovdqa		\src, \dst
-.else
-	vmovdqa64	\src, \dst
-.endif
-.endm
-
-// Broadcast a 128-bit value from memory to all 128-bit lanes of a vector
-// register.
-.macro	_vbroadcast128	src, dst
-.if VL == 16
-	vmovdqu		\src, \dst
-.elseif VL == 32
-	vbroadcasti128	\src, \dst
-.else
-	vbroadcasti32x4	\src, \dst
-.endif
-.endm
-
-// XOR two vectors together.
-.macro	_vpxor	src1, src2, dst
-.if VL < 64
-	vpxor		\src1, \src2, \dst
-.else
-	vpxord		\src1, \src2, \dst
-.endif
-.endm
-
-// Load 1 <= %ecx <= 15 bytes from the pointer \src into the xmm register \dst
-// and zeroize any remaining bytes.  Clobbers %rax, %rcx, and \tmp{64,32}.
-.macro	_load_partial_block	src, dst, tmp64, tmp32
-	sub		$8, %ecx		// LEN - 8
-	jle		.Lle8\@
-
-	// Load 9 <= LEN <= 15 bytes.
-	vmovq		(\src), \dst		// Load first 8 bytes
-	mov		(\src, %rcx), %rax	// Load last 8 bytes
-	neg		%ecx
-	shl		$3, %ecx
-	shr		%cl, %rax		// Discard overlapping bytes
-	vpinsrq		$1, %rax, \dst, \dst
-	jmp		.Ldone\@
-
-.Lle8\@:
-	add		$4, %ecx		// LEN - 4
-	jl		.Llt4\@
-
-	// Load 4 <= LEN <= 8 bytes.
-	mov		(\src), %eax		// Load first 4 bytes
-	mov		(\src, %rcx), \tmp32	// Load last 4 bytes
-	jmp		.Lcombine\@
-
-.Llt4\@:
-	// Load 1 <= LEN <= 3 bytes.
-	add		$2, %ecx		// LEN - 2
-	movzbl		(\src), %eax		// Load first byte
-	jl		.Lmovq\@
-	movzwl		(\src, %rcx), \tmp32	// Load last 2 bytes
-.Lcombine\@:
-	shl		$3, %ecx
-	shl		%cl, \tmp64
-	or		\tmp64, %rax		// Combine the two parts
-.Lmovq\@:
-	vmovq		%rax, \dst
-.Ldone\@:
-.endm
-
-// Store 1 <= %ecx <= 15 bytes from the xmm register \src to the pointer \dst.
-// Clobbers %rax, %rcx, and \tmp{64,32}.
-.macro	_store_partial_block	src, dst, tmp64, tmp32
-	sub		$8, %ecx		// LEN - 8
-	jl		.Llt8\@
-
-	// Store 8 <= LEN <= 15 bytes.
-	vpextrq		$1, \src, %rax
-	mov		%ecx, \tmp32
-	shl		$3, %ecx
-	ror		%cl, %rax
-	mov		%rax, (\dst, \tmp64)	// Store last LEN - 8 bytes
-	vmovq		\src, (\dst)		// Store first 8 bytes
-	jmp		.Ldone\@
-
-.Llt8\@:
-	add		$4, %ecx		// LEN - 4
-	jl		.Llt4\@
-
-	// Store 4 <= LEN <= 7 bytes.
-	vpextrd		$1, \src, %eax
-	mov		%ecx, \tmp32
-	shl		$3, %ecx
-	ror		%cl, %eax
-	mov		%eax, (\dst, \tmp64)	// Store last LEN - 4 bytes
-	vmovd		\src, (\dst)		// Store first 4 bytes
-	jmp		.Ldone\@
-
-.Llt4\@:
-	// Store 1 <= LEN <= 3 bytes.
-	vpextrb		$0, \src, 0(\dst)
-	cmp		$-2, %ecx		// LEN - 4 == -2, i.e. LEN == 2?
-	jl		.Ldone\@
-	vpextrb		$1, \src, 1(\dst)
-	je		.Ldone\@
-	vpextrb		$2, \src, 2(\dst)
-.Ldone\@:
-.endm
-
-// Prepare the next two vectors of AES inputs in AESDATA\i0 and AESDATA\i1, and
-// XOR each with the zero-th round key.  Also update LE_CTR if !\final.
-.macro	_prepare_2_ctr_vecs	is_xctr, i0, i1, final=0
-.if \is_xctr
-  .if USE_AVX512
-	vmovdqa64	LE_CTR, AESDATA\i0
-	vpternlogd	$0x96, XCTR_IV, RNDKEY0, AESDATA\i0
-  .else
-	vpxor		XCTR_IV, LE_CTR, AESDATA\i0
-	vpxor		RNDKEY0, AESDATA\i0, AESDATA\i0
-  .endif
-	vpaddq		LE_CTR_INC1, LE_CTR, AESDATA\i1
-
-  .if USE_AVX512
-	vpternlogd	$0x96, XCTR_IV, RNDKEY0, AESDATA\i1
-  .else
-	vpxor		XCTR_IV, AESDATA\i1, AESDATA\i1
-	vpxor		RNDKEY0, AESDATA\i1, AESDATA\i1
-  .endif
-.else
-	vpshufb		BSWAP_MASK, LE_CTR, AESDATA\i0
-	_vpxor		RNDKEY0, AESDATA\i0, AESDATA\i0
-	vpaddq		LE_CTR_INC1, LE_CTR, AESDATA\i1
-	vpshufb		BSWAP_MASK, AESDATA\i1, AESDATA\i1
-	_vpxor		RNDKEY0, AESDATA\i1, AESDATA\i1
-.endif
-.if !\final
-	vpaddq		LE_CTR_INC2, LE_CTR, LE_CTR
-.endif
-.endm
-
-// Do all AES rounds on the data in the given AESDATA vectors, excluding the
-// zero-th and last rounds.
-.macro	_aesenc_loop	vecs:vararg
-	mov		KEY, %rax
-1:
-	_vbroadcast128	(%rax), RNDKEY
-.irp i, \vecs
-	vaesenc		RNDKEY, AESDATA\i, AESDATA\i
-.endr
-	add		$16, %rax
-	cmp		%rax, RNDKEYLAST_PTR
-	jne		1b
-.endm
-
-// Finalize the keystream blocks in the given AESDATA vectors by doing the last
-// AES round, then XOR those keystream blocks with the corresponding data.
-// Reduce latency by doing the XOR before the vaesenclast, utilizing the
-// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a).
-.macro	_aesenclast_and_xor	vecs:vararg
-.irp i, \vecs
-	_vpxor		\i*VL(SRC), RNDKEYLAST, RNDKEY
-	vaesenclast	RNDKEY, AESDATA\i, AESDATA\i
-.endr
-.irp i, \vecs
-	_vmovdqu	AESDATA\i, \i*VL(DST)
-.endr
-.endm
-
-// XOR the keystream blocks in the specified AESDATA vectors with the
-// corresponding data.
-.macro	_xor_data	vecs:vararg
-.irp i, \vecs
-	_vpxor		\i*VL(SRC), AESDATA\i, AESDATA\i
-.endr
-.irp i, \vecs
-	_vmovdqu	AESDATA\i, \i*VL(DST)
-.endr
-.endm
-
-.macro	_aes_ctr_crypt		is_xctr
-
-	// Define register aliases V0-V15 that map to the xmm, ymm, or zmm
-	// registers according to the selected Vector Length (VL).
-.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
-  .if VL == 16
-	.set	V\i,		%xmm\i
-  .elseif VL == 32
-	.set	V\i,		%ymm\i
-  .elseif VL == 64
-	.set	V\i,		%zmm\i
-  .else
-	.error "Unsupported Vector Length (VL)"
-  .endif
-.endr
-
-	// Function arguments
-	.set	KEY,		%rdi	// Initially points to the start of the
-					// crypto_aes_ctx, then is advanced to
-					// point to the index 1 round key
-	.set	KEY32,		%edi	// Available as temp register after all
-					// keystream blocks have been generated
-	.set	SRC,		%rsi	// Pointer to next source data
-	.set	DST,		%rdx	// Pointer to next destination data
-	.set	LEN,		%ecx	// Remaining length in bytes.
-					// Note: _load_partial_block relies on
-					// this being in %ecx.
-	.set	LEN64,		%rcx	// Zero-extend LEN before using!
-	.set	LEN8,		%cl
-.if \is_xctr
-	.set	XCTR_IV_PTR,	%r8	// const u8 iv[AES_BLOCK_SIZE];
-	.set	XCTR_CTR,	%r9	// u64 ctr;
-.else
-	.set	LE_CTR_PTR,	%r8	// const u64 le_ctr[2];
-.endif
-
-	// Additional local variables
-	.set	RNDKEYLAST_PTR,	%r10
-	.set	AESDATA0,	V0
-	.set	AESDATA0_XMM,	%xmm0
-	.set	AESDATA1,	V1
-	.set	AESDATA1_XMM,	%xmm1
-	.set	AESDATA2,	V2
-	.set	AESDATA3,	V3
-	.set	AESDATA4,	V4
-	.set	AESDATA5,	V5
-	.set	AESDATA6,	V6
-	.set	AESDATA7,	V7
-.if \is_xctr
-	.set	XCTR_IV,	V8
-.else
-	.set	BSWAP_MASK,	V8
-.endif
-	.set	LE_CTR,		V9
-	.set	LE_CTR_XMM,	%xmm9
-	.set	LE_CTR_INC1,	V10
-	.set	LE_CTR_INC2,	V11
-	.set	RNDKEY0,	V12
-	.set	RNDKEYLAST,	V13
-	.set	RNDKEY,		V14
-
-	// Create the first vector of counters.
-.if \is_xctr
-  .if VL == 16
-	vmovq		XCTR_CTR, LE_CTR
-  .elseif VL == 32
-	vmovq		XCTR_CTR, LE_CTR_XMM
-	inc		XCTR_CTR
-	vmovq		XCTR_CTR, AESDATA0_XMM
-	vinserti128	$1, AESDATA0_XMM, LE_CTR, LE_CTR
-  .else
-	vpbroadcastq	XCTR_CTR, LE_CTR
-	vpsrldq		$8, LE_CTR, LE_CTR
-	vpaddq		.Lctr_pattern(%rip), LE_CTR, LE_CTR
-  .endif
-	_vbroadcast128	(XCTR_IV_PTR), XCTR_IV
-.else
-	_vbroadcast128	(LE_CTR_PTR), LE_CTR
-  .if VL > 16
-	vpaddq		.Lctr_pattern(%rip), LE_CTR, LE_CTR
-  .endif
-	_vbroadcast128	.Lbswap_mask(%rip), BSWAP_MASK
-.endif
-
-.if VL == 16
-	_vbroadcast128	.Lone(%rip), LE_CTR_INC1
-.elseif VL == 32
-	_vbroadcast128	.Ltwo(%rip), LE_CTR_INC1
-.else
-	_vbroadcast128	.Lfour(%rip), LE_CTR_INC1
-.endif
-	vpsllq		$1, LE_CTR_INC1, LE_CTR_INC2
-
-	// Load the AES key length: 16 (AES-128), 24 (AES-192), or 32 (AES-256).
-	movl		480(KEY), %eax
-
-	// Compute the pointer to the last round key.
-	lea		6*16(KEY, %rax, 4), RNDKEYLAST_PTR
-
-	// Load the zero-th and last round keys.
-	_vbroadcast128	(KEY), RNDKEY0
-	_vbroadcast128	(RNDKEYLAST_PTR), RNDKEYLAST
-
-	// Make KEY point to the first round key.
-	add		$16, KEY
-
-	// This is the main loop, which encrypts 8 vectors of data at a time.
-	add		$-8*VL, LEN
-	jl		.Lloop_8x_done\@
-.Lloop_8x\@:
-	_prepare_2_ctr_vecs	\is_xctr, 0, 1
-	_prepare_2_ctr_vecs	\is_xctr, 2, 3
-	_prepare_2_ctr_vecs	\is_xctr, 4, 5
-	_prepare_2_ctr_vecs	\is_xctr, 6, 7
-	_aesenc_loop	0,1,2,3,4,5,6,7
-	_aesenclast_and_xor 0,1,2,3,4,5,6,7
-	sub		$-8*VL, SRC
-	sub		$-8*VL, DST
-	add		$-8*VL, LEN
-	jge		.Lloop_8x\@
-.Lloop_8x_done\@:
-	sub		$-8*VL, LEN
-	jz		.Ldone\@
-
-	// 1 <= LEN < 8*VL.  Generate 2, 4, or 8 more vectors of keystream
-	// blocks, depending on the remaining LEN.
-
-	_prepare_2_ctr_vecs	\is_xctr, 0, 1
-	_prepare_2_ctr_vecs	\is_xctr, 2, 3
-	cmp		$4*VL, LEN
-	jle		.Lenc_tail_atmost4vecs\@
-
-	// 4*VL < LEN < 8*VL.  Generate 8 vectors of keystream blocks.  Use the
-	// first 4 to XOR 4 full vectors of data.  Then XOR the remaining data.
-	_prepare_2_ctr_vecs	\is_xctr, 4, 5
-	_prepare_2_ctr_vecs	\is_xctr, 6, 7, final=1
-	_aesenc_loop	0,1,2,3,4,5,6,7
-	_aesenclast_and_xor 0,1,2,3
-	vaesenclast	RNDKEYLAST, AESDATA4, AESDATA0
-	vaesenclast	RNDKEYLAST, AESDATA5, AESDATA1
-	vaesenclast	RNDKEYLAST, AESDATA6, AESDATA2
-	vaesenclast	RNDKEYLAST, AESDATA7, AESDATA3
-	sub		$-4*VL, SRC
-	sub		$-4*VL, DST
-	add		$-4*VL, LEN
-	cmp		$1*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_0\@
-	_xor_data	0
-	cmp		$2*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_1\@
-	_xor_data	1
-	cmp		$3*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_2\@
-	_xor_data	2
-	cmp		$4*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_3\@
-	_xor_data	3
-	jmp		.Ldone\@
-
-.Lenc_tail_atmost4vecs\@:
-	cmp		$2*VL, LEN
-	jle		.Lenc_tail_atmost2vecs\@
-
-	// 2*VL < LEN <= 4*VL.  Generate 4 vectors of keystream blocks.  Use the
-	// first 2 to XOR 2 full vectors of data.  Then XOR the remaining data.
-	_aesenc_loop	0,1,2,3
-	_aesenclast_and_xor 0,1
-	vaesenclast	RNDKEYLAST, AESDATA2, AESDATA0
-	vaesenclast	RNDKEYLAST, AESDATA3, AESDATA1
-	sub		$-2*VL, SRC
-	sub		$-2*VL, DST
-	add		$-2*VL, LEN
-	jmp		.Lxor_tail_upto2vecs\@
-
-.Lenc_tail_atmost2vecs\@:
-	// 1 <= LEN <= 2*VL.  Generate 2 vectors of keystream blocks.  Then XOR
-	// the remaining data.
-	_aesenc_loop	0,1
-	vaesenclast	RNDKEYLAST, AESDATA0, AESDATA0
-	vaesenclast	RNDKEYLAST, AESDATA1, AESDATA1
-
-.Lxor_tail_upto2vecs\@:
-	cmp		$1*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_0\@
-	_xor_data	0
-	cmp		$2*VL-1, LEN
-	jle		.Lxor_tail_partial_vec_1\@
-	_xor_data	1
-	jmp		.Ldone\@
-
-.Lxor_tail_partial_vec_1\@:
-	add		$-1*VL, LEN
-	jz		.Ldone\@
-	sub		$-1*VL, SRC
-	sub		$-1*VL, DST
-	_vmovdqa	AESDATA1, AESDATA0
-	jmp		.Lxor_tail_partial_vec_0\@
-
-.Lxor_tail_partial_vec_2\@:
-	add		$-2*VL, LEN
-	jz		.Ldone\@
-	sub		$-2*VL, SRC
-	sub		$-2*VL, DST
-	_vmovdqa	AESDATA2, AESDATA0
-	jmp		.Lxor_tail_partial_vec_0\@
-
-.Lxor_tail_partial_vec_3\@:
-	add		$-3*VL, LEN
-	jz		.Ldone\@
-	sub		$-3*VL, SRC
-	sub		$-3*VL, DST
-	_vmovdqa	AESDATA3, AESDATA0
-
-.Lxor_tail_partial_vec_0\@:
-	// XOR the remaining 1 <= LEN < VL bytes.  It's easy if masked
-	// loads/stores are available; otherwise it's a bit harder...
-.if USE_AVX512
-	mov		$-1, %rax
-	bzhi		LEN64, %rax, %rax
-	kmovq		%rax, %k1
-	vmovdqu8	(SRC), AESDATA1{%k1}{z}
-	vpxord		AESDATA1, AESDATA0, AESDATA0
-	vmovdqu8	AESDATA0, (DST){%k1}
-.else
-  .if VL == 32
-	cmp		$16, LEN
-	jl		1f
-	vpxor		(SRC), AESDATA0_XMM, AESDATA1_XMM
-	vmovdqu		AESDATA1_XMM, (DST)
-	add		$16, SRC
-	add		$16, DST
-	sub		$16, LEN
-	jz		.Ldone\@
-	vextracti128	$1, AESDATA0, AESDATA0_XMM
-1:
-  .endif
-	mov		LEN, %r10d
-	_load_partial_block	SRC, AESDATA1_XMM, KEY, KEY32
-	vpxor		AESDATA1_XMM, AESDATA0_XMM, AESDATA0_XMM
-	mov		%r10d, %ecx
-	_store_partial_block	AESDATA0_XMM, DST, KEY, KEY32
-.endif
-
-.Ldone\@:
-.if VL > 16
-	vzeroupper
-.endif
-	RET
-.endm
-
-// Below are the definitions of the functions generated by the above macro.
-// They have the following prototypes:
-//
-//
-// void aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,
-//				 const u8 *src, u8 *dst, int len,
-//				 const u64 le_ctr[2]);
-//
-// void aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,
-//				const u8 *src, u8 *dst, int len,
-//				const u8 iv[AES_BLOCK_SIZE], u64 ctr);
-//
-// Both functions generate |len| bytes of keystream, XOR it with the data from
-// |src|, and write the result to |dst|.  On non-final calls, |len| must be a
-// multiple of 16.  On the final call, |len| can be any value.
-//
-// aes_ctr64_crypt_* implement "regular" CTR, where the keystream is generated
-// from a 128-bit big endian counter that increments by 1 for each AES block.
-// HOWEVER, to keep the assembly code simple, some of the counter management is
-// left to the caller.  aes_ctr64_crypt_* take the counter in little endian
-// form, only increment the low 64 bits internally, do the conversion to big
-// endian internally, and don't write the updated counter back to memory.  The
-// caller is responsible for converting the starting IV to the little endian
-// le_ctr, detecting the (very rare) case of a carry out of the low 64 bits
-// being needed and splitting at that point with a carry done in between, and
-// updating le_ctr after each part if the message is multi-part.
-//
-// aes_xctr_crypt_* implement XCTR as specified in "Length-preserving encryption
-// with HCTR2" (https://eprint.iacr.org/2021/1441.pdf).  XCTR is an
-// easier-to-implement variant of CTR that uses little endian byte order and
-// eliminates carries.  |ctr| is the per-message block counter starting at 1.
-
-.set	VL, 16
-.set	USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_aesni_avx)
-	_aes_ctr_crypt	0
-SYM_FUNC_END(aes_ctr64_crypt_aesni_avx)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_aesni_avx)
-	_aes_ctr_crypt	1
-SYM_FUNC_END(aes_xctr_crypt_aesni_avx)
-
-.set	VL, 32
-.set	USE_AVX512, 0
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx2)
-	_aes_ctr_crypt	0
-SYM_FUNC_END(aes_ctr64_crypt_vaes_avx2)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx2)
-	_aes_ctr_crypt	1
-SYM_FUNC_END(aes_xctr_crypt_vaes_avx2)
-
-.set	VL, 64
-.set	USE_AVX512, 1
-SYM_TYPED_FUNC_START(aes_ctr64_crypt_vaes_avx512)
-	_aes_ctr_crypt	0
-SYM_FUNC_END(aes_ctr64_crypt_vaes_avx512)
-SYM_TYPED_FUNC_START(aes_xctr_crypt_vaes_avx512)
-	_aes_ctr_crypt	1
-SYM_FUNC_END(aes_xctr_crypt_vaes_avx512)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index a2d955336acd..3787b932f101 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -42,7 +42,6 @@
 #define AESNI_ALIGN	16
 #define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
 #define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
-#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
 
 struct aesni_xts_ctx {
@@ -60,11 +59,6 @@ static inline void *aes_align_addr(void *addr)
 asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
 			      unsigned int key_len);
 
-static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
-{
-	return aes_align_addr(raw_ctx);
-}
-
 static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
 {
 	return aes_align_addr(crypto_skcipher_ctx(tfm));
@@ -88,12 +82,6 @@ static int aes_set_key_common(struct crypto_aes_ctx *ctx,
 	return 0;
 }
 
-static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
-			         unsigned int len)
-{
-	return aes_set_key_common(aes_ctx(crypto_skcipher_ctx(tfm)), key, len);
-}
-
 static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
 			    unsigned int keylen)
 {
@@ -222,100 +210,6 @@ xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
 asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
 				   u8 iv[AES_BLOCK_SIZE]);
 
-/* __always_inline to avoid indirect call */
-static __always_inline int
-ctr_crypt(struct skcipher_request *req,
-	  void (*ctr64_func)(const struct crypto_aes_ctx *key,
-			     const u8 *src, u8 *dst, int len,
-			     const u64 le_ctr[2]))
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
-	unsigned int nbytes, p1_nbytes, nblocks;
-	struct skcipher_walk walk;
-	u64 le_ctr[2];
-	u64 ctr64;
-	int err;
-
-	ctr64 = le_ctr[0] = get_unaligned_be64(&req->iv[8]);
-	le_ctr[1] = get_unaligned_be64(&req->iv[0]);
-
-	err = skcipher_walk_virt(&walk, req, false);
-
-	while ((nbytes = walk.nbytes) != 0) {
-		if (nbytes < walk.total) {
-			/* Not the end yet, so keep the length block-aligned. */
-			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
-			nblocks = nbytes / AES_BLOCK_SIZE;
-		} else {
-			/* It's the end, so include any final partial block. */
-			nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
-		}
-		ctr64 += nblocks;
-
-		kernel_fpu_begin();
-		if (likely(ctr64 >= nblocks)) {
-			/* The low 64 bits of the counter won't overflow. */
-			(*ctr64_func)(key, walk.src.virt.addr,
-				      walk.dst.virt.addr, nbytes, le_ctr);
-		} else {
-			/*
-			 * The low 64 bits of the counter will overflow.  The
-			 * assembly doesn't handle this case, so split the
-			 * operation into two at the point where the overflow
-			 * will occur.  After the first part, add the carry bit.
-			 */
-			p1_nbytes = min(nbytes, (nblocks - ctr64) * AES_BLOCK_SIZE);
-			(*ctr64_func)(key, walk.src.virt.addr,
-				      walk.dst.virt.addr, p1_nbytes, le_ctr);
-			le_ctr[0] = 0;
-			le_ctr[1]++;
-			(*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
-				      walk.dst.virt.addr + p1_nbytes,
-				      nbytes - p1_nbytes, le_ctr);
-		}
-		kernel_fpu_end();
-		le_ctr[0] = ctr64;
-
-		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
-	}
-
-	put_unaligned_be64(ctr64, &req->iv[8]);
-	put_unaligned_be64(le_ctr[1], &req->iv[0]);
-
-	return err;
-}
-
-/* __always_inline to avoid indirect call */
-static __always_inline int
-xctr_crypt(struct skcipher_request *req,
-	   void (*xctr_func)(const struct crypto_aes_ctx *key,
-			     const u8 *src, u8 *dst, int len,
-			     const u8 iv[AES_BLOCK_SIZE], u64 ctr))
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct crypto_aes_ctx *key = aes_ctx(crypto_skcipher_ctx(tfm));
-	struct skcipher_walk walk;
-	unsigned int nbytes;
-	u64 ctr = 1;
-	int err;
-
-	err = skcipher_walk_virt(&walk, req, false);
-	while ((nbytes = walk.nbytes) != 0) {
-		if (nbytes < walk.total)
-			nbytes = round_down(nbytes, AES_BLOCK_SIZE);
-
-		kernel_fpu_begin();
-		(*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
-			     nbytes, req->iv, ctr);
-		kernel_fpu_end();
-
-		ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
-		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
-	}
-	return err;
-}
-
 #define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority)	       \
 									       \
 asmlinkage void								       \
@@ -335,25 +229,6 @@ static int xts_decrypt_##suffix(struct skcipher_request *req)		       \
 	return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix);   \
 }									       \
 									       \
-asmlinkage void								       \
-aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
-			 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
-									       \
-static int ctr_crypt_##suffix(struct skcipher_request *req)		       \
-{									       \
-	return ctr_crypt(req, aes_ctr64_crypt_##suffix);		       \
-}									       \
-									       \
-asmlinkage void								       \
-aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key,		       \
-			const u8 *src, u8 *dst, int len,		       \
-			const u8 iv[AES_BLOCK_SIZE], u64 ctr);		       \
-									       \
-static int xctr_crypt_##suffix(struct skcipher_request *req)		       \
-{									       \
-	return xctr_crypt(req, aes_xctr_crypt_##suffix);		       \
-}									       \
-									       \
 static struct skcipher_alg skcipher_algs_##suffix[] = {{		       \
 	.base.cra_name		= "xts(aes)",				       \
 	.base.cra_driver_name	= "xts-aes-" driver_name_suffix,	       \
@@ -368,34 +243,6 @@ static struct skcipher_alg skcipher_algs_##suffix[] = {{		       \
 	.setkey			= xts_setkey_aesni,			       \
 	.encrypt		= xts_encrypt_##suffix,			       \
 	.decrypt		= xts_decrypt_##suffix,			       \
-}, {									       \
-	.base.cra_name		= "ctr(aes)",				       \
-	.base.cra_driver_name	= "ctr-aes-" driver_name_suffix,	       \
-	.base.cra_priority	= priority,				       \
-	.base.cra_blocksize	= 1,					       \
-	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
-	.base.cra_module	= THIS_MODULE,				       \
-	.min_keysize		= AES_MIN_KEY_SIZE,			       \
-	.max_keysize		= AES_MAX_KEY_SIZE,			       \
-	.ivsize			= AES_BLOCK_SIZE,			       \
-	.chunksize		= AES_BLOCK_SIZE,			       \
-	.setkey			= aesni_skcipher_setkey,		       \
-	.encrypt		= ctr_crypt_##suffix,			       \
-	.decrypt		= ctr_crypt_##suffix,			       \
-}, {									       \
-	.base.cra_name		= "xctr(aes)",				       \
-	.base.cra_driver_name	= "xctr-aes-" driver_name_suffix,	       \
-	.base.cra_priority	= priority,				       \
-	.base.cra_blocksize	= 1,					       \
-	.base.cra_ctxsize	= CRYPTO_AES_CTX_SIZE,			       \
-	.base.cra_module	= THIS_MODULE,				       \
-	.min_keysize		= AES_MIN_KEY_SIZE,			       \
-	.max_keysize		= AES_MAX_KEY_SIZE,			       \
-	.ivsize			= AES_BLOCK_SIZE,			       \
-	.chunksize		= AES_BLOCK_SIZE,			       \
-	.setkey			= aesni_skcipher_setkey,		       \
-	.encrypt		= xctr_crypt_##suffix,			       \
-	.decrypt		= xctr_crypt_##suffix,			       \
 }}
 
 DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);