Merge tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library updates from Eric Biggers:

 - Add support for verifying ML-DSA signatures.

   ML-DSA (Module-Lattice-Based Digital Signature Algorithm) is a
   recently-standardized post-quantum (quantum-resistant) signature
   algorithm. It was known as Dilithium pre-standardization.

   The first use case in the kernel will be module signing. But there
   are also other users of RSA and ECDSA signatures in the kernel that
   might want to upgrade to ML-DSA eventually.

 - Improve the AES library:

     - Make the AES key expansion and single block encryption and
       decryption functions use the architecture-optimized AES code.
       Enable these optimizations by default.

     - Support preparing an AES key for encryption-only, using about
       half as much memory as a bidirectional key.

     - Replace the existing two generic implementations of AES with a
       single one.

 - Simplify how Adiantum message hashing is implemented. Remove the
   "nhpoly1305" crypto_shash in favor of direct lib/crypto/ support for
   NH hashing, and enable optimizations by default.

* tag 'libcrypto-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux: (53 commits)
  lib/crypto: mldsa: Clarify the documentation for mldsa_verify() slightly
  lib/crypto: aes: Drop 'volatile' from aes_sbox and aes_inv_sbox
  lib/crypto: aes: Remove old AES en/decryption functions
  lib/crypto: aesgcm: Use new AES library API
  lib/crypto: aescfb: Use new AES library API
  crypto: omap - Use new AES library API
  crypto: inside-secure - Use new AES library API
  crypto: drbg - Use new AES library API
  crypto: crypto4xx - Use new AES library API
  crypto: chelsio - Use new AES library API
  crypto: ccp - Use new AES library API
  crypto: x86/aes-gcm - Use new AES library API
  crypto: arm64/ghash - Use new AES library API
  crypto: arm/ghash - Use new AES library API
  staging: rtl8723bs: core: Use new AES library API
  net: phy: mscc: macsec: Use new AES library API
  chelsio: Use new AES library API
  Bluetooth: SMP: Use new AES library API
  crypto: x86/aes - Remove the superseded AES-NI crypto_cipher
  lib/crypto: x86/aes: Add AES-NI optimization
  ...

76 files changed, 150 insertions, 9553 deletions

diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig
index a2995eb390c6..77b69d672d40 100644
--- a/arch/arm/configs/milbeaut_m10v_defconfig
+++ b/arch/arm/configs/milbeaut_m10v_defconfig
@@ -98,7 +98,6 @@ CONFIG_CRYPTO_SELFTESTS=y
 CONFIG_CRYPTO_AES=y
 CONFIG_CRYPTO_SEQIV=m
 CONFIG_CRYPTO_GHASH_ARM_CE=m
-CONFIG_CRYPTO_AES_ARM=m
 CONFIG_CRYPTO_AES_ARM_BS=m
 CONFIG_CRYPTO_AES_ARM_CE=m
 # CONFIG_CRYPTO_HW is not set
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 7f1fa9dd88c9..b6d3e20926bb 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -1286,7 +1286,7 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
 CONFIG_CRYPTO_USER_API_AEAD=m
 CONFIG_CRYPTO_GHASH_ARM_CE=m
-CONFIG_CRYPTO_AES_ARM=m
+CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_AES_ARM_BS=m
 CONFIG_CRYPTO_AES_ARM_CE=m
 CONFIG_CRYPTO_DEV_SUN4I_SS=m
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index 4e53c331cd84..0464f6552169 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -706,7 +706,7 @@ CONFIG_NLS_ISO8859_1=y
 CONFIG_SECURITY=y
 CONFIG_CRYPTO_MICHAEL_MIC=y
 CONFIG_CRYPTO_GHASH_ARM_CE=m
-CONFIG_CRYPTO_AES_ARM=m
+CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_AES_ARM_BS=m
 CONFIG_CRYPTO_DEV_OMAP=m
 CONFIG_CRYPTO_DEV_OMAP_SHAM=m
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index 3ea189f1f42f..eacd08fd87ad 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -657,7 +657,7 @@ CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_DEFLATE=y
 CONFIG_CRYPTO_LZO=y
-CONFIG_CRYPTO_AES_ARM=m
+CONFIG_CRYPTO_AES=m
 CONFIG_FONTS=y
 CONFIG_FONT_8x8=y
 CONFIG_FONT_8x16=y
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index f30d743df264..b9c28c818b7c 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -23,38 +23,9 @@ config CRYPTO_GHASH_ARM_CE
 	  that is part of the ARMv8 Crypto Extensions, or a slower variant that
 	  uses the vmull.p8 instruction that is part of the basic NEON ISA.
 
-config CRYPTO_NHPOLY1305_NEON
-	tristate "Hash functions: NHPoly1305 (NEON)"
-	depends on KERNEL_MODE_NEON
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function (Adiantum)
-
-	  Architecture: arm using:
-	  - NEON (Advanced SIMD) extensions
-
-config CRYPTO_AES_ARM
-	tristate "Ciphers: AES"
-	select CRYPTO_ALGAPI
-	select CRYPTO_AES
-	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
-
-	  Architecture: arm
-
-	  On ARM processors without the Crypto Extensions, this is the
-	  fastest AES implementation for single blocks.  For multiple
-	  blocks, the NEON bit-sliced implementation is usually faster.
-
-	  This implementation may be vulnerable to cache timing attacks,
-	  since it uses lookup tables.  However, as countermeasures it
-	  disables IRQs and preloads the tables; it is hoped this makes
-	  such attacks very difficult.
-
 config CRYPTO_AES_ARM_BS
 	tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)"
 	depends on KERNEL_MODE_NEON
-	select CRYPTO_AES_ARM
 	select CRYPTO_SKCIPHER
 	select CRYPTO_LIB_AES
 	help
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index 86dd43313dbf..e73099e120b3 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -3,15 +3,11 @@
 # Arch-specific CryptoAPI modules.
 #
 
-obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o
 obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
 
 obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
 obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o
 
-aes-arm-y	:= aes-cipher-core.o aes-cipher-glue.o
 aes-arm-bs-y	:= aes-neonbs-core.o aes-neonbs-glue.o
 aes-arm-ce-y	:= aes-ce-core.o aes-ce-glue.o
 ghash-arm-ce-y	:= ghash-ce-core.o ghash-ce-glue.o
-nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
diff --git a/arch/arm/crypto/aes-cipher-core.S b/arch/arm/crypto/aes-cipher-core.S
deleted file mode 100644
index 1da3f41359aa..000000000000
--- a/arch/arm/crypto/aes-cipher-core.S
+++ /dev/null
@@ -1,201 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Scalar AES core transform
- *
- * Copyright (C) 2017 Linaro Ltd.
- * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-#include <asm/cache.h>
-
-	.text
-	.align		5
-
-	rk		.req	r0
-	rounds		.req	r1
-	in		.req	r2
-	out		.req	r3
-	ttab		.req	ip
-
-	t0		.req	lr
-	t1		.req	r2
-	t2		.req	r3
-
-	.macro		__select, out, in, idx
-	.if		__LINUX_ARM_ARCH__ < 7
-	and		\out, \in, #0xff << (8 * \idx)
-	.else
-	ubfx		\out, \in, #(8 * \idx), #8
-	.endif
-	.endm
-
-	.macro		__load, out, in, idx, sz, op
-	.if		__LINUX_ARM_ARCH__ < 7 && \idx > 0
-	ldr\op		\out, [ttab, \in, lsr #(8 * \idx) - \sz]
-	.else
-	ldr\op		\out, [ttab, \in, lsl #\sz]
-	.endif
-	.endm
-
-	.macro		__hround, out0, out1, in0, in1, in2, in3, t3, t4, enc, sz, op, oldcpsr
-	__select	\out0, \in0, 0
-	__select	t0, \in1, 1
-	__load		\out0, \out0, 0, \sz, \op
-	__load		t0, t0, 1, \sz, \op
-
-	.if		\enc
-	__select	\out1, \in1, 0
-	__select	t1, \in2, 1
-	.else
-	__select	\out1, \in3, 0
-	__select	t1, \in0, 1
-	.endif
-	__load		\out1, \out1, 0, \sz, \op
-	__select	t2, \in2, 2
-	__load		t1, t1, 1, \sz, \op
-	__load		t2, t2, 2, \sz, \op
-
-	eor		\out0, \out0, t0, ror #24
-
-	__select	t0, \in3, 3
-	.if		\enc
-	__select	\t3, \in3, 2
-	__select	\t4, \in0, 3
-	.else
-	__select	\t3, \in1, 2
-	__select	\t4, \in2, 3
-	.endif
-	__load		\t3, \t3, 2, \sz, \op
-	__load		t0, t0, 3, \sz, \op
-	__load		\t4, \t4, 3, \sz, \op
-
-	.ifnb		\oldcpsr
-	/*
-	 * This is the final round and we're done with all data-dependent table
-	 * lookups, so we can safely re-enable interrupts.
-	 */
-	restore_irqs	\oldcpsr
-	.endif
-
-	eor		\out1, \out1, t1, ror #24
-	eor		\out0, \out0, t2, ror #16
-	ldm		rk!, {t1, t2}
-	eor		\out1, \out1, \t3, ror #16
-	eor		\out0, \out0, t0, ror #8
-	eor		\out1, \out1, \t4, ror #8
-	eor		\out0, \out0, t1
-	eor		\out1, \out1, t2
-	.endm
-
-	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
-	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
-	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op, \oldcpsr
-	.endm
-
-	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op, oldcpsr
-	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
-	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op, \oldcpsr
-	.endm
-
-	.macro		do_crypt, round, ttab, ltab, bsz
-	push		{r3-r11, lr}
-
-	// Load keys first, to reduce latency in case they're not cached yet.
-	ldm		rk!, {r8-r11}
-
-	ldr		r4, [in]
-	ldr		r5, [in, #4]
-	ldr		r6, [in, #8]
-	ldr		r7, [in, #12]
-
-#ifdef CONFIG_CPU_BIG_ENDIAN
-	rev_l		r4, t0
-	rev_l		r5, t0
-	rev_l		r6, t0
-	rev_l		r7, t0
-#endif
-
-	eor		r4, r4, r8
-	eor		r5, r5, r9
-	eor		r6, r6, r10
-	eor		r7, r7, r11
-
-	mov_l		ttab, \ttab
-	/*
-	 * Disable interrupts and prefetch the 1024-byte 'ft' or 'it' table into
-	 * L1 cache, assuming cacheline size >= 32.  This is a hardening measure
-	 * intended to make cache-timing attacks more difficult.  They may not
-	 * be fully prevented, however; see the paper
-	 * https://cr.yp.to/antiforgery/cachetiming-20050414.pdf
-	 * ("Cache-timing attacks on AES") for a discussion of the many
-	 * difficulties involved in writing truly constant-time AES software.
-	 */
-	 save_and_disable_irqs	t0
-	.set		i, 0
-	.rept		1024 / 128
-	ldr		r8, [ttab, #i + 0]
-	ldr		r9, [ttab, #i + 32]
-	ldr		r10, [ttab, #i + 64]
-	ldr		r11, [ttab, #i + 96]
-	.set		i, i + 128
-	.endr
-	push		{t0}		// oldcpsr
-
-	tst		rounds, #2
-	bne		1f
-
-0:	\round		r8, r9, r10, r11, r4, r5, r6, r7
-	\round		r4, r5, r6, r7, r8, r9, r10, r11
-
-1:	subs		rounds, rounds, #4
-	\round		r8, r9, r10, r11, r4, r5, r6, r7
-	bls		2f
-	\round		r4, r5, r6, r7, r8, r9, r10, r11
-	b		0b
-
-2:	.ifb		\ltab
-	add		ttab, ttab, #1
-	.else
-	mov_l		ttab, \ltab
-	// Prefetch inverse S-box for final round; see explanation above
-	.set		i, 0
-	.rept		256 / 64
-	ldr		t0, [ttab, #i + 0]
-	ldr		t1, [ttab, #i + 32]
-	.set		i, i + 64
-	.endr
-	.endif
-
-	pop		{rounds}	// oldcpsr
-	\round		r4, r5, r6, r7, r8, r9, r10, r11, \bsz, b, rounds
-
-#ifdef CONFIG_CPU_BIG_ENDIAN
-	rev_l		r4, t0
-	rev_l		r5, t0
-	rev_l		r6, t0
-	rev_l		r7, t0
-#endif
-
-	ldr		out, [sp]
-
-	str		r4, [out]
-	str		r5, [out, #4]
-	str		r6, [out, #8]
-	str		r7, [out, #12]
-
-	pop		{r3-r11, pc}
-
-	.align		3
-	.ltorg
-	.endm
-
-ENTRY(__aes_arm_encrypt)
-	do_crypt	fround, crypto_ft_tab,, 2
-ENDPROC(__aes_arm_encrypt)
-
-	.align		5
-ENTRY(__aes_arm_decrypt)
-	do_crypt	iround, crypto_it_tab, crypto_aes_inv_sbox, 0
-ENDPROC(__aes_arm_decrypt)
diff --git a/arch/arm/crypto/aes-cipher-glue.c b/arch/arm/crypto/aes-cipher-glue.c
deleted file mode 100644
index 29efb7833960..000000000000
--- a/arch/arm/crypto/aes-cipher-glue.c
+++ /dev/null
@@ -1,69 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Scalar AES core transform
- *
- * Copyright (C) 2017 Linaro Ltd.
- * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org>
- */
-
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <linux/module.h>
-#include "aes-cipher.h"
-
-EXPORT_SYMBOL_GPL(__aes_arm_encrypt);
-EXPORT_SYMBOL_GPL(__aes_arm_decrypt);
-
-static void aes_arm_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	int rounds = 6 + ctx->key_length / 4;
-
-	__aes_arm_encrypt(ctx->key_enc, rounds, in, out);
-}
-
-static void aes_arm_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	int rounds = 6 + ctx->key_length / 4;
-
-	__aes_arm_decrypt(ctx->key_dec, rounds, in, out);
-}
-
-static struct crypto_alg aes_alg = {
-	.cra_name			= "aes",
-	.cra_driver_name		= "aes-arm",
-	.cra_priority			= 200,
-	.cra_flags			= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize			= AES_BLOCK_SIZE,
-	.cra_ctxsize			= sizeof(struct crypto_aes_ctx),
-	.cra_module			= THIS_MODULE,
-
-	.cra_cipher.cia_min_keysize	= AES_MIN_KEY_SIZE,
-	.cra_cipher.cia_max_keysize	= AES_MAX_KEY_SIZE,
-	.cra_cipher.cia_setkey		= crypto_aes_set_key,
-	.cra_cipher.cia_encrypt		= aes_arm_encrypt,
-	.cra_cipher.cia_decrypt		= aes_arm_decrypt,
-
-#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
-	.cra_alignmask			= 3,
-#endif
-};
-
-static int __init aes_init(void)
-{
-	return crypto_register_alg(&aes_alg);
-}
-
-static void __exit aes_fini(void)
-{
-	crypto_unregister_alg(&aes_alg);
-}
-
-module_init(aes_init);
-module_exit(aes_fini);
-
-MODULE_DESCRIPTION("Scalar AES cipher for ARM");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm/crypto/aes-cipher.h b/arch/arm/crypto/aes-cipher.h
deleted file mode 100644
index d5db2b87eb69..000000000000
--- a/arch/arm/crypto/aes-cipher.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef ARM_CRYPTO_AES_CIPHER_H
-#define ARM_CRYPTO_AES_CIPHER_H
-
-#include <linux/linkage.h>
-#include <linux/types.h>
-
-asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds,
-				  const u8 *in, u8 *out);
-asmlinkage void __aes_arm_decrypt(const u32 rk[], int rounds,
-				  const u8 *in, u8 *out);
-
-#endif /* ARM_CRYPTO_AES_CIPHER_H */
diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index df5afe601e4a..c49ddafc54f3 100644
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -12,7 +12,6 @@
 #include <crypto/scatterwalk.h>
 #include <crypto/xts.h>
 #include <linux/module.h>
-#include "aes-cipher.h"
 
 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 MODULE_DESCRIPTION("Bit sliced AES using NEON instructions");
@@ -48,13 +47,13 @@ struct aesbs_ctx {
 
 struct aesbs_cbc_ctx {
 	struct aesbs_ctx	key;
-	struct crypto_aes_ctx	fallback;
+	struct aes_enckey	fallback;
 };
 
 struct aesbs_xts_ctx {
 	struct aesbs_ctx	key;
-	struct crypto_aes_ctx	fallback;
-	struct crypto_aes_ctx	tweak_key;
+	struct aes_key		fallback;
+	struct aes_enckey	tweak_key;
 };
 
 static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
@@ -122,14 +121,19 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 	struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
 	int err;
 
-	err = aes_expandkey(&ctx->fallback, in_key, key_len);
+	err = aes_prepareenckey(&ctx->fallback, in_key, key_len);
 	if (err)
 		return err;
 
 	ctx->key.rounds = 6 + key_len / 4;
 
+	/*
+	 * Note: this assumes that the arm implementation of the AES library
+	 * stores the standard round keys in k.rndkeys.
+	 */
 	kernel_neon_begin();
-	aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds);
+	aesbs_convert_key(ctx->key.rk, ctx->fallback.k.rndkeys,
+			  ctx->key.rounds);
 	kernel_neon_end();
 
 	return 0;
@@ -152,8 +156,7 @@ static int cbc_encrypt(struct skcipher_request *req)
 
 		do {
 			crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE);
-			__aes_arm_encrypt(ctx->fallback.key_enc,
-					  ctx->key.rounds, dst, dst);
+			aes_encrypt(&ctx->fallback, dst, dst);
 			prev = dst;
 			src += AES_BLOCK_SIZE;
 			dst += AES_BLOCK_SIZE;
@@ -239,10 +242,10 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 		return err;
 
 	key_len /= 2;
-	err = aes_expandkey(&ctx->fallback, in_key, key_len);
+	err = aes_preparekey(&ctx->fallback, in_key, key_len);
 	if (err)
 		return err;
-	err = aes_expandkey(&ctx->tweak_key, in_key + key_len, key_len);
+	err = aes_prepareenckey(&ctx->tweak_key, in_key + key_len, key_len);
 	if (err)
 		return err;
 
@@ -279,7 +282,7 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt,
 	if (err)
 		return err;
 
-	__aes_arm_encrypt(ctx->tweak_key.key_enc, rounds, walk.iv, walk.iv);
+	aes_encrypt(&ctx->tweak_key, walk.iv, walk.iv);
 
 	while (walk.nbytes >= AES_BLOCK_SIZE) {
 		unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
@@ -311,9 +314,9 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt,
 	crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
 
 	if (encrypt)
-		__aes_arm_encrypt(ctx->fallback.key_enc, rounds, buf, buf);
+		aes_encrypt(&ctx->fallback, buf, buf);
 	else
-		__aes_arm_decrypt(ctx->fallback.key_dec, rounds, buf, buf);
+		aes_decrypt(&ctx->fallback, buf, buf);
 
 	crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
 
diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c
index a52dcc8c1e33..454adcc62cc6 100644
--- a/arch/arm/crypto/ghash-ce-glue.c
+++ b/arch/arm/crypto/ghash-ce-glue.c
@@ -204,20 +204,24 @@ static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
 			  unsigned int keylen)
 {
 	struct gcm_key *ctx = crypto_aead_ctx(tfm);
-	struct crypto_aes_ctx aes_ctx;
+	struct aes_enckey aes_key;
 	be128 h, k;
 	int ret;
 
-	ret = aes_expandkey(&aes_ctx, inkey, keylen);
+	ret = aes_prepareenckey(&aes_key, inkey, keylen);
 	if (ret)
 		return -EINVAL;
 
-	aes_encrypt(&aes_ctx, (u8 *)&k, (u8[AES_BLOCK_SIZE]){});
+	aes_encrypt(&aes_key, (u8 *)&k, (u8[AES_BLOCK_SIZE]){});
 
-	memcpy(ctx->rk, aes_ctx.key_enc, sizeof(ctx->rk));
+	/*
+	 * Note: this assumes that the arm implementation of the AES library
+	 * stores the standard round keys in k.rndkeys.
+	 */
+	memcpy(ctx->rk, aes_key.k.rndkeys, sizeof(ctx->rk));
 	ctx->rounds = 6 + keylen / 4;
 
-	memzero_explicit(&aes_ctx, sizeof(aes_ctx));
+	memzero_explicit(&aes_key, sizeof(aes_key));
 
 	ghash_reflect(ctx->h[0], &k);
 
diff --git a/arch/arm/crypto/nh-neon-core.S b/arch/arm/crypto/nh-neon-core.S
deleted file mode 100644
index 01620a0782ca..000000000000
--- a/arch/arm/crypto/nh-neon-core.S
+++ /dev/null
@@ -1,116 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, NEON accelerated version
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-
-	.text
-	.fpu		neon
-
-	KEY		.req	r0
-	MESSAGE		.req	r1
-	MESSAGE_LEN	.req	r2
-	HASH		.req	r3
-
-	PASS0_SUMS	.req	q0
-	PASS0_SUM_A	.req	d0
-	PASS0_SUM_B	.req	d1
-	PASS1_SUMS	.req	q1
-	PASS1_SUM_A	.req	d2
-	PASS1_SUM_B	.req	d3
-	PASS2_SUMS	.req	q2
-	PASS2_SUM_A	.req	d4
-	PASS2_SUM_B	.req	d5
-	PASS3_SUMS	.req	q3
-	PASS3_SUM_A	.req	d6
-	PASS3_SUM_B	.req	d7
-	K0		.req	q4
-	K1		.req	q5
-	K2		.req	q6
-	K3		.req	q7
-	T0		.req	q8
-	T0_L		.req	d16
-	T0_H		.req	d17
-	T1		.req	q9
-	T1_L		.req	d18
-	T1_H		.req	d19
-	T2		.req	q10
-	T2_L		.req	d20
-	T2_H		.req	d21
-	T3		.req	q11
-	T3_L		.req	d22
-	T3_H		.req	d23
-
-.macro _nh_stride	k0, k1, k2, k3
-
-	// Load next message stride
-	vld1.8		{T3}, [MESSAGE]!
-
-	// Load next key stride
-	vld1.32		{\k3}, [KEY]!
-
-	// Add message words to key words
-	vadd.u32	T0, T3, \k0
-	vadd.u32	T1, T3, \k1
-	vadd.u32	T2, T3, \k2
-	vadd.u32	T3, T3, \k3
-
-	// Multiply 32x32 => 64 and accumulate
-	vmlal.u32	PASS0_SUMS, T0_L, T0_H
-	vmlal.u32	PASS1_SUMS, T1_L, T1_H
-	vmlal.u32	PASS2_SUMS, T2_L, T2_H
-	vmlal.u32	PASS3_SUMS, T3_L, T3_H
-.endm
-
-/*
- * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-ENTRY(nh_neon)
-
-	vld1.32		{K0,K1}, [KEY]!
-	  vmov.u64	PASS0_SUMS, #0
-	  vmov.u64	PASS1_SUMS, #0
-	vld1.32		{K2}, [KEY]!
-	  vmov.u64	PASS2_SUMS, #0
-	  vmov.u64	PASS3_SUMS, #0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	blt		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3
-	_nh_stride	K1, K2, K3, K0
-	_nh_stride	K2, K3, K0, K1
-	_nh_stride	K3, K0, K1, K2
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	bge		.Lloop4
-
-.Lloop4_done:
-	ands		MESSAGE_LEN, MESSAGE_LEN, #63
-	beq		.Ldone
-	_nh_stride	K0, K1, K2, K3
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K1, K2, K3, K0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K2, K3, K0, K1
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	vadd.u64	T0_L, PASS0_SUM_A, PASS0_SUM_B
-	vadd.u64	T0_H, PASS1_SUM_A, PASS1_SUM_B
-	vadd.u64	T1_L, PASS2_SUM_A, PASS2_SUM_B
-	vadd.u64	T1_H, PASS3_SUM_A, PASS3_SUM_B
-	vst1.8		{T0-T1}, [HASH]
-	bx		lr
-ENDPROC(nh_neon)
diff --git a/arch/arm/crypto/nhpoly1305-neon-glue.c b/arch/arm/crypto/nhpoly1305-neon-glue.c
deleted file mode 100644
index 62cf7ccdde73..000000000000
--- a/arch/arm/crypto/nhpoly1305-neon-glue.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (NEON accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-
-asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_neon_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_neon_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
-		kernel_neon_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_neon_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_neon_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-neon",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_neon_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_neon_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!(elf_hwcap & HWCAP_NEON))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index bdd276a6e540..81ed892b3b72 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -15,16 +15,6 @@ config CRYPTO_GHASH_ARM64_CE
 	  Architecture: arm64 using:
 	  - ARMv8 Crypto Extensions
 
-config CRYPTO_NHPOLY1305_NEON
-	tristate "Hash functions: NHPoly1305 (NEON)"
-	depends on KERNEL_MODE_NEON
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function (Adiantum)
-
-	  Architecture: arm64 using:
-	  - NEON (Advanced SIMD) extensions
-
 config CRYPTO_SM3_NEON
 	tristate "Hash functions: SM3 (NEON)"
 	depends on KERNEL_MODE_NEON
@@ -47,35 +37,12 @@ config CRYPTO_SM3_ARM64_CE
 	  Architecture: arm64 using:
 	  - ARMv8.2 Crypto Extensions
 
-config CRYPTO_AES_ARM64
-	tristate "Ciphers: AES, modes: ECB, CBC, CTR, CTS, XCTR, XTS"
-	select CRYPTO_AES
-	select CRYPTO_LIB_SHA256
-	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
-	  Length-preserving ciphers: AES with ECB, CBC, CTR, CTS,
-	    XCTR, and XTS modes
-	  AEAD cipher: AES with CBC, ESSIV, and SHA-256
-	    for fscrypt and dm-crypt
-
-	  Architecture: arm64
-
-config CRYPTO_AES_ARM64_CE
-	tristate "Ciphers: AES (ARMv8 Crypto Extensions)"
-	depends on KERNEL_MODE_NEON
-	select CRYPTO_ALGAPI
-	select CRYPTO_LIB_AES
-	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
-
-	  Architecture: arm64 using:
-	  - ARMv8 Crypto Extensions
-
 config CRYPTO_AES_ARM64_CE_BLK
 	tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (ARMv8 Crypto Extensions)"
 	depends on KERNEL_MODE_NEON
 	select CRYPTO_SKCIPHER
-	select CRYPTO_AES_ARM64_CE
+	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_SHA256
 	help
 	  Length-preserving ciphers: AES cipher algorithms (FIPS-197)
 	  with block cipher modes:
@@ -93,6 +60,7 @@ config CRYPTO_AES_ARM64_NEON_BLK
 	depends on KERNEL_MODE_NEON
 	select CRYPTO_SKCIPHER
 	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_SHA256
 	help
 	  Length-preserving ciphers: AES cipher algorithms (FIPS-197)
 	  with block cipher modes:
@@ -174,7 +142,6 @@ config CRYPTO_AES_ARM64_CE_CCM
 	tristate "AEAD cipher: AES in CCM mode (ARMv8 Crypto Extensions)"
 	depends on KERNEL_MODE_NEON
 	select CRYPTO_ALGAPI
-	select CRYPTO_AES_ARM64_CE
 	select CRYPTO_AES_ARM64_CE_BLK
 	select CRYPTO_AEAD
 	select CRYPTO_LIB_AES
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 1e330aa08d3f..3574e917bc37 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -29,9 +29,6 @@ sm4-neon-y := sm4-neon-glue.o sm4-neon-core.o
 obj-$(CONFIG_CRYPTO_GHASH_ARM64_CE) += ghash-ce.o
 ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
 
-obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
-aes-ce-cipher-y := aes-ce-core.o aes-ce-glue.o
-
 obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
 aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
 
@@ -41,11 +38,5 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o
 obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
 aes-neon-blk-y := aes-glue-neon.o aes-neon.o
 
-obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
-nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o
-
-obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
-aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
-
 obj-$(CONFIG_CRYPTO_AES_ARM64_BS) += aes-neon-bs.o
 aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index c4fd648471f1..db371ac051fc 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -17,8 +17,6 @@
 
 #include <asm/simd.h>
 
-#include "aes-ce-setkey.h"
-
 MODULE_IMPORT_NS("CRYPTO_INTERNAL");
 
 static int num_rounds(struct crypto_aes_ctx *ctx)
diff --git a/arch/arm64/crypto/aes-ce-core.S b/arch/arm64/crypto/aes-ce-core.S
deleted file mode 100644
index e52e13eb8fdb..000000000000
--- a/arch/arm64/crypto/aes-ce-core.S
+++ /dev/null
@@ -1,84 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-
-	.arch		armv8-a+crypto
-
-SYM_FUNC_START(__aes_ce_encrypt)
-	sub		w3, w3, #2
-	ld1		{v0.16b}, [x2]
-	ld1		{v1.4s}, [x0], #16
-	cmp		w3, #10
-	bmi		0f
-	bne		3f
-	mov		v3.16b, v1.16b
-	b		2f
-0:	mov		v2.16b, v1.16b
-	ld1		{v3.4s}, [x0], #16
-1:	aese		v0.16b, v2.16b
-	aesmc		v0.16b, v0.16b
-2:	ld1		{v1.4s}, [x0], #16
-	aese		v0.16b, v3.16b
-	aesmc		v0.16b, v0.16b
-3:	ld1		{v2.4s}, [x0], #16
-	subs		w3, w3, #3
-	aese		v0.16b, v1.16b
-	aesmc		v0.16b, v0.16b
-	ld1		{v3.4s}, [x0], #16
-	bpl		1b
-	aese		v0.16b, v2.16b
-	eor		v0.16b, v0.16b, v3.16b
-	st1		{v0.16b}, [x1]
-	ret
-SYM_FUNC_END(__aes_ce_encrypt)
-
-SYM_FUNC_START(__aes_ce_decrypt)
-	sub		w3, w3, #2
-	ld1		{v0.16b}, [x2]
-	ld1		{v1.4s}, [x0], #16
-	cmp		w3, #10
-	bmi		0f
-	bne		3f
-	mov		v3.16b, v1.16b
-	b		2f
-0:	mov		v2.16b, v1.16b
-	ld1		{v3.4s}, [x0], #16
-1:	aesd		v0.16b, v2.16b
-	aesimc		v0.16b, v0.16b
-2:	ld1		{v1.4s}, [x0], #16
-	aesd		v0.16b, v3.16b
-	aesimc		v0.16b, v0.16b
-3:	ld1		{v2.4s}, [x0], #16
-	subs		w3, w3, #3
-	aesd		v0.16b, v1.16b
-	aesimc		v0.16b, v0.16b
-	ld1		{v3.4s}, [x0], #16
-	bpl		1b
-	aesd		v0.16b, v2.16b
-	eor		v0.16b, v0.16b, v3.16b
-	st1		{v0.16b}, [x1]
-	ret
-SYM_FUNC_END(__aes_ce_decrypt)
-
-/*
- * __aes_ce_sub() - use the aese instruction to perform the AES sbox
- *                  substitution on each byte in 'input'
- */
-SYM_FUNC_START(__aes_ce_sub)
-	dup		v1.4s, w0
-	movi		v0.16b, #0
-	aese		v0.16b, v1.16b
-	umov		w0, v0.s[0]
-	ret
-SYM_FUNC_END(__aes_ce_sub)
-
-SYM_FUNC_START(__aes_ce_invert)
-	ld1		{v0.4s}, [x1]
-	aesimc		v1.16b, v0.16b
-	st1		{v1.4s}, [x0]
-	ret
-SYM_FUNC_END(__aes_ce_invert)
diff --git a/arch/arm64/crypto/aes-ce-glue.c b/arch/arm64/crypto/aes-ce-glue.c
deleted file mode 100644
index a4dad370991d..000000000000
--- a/arch/arm64/crypto/aes-ce-glue.c
+++ /dev/null
@@ -1,178 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * aes-ce-cipher.c - core AES cipher using ARMv8 Crypto Extensions
- *
- * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <linux/unaligned.h>
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <crypto/internal/simd.h>
-#include <linux/cpufeature.h>
-#include <linux/module.h>
-
-#include "aes-ce-setkey.h"
-
-MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-
-struct aes_block {
-	u8 b[AES_BLOCK_SIZE];
-};
-
-asmlinkage void __aes_ce_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
-asmlinkage void __aes_ce_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
-
-asmlinkage u32 __aes_ce_sub(u32 l);
-asmlinkage void __aes_ce_invert(struct aes_block *out,
-				const struct aes_block *in);
-
-static int num_rounds(struct crypto_aes_ctx *ctx)
-{
-	/*
-	 * # of rounds specified by AES:
-	 * 128 bit key		10 rounds
-	 * 192 bit key		12 rounds
-	 * 256 bit key		14 rounds
-	 * => n byte key	=> 6 + (n/4) rounds
-	 */
-	return 6 + ctx->key_length / 4;
-}
-
-static void aes_cipher_encrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		aes_encrypt(ctx, dst, src);
-		return;
-	}
-
-	scoped_ksimd()
-		__aes_ce_encrypt(ctx->key_enc, dst, src, num_rounds(ctx));
-}
-
-static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[])
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		aes_decrypt(ctx, dst, src);
-		return;
-	}
-
-	scoped_ksimd()
-		__aes_ce_decrypt(ctx->key_dec, dst, src, num_rounds(ctx));
-}
-
-int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
-		     unsigned int key_len)
-{
-	/*
-	 * The AES key schedule round constants
-	 */
-	static u8 const rcon[] = {
-		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
-	};
-
-	u32 kwords = key_len / sizeof(u32);
-	struct aes_block *key_enc, *key_dec;
-	int i, j;
-
-	if (key_len != AES_KEYSIZE_128 &&
-	    key_len != AES_KEYSIZE_192 &&
-	    key_len != AES_KEYSIZE_256)
-		return -EINVAL;
-
-	ctx->key_length = key_len;
-	for (i = 0; i < kwords; i++)
-		ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
-
-	scoped_ksimd() {
-		for (i = 0; i < sizeof(rcon); i++) {
-			u32 *rki = ctx->key_enc + (i * kwords);
-			u32 *rko = rki + kwords;
-
-			rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^
-				 rcon[i] ^ rki[0];
-			rko[1] = rko[0] ^ rki[1];
-			rko[2] = rko[1] ^ rki[2];
-			rko[3] = rko[2] ^ rki[3];
-
-			if (key_len == AES_KEYSIZE_192) {
-				if (i >= 7)
-					break;
-				rko[4] = rko[3] ^ rki[4];
-				rko[5] = rko[4] ^ rki[5];
-			} else if (key_len == AES_KEYSIZE_256) {
-				if (i >= 6)
-					break;
-				rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
-				rko[5] = rko[4] ^ rki[5];
-				rko[6] = rko[5] ^ rki[6];
-				rko[7] = rko[6] ^ rki[7];
-			}
-		}
-
-		/*
-		 * Generate the decryption keys for the Equivalent Inverse
-		 * Cipher.  This involves reversing the order of the round
-		 * keys, and applying the Inverse Mix Columns transformation on
-		 * all but the first and the last one.
-		 */
-		key_enc = (struct aes_block *)ctx->key_enc;
-		key_dec = (struct aes_block *)ctx->key_dec;
-		j = num_rounds(ctx);
-
-		key_dec[0] = key_enc[j];
-		for (i = 1, j--; j > 0; i++, j--)
-			__aes_ce_invert(key_dec + i, key_enc + j);
-		key_dec[i] = key_enc[0];
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ce_aes_expandkey);
-
-int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-		  unsigned int key_len)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	return ce_aes_expandkey(ctx, in_key, key_len);
-}
-EXPORT_SYMBOL(ce_aes_setkey);
-
-static struct crypto_alg aes_alg = {
-	.cra_name		= "aes",
-	.cra_driver_name	= "aes-ce",
-	.cra_priority		= 250,
-	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_aes_ctx),
-	.cra_module		= THIS_MODULE,
-	.cra_cipher = {
-		.cia_min_keysize	= AES_MIN_KEY_SIZE,
-		.cia_max_keysize	= AES_MAX_KEY_SIZE,
-		.cia_setkey		= ce_aes_setkey,
-		.cia_encrypt		= aes_cipher_encrypt,
-		.cia_decrypt		= aes_cipher_decrypt
-	}
-};
-
-static int __init aes_mod_init(void)
-{
-	return crypto_register_alg(&aes_alg);
-}
-
-static void __exit aes_mod_exit(void)
-{
-	crypto_unregister_alg(&aes_alg);
-}
-
-module_cpu_feature_match(AES, aes_mod_init);
-module_exit(aes_mod_exit);
diff --git a/arch/arm64/crypto/aes-ce-setkey.h b/arch/arm64/crypto/aes-ce-setkey.h
deleted file mode 100644
index fd9ecf07d88c..000000000000
--- a/arch/arm64/crypto/aes-ce-setkey.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-		  unsigned int key_len);
-int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
-		     unsigned int key_len);
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
deleted file mode 100644
index c9d6955f8404..000000000000
--- a/arch/arm64/crypto/aes-cipher-core.S
+++ /dev/null
@@ -1,132 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Scalar AES core transform
- *
- * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/linkage.h>
-#include <asm/assembler.h>
-#include <asm/cache.h>
-
-	.text
-
-	rk		.req	x0
-	out		.req	x1
-	in		.req	x2
-	rounds		.req	x3
-	tt		.req	x2
-
-	.macro		__pair1, sz, op, reg0, reg1, in0, in1e, in1d, shift
-	.ifc		\op\shift, b0
-	ubfiz		\reg0, \in0, #2, #8
-	ubfiz		\reg1, \in1e, #2, #8
-	.else
-	ubfx		\reg0, \in0, #\shift, #8
-	ubfx		\reg1, \in1e, #\shift, #8
-	.endif
-
-	/*
-	 * AArch64 cannot do byte size indexed loads from a table containing
-	 * 32-bit quantities, i.e., 'ldrb w12, [tt, w12, uxtw #2]' is not a
-	 * valid instruction. So perform the shift explicitly first for the
-	 * high bytes (the low byte is shifted implicitly by using ubfiz rather
-	 * than ubfx above)
-	 */
-	.ifnc		\op, b
-	ldr		\reg0, [tt, \reg0, uxtw #2]
-	ldr		\reg1, [tt, \reg1, uxtw #2]
-	.else
-	.if		\shift > 0
-	lsl		\reg0, \reg0, #2
-	lsl		\reg1, \reg1, #2
-	.endif
-	ldrb		\reg0, [tt, \reg0, uxtw]
-	ldrb		\reg1, [tt, \reg1, uxtw]
-	.endif
-	.endm
-
-	.macro		__pair0, sz, op, reg0, reg1, in0, in1e, in1d, shift
-	ubfx		\reg0, \in0, #\shift, #8
-	ubfx		\reg1, \in1d, #\shift, #8
-	ldr\op		\reg0, [tt, \reg0, uxtw #\sz]
-	ldr\op		\reg1, [tt, \reg1, uxtw #\sz]
-	.endm
-
-	.macro		__hround, out0, out1, in0, in1, in2, in3, t0, t1, enc, sz, op
-	ldp		\out0, \out1, [rk], #8
-
-	__pair\enc	\sz, \op, w12, w13, \in0, \in1, \in3, 0
-	__pair\enc	\sz, \op, w14, w15, \in1, \in2, \in0, 8
-	__pair\enc	\sz, \op, w16, w17, \in2, \in3, \in1, 16
-	__pair\enc	\sz, \op, \t0, \t1, \in3, \in0, \in2, 24
-
-	eor		\out0, \out0, w12
-	eor		\out1, \out1, w13
-	eor		\out0, \out0, w14, ror #24
-	eor		\out1, \out1, w15, ror #24
-	eor		\out0, \out0, w16, ror #16
-	eor		\out1, \out1, w17, ror #16
-	eor		\out0, \out0, \t0, ror #8
-	eor		\out1, \out1, \t1, ror #8
-	.endm
-
-	.macro		fround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
-	__hround	\out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1, \sz, \op
-	__hround	\out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1, \sz, \op
-	.endm
-
-	.macro		iround, out0, out1, out2, out3, in0, in1, in2, in3, sz=2, op
-	__hround	\out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0, \sz, \op
-	__hround	\out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0, \sz, \op
-	.endm
-
-	.macro		do_crypt, round, ttab, ltab, bsz
-	ldp		w4, w5, [in]
-	ldp		w6, w7, [in, #8]
-	ldp		w8, w9, [rk], #16
-	ldp		w10, w11, [rk, #-8]
-
-CPU_BE(	rev		w4, w4		)
-CPU_BE(	rev		w5, w5		)
-CPU_BE(	rev		w6, w6		)
-CPU_BE(	rev		w7, w7		)
-
-	eor		w4, w4, w8
-	eor		w5, w5, w9
-	eor		w6, w6, w10
-	eor		w7, w7, w11
-
-	adr_l		tt, \ttab
-
-	tbnz		rounds, #1, 1f
-
-0:	\round		w8, w9, w10, w11, w4, w5, w6, w7
-	\round		w4, w5, w6, w7, w8, w9, w10, w11
-
-1:	subs		rounds, rounds, #4
-	\round		w8, w9, w10, w11, w4, w5, w6, w7
-	b.ls		3f
-2:	\round		w4, w5, w6, w7, w8, w9, w10, w11
-	b		0b
-3:	adr_l		tt, \ltab
-	\round		w4, w5, w6, w7, w8, w9, w10, w11, \bsz, b
-
-CPU_BE(	rev		w4, w4		)
-CPU_BE(	rev		w5, w5		)
-CPU_BE(	rev		w6, w6		)
-CPU_BE(	rev		w7, w7		)
-
-	stp		w4, w5, [out]
-	stp		w6, w7, [out, #8]
-	ret
-	.endm
-
-SYM_FUNC_START(__aes_arm64_encrypt)
-	do_crypt	fround, crypto_ft_tab, crypto_ft_tab + 1, 2
-SYM_FUNC_END(__aes_arm64_encrypt)
-
-	.align		5
-SYM_FUNC_START(__aes_arm64_decrypt)
-	do_crypt	iround, crypto_it_tab, crypto_aes_inv_sbox, 0
-SYM_FUNC_END(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-cipher-glue.c b/arch/arm64/crypto/aes-cipher-glue.c
deleted file mode 100644
index 4ec55e568941..000000000000
--- a/arch/arm64/crypto/aes-cipher-glue.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Scalar AES core transform
- *
- * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <crypto/aes.h>
-#include <crypto/algapi.h>
-#include <linux/module.h>
-
-asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
-asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
-
-static void aes_arm64_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	int rounds = 6 + ctx->key_length / 4;
-
-	__aes_arm64_encrypt(ctx->key_enc, out, in, rounds);
-}
-
-static void aes_arm64_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-	int rounds = 6 + ctx->key_length / 4;
-
-	__aes_arm64_decrypt(ctx->key_dec, out, in, rounds);
-}
-
-static struct crypto_alg aes_alg = {
-	.cra_name			= "aes",
-	.cra_driver_name		= "aes-arm64",
-	.cra_priority			= 200,
-	.cra_flags			= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize			= AES_BLOCK_SIZE,
-	.cra_ctxsize			= sizeof(struct crypto_aes_ctx),
-	.cra_module			= THIS_MODULE,
-
-	.cra_cipher.cia_min_keysize	= AES_MIN_KEY_SIZE,
-	.cra_cipher.cia_max_keysize	= AES_MAX_KEY_SIZE,
-	.cra_cipher.cia_setkey		= crypto_aes_set_key,
-	.cra_cipher.cia_encrypt		= aes_arm64_encrypt,
-	.cra_cipher.cia_decrypt		= aes_arm64_decrypt
-};
-
-static int __init aes_init(void)
-{
-	return crypto_register_alg(&aes_alg);
-}
-
-static void __exit aes_fini(void)
-{
-	crypto_unregister_alg(&aes_alg);
-}
-
-module_init(aes_init);
-module_exit(aes_fini);
-
-MODULE_DESCRIPTION("Scalar AES cipher for arm64");
-MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
-MODULE_LICENSE("GPL v2");
-MODULE_ALIAS_CRYPTO("aes");
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index c51d4487e9e9..92f43e1cd097 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -21,8 +21,6 @@
 #include <asm/hwcap.h>
 #include <asm/simd.h>
 
-#include "aes-ce-setkey.h"
-
 #ifdef USE_V8_CRYPTO_EXTENSIONS
 #define MODE			"ce"
 #define PRIO			300
diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c
index ef249d06c92c..63bb9e062251 100644
--- a/arch/arm64/crypto/ghash-ce-glue.c
+++ b/arch/arm64/crypto/ghash-ce-glue.c
@@ -40,7 +40,7 @@ struct arm_ghash_desc_ctx {
 };
 
 struct gcm_aes_ctx {
-	struct crypto_aes_ctx	aes_key;
+	struct aes_enckey	aes_key;
 	u8			nonce[RFC4106_NONCE_SIZE];
 	struct ghash_key	ghash_key;
 };
@@ -186,18 +186,6 @@ static struct shash_alg ghash_alg = {
 	.statesize		= sizeof(struct ghash_desc_ctx),
 };
 
-static int num_rounds(struct crypto_aes_ctx *ctx)
-{
-	/*
-	 * # of rounds specified by AES:
-	 * 128 bit key		10 rounds
-	 * 192 bit key		12 rounds
-	 * 256 bit key		14 rounds
-	 * => n byte key	=> 6 + (n/4) rounds
-	 */
-	return 6 + ctx->key_length / 4;
-}
-
 static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
 			  unsigned int keylen)
 {
@@ -206,7 +194,7 @@ static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
 	be128 h;
 	int ret;
 
-	ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
+	ret = aes_prepareenckey(&ctx->aes_key, inkey, keylen);
 	if (ret)
 		return -EINVAL;
 
@@ -296,7 +284,6 @@ static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
 {
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
-	int nrounds = num_rounds(&ctx->aes_key);
 	struct skcipher_walk walk;
 	u8 buf[AES_BLOCK_SIZE];
 	u64 dg[2] = {};
@@ -331,8 +318,8 @@ static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
 
 		scoped_ksimd()
 			pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
-					  dg, iv, ctx->aes_key.key_enc, nrounds,
-					  tag);
+					  dg, iv, ctx->aes_key.k.rndkeys,
+					  ctx->aes_key.nrounds, tag);
 
 		if (unlikely(!nbytes))
 			break;
@@ -359,7 +346,6 @@ static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen)
 	struct crypto_aead *aead = crypto_aead_reqtfm(req);
 	struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
 	unsigned int authsize = crypto_aead_authsize(aead);
-	int nrounds = num_rounds(&ctx->aes_key);
 	struct skcipher_walk walk;
 	u8 otag[AES_BLOCK_SIZE];
 	u8 buf[AES_BLOCK_SIZE];
@@ -401,8 +387,9 @@ static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen)
 		scoped_ksimd()
 			ret = pmull_gcm_decrypt(nbytes, dst, src,
 						ctx->ghash_key.h,
-						dg, iv, ctx->aes_key.key_enc,
-						nrounds, tag, otag, authsize);
+						dg, iv, ctx->aes_key.k.rndkeys,
+						ctx->aes_key.nrounds, tag, otag,
+						authsize);
 
 		if (unlikely(!nbytes))
 			break;
diff --git a/arch/arm64/crypto/nh-neon-core.S b/arch/arm64/crypto/nh-neon-core.S
deleted file mode 100644
index 13eda08fda1e..000000000000
--- a/arch/arm64/crypto/nh-neon-core.S
+++ /dev/null
@@ -1,104 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, ARM64 NEON accelerated version
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-	KEY		.req	x0
-	MESSAGE		.req	x1
-	MESSAGE_LEN	.req	x2
-	HASH		.req	x3
-
-	PASS0_SUMS	.req	v0
-	PASS1_SUMS	.req	v1
-	PASS2_SUMS	.req	v2
-	PASS3_SUMS	.req	v3
-	K0		.req	v4
-	K1		.req	v5
-	K2		.req	v6
-	K3		.req	v7
-	T0		.req	v8
-	T1		.req	v9
-	T2		.req	v10
-	T3		.req	v11
-	T4		.req	v12
-	T5		.req	v13
-	T6		.req	v14
-	T7		.req	v15
-
-.macro _nh_stride	k0, k1, k2, k3
-
-	// Load next message stride
-	ld1		{T3.16b}, [MESSAGE], #16
-
-	// Load next key stride
-	ld1		{\k3\().4s}, [KEY], #16
-
-	// Add message words to key words
-	add		T0.4s, T3.4s, \k0\().4s
-	add		T1.4s, T3.4s, \k1\().4s
-	add		T2.4s, T3.4s, \k2\().4s
-	add		T3.4s, T3.4s, \k3\().4s
-
-	// Multiply 32x32 => 64 and accumulate
-	mov		T4.d[0], T0.d[1]
-	mov		T5.d[0], T1.d[1]
-	mov		T6.d[0], T2.d[1]
-	mov		T7.d[0], T3.d[1]
-	umlal		PASS0_SUMS.2d, T0.2s, T4.2s
-	umlal		PASS1_SUMS.2d, T1.2s, T5.2s
-	umlal		PASS2_SUMS.2d, T2.2s, T6.2s
-	umlal		PASS3_SUMS.2d, T3.2s, T7.2s
-.endm
-
-/*
- * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_neon)
-
-	ld1		{K0.4s,K1.4s}, [KEY], #32
-	  movi		PASS0_SUMS.2d, #0
-	  movi		PASS1_SUMS.2d, #0
-	ld1		{K2.4s}, [KEY], #16
-	  movi		PASS2_SUMS.2d, #0
-	  movi		PASS3_SUMS.2d, #0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	blt		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3
-	_nh_stride	K1, K2, K3, K0
-	_nh_stride	K2, K3, K0, K1
-	_nh_stride	K3, K0, K1, K2
-	subs		MESSAGE_LEN, MESSAGE_LEN, #64
-	bge		.Lloop4
-
-.Lloop4_done:
-	ands		MESSAGE_LEN, MESSAGE_LEN, #63
-	beq		.Ldone
-	_nh_stride	K0, K1, K2, K3
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K1, K2, K3, K0
-
-	subs		MESSAGE_LEN, MESSAGE_LEN, #16
-	beq		.Ldone
-	_nh_stride	K2, K3, K0, K1
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	addp		T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
-	addp		T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
-	st1		{T0.16b,T1.16b}, [HASH]
-	ret
-SYM_FUNC_END(nh_neon)
diff --git a/arch/arm64/crypto/nhpoly1305-neon-glue.c b/arch/arm64/crypto/nhpoly1305-neon-glue.c
deleted file mode 100644
index 013de6ac569a..000000000000
--- a/arch/arm64/crypto/nhpoly1305-neon-glue.c
+++ /dev/null
@@ -1,79 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (ARM64 NEON accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <asm/neon.h>
-#include <asm/simd.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-
-asmlinkage void nh_neon(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_neon_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		scoped_ksimd()
-			crypto_nhpoly1305_update_helper(desc, src, n, nh_neon);
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_neon_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_neon_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-neon",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_neon_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_neon_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!cpu_have_named_feature(ASIMD))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (NEON-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-neon");
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index bfc1ee7c8158..bffcc417f44c 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -555,7 +555,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index d9d1f3c4c70d..3f894c20b132 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -512,7 +512,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index 523205adccc8..5c5603ca16aa 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -532,7 +532,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index 7b0a4ef0b010..37c747ee395e 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -504,7 +504,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 089c5c394c62..1a376c2b8c45 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -514,7 +514,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 5f2484c36733..2b26450692a5 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -531,7 +531,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 74f0a1f6d871..012e0e1f506f 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -618,7 +618,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 4bee18c820e4..37634b35bfbd 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -504,7 +504,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 322c17e55c9a..a0d2e0070afa 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -505,7 +505,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index 82f9baab8fea..62cc3964fc34 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -521,7 +521,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index f94ad226cb5b..13107aa4a1b4 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -502,7 +502,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index a5ecfc505ab2..eaab0ba08989 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -502,7 +502,6 @@ CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
 CONFIG_CRYPTO_AES=y
-CONFIG_CRYPTO_AES_TI=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig
index 662aed46f9c7..2d056f1fc90f 100644
--- a/arch/powerpc/crypto/Kconfig
+++ b/arch/powerpc/crypto/Kconfig
@@ -5,9 +5,9 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)"
 config CRYPTO_AES_PPC_SPE
 	tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)"
 	depends on SPE
+	select CRYPTO_LIB_AES
 	select CRYPTO_SKCIPHER
 	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
 	  Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes
 
 	  Architecture: powerpc using:
diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile
index 5960e5300db7..3ac0886282a2 100644
--- a/arch/powerpc/crypto/Makefile
+++ b/arch/powerpc/crypto/Makefile
@@ -9,9 +9,9 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o
 obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
 obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
 
-aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
+aes-ppc-spe-y := aes-spe-glue.o
 aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
-vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
+vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
 
 ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
 override flavour := linux-ppc64le
@@ -26,15 +26,14 @@ endif
 quiet_cmd_perl = PERL    $@
       cmd_perl = $(PERL) $< $(flavour) > $@
 
-targets += aesp10-ppc.S ghashp10-ppc.S aesp8-ppc.S ghashp8-ppc.S
+targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S
 
 $(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perl)
 
-$(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
+$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
 	$(call if_changed,perl)
 
 OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y
 OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y
-OBJECT_FILES_NON_STANDARD_aesp8-ppc.o := y
 OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y
diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c
index 85f4fd4b1bdc..f3417436d3f7 100644
--- a/arch/powerpc/crypto/aes-gcm-p10-glue.c
+++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c
@@ -44,7 +44,7 @@ asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable,
 			      unsigned char *aad, unsigned int alen);
 asmlinkage void gcm_update(u8 *iv, void *Xi);
 
-struct aes_key {
+struct p10_aes_key {
 	u8 key[AES_MAX_KEYLENGTH];
 	u64 rounds;
 };
@@ -63,7 +63,7 @@ struct Hash_ctx {
 };
 
 struct p10_aes_gcm_ctx {
-	struct aes_key enc_key;
+	struct p10_aes_key enc_key;
 	u8 nonce[RFC4106_NONCE_SIZE];
 };
 
diff --git a/arch/powerpc/crypto/aes-spe-core.S b/arch/powerpc/crypto/aes-spe-core.S
deleted file mode 100644
index 8e00eccc352b..000000000000
--- a/arch/powerpc/crypto/aes-spe-core.S
+++ /dev/null
@@ -1,346 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Fast AES implementation for SPE instruction set (PPC)
- *
- * This code makes use of the SPE SIMD instruction set as defined in
- * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
- * Implementation is based on optimization guide notes from
- * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-#include "aes-spe-regs.h"
-
-#define	EAD(in, bpos) \
-	rlwimi		rT0,in,28-((bpos+3)%4)*8,20,27;
-
-#define DAD(in, bpos) \
-	rlwimi		rT1,in,24-((bpos+3)%4)*8,24,31;
-
-#define LWH(out, off) \
-	evlwwsplat	out,off(rT0);	/* load word high		*/
-
-#define LWL(out, off) \
-	lwz		out,off(rT0);	/* load word low		*/
-
-#define LBZ(out, tab, off) \
-	lbz		out,off(tab);	/* load byte			*/
-
-#define LAH(out, in, bpos, off) \
-	EAD(in, bpos)			/* calc addr + load word high	*/ \
-	LWH(out, off)
-
-#define LAL(out, in, bpos, off) \
-	EAD(in, bpos)			/* calc addr + load word low	*/ \
-	LWL(out, off)
-
-#define LAE(out, in, bpos) \
-	EAD(in, bpos)			/* calc addr + load enc byte	*/ \
-	LBZ(out, rT0, 8)
-
-#define LBE(out) \
-	LBZ(out, rT0, 8)		/* load enc byte		*/
-
-#define LAD(out, in, bpos) \
-	DAD(in, bpos)			/* calc addr + load dec byte	*/ \
-	LBZ(out, rT1, 0)
-
-#define LBD(out) \
-	LBZ(out, rT1, 0)
-
-/*
- * ppc_encrypt_block: The central encryption function for a single 16 bytes
- * block. It does no stack handling or register saving to support fast calls
- * via bl/blr. It expects that caller has pre-xored input data with first
- * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
- * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
- * and rW0-rW3 and caller must execute a final xor on the output registers.
- * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
- *
- */
-_GLOBAL(ppc_encrypt_block)
-	LAH(rW4, rD1, 2, 4)
-	LAH(rW6, rD0, 3, 0)
-	LAH(rW3, rD0, 1, 8)
-ppc_encrypt_block_loop:
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,16(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 2)
-	evxor		rW3,rW3,rW5
-	LWH(rW4, 4)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,32(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,40(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 2)
-	evxor		rW3,rW3,rW5
-	LWH(rW4, 4)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	addi		rKP,rKP,32
-	bdnz		ppc_encrypt_block_loop
-	LAH(rW0, rD3, 0, 12)
-	LAL(rW0, rD0, 0, 12)
-	LAH(rW1, rD1, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAL(rW3, rD1, 1, 8)
-	LAL(rW4, rD2, 2, 4)
-	LAH(rW5, rD3, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAL(rW5, rD0, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	evldw		rD1,16(rKP)
-	EAD(rD3, 3)
-	evxor		rW2,rW2,rW4
-	LWL(rW7, 0)
-	evxor		rW2,rW2,rW6
-	EAD(rD2, 0)
-	evxor		rD1,rD1,rW2
-	LWL(rW1, 12)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW5
-	LBE(rW2)
-	evxor		rW3,rW3,rW7
-	EAD(rD0, 1)
-	evxor		rD3,rD3,rW3
-	LBE(rW6)
-	evxor		rD3,rD3,rW1
-	EAD(rD0, 0)
-	evmergehi	rD2,rD2,rD3
-	LBE(rW1)
-	LAE(rW0, rD3, 0)
-	LAE(rW1, rD0, 0)
-	LAE(rW4, rD2, 1)
-	LAE(rW5, rD3, 1)
-	LAE(rW3, rD2, 0)
-	LAE(rW7, rD1, 1)
-	rlwimi		rW0,rW4,8,16,23
-	rlwimi		rW1,rW5,8,16,23
-	LAE(rW4, rD1, 2)
-	LAE(rW5, rD2, 2)
-	rlwimi		rW2,rW6,8,16,23
-	rlwimi		rW3,rW7,8,16,23
-	LAE(rW6, rD3, 2)
-	LAE(rW7, rD0, 2)
-	rlwimi		rW0,rW4,16,8,15
-	rlwimi		rW1,rW5,16,8,15
-	LAE(rW4, rD0, 3)
-	LAE(rW5, rD1, 3)
-	rlwimi		rW2,rW6,16,8,15
-	lwz		rD0,32(rKP)
-	rlwimi		rW3,rW7,16,8,15
-	lwz		rD1,36(rKP)
-	LAE(rW6, rD2, 3)
-	LAE(rW7, rD3, 3)
-	rlwimi		rW0,rW4,24,0,7
-	lwz		rD2,40(rKP)
-	rlwimi		rW1,rW5,24,0,7
-	lwz		rD3,44(rKP)
-	rlwimi		rW2,rW6,24,0,7
-	rlwimi		rW3,rW7,24,0,7
-	blr
-
-/*
- * ppc_decrypt_block: The central decryption function for a single 16 bytes
- * block. It does no stack handling or register saving to support fast calls
- * via bl/blr. It expects that caller has pre-xored input data with first
- * 4 words of encryption key into rD0-rD3. Pointer/counter registers must
- * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3
- * and rW0-rW3 and caller must execute a final xor on the output registers.
- * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing.
- *
- */
-_GLOBAL(ppc_decrypt_block)
-	LAH(rW0, rD1, 0, 12)
-	LAH(rW6, rD0, 3, 0)
-	LAH(rW3, rD0, 1, 8)
-ppc_decrypt_block_loop:
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		rD1,16(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LWH(rW0, 12)
-	evxor		rW3,rW3,rW1
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW5
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		 rD1,32(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,40(rKP)
-	evmergehi	rD0,rD0,rD1
-	EAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LWH(rW0, 12)
-	evxor		rW3,rW3,rW1
-	EAD(rD0, 3)
-	evxor		rD3,rD3,rW3
-	LWH(rW6, 0)
-	evxor		rD3,rD3,rW5
-	EAD(rD0, 1)
-	evmergehi	rD2,rD2,rD3
-	LWH(rW3, 8)
-	addi		rKP,rKP,32
-	bdnz		ppc_decrypt_block_loop
-	LAH(rW1, rD3, 0, 12)
-	LAL(rW0, rD2, 0, 12)
-	LAH(rW2, rD2, 1, 8)
-	LAL(rW2, rD3, 1, 8)
-	LAH(rW4, rD3, 2, 4)
-	LAL(rW4, rD0, 2, 4)
-	LAL(rW6, rD1, 3, 0)
-	LAH(rW5, rD1, 2, 4)
-	LAH(rW7, rD2, 3, 0)
-	LAL(rW7, rD3, 3, 0)
-	LAL(rW3, rD1, 1, 8)
-	evldw		 rD1,16(rKP)
-	EAD(rD0, 0)
-	evxor		rW4,rW4,rW6
-	LWL(rW1, 12)
-	evxor		rW0,rW0,rW4
-	EAD(rD2, 2)
-	evxor		rW0,rW0,rW2
-	LWL(rW5, 4)
-	evxor		rD1,rD1,rW0
-	evldw		rD3,24(rKP)
-	evmergehi	rD0,rD0,rD1
-	DAD(rD1, 0)
-	evxor		rW3,rW3,rW7
-	LBD(rW0)
-	evxor		rW3,rW3,rW1
-	DAD(rD0, 1)
-	evxor		rD3,rD3,rW3
-	LBD(rW6)
-	evxor		rD3,rD3,rW5
-	DAD(rD0, 0)
-	evmergehi	rD2,rD2,rD3
-	LBD(rW3)
-	LAD(rW2, rD3, 0)
-	LAD(rW1, rD2, 0)
-	LAD(rW4, rD2, 1)
-	LAD(rW5, rD3, 1)
-	LAD(rW7, rD1, 1)
-	rlwimi		rW0,rW4,8,16,23
-	rlwimi		rW1,rW5,8,16,23
-	LAD(rW4, rD3, 2)
-	LAD(rW5, rD0, 2)
-	rlwimi		rW2,rW6,8,16,23
-	rlwimi		rW3,rW7,8,16,23
-	LAD(rW6, rD1, 2)
-	LAD(rW7, rD2, 2)
-	rlwimi		rW0,rW4,16,8,15
-	rlwimi		rW1,rW5,16,8,15
-	LAD(rW4, rD0, 3)
-	LAD(rW5, rD1, 3)
-	rlwimi		rW2,rW6,16,8,15
-	lwz		rD0,32(rKP)
-	rlwimi		rW3,rW7,16,8,15
-	lwz		rD1,36(rKP)
-	LAD(rW6, rD2, 3)
-	LAD(rW7, rD3, 3)
-	rlwimi		rW0,rW4,24,0,7
-	lwz		rD2,40(rKP)
-	rlwimi		rW1,rW5,24,0,7
-	lwz		rD3,44(rKP)
-	rlwimi		rW2,rW6,24,0,7
-	rlwimi		rW3,rW7,24,0,7
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c
index efab78a3a8f6..7d2827e65240 100644
--- a/arch/powerpc/crypto/aes-spe-glue.c
+++ b/arch/powerpc/crypto/aes-spe-glue.c
@@ -51,30 +51,6 @@ struct ppc_xts_ctx {
 	u32 rounds;
 };
 
-extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
-extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
-extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes);
-extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes);
-extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_crypt_ctr  (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv);
-extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
-			    u32 bytes, u8 *iv, u32 *key_twk);
-extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
-			    u32 bytes, u8 *iv, u32 *key_twk);
-
-extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
-extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
-
-extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
-				     unsigned int key_len);
-
 static void spe_begin(void)
 {
 	/* disable preemption and save users SPE registers if required */
@@ -89,10 +65,10 @@ static void spe_end(void)
 	preempt_enable();
 }
 
-static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
-		unsigned int key_len)
+static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
+				   const u8 *in_key, unsigned int key_len)
 {
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 
 	switch (key_len) {
 	case AES_KEYSIZE_128:
@@ -116,12 +92,6 @@ static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
-				   const u8 *in_key, unsigned int key_len)
-{
-	return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
-}
-
 static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 		   unsigned int key_len)
 {
@@ -159,24 +129,6 @@ static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
 	return 0;
 }
 
-static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	spe_begin();
-	ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
-	spe_end();
-}
-
-static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	spe_begin();
-	ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
-	spe_end();
-}
-
 static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -410,26 +362,6 @@ static int ppc_xts_decrypt(struct skcipher_request *req)
  * with kmalloc() in the crypto infrastructure
  */
 
-static struct crypto_alg aes_cipher_alg = {
-	.cra_name		=	"aes",
-	.cra_driver_name	=	"aes-ppc-spe",
-	.cra_priority		=	300,
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		=	AES_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct ppc_aes_ctx),
-	.cra_alignmask		=	0,
-	.cra_module		=	THIS_MODULE,
-	.cra_u			=	{
-		.cipher = {
-			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
-			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
-			.cia_setkey		=	ppc_aes_setkey,
-			.cia_encrypt		=	ppc_aes_encrypt,
-			.cia_decrypt		=	ppc_aes_decrypt
-		}
-	}
-};
-
 static struct skcipher_alg aes_skcipher_algs[] = {
 	{
 		.base.cra_name		=	"ecb(aes)",
@@ -488,22 +420,12 @@ static struct skcipher_alg aes_skcipher_algs[] = {
 
 static int __init ppc_aes_mod_init(void)
 {
-	int err;
-
-	err = crypto_register_alg(&aes_cipher_alg);
-	if (err)
-		return err;
-
-	err = crypto_register_skciphers(aes_skcipher_algs,
-					ARRAY_SIZE(aes_skcipher_algs));
-	if (err)
-		crypto_unregister_alg(&aes_cipher_alg);
-	return err;
+	return crypto_register_skciphers(aes_skcipher_algs,
+					 ARRAY_SIZE(aes_skcipher_algs));
 }
 
 static void __exit ppc_aes_mod_fini(void)
 {
-	crypto_unregister_alg(&aes_cipher_alg);
 	crypto_unregister_skciphers(aes_skcipher_algs,
 				    ARRAY_SIZE(aes_skcipher_algs));
 }
diff --git a/arch/powerpc/crypto/aes-spe-keys.S b/arch/powerpc/crypto/aes-spe-keys.S
deleted file mode 100644
index 2e1bc0d099bf..000000000000
--- a/arch/powerpc/crypto/aes-spe-keys.S
+++ /dev/null
@@ -1,278 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Key handling functions for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-
-#ifdef __BIG_ENDIAN__
-#define LOAD_KEY(d, s, off) \
-	lwz		d,off(s);
-#else
-#define LOAD_KEY(d, s, off) \
-	li		r0,off; \
-	lwbrx		d,s,r0;
-#endif
-
-#define INITIALIZE_KEY \
-	stwu		r1,-32(r1);	/* create stack frame		*/ \
-	stw		r14,8(r1);	/* save registers		*/ \
-	stw		r15,12(r1);					   \
-	stw		r16,16(r1);
-
-#define FINALIZE_KEY \
-	lwz		r14,8(r1);	/* restore registers		*/ \
-	lwz		r15,12(r1);					   \
-	lwz		r16,16(r1);					   \
-	xor		r5,r5,r5;	/* clear sensitive data		*/ \
-	xor		r6,r6,r6;					   \
-	xor		r7,r7,r7;					   \
-	xor		r8,r8,r8;					   \
-	xor		r9,r9,r9;					   \
-	xor		r10,r10,r10;					   \
-	xor		r11,r11,r11;					   \
-	xor		r12,r12,r12;					   \
-	addi		r1,r1,32;	/* cleanup stack		*/
-
-#define LS_BOX(r, t1, t2) \
-	lis		t2,PPC_AES_4K_ENCTAB@h;				   \
-	ori		t2,t2,PPC_AES_4K_ENCTAB@l;			   \
-	rlwimi		t2,r,4,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,0,24,31;					   \
-	rlwimi		t2,r,28,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,8,16,23;					   \
-	rlwimi		t2,r,20,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,16,8,15;					   \
-	rlwimi		t2,r,12,20,27;					   \
-	lbz		t1,8(t2);					   \
-	rlwimi		r,t1,24,0,7;
-
-#define GF8_MUL(out, in, t1, t2) \
-	lis t1,0x8080;			/* multiplication in GF8	*/ \
-	ori t1,t1,0x8080; 						   \
-	and t1,t1,in; 							   \
-	srwi t1,t1,7; 							   \
-	mulli t1,t1,0x1b; 						   \
-	lis t2,0x7f7f; 							   \
-	ori t2,t2,0x7f7f; 						   \
-	and t2,t2,in; 							   \
-	slwi t2,t2,1; 							   \
-	xor out,t1,t2;
-
-/*
- * ppc_expand_key_128(u32 *key_enc, const u8 *key)
- *
- * Expand 128 bit key into 176 bytes encryption key. It consists of
- * key itself plus 10 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_128)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	stw		r5,0(r3)	/* key[0..3] = input data	*/
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	li		r16,10		/* 10 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_128_loop:
-	addi		r3,r3,16
-	mr		r14,r8		/* apply LS_BOX to 4th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor next 4 keys		*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	stw		r5,0(r3)	/* store next 4 keys		*/
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO by 2 in GF	*/
-	subi		r16,r16,1
-	cmpwi		r16,0
-	bt		eq,ppc_expand_128_end
-	b		ppc_expand_128_loop
-ppc_expand_128_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_expand_key_192(u32 *key_enc, const u8 *key)
- *
- * Expand 192 bit key into 208 bytes encryption key. It consists of key
- * itself plus 12 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_192)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	LOAD_KEY(r9,r4,16)
-	LOAD_KEY(r10,r4,20)
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	li		r16,8		/* 8 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_192_loop:
-	addi		r3,r3,24
-	mr		r14,r10		/* apply LS_BOX to 6th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor next 6 keys		*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	xor		r9,r9,r8
-	xor		r10,r10,r9
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	subi		r16,r16,1
-	cmpwi		r16,0		/* last round early kick out	*/
-	bt		eq,ppc_expand_192_end
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	GF8_MUL(r0, r0, r4, r14)	/* multiply RCO GF8		*/
-	b		ppc_expand_192_loop
-ppc_expand_192_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_expand_key_256(u32 *key_enc, const u8 *key)
- *
- * Expand 256 bit key into 240 bytes encryption key. It consists of key
- * itself plus 14 rounds with 16 bytes each
- *
- */
-_GLOBAL(ppc_expand_key_256)
-	INITIALIZE_KEY
-	LOAD_KEY(r5,r4,0)
-	LOAD_KEY(r6,r4,4)
-	LOAD_KEY(r7,r4,8)
-	LOAD_KEY(r8,r4,12)
-	LOAD_KEY(r9,r4,16)
-	LOAD_KEY(r10,r4,20)
-	LOAD_KEY(r11,r4,24)
-	LOAD_KEY(r12,r4,28)
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	stw		r11,24(r3)
-	stw		r12,28(r3)
-	li		r16,7		/* 7 expansion rounds		*/
-	lis		r0,0x0100	/* RCO(1)			*/
-ppc_expand_256_loop:
-	addi		r3,r3,32
-	mr		r14,r12		/* apply LS_BOX to 8th temp	*/
-	rotlwi		r14,r14,8
-	LS_BOX(r14, r15, r4)
-	xor		r14,r14,r0
-	xor		r5,r5,r14	/* xor 4 keys			*/
-	xor		r6,r6,r5
-	xor		r7,r7,r6
-	xor		r8,r8,r7
-	mr		r14,r8
-	LS_BOX(r14, r15, r4)		/* apply LS_BOX to 4th temp	*/
-	xor		r9,r9,r14	/* xor 4 keys			*/
-	xor		r10,r10,r9
-	xor		r11,r11,r10
-	xor		r12,r12,r11
-	stw		r5,0(r3)
-	stw		r6,4(r3)
-	stw		r7,8(r3)
-	stw		r8,12(r3)
-	subi		r16,r16,1
-	cmpwi		r16,0		/* last round early kick out	*/
-	bt		eq,ppc_expand_256_end
-	stw		r9,16(r3)
-	stw		r10,20(r3)
-	stw		r11,24(r3)
-	stw		r12,28(r3)
-	GF8_MUL(r0, r0, r4, r14)
-	b		ppc_expand_256_loop
-ppc_expand_256_end:
-	FINALIZE_KEY
-	blr
-
-/*
- * ppc_generate_decrypt_key: derive decryption key from encryption key
- * number of bytes to handle are calculated from length of key (16/24/32)
- *
- */
-_GLOBAL(ppc_generate_decrypt_key)
-	addi		r6,r5,24
-	slwi		r6,r6,2
-	lwzx		r7,r4,r6	/* first/last 4 words are same	*/
-	stw		r7,0(r3)
-	lwz		r7,0(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,4(r3)
-	lwz		r7,4(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,8(r3)
-	lwz		r7,8(r4)
-	stwx		r7,r3,r6
-	addi		r6,r6,4
-	lwzx		r7,r4,r6
-	stw		r7,12(r3)
-	lwz		r7,12(r4)
-	stwx		r7,r3,r6
-	addi		r3,r3,16
-	add		r4,r4,r6
-	subi		r4,r4,28
-	addi		r5,r5,20
-	srwi		r5,r5,2
-ppc_generate_decrypt_block:
-	li	r6,4
-	mtctr	r6
-ppc_generate_decrypt_word:
-	lwz		r6,0(r4)
-	GF8_MUL(r7, r6, r0, r7)
-	GF8_MUL(r8, r7, r0, r8)
-	GF8_MUL(r9, r8, r0, r9)
-	xor		r10,r9,r6
-	xor		r11,r7,r8
-	xor		r11,r11,r9
-	xor		r12,r7,r10
-	rotrwi		r12,r12,24
-	xor		r11,r11,r12
-	xor		r12,r8,r10
-	rotrwi		r12,r12,16
-	xor		r11,r11,r12
-	rotrwi		r12,r10,8
-	xor		r11,r11,r12
-	stw		r11,0(r3)
-	addi		r3,r3,4
-	addi		r4,r4,4
-	bdnz		ppc_generate_decrypt_word
-	subi		r4,r4,32
-	subi		r5,r5,1
-	cmpwi		r5,0
-	bt		gt,ppc_generate_decrypt_block
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-modes.S b/arch/powerpc/crypto/aes-spe-modes.S
deleted file mode 100644
index 3f92a6a85785..000000000000
--- a/arch/powerpc/crypto/aes-spe-modes.S
+++ /dev/null
@@ -1,625 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#include <asm/ppc_asm.h>
-#include "aes-spe-regs.h"
-
-#ifdef __BIG_ENDIAN__			/* Macros for big endian builds	*/
-
-#define LOAD_DATA(reg, off) \
-	lwz		reg,off(rSP);	/* load with offset		*/
-#define SAVE_DATA(reg, off) \
-	stw		reg,off(rDP);	/* save with offset		*/
-#define NEXT_BLOCK \
-	addi		rSP,rSP,16;	/* increment pointers per bloc	*/ \
-	addi		rDP,rDP,16;
-#define LOAD_IV(reg, off) \
-	lwz		reg,off(rIP);	/* IV loading with offset	*/
-#define SAVE_IV(reg, off) \
-	stw		reg,off(rIP);	/* IV saving with offset	*/
-#define START_IV			/* nothing to reset		*/
-#define CBC_DEC 16			/* CBC decrement per block	*/
-#define CTR_DEC 1			/* CTR decrement one byte	*/
-
-#else					/* Macros for little endian	*/
-
-#define LOAD_DATA(reg, off) \
-	lwbrx		reg,0,rSP;	/* load reversed		*/ \
-	addi		rSP,rSP,4;	/* and increment pointer	*/
-#define SAVE_DATA(reg, off) \
-	stwbrx		reg,0,rDP;	/* save reversed		*/ \
-	addi		rDP,rDP,4;	/* and increment pointer	*/
-#define NEXT_BLOCK			/* nothing todo			*/
-#define LOAD_IV(reg, off) \
-	lwbrx		reg,0,rIP;	/* load reversed		*/ \
-	addi		rIP,rIP,4;	/* and increment pointer	*/
-#define SAVE_IV(reg, off) \
-	stwbrx		reg,0,rIP;	/* load reversed		*/ \
-	addi		rIP,rIP,4;	/* and increment pointer	*/
-#define START_IV \
-	subi		rIP,rIP,16;	/* must reset pointer		*/
-#define CBC_DEC 32			/* 2 blocks because of incs	*/
-#define CTR_DEC 17			/* 1 block because of incs	*/
-
-#endif
-
-#define SAVE_0_REGS
-#define LOAD_0_REGS
-
-#define SAVE_4_REGS \
-	stw		rI0,96(r1);	/* save 32 bit registers	*/ \
-	stw		rI1,100(r1);					   \
-	stw		rI2,104(r1);					   \
-	stw		rI3,108(r1);
-
-#define LOAD_4_REGS \
-	lwz		rI0,96(r1);	/* restore 32 bit registers	*/ \
-	lwz		rI1,100(r1);					   \
-	lwz		rI2,104(r1);					   \
-	lwz		rI3,108(r1);
-
-#define SAVE_8_REGS \
-	SAVE_4_REGS							   \
-	stw		rG0,112(r1);	/* save 32 bit registers	*/ \
-	stw		rG1,116(r1);					   \
-	stw		rG2,120(r1);					   \
-	stw		rG3,124(r1);
-
-#define LOAD_8_REGS \
-	LOAD_4_REGS							   \
-	lwz		rG0,112(r1);	/* restore 32 bit registers	*/ \
-	lwz		rG1,116(r1);					   \
-	lwz		rG2,120(r1);					   \
-	lwz		rG3,124(r1);
-
-#define INITIALIZE_CRYPT(tab,nr32bitregs) \
-	mflr		r0;						   \
-	stwu		r1,-160(r1);	/* create stack frame		*/ \
-	lis		rT0,tab@h;	/* en-/decryption table pointer	*/ \
-	stw		r0,8(r1);	/* save link register		*/ \
-	ori		rT0,rT0,tab@l;					   \
-	evstdw		r14,16(r1);					   \
-	mr		rKS,rKP;					   \
-	evstdw		r15,24(r1);	/* We must save non volatile	*/ \
-	evstdw		r16,32(r1);	/* registers. Take the chance	*/ \
-	evstdw		r17,40(r1);	/* and save the SPE part too	*/ \
-	evstdw		r18,48(r1);					   \
-	evstdw		r19,56(r1);					   \
-	evstdw		r20,64(r1);					   \
-	evstdw		r21,72(r1);					   \
-	evstdw		r22,80(r1);					   \
-	evstdw		r23,88(r1);					   \
-	SAVE_##nr32bitregs##_REGS
-
-#define FINALIZE_CRYPT(nr32bitregs) \
-	lwz		r0,8(r1);					   \
-	evldw		r14,16(r1);	/* restore SPE registers	*/ \
-	evldw		r15,24(r1);					   \
-	evldw		r16,32(r1);					   \
-	evldw		r17,40(r1);					   \
-	evldw		r18,48(r1);					   \
-	evldw		r19,56(r1);					   \
-	evldw		r20,64(r1);					   \
-	evldw		r21,72(r1);					   \
-	evldw		r22,80(r1);					   \
-	evldw		r23,88(r1);					   \
-	LOAD_##nr32bitregs##_REGS					   \
-	mtlr		r0;		/* restore link register	*/ \
-	xor		r0,r0,r0;					   \
-	stw		r0,16(r1);	/* delete sensitive data	*/ \
-	stw		r0,24(r1);	/* that we might have pushed	*/ \
-	stw		r0,32(r1);	/* from other context that runs	*/ \
-	stw		r0,40(r1);	/* the same code		*/ \
-	stw		r0,48(r1);					   \
-	stw		r0,56(r1);					   \
-	stw		r0,64(r1);					   \
-	stw		r0,72(r1);					   \
-	stw		r0,80(r1);					   \
-	stw		r0,88(r1);					   \
-	addi		r1,r1,160;	/* cleanup stack frame		*/
-
-#define ENDIAN_SWAP(t0, t1, s0, s1) \
-	rotrwi		t0,s0,8;	/* swap endianness for 2 GPRs	*/ \
-	rotrwi		t1,s1,8;					   \
-	rlwimi		t0,s0,8,8,15;					   \
-	rlwimi		t1,s1,8,8,15;					   \
-	rlwimi		t0,s0,8,24,31;					   \
-	rlwimi		t1,s1,8,24,31;
-
-#define GF128_MUL(d0, d1, d2, d3, t0) \
-	li		t0,0x87;	/* multiplication in GF128	*/ \
-	cmpwi		d3,-1;						   \
-	iselgt		t0,0,t0;					   \
-	rlwimi		d3,d2,0,0,0;	/* propagate "carry" bits	*/ \
-	rotlwi		d3,d3,1;					   \
-	rlwimi		d2,d1,0,0,0;					   \
-	rotlwi		d2,d2,1;					   \
-	rlwimi		d1,d0,0,0,0;					   \
-	slwi		d0,d0,1;	/* shift left 128 bit		*/ \
-	rotlwi		d1,d1,1;					   \
-	xor		d0,d0,t0;
-
-#define START_KEY(d0, d1, d2, d3) \
-	lwz		rW0,0(rKP);					   \
-	mtctr		rRR;						   \
-	lwz		rW1,4(rKP);					   \
-	lwz		rW2,8(rKP);					   \
-	lwz		rW3,12(rKP);					   \
-	xor		rD0,d0,rW0;					   \
-	xor		rD1,d1,rW1;					   \
-	xor		rD2,d2,rW2;					   \
-	xor		rD3,d3,rW3;
-
-/*
- * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds)
- *
- * called from glue layer to encrypt a single 16 byte block
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_aes)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
-	LOAD_DATA(rD0, 0)
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds)
- *
- * called from glue layer to decrypt a single 16 byte block
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_aes)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0)
-	LOAD_DATA(rD0, 0)
-	addi		rT1,rT0,4096
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds, u32 bytes);
- *
- * called from glue layer to encrypt multiple blocks via ECB
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_ecb)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0)
-ppc_encrypt_ecb_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_ecb_loop
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 bytes);
- *
- * called from glue layer to decrypt multiple blocks via ECB
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_ecb)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0)
-	addi		rT1,rT0,4096
-ppc_decrypt_ecb_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_decrypt_ecb_loop
-	FINALIZE_CRYPT(0)
-	blr
-
-/*
- * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc,
- *		   32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to encrypt multiple blocks via CBC
- * Bytes must be larger or equal 16 and only whole blocks are
- * processed. round values are AES128 = 4, AES192 = 5 and
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_cbc)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	LOAD_IV(rI3, 12)
-ppc_encrypt_cbc_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	cmpwi		rLN,15
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	SAVE_DATA(rI0, 0)
-	xor		rI1,rD1,rW1
-	SAVE_DATA(rI1, 4)
-	xor		rI2,rD2,rW2
-	SAVE_DATA(rI2, 8)
-	xor		rI3,rD3,rW3
-	SAVE_DATA(rI3, 12)
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_cbc_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to decrypt multiple blocks via CBC
- * round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_cbc)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4)
-	li		rT1,15
-	LOAD_IV(rI0, 0)
-	andc		rLN,rLN,rT1
-	LOAD_IV(rI1, 4)
-	subi		rLN,rLN,16
-	LOAD_IV(rI2, 8)
-	add		rSP,rSP,rLN	/* reverse processing		*/
-	LOAD_IV(rI3, 12)
-	add		rDP,rDP,rLN
-	LOAD_DATA(rD0, 0)
-	addi		rT1,rT0,4096
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	START_IV
-	SAVE_IV(rD0, 0)
-	SAVE_IV(rD1, 4)
-	SAVE_IV(rD2, 8)
-	cmpwi		rLN,16
-	SAVE_IV(rD3, 12)
-	bt		lt,ppc_decrypt_cbc_end
-ppc_decrypt_cbc_loop:
-	mr		rKP,rKS
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	subi		rLN,rLN,16
-	subi		rSP,rSP,CBC_DEC
-	xor		rW0,rD0,rW0
-	LOAD_DATA(rD0, 0)
-	xor		rW1,rD1,rW1
-	LOAD_DATA(rD1, 4)
-	xor		rW2,rD2,rW2
-	LOAD_DATA(rD2, 8)
-	xor		rW3,rD3,rW3
-	LOAD_DATA(rD3, 12)
-	xor		rW0,rW0,rD0
-	SAVE_DATA(rW0, 0)
-	xor		rW1,rW1,rD1
-	SAVE_DATA(rW1, 4)
-	xor		rW2,rW2,rD2
-	SAVE_DATA(rW2, 8)
-	xor		rW3,rW3,rD3
-	SAVE_DATA(rW3, 12)
-	cmpwi		rLN,15
-	subi		rDP,rDP,CBC_DEC
-	bt		gt,ppc_decrypt_cbc_loop
-ppc_decrypt_cbc_end:
-	mr		rKP,rKS
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rW0,rW0,rD0
-	xor		rW1,rW1,rD1
-	xor		rW2,rW2,rD2
-	xor		rW3,rW3,rD3
-	xor		rW0,rW0,rI0	/* decrypt with initial IV	*/
-	SAVE_DATA(rW0, 0)
-	xor		rW1,rW1,rI1
-	SAVE_DATA(rW1, 4)
-	xor		rW2,rW2,rI2
-	SAVE_DATA(rW2, 8)
-	xor		rW3,rW3,rI3
-	SAVE_DATA(rW3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc,
- *		 u32 rounds, u32 bytes, u8 *iv);
- *
- * called from glue layer to encrypt/decrypt multiple blocks
- * via CTR. Number of bytes does not need to be a multiple of
- * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6
- *
- */
-_GLOBAL(ppc_crypt_ctr)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rLN,16
-	LOAD_IV(rI3, 12)
-	START_IV
-	bt		lt,ppc_crypt_ctr_partial
-ppc_crypt_ctr_loop:
-	mr		rKP,rKS
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rW0,rD0,rW0
-	xor		rW1,rD1,rW1
-	xor		rW2,rD2,rW2
-	xor		rW3,rD3,rW3
-	LOAD_DATA(rD0, 0)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD1, 4)
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rW0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rW1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rW2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rW3
-	SAVE_DATA(rD3, 12)
-	addic		rI3,rI3,1	/* increase counter			*/
-	addze		rI2,rI2
-	addze		rI1,rI1
-	addze		rI0,rI0
-	NEXT_BLOCK
-	cmpwi		rLN,15
-	bt		gt,ppc_crypt_ctr_loop
-ppc_crypt_ctr_partial:
-	cmpwi		rLN,0
-	bt		eq,ppc_crypt_ctr_end
-	mr		rKP,rKS
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rW0,rD0,rW0
-	SAVE_IV(rW0, 0)
-	xor		rW1,rD1,rW1
-	SAVE_IV(rW1, 4)
-	xor		rW2,rD2,rW2
-	SAVE_IV(rW2, 8)
-	xor		rW3,rD3,rW3
-	SAVE_IV(rW3, 12)
-	mtctr		rLN
-	subi		rIP,rIP,CTR_DEC
-	subi		rSP,rSP,1
-	subi		rDP,rDP,1
-ppc_crypt_ctr_xorbyte:
-	lbzu		rW4,1(rIP)	/* bytewise xor for partial block	*/
-	lbzu		rW5,1(rSP)
-	xor		rW4,rW4,rW5
-	stbu		rW4,1(rDP)
-	bdnz		ppc_crypt_ctr_xorbyte
-	subf		rIP,rLN,rIP
-	addi		rIP,rIP,1
-	addic		rI3,rI3,1
-	addze		rI2,rI2
-	addze		rI1,rI1
-	addze		rI0,rI0
-ppc_crypt_ctr_end:
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(4)
-	blr
-
-/*
- * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc,
- *		   u32 rounds, u32 bytes, u8 *iv, u32 *key_twk);
- *
- * called from glue layer to encrypt multiple blocks via XTS
- * If key_twk is given, the initial IV encryption will be
- * processed too. Round values are AES128 = 4, AES192 = 5,
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_encrypt_xts)
-	INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8)
-	LOAD_IV(rI0, 0)
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rKT,0
-	LOAD_IV(rI3, 12)
-	bt		eq,ppc_encrypt_xts_notweak
-	mr		rKP,rKT
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	xor		rI1,rD1,rW1
-	xor		rI2,rD2,rW2
-	xor		rI3,rD3,rW3
-ppc_encrypt_xts_notweak:
-	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
-	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
-ppc_encrypt_xts_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_encrypt_block
-	xor		rD0,rD0,rW0
-	xor		rD1,rD1,rW1
-	xor		rD2,rD2,rW2
-	xor		rD3,rD3,rW3
-	xor		rD0,rD0,rI0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rI1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rI2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rI3
-	SAVE_DATA(rD3, 12)
-	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
-	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
-	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
-	cmpwi		rLN,0
-	NEXT_BLOCK
-	bt		gt,ppc_encrypt_xts_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(8)
-	blr
-
-/*
- * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec,
- *		   u32 rounds, u32 blocks, u8 *iv, u32 *key_twk);
- *
- * called from glue layer to decrypt multiple blocks via XTS
- * If key_twk is given, the initial IV encryption will be
- * processed too. Round values are AES128 = 4, AES192 = 5,
- * AES256 = 6
- *
- */
-_GLOBAL(ppc_decrypt_xts)
-	INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8)
-	LOAD_IV(rI0, 0)
-	addi		rT1,rT0,4096
-	LOAD_IV(rI1, 4)
-	LOAD_IV(rI2, 8)
-	cmpwi		rKT,0
-	LOAD_IV(rI3, 12)
-	bt		eq,ppc_decrypt_xts_notweak
-	subi		rT0,rT0,4096
-	mr		rKP,rKT
-	START_KEY(rI0, rI1, rI2, rI3)
-	bl		ppc_encrypt_block
-	xor		rI0,rD0,rW0
-	xor		rI1,rD1,rW1
-	xor		rI2,rD2,rW2
-	xor		rI3,rD3,rW3
-	addi		rT0,rT0,4096
-ppc_decrypt_xts_notweak:
-	ENDIAN_SWAP(rG0, rG1, rI0, rI1)
-	ENDIAN_SWAP(rG2, rG3, rI2, rI3)
-ppc_decrypt_xts_loop:
-	LOAD_DATA(rD0, 0)
-	mr		rKP,rKS
-	LOAD_DATA(rD1, 4)
-	subi		rLN,rLN,16
-	LOAD_DATA(rD2, 8)
-	LOAD_DATA(rD3, 12)
-	xor		rD0,rD0,rI0
-	xor		rD1,rD1,rI1
-	xor		rD2,rD2,rI2
-	xor		rD3,rD3,rI3
-	START_KEY(rD0, rD1, rD2, rD3)
-	bl		ppc_decrypt_block
-	xor		rD0,rD0,rW0
-	xor		rD1,rD1,rW1
-	xor		rD2,rD2,rW2
-	xor		rD3,rD3,rW3
-	xor		rD0,rD0,rI0
-	SAVE_DATA(rD0, 0)
-	xor		rD1,rD1,rI1
-	SAVE_DATA(rD1, 4)
-	xor		rD2,rD2,rI2
-	SAVE_DATA(rD2, 8)
-	xor		rD3,rD3,rI3
-	SAVE_DATA(rD3, 12)
-	GF128_MUL(rG0, rG1, rG2, rG3, rW0)
-	ENDIAN_SWAP(rI0, rI1, rG0, rG1)
-	ENDIAN_SWAP(rI2, rI3, rG2, rG3)
-	cmpwi		rLN,0
-	NEXT_BLOCK
-	bt		gt,ppc_decrypt_xts_loop
-	START_IV
-	SAVE_IV(rI0, 0)
-	SAVE_IV(rI1, 4)
-	SAVE_IV(rI2, 8)
-	SAVE_IV(rI3, 12)
-	FINALIZE_CRYPT(8)
-	blr
diff --git a/arch/powerpc/crypto/aes-spe-regs.h b/arch/powerpc/crypto/aes-spe-regs.h
deleted file mode 100644
index 2eb4c9b94152..000000000000
--- a/arch/powerpc/crypto/aes-spe-regs.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Common registers for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-#define rKS r0	/* copy of en-/decryption key pointer			*/
-#define rDP r3	/* destination pointer					*/
-#define rSP r4	/* source pointer					*/
-#define rKP r5	/* pointer to en-/decryption key pointer		*/
-#define rRR r6	/* en-/decryption rounds				*/
-#define rLN r7	/* length of data to be processed			*/
-#define rIP r8	/* potiner to IV (CBC/CTR/XTS modes)			*/
-#define rKT r9	/* pointer to tweak key (XTS mode)			*/
-#define rT0 r11	/* pointers to en-/decryption tables			*/
-#define rT1 r10
-#define rD0 r9	/* data 						*/
-#define rD1 r14
-#define rD2 r12
-#define rD3 r15
-#define rW0 r16	/* working registers					*/
-#define rW1 r17
-#define rW2 r18
-#define rW3 r19
-#define rW4 r20
-#define rW5 r21
-#define rW6 r22
-#define rW7 r23
-#define rI0 r24	/* IV							*/
-#define rI1 r25
-#define rI2 r26
-#define rI3 r27
-#define rG0 r28	/* endian reversed tweak (XTS mode)			*/
-#define rG1 r29
-#define rG2 r30
-#define rG3 r31
diff --git a/arch/powerpc/crypto/aes-tab-4k.S b/arch/powerpc/crypto/aes-tab-4k.S
deleted file mode 100644
index ceb604bc6f72..000000000000
--- a/arch/powerpc/crypto/aes-tab-4k.S
+++ /dev/null
@@ -1,326 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * 4K AES tables for PPC AES implementation
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-
-/*
- * These big endian AES encryption/decryption tables have been taken from
- * crypto/aes_generic.c and are designed to be simply accessed by a combination
- * of rlwimi/lwz instructions with a minimum of table registers (usually only
- * one required). Thus they are aligned to 4K. The locality of rotated values
- * is derived from the reduced offsets that are available in the SPE load
- * instructions. E.g. evldw, evlwwsplat, ...
- *
- * For the safety-conscious it has to be noted that they might be vulnerable
- * to cache timing attacks because of their size. Nevertheless in contrast to
- * the generic tables they have been reduced from 16KB to 8KB + 256 bytes.
- * This is a quite good tradeoff for low power devices (e.g. routers) without
- * dedicated encryption hardware where we usually have no multiuser
- * environment.
- *
- */
-
-#define R(a, b, c, d) \
-	0x##a##b##c##d, 0x##d##a##b##c, 0x##c##d##a##b, 0x##b##c##d##a
-
-.data
-.align 12
-.globl PPC_AES_4K_ENCTAB
-PPC_AES_4K_ENCTAB:
-/* encryption table, same as crypto_ft_tab in crypto/aes-generic.c */
-	.long R(c6, 63, 63, a5), R(f8, 7c, 7c, 84)
-	.long R(ee, 77, 77, 99), R(f6, 7b, 7b, 8d)
-	.long R(ff, f2, f2, 0d), R(d6, 6b, 6b, bd)
-	.long R(de, 6f, 6f, b1), R(91, c5, c5, 54)
-	.long R(60, 30, 30, 50), R(02, 01, 01, 03)
-	.long R(ce, 67, 67, a9), R(56, 2b, 2b, 7d)
-	.long R(e7, fe, fe, 19), R(b5, d7, d7, 62)
-	.long R(4d, ab, ab, e6), R(ec, 76, 76, 9a)
-	.long R(8f, ca, ca, 45), R(1f, 82, 82, 9d)
-	.long R(89, c9, c9, 40), R(fa, 7d, 7d, 87)
-	.long R(ef, fa, fa, 15), R(b2, 59, 59, eb)
-	.long R(8e, 47, 47, c9), R(fb, f0, f0, 0b)
-	.long R(41, ad, ad, ec), R(b3, d4, d4, 67)
-	.long R(5f, a2, a2, fd), R(45, af, af, ea)
-	.long R(23, 9c, 9c, bf), R(53, a4, a4, f7)
-	.long R(e4, 72, 72, 96), R(9b, c0, c0, 5b)
-	.long R(75, b7, b7, c2), R(e1, fd, fd, 1c)
-	.long R(3d, 93, 93, ae), R(4c, 26, 26, 6a)
-	.long R(6c, 36, 36, 5a), R(7e, 3f, 3f, 41)
-	.long R(f5, f7, f7, 02), R(83, cc, cc, 4f)
-	.long R(68, 34, 34, 5c), R(51, a5, a5, f4)
-	.long R(d1, e5, e5, 34), R(f9, f1, f1, 08)
-	.long R(e2, 71, 71, 93), R(ab, d8, d8, 73)
-	.long R(62, 31, 31, 53), R(2a, 15, 15, 3f)
-	.long R(08, 04, 04, 0c), R(95, c7, c7, 52)
-	.long R(46, 23, 23, 65), R(9d, c3, c3, 5e)
-	.long R(30, 18, 18, 28), R(37, 96, 96, a1)
-	.long R(0a, 05, 05, 0f), R(2f, 9a, 9a, b5)
-	.long R(0e, 07, 07, 09), R(24, 12, 12, 36)
-	.long R(1b, 80, 80, 9b), R(df, e2, e2, 3d)
-	.long R(cd, eb, eb, 26), R(4e, 27, 27, 69)
-	.long R(7f, b2, b2, cd), R(ea, 75, 75, 9f)
-	.long R(12, 09, 09, 1b), R(1d, 83, 83, 9e)
-	.long R(58, 2c, 2c, 74), R(34, 1a, 1a, 2e)
-	.long R(36, 1b, 1b, 2d), R(dc, 6e, 6e, b2)
-	.long R(b4, 5a, 5a, ee), R(5b, a0, a0, fb)
-	.long R(a4, 52, 52, f6), R(76, 3b, 3b, 4d)
-	.long R(b7, d6, d6, 61), R(7d, b3, b3, ce)
-	.long R(52, 29, 29, 7b), R(dd, e3, e3, 3e)
-	.long R(5e, 2f, 2f, 71), R(13, 84, 84, 97)
-	.long R(a6, 53, 53, f5), R(b9, d1, d1, 68)
-	.long R(00, 00, 00, 00), R(c1, ed, ed, 2c)
-	.long R(40, 20, 20, 60), R(e3, fc, fc, 1f)
-	.long R(79, b1, b1, c8), R(b6, 5b, 5b, ed)
-	.long R(d4, 6a, 6a, be), R(8d, cb, cb, 46)
-	.long R(67, be, be, d9), R(72, 39, 39, 4b)
-	.long R(94, 4a, 4a, de), R(98, 4c, 4c, d4)
-	.long R(b0, 58, 58, e8), R(85, cf, cf, 4a)
-	.long R(bb, d0, d0, 6b), R(c5, ef, ef, 2a)
-	.long R(4f, aa, aa, e5), R(ed, fb, fb, 16)
-	.long R(86, 43, 43, c5), R(9a, 4d, 4d, d7)
-	.long R(66, 33, 33, 55), R(11, 85, 85, 94)
-	.long R(8a, 45, 45, cf), R(e9, f9, f9, 10)
-	.long R(04, 02, 02, 06), R(fe, 7f, 7f, 81)
-	.long R(a0, 50, 50, f0), R(78, 3c, 3c, 44)
-	.long R(25, 9f, 9f, ba), R(4b, a8, a8, e3)
-	.long R(a2, 51, 51, f3), R(5d, a3, a3, fe)
-	.long R(80, 40, 40, c0), R(05, 8f, 8f, 8a)
-	.long R(3f, 92, 92, ad), R(21, 9d, 9d, bc)
-	.long R(70, 38, 38, 48), R(f1, f5, f5, 04)
-	.long R(63, bc, bc, df), R(77, b6, b6, c1)
-	.long R(af, da, da, 75), R(42, 21, 21, 63)
-	.long R(20, 10, 10, 30), R(e5, ff, ff, 1a)
-	.long R(fd, f3, f3, 0e), R(bf, d2, d2, 6d)
-	.long R(81, cd, cd, 4c), R(18, 0c, 0c, 14)
-	.long R(26, 13, 13, 35), R(c3, ec, ec, 2f)
-	.long R(be, 5f, 5f, e1), R(35, 97, 97, a2)
-	.long R(88, 44, 44, cc), R(2e, 17, 17, 39)
-	.long R(93, c4, c4, 57), R(55, a7, a7, f2)
-	.long R(fc, 7e, 7e, 82), R(7a, 3d, 3d, 47)
-	.long R(c8, 64, 64, ac), R(ba, 5d, 5d, e7)
-	.long R(32, 19, 19, 2b), R(e6, 73, 73, 95)
-	.long R(c0, 60, 60, a0), R(19, 81, 81, 98)
-	.long R(9e, 4f, 4f, d1), R(a3, dc, dc, 7f)
-	.long R(44, 22, 22, 66), R(54, 2a, 2a, 7e)
-	.long R(3b, 90, 90, ab), R(0b, 88, 88, 83)
-	.long R(8c, 46, 46, ca), R(c7, ee, ee, 29)
-	.long R(6b, b8, b8, d3), R(28, 14, 14, 3c)
-	.long R(a7, de, de, 79), R(bc, 5e, 5e, e2)
-	.long R(16, 0b, 0b, 1d), R(ad, db, db, 76)
-	.long R(db, e0, e0, 3b), R(64, 32, 32, 56)
-	.long R(74, 3a, 3a, 4e), R(14, 0a, 0a, 1e)
-	.long R(92, 49, 49, db), R(0c, 06, 06, 0a)
-	.long R(48, 24, 24, 6c), R(b8, 5c, 5c, e4)
-	.long R(9f, c2, c2, 5d), R(bd, d3, d3, 6e)
-	.long R(43, ac, ac, ef), R(c4, 62, 62, a6)
-	.long R(39, 91, 91, a8), R(31, 95, 95, a4)
-	.long R(d3, e4, e4, 37), R(f2, 79, 79, 8b)
-	.long R(d5, e7, e7, 32), R(8b, c8, c8, 43)
-	.long R(6e, 37, 37, 59), R(da, 6d, 6d, b7)
-	.long R(01, 8d, 8d, 8c), R(b1, d5, d5, 64)
-	.long R(9c, 4e, 4e, d2), R(49, a9, a9, e0)
-	.long R(d8, 6c, 6c, b4), R(ac, 56, 56, fa)
-	.long R(f3, f4, f4, 07), R(cf, ea, ea, 25)
-	.long R(ca, 65, 65, af), R(f4, 7a, 7a, 8e)
-	.long R(47, ae, ae, e9), R(10, 08, 08, 18)
-	.long R(6f, ba, ba, d5), R(f0, 78, 78, 88)
-	.long R(4a, 25, 25, 6f), R(5c, 2e, 2e, 72)
-	.long R(38, 1c, 1c, 24), R(57, a6, a6, f1)
-	.long R(73, b4, b4, c7), R(97, c6, c6, 51)
-	.long R(cb, e8, e8, 23), R(a1, dd, dd, 7c)
-	.long R(e8, 74, 74, 9c), R(3e, 1f, 1f, 21)
-	.long R(96, 4b, 4b, dd), R(61, bd, bd, dc)
-	.long R(0d, 8b, 8b, 86), R(0f, 8a, 8a, 85)
-	.long R(e0, 70, 70, 90), R(7c, 3e, 3e, 42)
-	.long R(71, b5, b5, c4), R(cc, 66, 66, aa)
-	.long R(90, 48, 48, d8), R(06, 03, 03, 05)
-	.long R(f7, f6, f6, 01), R(1c, 0e, 0e, 12)
-	.long R(c2, 61, 61, a3), R(6a, 35, 35, 5f)
-	.long R(ae, 57, 57, f9), R(69, b9, b9, d0)
-	.long R(17, 86, 86, 91), R(99, c1, c1, 58)
-	.long R(3a, 1d, 1d, 27), R(27, 9e, 9e, b9)
-	.long R(d9, e1, e1, 38), R(eb, f8, f8, 13)
-	.long R(2b, 98, 98, b3), R(22, 11, 11, 33)
-	.long R(d2, 69, 69, bb), R(a9, d9, d9, 70)
-	.long R(07, 8e, 8e, 89), R(33, 94, 94, a7)
-	.long R(2d, 9b, 9b, b6), R(3c, 1e, 1e, 22)
-	.long R(15, 87, 87, 92), R(c9, e9, e9, 20)
-	.long R(87, ce, ce, 49), R(aa, 55, 55, ff)
-	.long R(50, 28, 28, 78), R(a5, df, df, 7a)
-	.long R(03, 8c, 8c, 8f), R(59, a1, a1, f8)
-	.long R(09, 89, 89, 80), R(1a, 0d, 0d, 17)
-	.long R(65, bf, bf, da), R(d7, e6, e6, 31)
-	.long R(84, 42, 42, c6), R(d0, 68, 68, b8)
-	.long R(82, 41, 41, c3), R(29, 99, 99, b0)
-	.long R(5a, 2d, 2d, 77), R(1e, 0f, 0f, 11)
-	.long R(7b, b0, b0, cb), R(a8, 54, 54, fc)
-	.long R(6d, bb, bb, d6), R(2c, 16, 16, 3a)
-.globl PPC_AES_4K_DECTAB
-PPC_AES_4K_DECTAB:
-/* decryption table, same as crypto_it_tab in crypto/aes-generic.c */
-	.long R(51, f4, a7, 50), R(7e, 41, 65, 53)
-	.long R(1a, 17, a4, c3), R(3a, 27, 5e, 96)
-	.long R(3b, ab, 6b, cb), R(1f, 9d, 45, f1)
-	.long R(ac, fa, 58, ab), R(4b, e3, 03, 93)
-	.long R(20, 30, fa, 55), R(ad, 76, 6d, f6)
-	.long R(88, cc, 76, 91), R(f5, 02, 4c, 25)
-	.long R(4f, e5, d7, fc), R(c5, 2a, cb, d7)
-	.long R(26, 35, 44, 80), R(b5, 62, a3, 8f)
-	.long R(de, b1, 5a, 49), R(25, ba, 1b, 67)
-	.long R(45, ea, 0e, 98), R(5d, fe, c0, e1)
-	.long R(c3, 2f, 75, 02), R(81, 4c, f0, 12)
-	.long R(8d, 46, 97, a3), R(6b, d3, f9, c6)
-	.long R(03, 8f, 5f, e7), R(15, 92, 9c, 95)
-	.long R(bf, 6d, 7a, eb), R(95, 52, 59, da)
-	.long R(d4, be, 83, 2d), R(58, 74, 21, d3)
-	.long R(49, e0, 69, 29), R(8e, c9, c8, 44)
-	.long R(75, c2, 89, 6a), R(f4, 8e, 79, 78)
-	.long R(99, 58, 3e, 6b), R(27, b9, 71, dd)
-	.long R(be, e1, 4f, b6), R(f0, 88, ad, 17)
-	.long R(c9, 20, ac, 66), R(7d, ce, 3a, b4)
-	.long R(63, df, 4a, 18), R(e5, 1a, 31, 82)
-	.long R(97, 51, 33, 60), R(62, 53, 7f, 45)
-	.long R(b1, 64, 77, e0), R(bb, 6b, ae, 84)
-	.long R(fe, 81, a0, 1c), R(f9, 08, 2b, 94)
-	.long R(70, 48, 68, 58), R(8f, 45, fd, 19)
-	.long R(94, de, 6c, 87), R(52, 7b, f8, b7)
-	.long R(ab, 73, d3, 23), R(72, 4b, 02, e2)
-	.long R(e3, 1f, 8f, 57), R(66, 55, ab, 2a)
-	.long R(b2, eb, 28, 07), R(2f, b5, c2, 03)
-	.long R(86, c5, 7b, 9a), R(d3, 37, 08, a5)
-	.long R(30, 28, 87, f2), R(23, bf, a5, b2)
-	.long R(02, 03, 6a, ba), R(ed, 16, 82, 5c)
-	.long R(8a, cf, 1c, 2b), R(a7, 79, b4, 92)
-	.long R(f3, 07, f2, f0), R(4e, 69, e2, a1)
-	.long R(65, da, f4, cd), R(06, 05, be, d5)
-	.long R(d1, 34, 62, 1f), R(c4, a6, fe, 8a)
-	.long R(34, 2e, 53, 9d), R(a2, f3, 55, a0)
-	.long R(05, 8a, e1, 32), R(a4, f6, eb, 75)
-	.long R(0b, 83, ec, 39), R(40, 60, ef, aa)
-	.long R(5e, 71, 9f, 06), R(bd, 6e, 10, 51)
-	.long R(3e, 21, 8a, f9), R(96, dd, 06, 3d)
-	.long R(dd, 3e, 05, ae), R(4d, e6, bd, 46)
-	.long R(91, 54, 8d, b5), R(71, c4, 5d, 05)
-	.long R(04, 06, d4, 6f), R(60, 50, 15, ff)
-	.long R(19, 98, fb, 24), R(d6, bd, e9, 97)
-	.long R(89, 40, 43, cc), R(67, d9, 9e, 77)
-	.long R(b0, e8, 42, bd), R(07, 89, 8b, 88)
-	.long R(e7, 19, 5b, 38), R(79, c8, ee, db)
-	.long R(a1, 7c, 0a, 47), R(7c, 42, 0f, e9)
-	.long R(f8, 84, 1e, c9), R(00, 00, 00, 00)
-	.long R(09, 80, 86, 83), R(32, 2b, ed, 48)
-	.long R(1e, 11, 70, ac), R(6c, 5a, 72, 4e)
-	.long R(fd, 0e, ff, fb), R(0f, 85, 38, 56)
-	.long R(3d, ae, d5, 1e), R(36, 2d, 39, 27)
-	.long R(0a, 0f, d9, 64), R(68, 5c, a6, 21)
-	.long R(9b, 5b, 54, d1), R(24, 36, 2e, 3a)
-	.long R(0c, 0a, 67, b1), R(93, 57, e7, 0f)
-	.long R(b4, ee, 96, d2), R(1b, 9b, 91, 9e)
-	.long R(80, c0, c5, 4f), R(61, dc, 20, a2)
-	.long R(5a, 77, 4b, 69), R(1c, 12, 1a, 16)
-	.long R(e2, 93, ba, 0a), R(c0, a0, 2a, e5)
-	.long R(3c, 22, e0, 43), R(12, 1b, 17, 1d)
-	.long R(0e, 09, 0d, 0b), R(f2, 8b, c7, ad)
-	.long R(2d, b6, a8, b9), R(14, 1e, a9, c8)
-	.long R(57, f1, 19, 85), R(af, 75, 07, 4c)
-	.long R(ee, 99, dd, bb), R(a3, 7f, 60, fd)
-	.long R(f7, 01, 26, 9f), R(5c, 72, f5, bc)
-	.long R(44, 66, 3b, c5), R(5b, fb, 7e, 34)
-	.long R(8b, 43, 29, 76), R(cb, 23, c6, dc)
-	.long R(b6, ed, fc, 68), R(b8, e4, f1, 63)
-	.long R(d7, 31, dc, ca), R(42, 63, 85, 10)
-	.long R(13, 97, 22, 40), R(84, c6, 11, 20)
-	.long R(85, 4a, 24, 7d), R(d2, bb, 3d, f8)
-	.long R(ae, f9, 32, 11), R(c7, 29, a1, 6d)
-	.long R(1d, 9e, 2f, 4b), R(dc, b2, 30, f3)
-	.long R(0d, 86, 52, ec), R(77, c1, e3, d0)
-	.long R(2b, b3, 16, 6c), R(a9, 70, b9, 99)
-	.long R(11, 94, 48, fa), R(47, e9, 64, 22)
-	.long R(a8, fc, 8c, c4), R(a0, f0, 3f, 1a)
-	.long R(56, 7d, 2c, d8), R(22, 33, 90, ef)
-	.long R(87, 49, 4e, c7), R(d9, 38, d1, c1)
-	.long R(8c, ca, a2, fe), R(98, d4, 0b, 36)
-	.long R(a6, f5, 81, cf), R(a5, 7a, de, 28)
-	.long R(da, b7, 8e, 26), R(3f, ad, bf, a4)
-	.long R(2c, 3a, 9d, e4), R(50, 78, 92, 0d)
-	.long R(6a, 5f, cc, 9b), R(54, 7e, 46, 62)
-	.long R(f6, 8d, 13, c2), R(90, d8, b8, e8)
-	.long R(2e, 39, f7, 5e), R(82, c3, af, f5)
-	.long R(9f, 5d, 80, be), R(69, d0, 93, 7c)
-	.long R(6f, d5, 2d, a9), R(cf, 25, 12, b3)
-	.long R(c8, ac, 99, 3b), R(10, 18, 7d, a7)
-	.long R(e8, 9c, 63, 6e), R(db, 3b, bb, 7b)
-	.long R(cd, 26, 78, 09), R(6e, 59, 18, f4)
-	.long R(ec, 9a, b7, 01), R(83, 4f, 9a, a8)
-	.long R(e6, 95, 6e, 65), R(aa, ff, e6, 7e)
-	.long R(21, bc, cf, 08), R(ef, 15, e8, e6)
-	.long R(ba, e7, 9b, d9), R(4a, 6f, 36, ce)
-	.long R(ea, 9f, 09, d4), R(29, b0, 7c, d6)
-	.long R(31, a4, b2, af), R(2a, 3f, 23, 31)
-	.long R(c6, a5, 94, 30), R(35, a2, 66, c0)
-	.long R(74, 4e, bc, 37), R(fc, 82, ca, a6)
-	.long R(e0, 90, d0, b0), R(33, a7, d8, 15)
-	.long R(f1, 04, 98, 4a), R(41, ec, da, f7)
-	.long R(7f, cd, 50, 0e), R(17, 91, f6, 2f)
-	.long R(76, 4d, d6, 8d), R(43, ef, b0, 4d)
-	.long R(cc, aa, 4d, 54), R(e4, 96, 04, df)
-	.long R(9e, d1, b5, e3), R(4c, 6a, 88, 1b)
-	.long R(c1, 2c, 1f, b8), R(46, 65, 51, 7f)
-	.long R(9d, 5e, ea, 04), R(01, 8c, 35, 5d)
-	.long R(fa, 87, 74, 73), R(fb, 0b, 41, 2e)
-	.long R(b3, 67, 1d, 5a), R(92, db, d2, 52)
-	.long R(e9, 10, 56, 33), R(6d, d6, 47, 13)
-	.long R(9a, d7, 61, 8c), R(37, a1, 0c, 7a)
-	.long R(59, f8, 14, 8e), R(eb, 13, 3c, 89)
-	.long R(ce, a9, 27, ee), R(b7, 61, c9, 35)
-	.long R(e1, 1c, e5, ed), R(7a, 47, b1, 3c)
-	.long R(9c, d2, df, 59), R(55, f2, 73, 3f)
-	.long R(18, 14, ce, 79), R(73, c7, 37, bf)
-	.long R(53, f7, cd, ea), R(5f, fd, aa, 5b)
-	.long R(df, 3d, 6f, 14), R(78, 44, db, 86)
-	.long R(ca, af, f3, 81), R(b9, 68, c4, 3e)
-	.long R(38, 24, 34, 2c), R(c2, a3, 40, 5f)
-	.long R(16, 1d, c3, 72), R(bc, e2, 25, 0c)
-	.long R(28, 3c, 49, 8b), R(ff, 0d, 95, 41)
-	.long R(39, a8, 01, 71), R(08, 0c, b3, de)
-	.long R(d8, b4, e4, 9c), R(64, 56, c1, 90)
-	.long R(7b, cb, 84, 61), R(d5, 32, b6, 70)
-	.long R(48, 6c, 5c, 74), R(d0, b8, 57, 42)
-.globl PPC_AES_4K_DECTAB2
-PPC_AES_4K_DECTAB2:
-/* decryption table, same as crypto_il_tab in crypto/aes-generic.c */
-	.byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38
-	.byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb
-	.byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87
-	.byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb
-	.byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d
-	.byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e
-	.byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2
-	.byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25
-	.byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16
-	.byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92
-	.byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda
-	.byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84
-	.byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a
-	.byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06
-	.byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02
-	.byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b
-	.byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea
-	.byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73
-	.byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85
-	.byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e
-	.byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89
-	.byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b
-	.byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20
-	.byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4
-	.byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31
-	.byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f
-	.byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d
-	.byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef
-	.byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0
-	.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
-	.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
-	.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
diff --git a/arch/powerpc/crypto/aes.c b/arch/powerpc/crypto/aes.c
deleted file mode 100644
index 3f1e5e894902..000000000000
--- a/arch/powerpc/crypto/aes.c
+++ /dev/null
@@ -1,134 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * AES routines supporting VMX instructions on the Power 8
- *
- * Copyright (C) 2015 International Business Machines Inc.
- *
- * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
- */
-
-#include <asm/simd.h>
-#include <asm/switch_to.h>
-#include <crypto/aes.h>
-#include <crypto/internal/cipher.h>
-#include <crypto/internal/simd.h>
-#include <linux/err.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/uaccess.h>
-
-#include "aesp8-ppc.h"
-
-struct p8_aes_ctx {
-	struct crypto_cipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
-};
-
-static int p8_aes_init(struct crypto_tfm *tfm)
-{
-	const char *alg = crypto_tfm_alg_name(tfm);
-	struct crypto_cipher *fallback;
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
-	if (IS_ERR(fallback)) {
-		printk(KERN_ERR
-		       "Failed to allocate transformation for '%s': %ld\n",
-		       alg, PTR_ERR(fallback));
-		return PTR_ERR(fallback);
-	}
-
-	crypto_cipher_set_flags(fallback,
-				crypto_cipher_get_flags((struct
-							 crypto_cipher *)
-							tfm));
-	ctx->fallback = fallback;
-
-	return 0;
-}
-
-static void p8_aes_exit(struct crypto_tfm *tfm)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (ctx->fallback) {
-		crypto_free_cipher(ctx->fallback);
-		ctx->fallback = NULL;
-	}
-}
-
-static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,
-			 unsigned int keylen)
-{
-	int ret;
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	preempt_disable();
-	pagefault_disable();
-	enable_kernel_vsx();
-	ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
-	ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
-	disable_kernel_vsx();
-	pagefault_enable();
-	preempt_enable();
-
-	ret |= crypto_cipher_setkey(ctx->fallback, key, keylen);
-
-	return ret ? -EINVAL : 0;
-}
-
-static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		crypto_cipher_encrypt_one(ctx->fallback, dst, src);
-	} else {
-		preempt_disable();
-		pagefault_disable();
-		enable_kernel_vsx();
-		aes_p8_encrypt(src, dst, &ctx->enc_key);
-		disable_kernel_vsx();
-		pagefault_enable();
-		preempt_enable();
-	}
-}
-
-static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (!crypto_simd_usable()) {
-		crypto_cipher_decrypt_one(ctx->fallback, dst, src);
-	} else {
-		preempt_disable();
-		pagefault_disable();
-		enable_kernel_vsx();
-		aes_p8_decrypt(src, dst, &ctx->dec_key);
-		disable_kernel_vsx();
-		pagefault_enable();
-		preempt_enable();
-	}
-}
-
-struct crypto_alg p8_aes_alg = {
-	.cra_name = "aes",
-	.cra_driver_name = "p8_aes",
-	.cra_module = THIS_MODULE,
-	.cra_priority = 1000,
-	.cra_type = NULL,
-	.cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK,
-	.cra_alignmask = 0,
-	.cra_blocksize = AES_BLOCK_SIZE,
-	.cra_ctxsize = sizeof(struct p8_aes_ctx),
-	.cra_init = p8_aes_init,
-	.cra_exit = p8_aes_exit,
-	.cra_cipher = {
-		       .cia_min_keysize = AES_MIN_KEY_SIZE,
-		       .cia_max_keysize = AES_MAX_KEY_SIZE,
-		       .cia_setkey = p8_aes_setkey,
-		       .cia_encrypt = p8_aes_encrypt,
-		       .cia_decrypt = p8_aes_decrypt,
-	},
-};
diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c
index 5f2a4f375eef..4a9f285f0970 100644
--- a/arch/powerpc/crypto/aes_cbc.c
+++ b/arch/powerpc/crypto/aes_cbc.c
@@ -21,8 +21,8 @@
 
 struct p8_aes_cbc_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
+	struct p8_aes_key enc_key;
+	struct p8_aes_key dec_key;
 };
 
 static int p8_aes_cbc_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c
index e27c4036e711..7dbd06f442db 100644
--- a/arch/powerpc/crypto/aes_ctr.c
+++ b/arch/powerpc/crypto/aes_ctr.c
@@ -21,7 +21,7 @@
 
 struct p8_aes_ctr_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
+	struct p8_aes_key enc_key;
 };
 
 static int p8_aes_ctr_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aes_xts.c b/arch/powerpc/crypto/aes_xts.c
index 9440e771cede..b4c760e465ea 100644
--- a/arch/powerpc/crypto/aes_xts.c
+++ b/arch/powerpc/crypto/aes_xts.c
@@ -22,9 +22,9 @@
 
 struct p8_aes_xts_ctx {
 	struct crypto_skcipher *fallback;
-	struct aes_key enc_key;
-	struct aes_key dec_key;
-	struct aes_key tweak_key;
+	struct p8_aes_key enc_key;
+	struct p8_aes_key dec_key;
+	struct p8_aes_key tweak_key;
 };
 
 static int p8_aes_xts_init(struct crypto_skcipher *tfm)
diff --git a/arch/powerpc/crypto/aesp8-ppc.h b/arch/powerpc/crypto/aesp8-ppc.h
index 5764d4438388..6862c605cc33 100644
--- a/arch/powerpc/crypto/aesp8-ppc.h
+++ b/arch/powerpc/crypto/aesp8-ppc.h
@@ -2,29 +2,7 @@
 #include <linux/types.h>
 #include <crypto/aes.h>
 
-struct aes_key {
-	u8 key[AES_MAX_KEYLENGTH];
-	int rounds;
-};
-
 extern struct shash_alg p8_ghash_alg;
-extern struct crypto_alg p8_aes_alg;
 extern struct skcipher_alg p8_aes_cbc_alg;
 extern struct skcipher_alg p8_aes_ctr_alg;
 extern struct skcipher_alg p8_aes_xts_alg;
-
-int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,
-			   struct aes_key *key);
-int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
-			   struct aes_key *key);
-void aes_p8_encrypt(const u8 *in, u8 *out, const struct aes_key *key);
-void aes_p8_decrypt(const u8 *in, u8 *out, const struct aes_key *key);
-void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key, u8 *iv, const int enc);
-void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out,
-				 size_t len, const struct aes_key *key,
-				 const u8 *iv);
-void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
-void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
-			const struct aes_key *key1, const struct aes_key *key2, u8 *iv);
diff --git a/arch/powerpc/crypto/aesp8-ppc.pl b/arch/powerpc/crypto/aesp8-ppc.pl
deleted file mode 100644
index f729589d792e..000000000000
--- a/arch/powerpc/crypto/aesp8-ppc.pl
+++ /dev/null
@@ -1,3889 +0,0 @@
-#! /usr/bin/env perl
-# SPDX-License-Identifier: GPL-2.0
-
-# This code is taken from CRYPTOGAMs[1] and is included here using the option
-# in the license to distribute the code under the GPL. Therefore this program
-# is free software; you can redistribute it and/or modify it under the terms of
-# the GNU General Public License version 2 as published by the Free Software
-# Foundation.
-#
-# [1] https://www.openssl.org/~appro/cryptogams/
-
-# Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-#       * Redistributions of source code must retain copyright notices,
-#         this list of conditions and the following disclaimer.
-#
-#       * Redistributions in binary form must reproduce the above
-#         copyright notice, this list of conditions and the following
-#         disclaimer in the documentation and/or other materials
-#         provided with the distribution.
-#
-#       * Neither the name of the CRYPTOGAMS nor the names of its
-#         copyright holder and contributors may be used to endorse or
-#         promote products derived from this software without specific
-#         prior written permission.
-#
-# ALTERNATIVELY, provided that this notice is retained in full, this
-# product may be distributed under the terms of the GNU General Public
-# License (GPL), in which case the provisions of the GPL apply INSTEAD OF
-# those given above.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# ====================================================================
-# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
-# project. The module is, however, dual licensed under OpenSSL and
-# CRYPTOGAMS licenses depending on where you obtain it. For further
-# details see https://www.openssl.org/~appro/cryptogams/.
-# ====================================================================
-#
-# This module implements support for AES instructions as per PowerISA
-# specification version 2.07, first implemented by POWER8 processor.
-# The module is endian-agnostic in sense that it supports both big-
-# and little-endian cases. Data alignment in parallelizable modes is
-# handled with VSX loads and stores, which implies MSR.VSX flag being
-# set. It should also be noted that ISA specification doesn't prohibit
-# alignment exceptions for these instructions on page boundaries.
-# Initially alignment was handled in pure AltiVec/VMX way [when data
-# is aligned programmatically, which in turn guarantees exception-
-# free execution], but it turned to hamper performance when vcipher
-# instructions are interleaved. It's reckoned that eventual
-# misalignment penalties at page boundaries are in average lower
-# than additional overhead in pure AltiVec approach.
-#
-# May 2016
-#
-# Add XTS subroutine, 9x on little- and 12x improvement on big-endian
-# systems were measured.
-#
-######################################################################
-# Current large-block performance in cycles per byte processed with
-# 128-bit key (less is better).
-#
-#		CBC en-/decrypt	CTR	XTS
-# POWER8[le]	3.96/0.72	0.74	1.1
-# POWER8[be]	3.75/0.65	0.66	1.0
-
-$flavour = shift;
-
-if ($flavour =~ /64/) {
-	$SIZE_T	=8;
-	$LRSAVE	=2*$SIZE_T;
-	$STU	="stdu";
-	$POP	="ld";
-	$PUSH	="std";
-	$UCMP	="cmpld";
-	$SHL	="sldi";
-} elsif ($flavour =~ /32/) {
-	$SIZE_T	=4;
-	$LRSAVE	=$SIZE_T;
-	$STU	="stwu";
-	$POP	="lwz";
-	$PUSH	="stw";
-	$UCMP	="cmplw";
-	$SHL	="slwi";
-} else { die "nonsense $flavour"; }
-
-$LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
-
-$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
-( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
-( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
-die "can't locate ppc-xlate.pl";
-
-open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
-
-$FRAME=8*$SIZE_T;
-$prefix="aes_p8";
-
-$sp="r1";
-$vrsave="r12";
-
-#########################################################################
-{{{	# Key setup procedures						#
-my ($inp,$bits,$out,$ptr,$cnt,$rounds)=map("r$_",(3..8));
-my ($zero,$in0,$in1,$key,$rcon,$mask,$tmp)=map("v$_",(0..6));
-my ($stage,$outperm,$outmask,$outhead,$outtail)=map("v$_",(7..11));
-
-$code.=<<___;
-.machine	"any"
-
-.text
-
-.align	7
-rcon:
-.long	0x01000000, 0x01000000, 0x01000000, 0x01000000	?rev
-.long	0x1b000000, 0x1b000000, 0x1b000000, 0x1b000000	?rev
-.long	0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c, 0x0d0e0f0c	?rev
-.long	0,0,0,0						?asis
-.long	0x0f102132, 0x43546576, 0x8798a9ba, 0xcbdcedfe
-Lconsts:
-	mflr	r0
-	bcl	20,31,\$+4
-	mflr	$ptr	 #vvvvv "distance between . and rcon
-	addi	$ptr,$ptr,-0x58
-	mtlr	r0
-	blr
-	.long	0
-	.byte	0,12,0x14,0,0,0,0,0
-.asciz	"AES for PowerISA 2.07, CRYPTOGAMS by <appro\@openssl.org>"
-
-.globl	.${prefix}_set_encrypt_key
-Lset_encrypt_key:
-	mflr		r11
-	$PUSH		r11,$LRSAVE($sp)
-
-	li		$ptr,-1
-	${UCMP}i	$inp,0
-	beq-		Lenc_key_abort		# if ($inp==0) return -1;
-	${UCMP}i	$out,0
-	beq-		Lenc_key_abort		# if ($out==0) return -1;
-	li		$ptr,-2
-	cmpwi		$bits,128
-	blt-		Lenc_key_abort
-	cmpwi		$bits,256
-	bgt-		Lenc_key_abort
-	andi.		r0,$bits,0x3f
-	bne-		Lenc_key_abort
-
-	lis		r0,0xfff0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	bl		Lconsts
-	mtlr		r11
-
-	neg		r9,$inp
-	lvx		$in0,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	lvsr		$key,0,r9		# borrow $key
-	li		r8,0x20
-	cmpwi		$bits,192
-	lvx		$in1,0,$inp
-	le?vspltisb	$mask,0x0f		# borrow $mask
-	lvx		$rcon,0,$ptr
-	le?vxor		$key,$key,$mask		# adjust for byte swap
-	lvx		$mask,r8,$ptr
-	addi		$ptr,$ptr,0x10
-	vperm		$in0,$in0,$in1,$key	# align [and byte swap in LE]
-	li		$cnt,8
-	vxor		$zero,$zero,$zero
-	mtctr		$cnt
-
-	?lvsr		$outperm,0,$out
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$zero,$outmask,$outperm
-
-	blt		Loop128
-	addi		$inp,$inp,8
-	beq		L192
-	addi		$inp,$inp,8
-	b		L256
-
-.align	4
-Loop128:
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-	bdnz		Loop128
-
-	lvx		$rcon,0,$ptr		# last two round keys
-
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-
-	vperm		$key,$in0,$in0,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vxor		$in0,$in0,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-
-	addi		$inp,$out,15		# 15 is not typo
-	addi		$out,$out,0x50
-
-	li		$rounds,10
-	b		Ldone
-
-.align	4
-L192:
-	lvx		$tmp,0,$inp
-	li		$cnt,4
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
-	vspltisb	$key,8			# borrow $key
-	mtctr		$cnt
-	vsububm		$mask,$mask,$key	# adjust the mask
-
-Loop192:
-	vperm		$key,$in1,$in1,$mask	# roate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	vcipherlast	$key,$key,$rcon
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-
-	 vsldoi		$stage,$zero,$in1,8
-	vspltw		$tmp,$in0,3
-	vxor		$tmp,$tmp,$in1
-	vsldoi		$in1,$zero,$in1,12	# >>32
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in1,$in1,$tmp
-	vxor		$in0,$in0,$key
-	vxor		$in1,$in1,$key
-	 vsldoi		$stage,$stage,$in0,8
-
-	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$stage,$stage,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	 vsldoi		$stage,$in0,$in1,8
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	 vperm		$outtail,$stage,$stage,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vspltw		$tmp,$in0,3
-	vxor		$tmp,$tmp,$in1
-	vsldoi		$in1,$zero,$in1,12	# >>32
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in1,$in1,$tmp
-	vxor		$in0,$in0,$key
-	vxor		$in1,$in1,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$inp,$out,15		# 15 is not typo
-	 addi		$out,$out,16
-	bdnz		Loop192
-
-	li		$rounds,12
-	addi		$out,$out,0x20
-	b		Ldone
-
-.align	4
-L256:
-	lvx		$tmp,0,$inp
-	li		$cnt,7
-	li		$rounds,14
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-	vperm		$in1,$in1,$tmp,$key	# align [and byte swap in LE]
-	mtctr		$cnt
-
-Loop256:
-	vperm		$key,$in1,$in1,$mask	# rotate-n-splat
-	vsldoi		$tmp,$zero,$in0,12	# >>32
-	 vperm		$outtail,$in1,$in1,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	vcipherlast	$key,$key,$rcon
-	 stvx		$stage,0,$out
-	 addi		$out,$out,16
-
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in0,$in0,$tmp
-	 vadduwm	$rcon,$rcon,$rcon
-	vxor		$in0,$in0,$key
-	 vperm		$outtail,$in0,$in0,$outperm	# rotate
-	 vsel		$stage,$outhead,$outtail,$outmask
-	 vmr		$outhead,$outtail
-	 stvx		$stage,0,$out
-	 addi		$inp,$out,15		# 15 is not typo
-	 addi		$out,$out,16
-	bdz		Ldone
-
-	vspltw		$key,$in0,3		# just splat
-	vsldoi		$tmp,$zero,$in1,12	# >>32
-	vsbox		$key,$key
-
-	vxor		$in1,$in1,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in1,$in1,$tmp
-	vsldoi		$tmp,$zero,$tmp,12	# >>32
-	vxor		$in1,$in1,$tmp
-
-	vxor		$in1,$in1,$key
-	b		Loop256
-
-.align	4
-Ldone:
-	lvx		$in1,0,$inp		# redundant in aligned case
-	vsel		$in1,$outhead,$in1,$outmask
-	stvx		$in1,0,$inp
-	li		$ptr,0
-	mtspr		256,$vrsave
-	stw		$rounds,0($out)
-
-Lenc_key_abort:
-	mr		r3,$ptr
-	blr
-	.long		0
-	.byte		0,12,0x14,1,0,0,3,0
-	.long		0
-.size	.${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
-
-.globl	.${prefix}_set_decrypt_key
-	$STU		$sp,-$FRAME($sp)
-	mflr		r10
-	$PUSH		r10,$FRAME+$LRSAVE($sp)
-	bl		Lset_encrypt_key
-	mtlr		r10
-
-	cmpwi		r3,0
-	bne-		Ldec_key_abort
-
-	slwi		$cnt,$rounds,4
-	subi		$inp,$out,240		# first round key
-	srwi		$rounds,$rounds,1
-	add		$out,$inp,$cnt		# last round key
-	mtctr		$rounds
-
-Ldeckey:
-	lwz		r0, 0($inp)
-	lwz		r6, 4($inp)
-	lwz		r7, 8($inp)
-	lwz		r8, 12($inp)
-	addi		$inp,$inp,16
-	lwz		r9, 0($out)
-	lwz		r10,4($out)
-	lwz		r11,8($out)
-	lwz		r12,12($out)
-	stw		r0, 0($out)
-	stw		r6, 4($out)
-	stw		r7, 8($out)
-	stw		r8, 12($out)
-	subi		$out,$out,16
-	stw		r9, -16($inp)
-	stw		r10,-12($inp)
-	stw		r11,-8($inp)
-	stw		r12,-4($inp)
-	bdnz		Ldeckey
-
-	xor		r3,r3,r3		# return value
-Ldec_key_abort:
-	addi		$sp,$sp,$FRAME
-	blr
-	.long		0
-	.byte		0,12,4,1,0x80,0,3,0
-	.long		0
-.size	.${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
-___
-}}}
-#########################################################################
-{{{	# Single block en- and decrypt procedures			#
-sub gen_block () {
-my $dir = shift;
-my $n   = $dir eq "de" ? "n" : "";
-my ($inp,$out,$key,$rounds,$idx)=map("r$_",(3..7));
-
-$code.=<<___;
-.globl	.${prefix}_${dir}crypt
-	lwz		$rounds,240($key)
-	lis		r0,0xfc00
-	mfspr		$vrsave,256
-	li		$idx,15			# 15 is not typo
-	mtspr		256,r0
-
-	lvx		v0,0,$inp
-	neg		r11,$out
-	lvx		v1,$idx,$inp
-	lvsl		v2,0,$inp		# inpperm
-	le?vspltisb	v4,0x0f
-	?lvsl		v3,0,r11		# outperm
-	le?vxor		v2,v2,v4
-	li		$idx,16
-	vperm		v0,v0,v1,v2		# align [and byte swap in LE]
-	lvx		v1,0,$key
-	?lvsl		v5,0,$key		# keyperm
-	srwi		$rounds,$rounds,1
-	lvx		v2,$idx,$key
-	addi		$idx,$idx,16
-	subi		$rounds,$rounds,1
-	?vperm		v1,v1,v2,v5		# align round key
-
-	vxor		v0,v0,v1
-	lvx		v1,$idx,$key
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Loop_${dir}c:
-	?vperm		v2,v2,v1,v5
-	v${n}cipher	v0,v0,v2
-	lvx		v2,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		v1,v1,v2,v5
-	v${n}cipher	v0,v0,v1
-	lvx		v1,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_${dir}c
-
-	?vperm		v2,v2,v1,v5
-	v${n}cipher	v0,v0,v2
-	lvx		v2,$idx,$key
-	?vperm		v1,v1,v2,v5
-	v${n}cipherlast	v0,v0,v1
-
-	vspltisb	v2,-1
-	vxor		v1,v1,v1
-	li		$idx,15			# 15 is not typo
-	?vperm		v2,v1,v2,v3		# outmask
-	le?vxor		v3,v3,v4
-	lvx		v1,0,$out		# outhead
-	vperm		v0,v0,v0,v3		# rotate [and byte swap in LE]
-	vsel		v1,v1,v0,v2
-	lvx		v4,$idx,$out
-	stvx		v1,0,$out
-	vsel		v0,v0,v4,v2
-	stvx		v0,$idx,$out
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,3,0
-	.long		0
-.size	.${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
-___
-}
-&gen_block("en");
-&gen_block("de");
-}}}
-#########################################################################
-{{{	# CBC en- and decrypt procedures				#
-my ($inp,$out,$len,$key,$ivp,$enc,$rounds,$idx)=map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
-my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm)=
-						map("v$_",(4..10));
-$code.=<<___;
-.globl	.${prefix}_cbc_encrypt
-	${UCMP}i	$len,16
-	bltlr-
-
-	cmpwi		$enc,0			# test direction
-	lis		r0,0xffe0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	li		$idx,15
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	le?vspltisb	$tmp,0x0f
-
-	lvx		$ivec,0,$ivp		# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$ivec,$ivec,$inptail,$inpperm
-
-	neg		r11,$inp
-	?lvsl		$keyperm,0,$key		# prepare for unaligned key
-	lwz		$rounds,240($key)
-
-	lvsr		$inpperm,0,r11		# prepare for unaligned load
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	?lvsr		$outperm,0,$out		# prepare for unaligned store
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-
-	srwi		$rounds,$rounds,1
-	li		$idx,16
-	subi		$rounds,$rounds,1
-	beq		Lcbc_dec
-
-Lcbc_enc:
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	mtctr		$rounds
-	subi		$len,$len,16		# len-=16
-
-	lvx		$rndkey0,0,$key
-	 vperm		$inout,$inout,$inptail,$inpperm
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	vxor		$inout,$inout,$ivec
-
-Loop_cbc_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_cbc_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$ivec,$inout,$rndkey0
-	${UCMP}i	$len,16
-
-	vperm		$tmp,$ivec,$ivec,$outperm
-	vsel		$inout,$outhead,$tmp,$outmask
-	vmr		$outhead,$tmp
-	stvx		$inout,0,$out
-	addi		$out,$out,16
-	bge		Lcbc_enc
-
-	b		Lcbc_done
-
-.align	4
-Lcbc_dec:
-	${UCMP}i	$len,128
-	bge		_aesp8_cbc_decrypt8x
-	vmr		$tmp,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	mtctr		$rounds
-	subi		$len,$len,16		# len-=16
-
-	lvx		$rndkey0,0,$key
-	 vperm		$tmp,$tmp,$inptail,$inpperm
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$tmp,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-
-Loop_cbc_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_cbc_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipherlast	$inout,$inout,$rndkey0
-	${UCMP}i	$len,16
-
-	vxor		$inout,$inout,$ivec
-	vmr		$ivec,$tmp
-	vperm		$tmp,$inout,$inout,$outperm
-	vsel		$inout,$outhead,$tmp,$outmask
-	vmr		$outhead,$tmp
-	stvx		$inout,0,$out
-	addi		$out,$out,16
-	bge		Lcbc_dec
-
-Lcbc_done:
-	addi		$out,$out,-1
-	lvx		$inout,0,$out		# redundant in aligned case
-	vsel		$inout,$outhead,$inout,$outmask
-	stvx		$inout,0,$out
-
-	neg		$enc,$ivp		# write [unaligned] iv
-	li		$idx,15			# 15 is not typo
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	vspltisb	$outmask,-1
-	le?vspltisb	$tmp,0x0f
-	?lvsl		$outperm,0,$enc
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-	lvx		$outhead,0,$ivp
-	vperm		$ivec,$ivec,$ivec,$outperm
-	vsel		$inout,$outhead,$ivec,$outmask
-	lvx		$inptail,$idx,$ivp
-	stvx		$inout,0,$ivp
-	vsel		$inout,$ivec,$inptail,$outmask
-	stvx		$inout,$idx,$ivp
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,6,0
-	.long		0
-___
-#########################################################################
-{{	# Optimized CBC decrypt procedure				#
-my $key_="r11";
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
-my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10..13));
-my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(14..21));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
-
-$code.=<<___;
-.align	5
-_aesp8_cbc_decrypt8x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	li		r10,`$FRAME+8*16+15`
-	li		r11,`$FRAME+8*16+31`
-	stvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	stvx		v21,r11,$sp
-	addi		r11,r11,32
-	stvx		v22,r10,$sp
-	addi		r10,r10,32
-	stvx		v23,r11,$sp
-	addi		r11,r11,32
-	stvx		v24,r10,$sp
-	addi		r10,r10,32
-	stvx		v25,r11,$sp
-	addi		r11,r11,32
-	stvx		v26,r10,$sp
-	addi		r10,r10,32
-	stvx		v27,r11,$sp
-	addi		r11,r11,32
-	stvx		v28,r10,$sp
-	addi		r10,r10,32
-	stvx		v29,r11,$sp
-	addi		r11,r11,32
-	stvx		v30,r10,$sp
-	stvx		v31,r11,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	subi		$rounds,$rounds,3	# -4 in total
-	subi		$len,$len,128		# bias
-
-	lvx		$rndkey0,$x00,$key	# load key schedule
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	lvx		v31,$x00,$key
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_cbc_dec_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_cbc_dec_key
-
-	lvx		v26,$x10,$key
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$out0,$x70,$key		# borrow $out0
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$out0,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	#lvx		$inptail,0,$inp		# "caller" already did this
-	#addi		$inp,$inp,15		# 15 is not typo
-	subi		$inp,$inp,15		# undo "caller"
-
-	 le?li		$idx,8
-	lvx_u		$in0,$x00,$inp		# load first 8 "words"
-	 le?lvsl	$inpperm,0,$idx
-	 le?vspltisb	$tmp,0x0f
-	lvx_u		$in1,$x10,$inp
-	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
-	lvx_u		$in2,$x20,$inp
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	lvx_u		$in3,$x30,$inp
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	lvx_u		$in4,$x40,$inp
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	vxor		$out0,$in0,$rndkey0
-	lvx_u		$in5,$x50,$inp
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	vxor		$out1,$in1,$rndkey0
-	lvx_u		$in6,$x60,$inp
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	vxor		$out2,$in2,$rndkey0
-	lvx_u		$in7,$x70,$inp
-	addi		$inp,$inp,0x80
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	vxor		$out3,$in3,$rndkey0
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	vxor		$out4,$in4,$rndkey0
-	 le?vperm	$in7,$in7,$in7,$inpperm
-	vxor		$out5,$in5,$rndkey0
-	vxor		$out6,$in6,$rndkey0
-	vxor		$out7,$in7,$rndkey0
-
-	mtctr		$rounds
-	b		Loop_cbc_dec8x
-.align	5
-Loop_cbc_dec8x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_cbc_dec8x
-
-	subic		$len,$len,128		# $len-=128
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-
-	and		r0,r0,$len
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-	vncipher	$out6,$out6,v26
-	vncipher	$out7,$out7,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in7 are loaded
-						# with last "words"
-	vncipher	$out0,$out0,v27
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-	vncipher	$out6,$out6,v27
-	vncipher	$out7,$out7,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	vncipher	$out6,$out6,v28
-	vncipher	$out7,$out7,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	vncipher	$out6,$out6,v29
-	vncipher	$out7,$out7,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-
-	vncipher	$out0,$out0,v30
-	 vxor		$ivec,$ivec,v31		# xor with last round key
-	vncipher	$out1,$out1,v30
-	 vxor		$in0,$in0,v31
-	vncipher	$out2,$out2,v30
-	 vxor		$in1,$in1,v31
-	vncipher	$out3,$out3,v30
-	 vxor		$in2,$in2,v31
-	vncipher	$out4,$out4,v30
-	 vxor		$in3,$in3,v31
-	vncipher	$out5,$out5,v30
-	 vxor		$in4,$in4,v31
-	vncipher	$out6,$out6,v30
-	 vxor		$in5,$in5,v31
-	vncipher	$out7,$out7,v30
-	 vxor		$in6,$in6,v31
-
-	vncipherlast	$out0,$out0,$ivec
-	vncipherlast	$out1,$out1,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	vncipherlast	$out2,$out2,$in1
-	 lvx_u		$in1,$x10,$inp
-	vncipherlast	$out3,$out3,$in2
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	 lvx_u		$in2,$x20,$inp
-	vncipherlast	$out4,$out4,$in3
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	 lvx_u		$in3,$x30,$inp
-	vncipherlast	$out5,$out5,$in4
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	 lvx_u		$in4,$x40,$inp
-	vncipherlast	$out6,$out6,$in5
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	 lvx_u		$in5,$x50,$inp
-	vncipherlast	$out7,$out7,$in6
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	 lvx_u		$in6,$x60,$inp
-	vmr		$ivec,$in7
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	 lvx_u		$in7,$x70,$inp
-	 addi		$inp,$inp,0x80
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	 vxor		$out0,$in0,$rndkey0
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	 le?vperm	$in7,$in7,$in7,$inpperm
-	 vxor		$out1,$in1,$rndkey0
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$rndkey0
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$rndkey0
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$rndkey0
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	 vxor		$out5,$in5,$rndkey0
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x60,$out
-	 vxor		$out6,$in6,$rndkey0
-	stvx_u		$out7,$x70,$out
-	addi		$out,$out,0x80
-	 vxor		$out7,$in7,$rndkey0
-
-	mtctr		$rounds
-	beq		Loop_cbc_dec8x		# did $len-=128 borrow?
-
-	addic.		$len,$len,128
-	beq		Lcbc_dec8x_done
-	nop
-	nop
-
-Loop_cbc_dec8x_tail:				# up to 7 "words" tail...
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_cbc_dec8x_tail
-
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	vncipher	$out6,$out6,v24
-	vncipher	$out7,$out7,v24
-
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	vncipher	$out6,$out6,v25
-	vncipher	$out7,$out7,v25
-
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-	vncipher	$out6,$out6,v26
-	vncipher	$out7,$out7,v26
-
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-	vncipher	$out6,$out6,v27
-	vncipher	$out7,$out7,v27
-
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	vncipher	$out6,$out6,v28
-	vncipher	$out7,$out7,v28
-
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	vncipher	$out6,$out6,v29
-	vncipher	$out7,$out7,v29
-
-	vncipher	$out1,$out1,v30
-	 vxor		$ivec,$ivec,v31		# last round key
-	vncipher	$out2,$out2,v30
-	 vxor		$in1,$in1,v31
-	vncipher	$out3,$out3,v30
-	 vxor		$in2,$in2,v31
-	vncipher	$out4,$out4,v30
-	 vxor		$in3,$in3,v31
-	vncipher	$out5,$out5,v30
-	 vxor		$in4,$in4,v31
-	vncipher	$out6,$out6,v30
-	 vxor		$in5,$in5,v31
-	vncipher	$out7,$out7,v30
-	 vxor		$in6,$in6,v31
-
-	cmplwi		$len,32			# switch($len)
-	blt		Lcbc_dec8x_one
-	nop
-	beq		Lcbc_dec8x_two
-	cmplwi		$len,64
-	blt		Lcbc_dec8x_three
-	nop
-	beq		Lcbc_dec8x_four
-	cmplwi		$len,96
-	blt		Lcbc_dec8x_five
-	nop
-	beq		Lcbc_dec8x_six
-
-Lcbc_dec8x_seven:
-	vncipherlast	$out1,$out1,$ivec
-	vncipherlast	$out2,$out2,$in1
-	vncipherlast	$out3,$out3,$in2
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out1,$out1,$out1,$inpperm
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x00,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x10,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x20,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x30,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x40,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x50,$out
-	stvx_u		$out7,$x60,$out
-	addi		$out,$out,0x70
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_six:
-	vncipherlast	$out2,$out2,$ivec
-	vncipherlast	$out3,$out3,$in2
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out2,$out2,$out2,$inpperm
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x00,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x10,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x20,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x30,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x40,$out
-	stvx_u		$out7,$x50,$out
-	addi		$out,$out,0x60
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_five:
-	vncipherlast	$out3,$out3,$ivec
-	vncipherlast	$out4,$out4,$in3
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out3,$out3,$out3,$inpperm
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x00,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x10,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x20,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x30,$out
-	stvx_u		$out7,$x40,$out
-	addi		$out,$out,0x50
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_four:
-	vncipherlast	$out4,$out4,$ivec
-	vncipherlast	$out5,$out5,$in4
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out4,$out4,$out4,$inpperm
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x00,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x10,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x20,$out
-	stvx_u		$out7,$x30,$out
-	addi		$out,$out,0x40
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_three:
-	vncipherlast	$out5,$out5,$ivec
-	vncipherlast	$out6,$out6,$in5
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out5,$out5,$out5,$inpperm
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x00,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x10,$out
-	stvx_u		$out7,$x20,$out
-	addi		$out,$out,0x30
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_two:
-	vncipherlast	$out6,$out6,$ivec
-	vncipherlast	$out7,$out7,$in6
-	vmr		$ivec,$in7
-
-	le?vperm	$out6,$out6,$out6,$inpperm
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x00,$out
-	stvx_u		$out7,$x10,$out
-	addi		$out,$out,0x20
-	b		Lcbc_dec8x_done
-
-.align	5
-Lcbc_dec8x_one:
-	vncipherlast	$out7,$out7,$ivec
-	vmr		$ivec,$in7
-
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out7,0,$out
-	addi		$out,$out,0x10
-
-Lcbc_dec8x_done:
-	le?vperm	$ivec,$ivec,$ivec,$inpperm
-	stvx_u		$ivec,0,$ivp		# write [unaligned] iv
-
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$inpperm,r10,$sp	# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
-___
-}}	}}}
-
-#########################################################################
-{{{	# CTR procedure[s]						#
-
-####################### WARNING: Here be dragons! #######################
-#
-# This code is written as 'ctr32', based on a 32-bit counter used
-# upstream. The kernel does *not* use a 32-bit counter. The kernel uses
-# a 128-bit counter.
-#
-# This leads to subtle changes from the upstream code: the counter
-# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in
-# both the bulk (8 blocks at a time) path, and in the individual block
-# path. Be aware of this when doing updates.
-#
-# See:
-# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
-# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
-# https://github.com/openssl/openssl/pull/8942
-#
-#########################################################################
-my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout,$tmp)=		map("v$_",(0..3));
-my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)=
-						map("v$_",(4..11));
-my $dat=$tmp;
-
-$code.=<<___;
-.globl	.${prefix}_ctr32_encrypt_blocks
-	${UCMP}i	$len,1
-	bltlr-
-
-	lis		r0,0xfff0
-	mfspr		$vrsave,256
-	mtspr		256,r0
-
-	li		$idx,15
-	vxor		$rndkey0,$rndkey0,$rndkey0
-	le?vspltisb	$tmp,0x0f
-
-	lvx		$ivec,0,$ivp		# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	 vspltisb	$one,1
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$ivec,$ivec,$inptail,$inpperm
-	 vsldoi		$one,$rndkey0,$one,1
-
-	neg		r11,$inp
-	?lvsl		$keyperm,0,$key		# prepare for unaligned key
-	lwz		$rounds,240($key)
-
-	lvsr		$inpperm,0,r11		# prepare for unaligned load
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,15		# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	srwi		$rounds,$rounds,1
-	li		$idx,16
-	subi		$rounds,$rounds,1
-
-	${UCMP}i	$len,8
-	bge		_aesp8_ctr32_encrypt8x
-
-	?lvsr		$outperm,0,$out		# prepare for unaligned store
-	vspltisb	$outmask,-1
-	lvx		$outhead,0,$out
-	?vperm		$outmask,$rndkey0,$outmask,$outperm
-	le?vxor		$outperm,$outperm,$tmp
-
-	lvx		$rndkey0,0,$key
-	mtctr		$rounds
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$ivec,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	b		Loop_ctr32_enc
-
-.align	5
-Loop_ctr32_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key
-	addi		$idx,$idx,16
-	bdnz		Loop_ctr32_enc
-
-	vadduqm		$ivec,$ivec,$one	# Kernel change for 128-bit
-	 vmr		$dat,$inptail
-	 lvx		$inptail,0,$inp
-	 addi		$inp,$inp,16
-	 subic.		$len,$len,1		# blocks--
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key
-	 vperm		$dat,$dat,$inptail,$inpperm
-	 li		$idx,16
-	?vperm		$rndkey1,$rndkey0,$rndkey1,$keyperm
-	 lvx		$rndkey0,0,$key
-	vxor		$dat,$dat,$rndkey1	# last round key
-	vcipherlast	$inout,$inout,$dat
-
-	 lvx		$rndkey1,$idx,$key
-	 addi		$idx,$idx,16
-	vperm		$inout,$inout,$inout,$outperm
-	vsel		$dat,$outhead,$inout,$outmask
-	 mtctr		$rounds
-	 ?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vmr		$outhead,$inout
-	 vxor		$inout,$ivec,$rndkey0
-	 lvx		$rndkey0,$idx,$key
-	 addi		$idx,$idx,16
-	stvx		$dat,0,$out
-	addi		$out,$out,16
-	bne		Loop_ctr32_enc
-
-	addi		$out,$out,-1
-	lvx		$inout,0,$out		# redundant in aligned case
-	vsel		$inout,$outhead,$inout,$outmask
-	stvx		$inout,0,$out
-
-	mtspr		256,$vrsave
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0,0,6,0
-	.long		0
-___
-#########################################################################
-{{	# Optimized CTR procedure					#
-my $key_="r11";
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,8,26..31));
-my ($in0, $in1, $in2, $in3, $in4, $in5, $in6, $in7 )=map("v$_",(0..3,10,12..14));
-my ($out0,$out1,$out2,$out3,$out4,$out5,$out6,$out7)=map("v$_",(15..22));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($tmp,$keyperm)=($in3,$in4);	# aliases with "caller", redundant assignment
-my ($two,$three,$four)=($outhead,$outperm,$outmask);
-
-$code.=<<___;
-.align	5
-_aesp8_ctr32_encrypt8x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	li		r10,`$FRAME+8*16+15`
-	li		r11,`$FRAME+8*16+31`
-	stvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	stvx		v21,r11,$sp
-	addi		r11,r11,32
-	stvx		v22,r10,$sp
-	addi		r10,r10,32
-	stvx		v23,r11,$sp
-	addi		r11,r11,32
-	stvx		v24,r10,$sp
-	addi		r10,r10,32
-	stvx		v25,r11,$sp
-	addi		r11,r11,32
-	stvx		v26,r10,$sp
-	addi		r10,r10,32
-	stvx		v27,r11,$sp
-	addi		r11,r11,32
-	stvx		v28,r10,$sp
-	addi		r10,r10,32
-	stvx		v29,r11,$sp
-	addi		r11,r11,32
-	stvx		v30,r10,$sp
-	stvx		v31,r11,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key	# load key schedule
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	lvx		v31,$x00,$key
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_ctr32_enc_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key
-	addi		$key,$key,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_ctr32_enc_key
-
-	lvx		v26,$x10,$key
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$out0,$x70,$key		# borrow $out0
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$out0,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	vadduqm		$two,$one,$one
-	subi		$inp,$inp,15		# undo "caller"
-	$SHL		$len,$len,4
-
-	vadduqm		$out1,$ivec,$one	# counter values ...
-	vadduqm		$out2,$ivec,$two	# (do all ctr adds as 128-bit)
-	vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
-	 le?li		$idx,8
-	vadduqm		$out3,$out1,$two
-	vxor		$out1,$out1,$rndkey0
-	 le?lvsl	$inpperm,0,$idx
-	vadduqm		$out4,$out2,$two
-	vxor		$out2,$out2,$rndkey0
-	 le?vspltisb	$tmp,0x0f
-	vadduqm		$out5,$out3,$two
-	vxor		$out3,$out3,$rndkey0
-	 le?vxor	$inpperm,$inpperm,$tmp	# transform for lvx_u/stvx_u
-	vadduqm		$out6,$out4,$two
-	vxor		$out4,$out4,$rndkey0
-	vadduqm		$out7,$out5,$two
-	vxor		$out5,$out5,$rndkey0
-	vadduqm		$ivec,$out6,$two	# next counter value
-	vxor		$out6,$out6,$rndkey0
-	vxor		$out7,$out7,$rndkey0
-
-	mtctr		$rounds
-	b		Loop_ctr32_enc8x
-.align	5
-Loop_ctr32_enc8x:
-	vcipher 	$out0,$out0,v24
-	vcipher 	$out1,$out1,v24
-	vcipher 	$out2,$out2,v24
-	vcipher 	$out3,$out3,v24
-	vcipher 	$out4,$out4,v24
-	vcipher 	$out5,$out5,v24
-	vcipher 	$out6,$out6,v24
-	vcipher 	$out7,$out7,v24
-Loop_ctr32_enc8x_middle:
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher 	$out0,$out0,v25
-	vcipher 	$out1,$out1,v25
-	vcipher 	$out2,$out2,v25
-	vcipher 	$out3,$out3,v25
-	vcipher 	$out4,$out4,v25
-	vcipher 	$out5,$out5,v25
-	vcipher 	$out6,$out6,v25
-	vcipher 	$out7,$out7,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_ctr32_enc8x
-
-	subic		r11,$len,256		# $len-256, borrow $key_
-	vcipher 	$out0,$out0,v24
-	vcipher 	$out1,$out1,v24
-	vcipher 	$out2,$out2,v24
-	vcipher 	$out3,$out3,v24
-	vcipher 	$out4,$out4,v24
-	vcipher 	$out5,$out5,v24
-	vcipher 	$out6,$out6,v24
-	vcipher 	$out7,$out7,v24
-
-	subfe		r0,r0,r0		# borrow?-1:0
-	vcipher 	$out0,$out0,v25
-	vcipher 	$out1,$out1,v25
-	vcipher 	$out2,$out2,v25
-	vcipher 	$out3,$out3,v25
-	vcipher 	$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-	vcipher		$out6,$out6,v25
-	vcipher		$out7,$out7,v25
-
-	and		r0,r0,r11
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v26
-	vcipher		$out1,$out1,v26
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	vcipher		$out4,$out4,v26
-	vcipher		$out5,$out5,v26
-	vcipher		$out6,$out6,v26
-	vcipher		$out7,$out7,v26
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	subic		$len,$len,129		# $len-=129
-	vcipher		$out0,$out0,v27
-	addi		$len,$len,1		# $len-=128 really
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	vcipher		$out4,$out4,v27
-	vcipher		$out5,$out5,v27
-	vcipher		$out6,$out6,v27
-	vcipher		$out7,$out7,v27
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-
-	vcipher		$out0,$out0,v28
-	 lvx_u		$in0,$x00,$inp		# load input
-	vcipher		$out1,$out1,v28
-	 lvx_u		$in1,$x10,$inp
-	vcipher		$out2,$out2,v28
-	 lvx_u		$in2,$x20,$inp
-	vcipher		$out3,$out3,v28
-	 lvx_u		$in3,$x30,$inp
-	vcipher		$out4,$out4,v28
-	 lvx_u		$in4,$x40,$inp
-	vcipher		$out5,$out5,v28
-	 lvx_u		$in5,$x50,$inp
-	vcipher		$out6,$out6,v28
-	 lvx_u		$in6,$x60,$inp
-	vcipher		$out7,$out7,v28
-	 lvx_u		$in7,$x70,$inp
-	 addi		$inp,$inp,0x80
-
-	vcipher		$out0,$out0,v29
-	 le?vperm	$in0,$in0,$in0,$inpperm
-	vcipher		$out1,$out1,v29
-	 le?vperm	$in1,$in1,$in1,$inpperm
-	vcipher		$out2,$out2,v29
-	 le?vperm	$in2,$in2,$in2,$inpperm
-	vcipher		$out3,$out3,v29
-	 le?vperm	$in3,$in3,$in3,$inpperm
-	vcipher		$out4,$out4,v29
-	 le?vperm	$in4,$in4,$in4,$inpperm
-	vcipher		$out5,$out5,v29
-	 le?vperm	$in5,$in5,$in5,$inpperm
-	vcipher		$out6,$out6,v29
-	 le?vperm	$in6,$in6,$in6,$inpperm
-	vcipher		$out7,$out7,v29
-	 le?vperm	$in7,$in7,$in7,$inpperm
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in7 are loaded
-						# with last "words"
-	subfe.		r0,r0,r0		# borrow?-1:0
-	vcipher		$out0,$out0,v30
-	 vxor		$in0,$in0,v31		# xor with last round key
-	vcipher		$out1,$out1,v30
-	 vxor		$in1,$in1,v31
-	vcipher		$out2,$out2,v30
-	 vxor		$in2,$in2,v31
-	vcipher		$out3,$out3,v30
-	 vxor		$in3,$in3,v31
-	vcipher		$out4,$out4,v30
-	 vxor		$in4,$in4,v31
-	vcipher		$out5,$out5,v30
-	 vxor		$in5,$in5,v31
-	vcipher		$out6,$out6,v30
-	 vxor		$in6,$in6,v31
-	vcipher		$out7,$out7,v30
-	 vxor		$in7,$in7,v31
-
-	bne		Lctr32_enc8x_break	# did $len-129 borrow?
-
-	vcipherlast	$in0,$out0,$in0
-	vcipherlast	$in1,$out1,$in1
-	 vadduqm	$out1,$ivec,$one	# counter values ...
-	vcipherlast	$in2,$out2,$in2
-	 vadduqm	$out2,$ivec,$two
-	 vxor		$out0,$ivec,$rndkey0	# ... xored with rndkey[0]
-	vcipherlast	$in3,$out3,$in3
-	 vadduqm	$out3,$out1,$two
-	 vxor		$out1,$out1,$rndkey0
-	vcipherlast	$in4,$out4,$in4
-	 vadduqm	$out4,$out2,$two
-	 vxor		$out2,$out2,$rndkey0
-	vcipherlast	$in5,$out5,$in5
-	 vadduqm	$out5,$out3,$two
-	 vxor		$out3,$out3,$rndkey0
-	vcipherlast	$in6,$out6,$in6
-	 vadduqm	$out6,$out4,$two
-	 vxor		$out4,$out4,$rndkey0
-	vcipherlast	$in7,$out7,$in7
-	 vadduqm	$out7,$out5,$two
-	 vxor		$out5,$out5,$rndkey0
-	le?vperm	$in0,$in0,$in0,$inpperm
-	 vadduqm	$ivec,$out6,$two	# next counter value
-	 vxor		$out6,$out6,$rndkey0
-	le?vperm	$in1,$in1,$in1,$inpperm
-	 vxor		$out7,$out7,$rndkey0
-	mtctr		$rounds
-
-	 vcipher	$out0,$out0,v24
-	stvx_u		$in0,$x00,$out
-	le?vperm	$in2,$in2,$in2,$inpperm
-	 vcipher	$out1,$out1,v24
-	stvx_u		$in1,$x10,$out
-	le?vperm	$in3,$in3,$in3,$inpperm
-	 vcipher	$out2,$out2,v24
-	stvx_u		$in2,$x20,$out
-	le?vperm	$in4,$in4,$in4,$inpperm
-	 vcipher	$out3,$out3,v24
-	stvx_u		$in3,$x30,$out
-	le?vperm	$in5,$in5,$in5,$inpperm
-	 vcipher	$out4,$out4,v24
-	stvx_u		$in4,$x40,$out
-	le?vperm	$in6,$in6,$in6,$inpperm
-	 vcipher	$out5,$out5,v24
-	stvx_u		$in5,$x50,$out
-	le?vperm	$in7,$in7,$in7,$inpperm
-	 vcipher	$out6,$out6,v24
-	stvx_u		$in6,$x60,$out
-	 vcipher	$out7,$out7,v24
-	stvx_u		$in7,$x70,$out
-	addi		$out,$out,0x80
-
-	b		Loop_ctr32_enc8x_middle
-
-.align	5
-Lctr32_enc8x_break:
-	cmpwi		$len,-0x60
-	blt		Lctr32_enc8x_one
-	nop
-	beq		Lctr32_enc8x_two
-	cmpwi		$len,-0x40
-	blt		Lctr32_enc8x_three
-	nop
-	beq		Lctr32_enc8x_four
-	cmpwi		$len,-0x20
-	blt		Lctr32_enc8x_five
-	nop
-	beq		Lctr32_enc8x_six
-	cmpwi		$len,0x00
-	blt		Lctr32_enc8x_seven
-
-Lctr32_enc8x_eight:
-	vcipherlast	$out0,$out0,$in0
-	vcipherlast	$out1,$out1,$in1
-	vcipherlast	$out2,$out2,$in2
-	vcipherlast	$out3,$out3,$in3
-	vcipherlast	$out4,$out4,$in4
-	vcipherlast	$out5,$out5,$in5
-	vcipherlast	$out6,$out6,$in6
-	vcipherlast	$out7,$out7,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	le?vperm	$out7,$out7,$out7,$inpperm
-	stvx_u		$out6,$x60,$out
-	stvx_u		$out7,$x70,$out
-	addi		$out,$out,0x80
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_seven:
-	vcipherlast	$out0,$out0,$in1
-	vcipherlast	$out1,$out1,$in2
-	vcipherlast	$out2,$out2,$in3
-	vcipherlast	$out3,$out3,$in4
-	vcipherlast	$out4,$out4,$in5
-	vcipherlast	$out5,$out5,$in6
-	vcipherlast	$out6,$out6,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	le?vperm	$out6,$out6,$out6,$inpperm
-	stvx_u		$out5,$x50,$out
-	stvx_u		$out6,$x60,$out
-	addi		$out,$out,0x70
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_six:
-	vcipherlast	$out0,$out0,$in2
-	vcipherlast	$out1,$out1,$in3
-	vcipherlast	$out2,$out2,$in4
-	vcipherlast	$out3,$out3,$in5
-	vcipherlast	$out4,$out4,$in6
-	vcipherlast	$out5,$out5,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	le?vperm	$out5,$out5,$out5,$inpperm
-	stvx_u		$out4,$x40,$out
-	stvx_u		$out5,$x50,$out
-	addi		$out,$out,0x60
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_five:
-	vcipherlast	$out0,$out0,$in3
-	vcipherlast	$out1,$out1,$in4
-	vcipherlast	$out2,$out2,$in5
-	vcipherlast	$out3,$out3,$in6
-	vcipherlast	$out4,$out4,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$inpperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_four:
-	vcipherlast	$out0,$out0,$in4
-	vcipherlast	$out1,$out1,$in5
-	vcipherlast	$out2,$out2,$in6
-	vcipherlast	$out3,$out3,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$inpperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_three:
-	vcipherlast	$out0,$out0,$in5
-	vcipherlast	$out1,$out1,$in6
-	vcipherlast	$out2,$out2,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	le?vperm	$out2,$out2,$out2,$inpperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_two:
-	vcipherlast	$out0,$out0,$in6
-	vcipherlast	$out1,$out1,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	le?vperm	$out1,$out1,$out1,$inpperm
-	stvx_u		$out0,$x00,$out
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	b		Lctr32_enc8x_done
-
-.align	5
-Lctr32_enc8x_one:
-	vcipherlast	$out0,$out0,$in7
-
-	le?vperm	$out0,$out0,$out0,$inpperm
-	stvx_u		$out0,0,$out
-	addi		$out,$out,0x10
-
-Lctr32_enc8x_done:
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$inpperm,r10,$sp	# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-	stvx		$inpperm,r10,$sp
-	addi		r10,r10,32
-	stvx		$inpperm,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x14,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
-___
-}}	}}}
-
-#########################################################################
-{{{	# XTS procedures						#
-# int aes_p8_xts_[en|de]crypt(const char *inp, char *out, size_t len,	#
-#                             const AES_KEY *key1, const AES_KEY *key2,	#
-#                             [const] unsigned char iv[16]);		#
-# If $key2 is NULL, then a "tweak chaining" mode is engaged, in which	#
-# input tweak value is assumed to be encrypted already, and last tweak	#
-# value, one suitable for consecutive call on same chunk of data, is	#
-# written back to original buffer. In addition, in "tweak chaining"	#
-# mode only complete input blocks are processed.			#
-
-my ($inp,$out,$len,$key1,$key2,$ivp,$rounds,$idx) =	map("r$_",(3..10));
-my ($rndkey0,$rndkey1,$inout) =				map("v$_",(0..2));
-my ($output,$inptail,$inpperm,$leperm,$keyperm) =	map("v$_",(3..7));
-my ($tweak,$seven,$eighty7,$tmp,$tweak1) =		map("v$_",(8..12));
-my $taillen = $key2;
-
-   ($inp,$idx) = ($idx,$inp);				# reassign
-
-$code.=<<___;
-.globl	.${prefix}_xts_encrypt
-	mr		$inp,r3				# reassign
-	li		r3,-1
-	${UCMP}i	$len,16
-	bltlr-
-
-	lis		r0,0xfff0
-	mfspr		r12,256				# save vrsave
-	li		r11,0
-	mtspr		256,r0
-
-	vspltisb	$seven,0x07			# 0x070707..07
-	le?lvsl		$leperm,r11,r11
-	le?vspltisb	$tmp,0x0f
-	le?vxor		$leperm,$leperm,$seven
-
-	li		$idx,15
-	lvx		$tweak,0,$ivp			# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$tweak,$tweak,$inptail,$inpperm
-
-	neg		r11,$inp
-	lvsr		$inpperm,0,r11			# prepare for unaligned load
-	lvx		$inout,0,$inp
-	addi		$inp,$inp,15			# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	${UCMP}i	$key2,0				# key2==NULL?
-	beq		Lxts_enc_no_key2
-
-	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
-	lwz		$rounds,240($key2)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	lvx		$rndkey0,0,$key2
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Ltweak_xts_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	bdnz		Ltweak_xts_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$tweak,$tweak,$rndkey0
-
-	li		$ivp,0				# don't chain the tweak
-	b		Lxts_enc
-
-Lxts_enc_no_key2:
-	li		$idx,-16
-	and		$len,$len,$idx			# in "tweak chaining"
-							# mode only complete
-							# blocks are processed
-Lxts_enc:
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-
-	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
-	lwz		$rounds,240($key1)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	vslb		$eighty7,$seven,$seven		# 0x808080..80
-	vor		$eighty7,$eighty7,$seven	# 0x878787..87
-	vspltisb	$tmp,1				# 0x010101..01
-	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
-
-	${UCMP}i	$len,96
-	bge		_aesp8_xts_encrypt6x
-
-	andi.		$taillen,$len,15
-	subic		r0,$len,32
-	subi		$taillen,$taillen,16
-	subfe		r0,r0,r0
-	and		r0,r0,$taillen
-	add		$inp,$inp,r0
-
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	mtctr		$rounds
-	b		Loop_xts_enc
-
-.align	5
-Loop_xts_enc:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_enc
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak
-	vcipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-	addi		$out,$out,16
-
-	subic.		$len,$len,16
-	beq		Lxts_enc_done
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-
-	subic		r0,$len,32
-	subfe		r0,r0,r0
-	and		r0,r0,$taillen
-	add		$inp,$inp,r0
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$output,$output,$rndkey0	# just in case $len<16
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	mtctr		$rounds
-	${UCMP}i	$len,16
-	bge		Loop_xts_enc
-
-	vxor		$output,$output,$tweak
-	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
-	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
-	vspltisb	$tmp,-1
-	vperm		$inptail,$inptail,$tmp,$inpperm
-	vsel		$inout,$inout,$output,$inptail
-
-	subi		r11,$out,17
-	subi		$out,$out,16
-	mtctr		$len
-	li		$len,16
-Loop_xts_enc_steal:
-	lbzu		r0,1(r11)
-	stb		r0,16(r11)
-	bdnz		Loop_xts_enc_steal
-
-	mtctr		$rounds
-	b		Loop_xts_enc			# one more time...
-
-Lxts_enc_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_enc_ret
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_enc_ret:
-	mtspr		256,r12				# restore vrsave
-	li		r3,0
-	blr
-	.long		0
-	.byte		0,12,0x04,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
-
-.globl	.${prefix}_xts_decrypt
-	mr		$inp,r3				# reassign
-	li		r3,-1
-	${UCMP}i	$len,16
-	bltlr-
-
-	lis		r0,0xfff8
-	mfspr		r12,256				# save vrsave
-	li		r11,0
-	mtspr		256,r0
-
-	andi.		r0,$len,15
-	neg		r0,r0
-	andi.		r0,r0,16
-	sub		$len,$len,r0
-
-	vspltisb	$seven,0x07			# 0x070707..07
-	le?lvsl		$leperm,r11,r11
-	le?vspltisb	$tmp,0x0f
-	le?vxor		$leperm,$leperm,$seven
-
-	li		$idx,15
-	lvx		$tweak,0,$ivp			# load [unaligned] iv
-	lvsl		$inpperm,0,$ivp
-	lvx		$inptail,$idx,$ivp
-	le?vxor		$inpperm,$inpperm,$tmp
-	vperm		$tweak,$tweak,$inptail,$inpperm
-
-	neg		r11,$inp
-	lvsr		$inpperm,0,r11			# prepare for unaligned load
-	lvx		$inout,0,$inp
-	addi		$inp,$inp,15			# 15 is not typo
-	le?vxor		$inpperm,$inpperm,$tmp
-
-	${UCMP}i	$key2,0				# key2==NULL?
-	beq		Lxts_dec_no_key2
-
-	?lvsl		$keyperm,0,$key2		# prepare for unaligned key
-	lwz		$rounds,240($key2)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	lvx		$rndkey0,0,$key2
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-Ltweak_xts_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipher		$tweak,$tweak,$rndkey0
-	lvx		$rndkey0,$idx,$key2
-	addi		$idx,$idx,16
-	bdnz		Ltweak_xts_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vcipher		$tweak,$tweak,$rndkey1
-	lvx		$rndkey1,$idx,$key2
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vcipherlast	$tweak,$tweak,$rndkey0
-
-	li		$ivp,0				# don't chain the tweak
-	b		Lxts_dec
-
-Lxts_dec_no_key2:
-	neg		$idx,$len
-	andi.		$idx,$idx,15
-	add		$len,$len,$idx			# in "tweak chaining"
-							# mode only complete
-							# blocks are processed
-Lxts_dec:
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-
-	?lvsl		$keyperm,0,$key1		# prepare for unaligned key
-	lwz		$rounds,240($key1)
-	srwi		$rounds,$rounds,1
-	subi		$rounds,$rounds,1
-	li		$idx,16
-
-	vslb		$eighty7,$seven,$seven		# 0x808080..80
-	vor		$eighty7,$eighty7,$seven	# 0x878787..87
-	vspltisb	$tmp,1				# 0x010101..01
-	vsldoi		$eighty7,$eighty7,$tmp,15	# 0x870101..01
-
-	${UCMP}i	$len,96
-	bge		_aesp8_xts_decrypt6x
-
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	mtctr		$rounds
-
-	${UCMP}i	$len,16
-	blt		Ltail_xts_dec
-	be?b		Loop_xts_dec
-
-.align	5
-Loop_xts_dec:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_dec
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak
-	vncipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-	addi		$out,$out,16
-
-	subic.		$len,$len,16
-	beq		Lxts_dec_done
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$inout,$inout,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	mtctr		$rounds
-	${UCMP}i	$len,16
-	bge		Loop_xts_dec
-
-Ltail_xts_dec:
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak1,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak1,$tweak1,$tmp
-
-	subi		$inp,$inp,16
-	add		$inp,$inp,$len
-
-	vxor		$inout,$inout,$tweak		# :-(
-	vxor		$inout,$inout,$tweak1		# :-)
-
-Loop_xts_dec_short:
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vncipher	$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-	bdnz		Loop_xts_dec_short
-
-	?vperm		$rndkey1,$rndkey1,$rndkey0,$keyperm
-	vncipher	$inout,$inout,$rndkey1
-	lvx		$rndkey1,$idx,$key1
-	li		$idx,16
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-	vxor		$rndkey0,$rndkey0,$tweak1
-	vncipherlast	$output,$inout,$rndkey0
-
-	le?vperm	$tmp,$output,$output,$leperm
-	be?nop
-	le?stvx_u	$tmp,0,$out
-	be?stvx_u	$output,0,$out
-
-	vmr		$inout,$inptail
-	lvx		$inptail,0,$inp
-	#addi		$inp,$inp,16
-	lvx		$rndkey0,0,$key1
-	lvx		$rndkey1,$idx,$key1
-	addi		$idx,$idx,16
-	vperm		$inout,$inout,$inptail,$inpperm
-	?vperm		$rndkey0,$rndkey0,$rndkey1,$keyperm
-
-	lvsr		$inpperm,0,$len			# $inpperm is no longer needed
-	vxor		$inptail,$inptail,$inptail	# $inptail is no longer needed
-	vspltisb	$tmp,-1
-	vperm		$inptail,$inptail,$tmp,$inpperm
-	vsel		$inout,$inout,$output,$inptail
-
-	vxor		$rndkey0,$rndkey0,$tweak
-	vxor		$inout,$inout,$rndkey0
-	lvx		$rndkey0,$idx,$key1
-	addi		$idx,$idx,16
-
-	subi		r11,$out,1
-	mtctr		$len
-	li		$len,16
-Loop_xts_dec_steal:
-	lbzu		r0,1(r11)
-	stb		r0,16(r11)
-	bdnz		Loop_xts_dec_steal
-
-	mtctr		$rounds
-	b		Loop_xts_dec			# one more time...
-
-Lxts_dec_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_dec_ret
-
-	vsrab		$tmp,$tweak,$seven		# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vsldoi		$tmp,$tmp,$tmp,15
-	vand		$tmp,$tmp,$eighty7
-	vxor		$tweak,$tweak,$tmp
-
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_dec_ret:
-	mtspr		256,r12				# restore vrsave
-	li		r3,0
-	blr
-	.long		0
-	.byte		0,12,0x04,0,0x80,6,6,0
-	.long		0
-.size	.${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
-___
-#########################################################################
-{{	# Optimized XTS procedures					#
-my $key_=$key2;
-my ($x00,$x10,$x20,$x30,$x40,$x50,$x60,$x70)=map("r$_",(0,3,26..31));
-    $x00=0 if ($flavour =~ /osx/);
-my ($in0,  $in1,  $in2,  $in3,  $in4,  $in5 )=map("v$_",(0..5));
-my ($out0, $out1, $out2, $out3, $out4, $out5)=map("v$_",(7,12..16));
-my ($twk0, $twk1, $twk2, $twk3, $twk4, $twk5)=map("v$_",(17..22));
-my $rndkey0="v23";	# v24-v25 rotating buffer for first found keys
-			# v26-v31 last 6 round keys
-my ($keyperm)=($out0);	# aliases with "caller", redundant assignment
-my $taillen=$x70;
-
-$code.=<<___;
-.align	5
-_aesp8_xts_encrypt6x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	mflr		r11
-	li		r7,`$FRAME+8*16+15`
-	li		r3,`$FRAME+8*16+31`
-	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
-	stvx		v20,r7,$sp		# ABI says so
-	addi		r7,r7,32
-	stvx		v21,r3,$sp
-	addi		r3,r3,32
-	stvx		v22,r7,$sp
-	addi		r7,r7,32
-	stvx		v23,r3,$sp
-	addi		r3,r3,32
-	stvx		v24,r7,$sp
-	addi		r7,r7,32
-	stvx		v25,r3,$sp
-	addi		r3,r3,32
-	stvx		v26,r7,$sp
-	addi		r7,r7,32
-	stvx		v27,r3,$sp
-	addi		r3,r3,32
-	stvx		v28,r7,$sp
-	addi		r7,r7,32
-	stvx		v29,r3,$sp
-	addi		r3,r3,32
-	stvx		v30,r7,$sp
-	stvx		v31,r3,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	xxlor		2, 32+$eighty7, 32+$eighty7
-	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
-	xxlor		1, 32+$eighty7, 32+$eighty7
-
-	# Load XOR Lconsts.
-	mr		$x70, r6
-	bl		Lconsts
-	lxvw4x		0, $x40, r6		# load XOR contents
-	mr		r6, $x70
-	li		$x70,0x70
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key1	# load key schedule
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	lvx		v31,$x00,$key1
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_xts_enc_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key1
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_xts_enc_key
-
-	lvx		v26,$x10,$key1
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key1
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key1
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key1
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key1
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key1
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$twk5,$x70,$key1	# borrow $twk5
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$twk5,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	# Switch to use the following codes with 0x010101..87 to generate tweak.
-	#     eighty7 = 0x010101..87
-	# vsrab         tmp, tweak, seven       # next tweak value, right shift 7 bits
-	# vand          tmp, tmp, eighty7       # last byte with carry
-	# vaddubm       tweak, tweak, tweak     # left shift 1 bit (x2)
-	# xxlor         vsx, 0, 0
-	# vpermxor      tweak, tweak, tmp, vsx
-
-	 vperm		$in0,$inout,$inptail,$inpperm
-	 subi		$inp,$inp,31		# undo "caller"
-	vxor		$twk0,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out0,$in0,$twk0
-	xxlor		32+$in1, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in1
-
-	 lvx_u		$in1,$x10,$inp
-	vxor		$twk1,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in1,$in1,$in1,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out1,$in1,$twk1
-	xxlor		32+$in2, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in2
-
-	 lvx_u		$in2,$x20,$inp
-	 andi.		$taillen,$len,15
-	vxor		$twk2,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in2,$in2,$in2,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out2,$in2,$twk2
-	xxlor		32+$in3, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in3
-
-	 lvx_u		$in3,$x30,$inp
-	 sub		$len,$len,$taillen
-	vxor		$twk3,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in3,$in3,$in3,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out3,$in3,$twk3
-	xxlor		32+$in4, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in4
-
-	 lvx_u		$in4,$x40,$inp
-	 subi		$len,$len,0x60
-	vxor		$twk4,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in4,$in4,$in4,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out4,$in4,$twk4
-	xxlor		32+$in5, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in5
-
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	vxor		$twk5,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in5,$in5,$in5,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out5,$in5,$twk5
-	xxlor		32+$in0, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in0
-
-	vxor		v31,v31,$rndkey0
-	mtctr		$rounds
-	b		Loop_xts_enc6x
-
-.align	5
-Loop_xts_enc6x:
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	vcipher		$out5,$out5,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_enc6x
-
-	xxlor		32+$eighty7, 1, 1	# 0x010101..87
-
-	subic		$len,$len,96		# $len-=96
-	 vxor		$in0,$twk0,v31		# xor with last round key
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk0,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	vcipher		$out5,$out5,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	 xxlor		32+$in1, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in1
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	 vxor		$in1,$twk1,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk1,$tweak,$rndkey0
-	vcipher		$out4,$out4,v25
-	vcipher		$out5,$out5,v25
-
-	and		r0,r0,$len
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out0,$out0,v26
-	vcipher		$out1,$out1,v26
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	 xxlor		32+$in2, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in2
-	vcipher		$out4,$out4,v26
-	vcipher		$out5,$out5,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in5 are loaded
-						# with last "words"
-	 vxor		$in2,$twk2,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk2,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out0,$out0,v27
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out4,$out4,v27
-	vcipher		$out5,$out5,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	 xxlor		32+$in3, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in3
-	vcipher		$out0,$out0,v28
-	vcipher		$out1,$out1,v28
-	 vxor		$in3,$twk3,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk3,$tweak,$rndkey0
-	vcipher		$out2,$out2,v28
-	vcipher		$out3,$out3,v28
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipher		$out4,$out4,v28
-	vcipher		$out5,$out5,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vand		$tmp,$tmp,$eighty7
-
-	vcipher		$out0,$out0,v29
-	vcipher		$out1,$out1,v29
-	 xxlor		32+$in4, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in4
-	vcipher		$out2,$out2,v29
-	vcipher		$out3,$out3,v29
-	 vxor		$in4,$twk4,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk4,$tweak,$rndkey0
-	vcipher		$out4,$out4,v29
-	vcipher		$out5,$out5,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vaddubm	$tweak,$tweak,$tweak
-
-	vcipher		$out0,$out0,v30
-	vcipher		$out1,$out1,v30
-	 vand		$tmp,$tmp,$eighty7
-	vcipher		$out2,$out2,v30
-	vcipher		$out3,$out3,v30
-	 xxlor		32+$in5, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in5
-	vcipher		$out4,$out4,v30
-	vcipher		$out5,$out5,v30
-	 vxor		$in5,$twk5,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk5,$tweak,$rndkey0
-
-	vcipherlast	$out0,$out0,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	 vaddubm	$tweak,$tweak,$tweak
-	vcipherlast	$out1,$out1,$in1
-	 lvx_u		$in1,$x10,$inp
-	vcipherlast	$out2,$out2,$in2
-	 le?vperm	$in0,$in0,$in0,$leperm
-	 lvx_u		$in2,$x20,$inp
-	 vand		$tmp,$tmp,$eighty7
-	vcipherlast	$out3,$out3,$in3
-	 le?vperm	$in1,$in1,$in1,$leperm
-	 lvx_u		$in3,$x30,$inp
-	vcipherlast	$out4,$out4,$in4
-	 le?vperm	$in2,$in2,$in2,$leperm
-	 lvx_u		$in4,$x40,$inp
-	 xxlor		10, 32+$in0, 32+$in0
-	 xxlor		32+$in0, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in0
-	 xxlor		32+$in0, 10, 10
-	vcipherlast	$tmp,$out5,$in5		# last block might be needed
-						# in stealing mode
-	 le?vperm	$in3,$in3,$in3,$leperm
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	 le?vperm	$in4,$in4,$in4,$leperm
-	 le?vperm	$in5,$in5,$in5,$leperm
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	 vxor		$out0,$in0,$twk0
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	 vxor		$out1,$in1,$twk1
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$twk2
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$twk3
-	le?vperm	$out5,$tmp,$tmp,$leperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$twk4
-	le?stvx_u	$out5,$x50,$out
-	be?stvx_u	$tmp, $x50,$out
-	 vxor		$out5,$in5,$twk5
-	addi		$out,$out,0x60
-
-	mtctr		$rounds
-	beq		Loop_xts_enc6x		# did $len-=96 borrow?
-
-	xxlor		32+$eighty7, 2, 2	# 0x010101..87
-
-	addic.		$len,$len,0x60
-	beq		Lxts_enc6x_zero
-	cmpwi		$len,0x20
-	blt		Lxts_enc6x_one
-	nop
-	beq		Lxts_enc6x_two
-	cmpwi		$len,0x40
-	blt		Lxts_enc6x_three
-	nop
-	beq		Lxts_enc6x_four
-
-Lxts_enc6x_five:
-	vxor		$out0,$in1,$twk0
-	vxor		$out1,$in2,$twk1
-	vxor		$out2,$in3,$twk2
-	vxor		$out3,$in4,$twk3
-	vxor		$out4,$in5,$twk4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk5		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	vxor		$tmp,$out4,$twk5	# last block prep for stealing
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_four:
-	vxor		$out0,$in2,$twk0
-	vxor		$out1,$in3,$twk1
-	vxor		$out2,$in4,$twk2
-	vxor		$out3,$in5,$twk3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk4		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	vxor		$tmp,$out3,$twk4	# last block prep for stealing
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_three:
-	vxor		$out0,$in3,$twk0
-	vxor		$out1,$in4,$twk1
-	vxor		$out2,$in5,$twk2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk3		# unused tweak
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$tmp,$out2,$twk3	# last block prep for stealing
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_two:
-	vxor		$out0,$in4,$twk0
-	vxor		$out1,$in5,$twk1
-	vxor		$out2,$out2,$out2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_enc5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk2		# unused tweak
-	vxor		$tmp,$out1,$twk2	# last block prep for stealing
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_one:
-	vxor		$out0,$in5,$twk0
-	nop
-Loop_xts_enc1x:
-	vcipher		$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_enc1x
-
-	add		$inp,$inp,$taillen
-	cmpwi		$taillen,0
-	vcipher		$out0,$out0,v24
-
-	subi		$inp,$inp,16
-	vcipher		$out0,$out0,v25
-
-	lvsr		$inpperm,0,$taillen
-	vcipher		$out0,$out0,v26
-
-	lvx_u		$in0,0,$inp
-	vcipher		$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vcipher		$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk0,$twk0,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vcipher		$out0,$out0,v30
-
-	vperm		$in0,$in0,$in0,$inpperm
-	vcipherlast	$out0,$out0,$twk0
-
-	vmr		$twk0,$twk1		# unused tweak
-	vxor		$tmp,$out0,$twk1	# last block prep for stealing
-	le?vperm	$out0,$out0,$out0,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	addi		$out,$out,0x10
-	bne		Lxts_enc6x_steal
-	b		Lxts_enc6x_done
-
-.align	4
-Lxts_enc6x_zero:
-	cmpwi		$taillen,0
-	beq		Lxts_enc6x_done
-
-	add		$inp,$inp,$taillen
-	subi		$inp,$inp,16
-	lvx_u		$in0,0,$inp
-	lvsr		$inpperm,0,$taillen	# $in5 is no more
-	le?vperm	$in0,$in0,$in0,$leperm
-	vperm		$in0,$in0,$in0,$inpperm
-	vxor		$tmp,$tmp,$twk0
-Lxts_enc6x_steal:
-	vxor		$in0,$in0,$twk0
-	vxor		$out0,$out0,$out0
-	vspltisb	$out1,-1
-	vperm		$out0,$out0,$out1,$inpperm
-	vsel		$out0,$in0,$tmp,$out0	# $tmp is last block, remember?
-
-	subi		r30,$out,17
-	subi		$out,$out,16
-	mtctr		$taillen
-Loop_xts_enc6x_steal:
-	lbzu		r0,1(r30)
-	stb		r0,16(r30)
-	bdnz		Loop_xts_enc6x_steal
-
-	li		$taillen,0
-	mtctr		$rounds
-	b		Loop_xts_enc1x		# one more time...
-
-.align	4
-Lxts_enc6x_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_enc6x_ret
-
-	vxor		$tweak,$twk0,$rndkey0
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_enc6x_ret:
-	mtlr		r11
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$seven,r10,$sp		# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x04,1,0x80,6,6,0
-	.long		0
-
-.align	5
-_aesp8_xts_enc5x:
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		_aesp8_xts_enc5x
-
-	add		$inp,$inp,$taillen
-	cmpwi		$taillen,0
-	vcipher		$out0,$out0,v24
-	vcipher		$out1,$out1,v24
-	vcipher		$out2,$out2,v24
-	vcipher		$out3,$out3,v24
-	vcipher		$out4,$out4,v24
-
-	subi		$inp,$inp,16
-	vcipher		$out0,$out0,v25
-	vcipher		$out1,$out1,v25
-	vcipher		$out2,$out2,v25
-	vcipher		$out3,$out3,v25
-	vcipher		$out4,$out4,v25
-	 vxor		$twk0,$twk0,v31
-
-	vcipher		$out0,$out0,v26
-	lvsr		$inpperm,r0,$taillen	# $in5 is no more
-	vcipher		$out1,$out1,v26
-	vcipher		$out2,$out2,v26
-	vcipher		$out3,$out3,v26
-	vcipher		$out4,$out4,v26
-	 vxor		$in1,$twk1,v31
-
-	vcipher		$out0,$out0,v27
-	lvx_u		$in0,0,$inp
-	vcipher		$out1,$out1,v27
-	vcipher		$out2,$out2,v27
-	vcipher		$out3,$out3,v27
-	vcipher		$out4,$out4,v27
-	 vxor		$in2,$twk2,v31
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vcipher		$out0,$out0,v28
-	vcipher		$out1,$out1,v28
-	vcipher		$out2,$out2,v28
-	vcipher		$out3,$out3,v28
-	vcipher		$out4,$out4,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vxor		$in3,$twk3,v31
-
-	vcipher		$out0,$out0,v29
-	le?vperm	$in0,$in0,$in0,$leperm
-	vcipher		$out1,$out1,v29
-	vcipher		$out2,$out2,v29
-	vcipher		$out3,$out3,v29
-	vcipher		$out4,$out4,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$in4,$twk4,v31
-
-	vcipher		$out0,$out0,v30
-	vperm		$in0,$in0,$in0,$inpperm
-	vcipher		$out1,$out1,v30
-	vcipher		$out2,$out2,v30
-	vcipher		$out3,$out3,v30
-	vcipher		$out4,$out4,v30
-
-	vcipherlast	$out0,$out0,$twk0
-	vcipherlast	$out1,$out1,$in1
-	vcipherlast	$out2,$out2,$in2
-	vcipherlast	$out3,$out3,$in3
-	vcipherlast	$out4,$out4,$in4
-	blr
-        .long   	0
-        .byte   	0,12,0x14,0,0,0,0,0
-
-.align	5
-_aesp8_xts_decrypt6x:
-	$STU		$sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
-	mflr		r11
-	li		r7,`$FRAME+8*16+15`
-	li		r3,`$FRAME+8*16+31`
-	$PUSH		r11,`$FRAME+21*16+6*$SIZE_T+$LRSAVE`($sp)
-	stvx		v20,r7,$sp		# ABI says so
-	addi		r7,r7,32
-	stvx		v21,r3,$sp
-	addi		r3,r3,32
-	stvx		v22,r7,$sp
-	addi		r7,r7,32
-	stvx		v23,r3,$sp
-	addi		r3,r3,32
-	stvx		v24,r7,$sp
-	addi		r7,r7,32
-	stvx		v25,r3,$sp
-	addi		r3,r3,32
-	stvx		v26,r7,$sp
-	addi		r7,r7,32
-	stvx		v27,r3,$sp
-	addi		r3,r3,32
-	stvx		v28,r7,$sp
-	addi		r7,r7,32
-	stvx		v29,r3,$sp
-	addi		r3,r3,32
-	stvx		v30,r7,$sp
-	stvx		v31,r3,$sp
-	li		r0,-1
-	stw		$vrsave,`$FRAME+21*16-4`($sp)	# save vrsave
-	li		$x10,0x10
-	$PUSH		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	li		$x20,0x20
-	$PUSH		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	li		$x30,0x30
-	$PUSH		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	li		$x40,0x40
-	$PUSH		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	li		$x50,0x50
-	$PUSH		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	li		$x60,0x60
-	$PUSH		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	li		$x70,0x70
-	mtspr		256,r0
-
-	xxlor		2, 32+$eighty7, 32+$eighty7
-	vsldoi		$eighty7,$tmp,$eighty7,1        # 0x010101..87
-	xxlor		1, 32+$eighty7, 32+$eighty7
-
-	# Load XOR Lconsts.
-	mr		$x70, r6
-	bl		Lconsts
-	lxvw4x		0, $x40, r6		# load XOR contents
-	mr		r6, $x70
-	li		$x70,0x70
-
-	subi		$rounds,$rounds,3	# -4 in total
-
-	lvx		$rndkey0,$x00,$key1	# load key schedule
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	lvx		v31,$x00,$key1
-	?vperm		$rndkey0,$rndkey0,v30,$keyperm
-	addi		$key_,$sp,$FRAME+15
-	mtctr		$rounds
-
-Load_xts_dec_key:
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v30,$x10,$key1
-	addi		$key1,$key1,0x20
-	stvx		v24,$x00,$key_		# off-load round[1]
-	?vperm		v25,v31,v30,$keyperm
-	lvx		v31,$x00,$key1
-	stvx		v25,$x10,$key_		# off-load round[2]
-	addi		$key_,$key_,0x20
-	bdnz		Load_xts_dec_key
-
-	lvx		v26,$x10,$key1
-	?vperm		v24,v30,v31,$keyperm
-	lvx		v27,$x20,$key1
-	stvx		v24,$x00,$key_		# off-load round[3]
-	?vperm		v25,v31,v26,$keyperm
-	lvx		v28,$x30,$key1
-	stvx		v25,$x10,$key_		# off-load round[4]
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	?vperm		v26,v26,v27,$keyperm
-	lvx		v29,$x40,$key1
-	?vperm		v27,v27,v28,$keyperm
-	lvx		v30,$x50,$key1
-	?vperm		v28,v28,v29,$keyperm
-	lvx		v31,$x60,$key1
-	?vperm		v29,v29,v30,$keyperm
-	lvx		$twk5,$x70,$key1	# borrow $twk5
-	?vperm		v30,v30,v31,$keyperm
-	lvx		v24,$x00,$key_		# pre-load round[1]
-	?vperm		v31,v31,$twk5,$keyperm
-	lvx		v25,$x10,$key_		# pre-load round[2]
-
-	 vperm		$in0,$inout,$inptail,$inpperm
-	 subi		$inp,$inp,31		# undo "caller"
-	vxor		$twk0,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out0,$in0,$twk0
-	xxlor		32+$in1, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in1
-
-	 lvx_u		$in1,$x10,$inp
-	vxor		$twk1,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in1,$in1,$in1,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out1,$in1,$twk1
-	xxlor		32+$in2, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in2
-
-	 lvx_u		$in2,$x20,$inp
-	 andi.		$taillen,$len,15
-	vxor		$twk2,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in2,$in2,$in2,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out2,$in2,$twk2
-	xxlor		32+$in3, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in3
-
-	 lvx_u		$in3,$x30,$inp
-	 sub		$len,$len,$taillen
-	vxor		$twk3,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in3,$in3,$in3,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out3,$in3,$twk3
-	xxlor		32+$in4, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in4
-
-	 lvx_u		$in4,$x40,$inp
-	 subi		$len,$len,0x60
-	vxor		$twk4,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in4,$in4,$in4,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out4,$in4,$twk4
-	xxlor		32+$in5, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in5
-
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	vxor		$twk5,$tweak,$rndkey0
-	vsrab		$tmp,$tweak,$seven	# next tweak value
-	vaddubm		$tweak,$tweak,$tweak
-	 le?vperm	$in5,$in5,$in5,$leperm
-	vand		$tmp,$tmp,$eighty7
-	 vxor		$out5,$in5,$twk5
-	xxlor		32+$in0, 0, 0
-	vpermxor	$tweak, $tweak, $tmp, $in0
-
-	vxor		v31,v31,$rndkey0
-	mtctr		$rounds
-	b		Loop_xts_dec6x
-
-.align	5
-Loop_xts_dec6x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_dec6x
-
-	xxlor		32+$eighty7, 1, 1	# 0x010101..87
-
-	subic		$len,$len,96		# $len-=96
-	 vxor		$in0,$twk0,v31		# xor with last round key
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk0,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	vncipher	$out5,$out5,v24
-
-	subfe.		r0,r0,r0		# borrow?-1:0
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	 xxlor		32+$in1, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in1
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	 vxor		$in1,$twk1,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk1,$tweak,$rndkey0
-	vncipher	$out4,$out4,v25
-	vncipher	$out5,$out5,v25
-
-	and		r0,r0,$len
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	 xxlor		32+$in2, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in2
-	vncipher	$out4,$out4,v26
-	vncipher	$out5,$out5,v26
-
-	add		$inp,$inp,r0		# $inp is adjusted in such
-						# way that at exit from the
-						# loop inX-in5 are loaded
-						# with last "words"
-	 vxor		$in2,$twk2,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk2,$tweak,$rndkey0
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out0,$out0,v27
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out4,$out4,v27
-	vncipher	$out5,$out5,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	 xxlor		32+$in3, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in3
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	 vxor		$in3,$twk3,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk3,$tweak,$rndkey0
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipher	$out4,$out4,v28
-	vncipher	$out5,$out5,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vand		$tmp,$tmp,$eighty7
-
-	vncipher	$out0,$out0,v29
-	vncipher	$out1,$out1,v29
-	 xxlor		32+$in4, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in4
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	 vxor		$in4,$twk4,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk4,$tweak,$rndkey0
-	vncipher	$out4,$out4,v29
-	vncipher	$out5,$out5,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vaddubm	$tweak,$tweak,$tweak
-
-	vncipher	$out0,$out0,v30
-	vncipher	$out1,$out1,v30
-	 vand		$tmp,$tmp,$eighty7
-	vncipher	$out2,$out2,v30
-	vncipher	$out3,$out3,v30
-	 xxlor		32+$in5, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in5
-	vncipher	$out4,$out4,v30
-	vncipher	$out5,$out5,v30
-	 vxor		$in5,$twk5,v31
-	 vsrab		$tmp,$tweak,$seven	# next tweak value
-	 vxor		$twk5,$tweak,$rndkey0
-
-	vncipherlast	$out0,$out0,$in0
-	 lvx_u		$in0,$x00,$inp		# load next input block
-	 vaddubm	$tweak,$tweak,$tweak
-	vncipherlast	$out1,$out1,$in1
-	 lvx_u		$in1,$x10,$inp
-	vncipherlast	$out2,$out2,$in2
-	 le?vperm	$in0,$in0,$in0,$leperm
-	 lvx_u		$in2,$x20,$inp
-	 vand		$tmp,$tmp,$eighty7
-	vncipherlast	$out3,$out3,$in3
-	 le?vperm	$in1,$in1,$in1,$leperm
-	 lvx_u		$in3,$x30,$inp
-	vncipherlast	$out4,$out4,$in4
-	 le?vperm	$in2,$in2,$in2,$leperm
-	 lvx_u		$in4,$x40,$inp
-	 xxlor		10, 32+$in0, 32+$in0
-	 xxlor		32+$in0, 0, 0
-	 vpermxor	$tweak, $tweak, $tmp, $in0
-	 xxlor		32+$in0, 10, 10
-	vncipherlast	$out5,$out5,$in5
-	 le?vperm	$in3,$in3,$in3,$leperm
-	 lvx_u		$in5,$x50,$inp
-	 addi		$inp,$inp,0x60
-	 le?vperm	$in4,$in4,$in4,$leperm
-	 le?vperm	$in5,$in5,$in5,$leperm
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	 vxor		$out0,$in0,$twk0
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	 vxor		$out1,$in1,$twk1
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	 vxor		$out2,$in2,$twk2
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	 vxor		$out3,$in3,$twk3
-	le?vperm	$out5,$out5,$out5,$leperm
-	stvx_u		$out4,$x40,$out
-	 vxor		$out4,$in4,$twk4
-	stvx_u		$out5,$x50,$out
-	 vxor		$out5,$in5,$twk5
-	addi		$out,$out,0x60
-
-	mtctr		$rounds
-	beq		Loop_xts_dec6x		# did $len-=96 borrow?
-
-	xxlor		32+$eighty7, 2, 2	# 0x010101..87
-
-	addic.		$len,$len,0x60
-	beq		Lxts_dec6x_zero
-	cmpwi		$len,0x20
-	blt		Lxts_dec6x_one
-	nop
-	beq		Lxts_dec6x_two
-	cmpwi		$len,0x40
-	blt		Lxts_dec6x_three
-	nop
-	beq		Lxts_dec6x_four
-
-Lxts_dec6x_five:
-	vxor		$out0,$in1,$twk0
-	vxor		$out1,$in2,$twk1
-	vxor		$out2,$in3,$twk2
-	vxor		$out3,$in4,$twk3
-	vxor		$out4,$in5,$twk4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk5		# unused tweak
-	vxor		$twk1,$tweak,$rndkey0
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk1
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	le?vperm	$out4,$out4,$out4,$leperm
-	stvx_u		$out3,$x30,$out
-	stvx_u		$out4,$x40,$out
-	addi		$out,$out,0x50
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_four:
-	vxor		$out0,$in2,$twk0
-	vxor		$out1,$in3,$twk1
-	vxor		$out2,$in4,$twk2
-	vxor		$out3,$in5,$twk3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk4		# unused tweak
-	vmr		$twk1,$twk5
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk5
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	le?vperm	$out3,$out3,$out3,$leperm
-	stvx_u		$out2,$x20,$out
-	stvx_u		$out3,$x30,$out
-	addi		$out,$out,0x40
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_three:
-	vxor		$out0,$in3,$twk0
-	vxor		$out1,$in4,$twk1
-	vxor		$out2,$in5,$twk2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk3		# unused tweak
-	vmr		$twk1,$twk4
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk4
-	le?vperm	$out2,$out2,$out2,$leperm
-	stvx_u		$out1,$x10,$out
-	stvx_u		$out2,$x20,$out
-	addi		$out,$out,0x30
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_two:
-	vxor		$out0,$in4,$twk0
-	vxor		$out1,$in5,$twk1
-	vxor		$out2,$out2,$out2
-	vxor		$out3,$out3,$out3
-	vxor		$out4,$out4,$out4
-
-	bl		_aesp8_xts_dec5x
-
-	le?vperm	$out0,$out0,$out0,$leperm
-	vmr		$twk0,$twk2		# unused tweak
-	vmr		$twk1,$twk3
-	le?vperm	$out1,$out1,$out1,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	vxor		$out0,$in0,$twk3
-	stvx_u		$out1,$x10,$out
-	addi		$out,$out,0x20
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_one:
-	vxor		$out0,$in5,$twk0
-	nop
-Loop_xts_dec1x:
-	vncipher	$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Loop_xts_dec1x
-
-	subi		r0,$taillen,1
-	vncipher	$out0,$out0,v24
-
-	andi.		r0,r0,16
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-
-	sub		$inp,$inp,r0
-	vncipher	$out0,$out0,v26
-
-	lvx_u		$in0,0,$inp
-	vncipher	$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk0,$twk0,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out0,$out0,v30
-
-	mtctr		$rounds
-	vncipherlast	$out0,$out0,$twk0
-
-	vmr		$twk0,$twk1		# unused tweak
-	vmr		$twk1,$twk2
-	le?vperm	$out0,$out0,$out0,$leperm
-	stvx_u		$out0,$x00,$out		# store output
-	addi		$out,$out,0x10
-	vxor		$out0,$in0,$twk2
-	bne		Lxts_dec6x_steal
-	b		Lxts_dec6x_done
-
-.align	4
-Lxts_dec6x_zero:
-	cmpwi		$taillen,0
-	beq		Lxts_dec6x_done
-
-	lvx_u		$in0,0,$inp
-	le?vperm	$in0,$in0,$in0,$leperm
-	vxor		$out0,$in0,$twk1
-Lxts_dec6x_steal:
-	vncipher	$out0,$out0,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		Lxts_dec6x_steal
-
-	add		$inp,$inp,$taillen
-	vncipher	$out0,$out0,v24
-
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-
-	lvx_u		$in0,0,$inp
-	vncipher	$out0,$out0,v26
-
-	lvsr		$inpperm,0,$taillen	# $in5 is no more
-	vncipher	$out0,$out0,v27
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-
-	vncipher	$out0,$out0,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$twk1,$twk1,v31
-
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out0,$out0,v30
-
-	vperm		$in0,$in0,$in0,$inpperm
-	vncipherlast	$tmp,$out0,$twk1
-
-	le?vperm	$out0,$tmp,$tmp,$leperm
-	le?stvx_u	$out0,0,$out
-	be?stvx_u	$tmp,0,$out
-
-	vxor		$out0,$out0,$out0
-	vspltisb	$out1,-1
-	vperm		$out0,$out0,$out1,$inpperm
-	vsel		$out0,$in0,$tmp,$out0
-	vxor		$out0,$out0,$twk0
-
-	subi		r30,$out,1
-	mtctr		$taillen
-Loop_xts_dec6x_steal:
-	lbzu		r0,1(r30)
-	stb		r0,16(r30)
-	bdnz		Loop_xts_dec6x_steal
-
-	li		$taillen,0
-	mtctr		$rounds
-	b		Loop_xts_dec1x		# one more time...
-
-.align	4
-Lxts_dec6x_done:
-	${UCMP}i	$ivp,0
-	beq		Lxts_dec6x_ret
-
-	vxor		$tweak,$twk0,$rndkey0
-	le?vperm	$tweak,$tweak,$tweak,$leperm
-	stvx_u		$tweak,0,$ivp
-
-Lxts_dec6x_ret:
-	mtlr		r11
-	li		r10,`$FRAME+15`
-	li		r11,`$FRAME+31`
-	stvx		$seven,r10,$sp		# wipe copies of round keys
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-	stvx		$seven,r10,$sp
-	addi		r10,r10,32
-	stvx		$seven,r11,$sp
-	addi		r11,r11,32
-
-	mtspr		256,$vrsave
-	lvx		v20,r10,$sp		# ABI says so
-	addi		r10,r10,32
-	lvx		v21,r11,$sp
-	addi		r11,r11,32
-	lvx		v22,r10,$sp
-	addi		r10,r10,32
-	lvx		v23,r11,$sp
-	addi		r11,r11,32
-	lvx		v24,r10,$sp
-	addi		r10,r10,32
-	lvx		v25,r11,$sp
-	addi		r11,r11,32
-	lvx		v26,r10,$sp
-	addi		r10,r10,32
-	lvx		v27,r11,$sp
-	addi		r11,r11,32
-	lvx		v28,r10,$sp
-	addi		r10,r10,32
-	lvx		v29,r11,$sp
-	addi		r11,r11,32
-	lvx		v30,r10,$sp
-	lvx		v31,r11,$sp
-	$POP		r26,`$FRAME+21*16+0*$SIZE_T`($sp)
-	$POP		r27,`$FRAME+21*16+1*$SIZE_T`($sp)
-	$POP		r28,`$FRAME+21*16+2*$SIZE_T`($sp)
-	$POP		r29,`$FRAME+21*16+3*$SIZE_T`($sp)
-	$POP		r30,`$FRAME+21*16+4*$SIZE_T`($sp)
-	$POP		r31,`$FRAME+21*16+5*$SIZE_T`($sp)
-	addi		$sp,$sp,`$FRAME+21*16+6*$SIZE_T`
-	blr
-	.long		0
-	.byte		0,12,0x04,1,0x80,6,6,0
-	.long		0
-
-.align	5
-_aesp8_xts_dec5x:
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-	lvx		v24,$x20,$key_		# round[3]
-	addi		$key_,$key_,0x20
-
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	lvx		v25,$x10,$key_		# round[4]
-	bdnz		_aesp8_xts_dec5x
-
-	subi		r0,$taillen,1
-	vncipher	$out0,$out0,v24
-	vncipher	$out1,$out1,v24
-	vncipher	$out2,$out2,v24
-	vncipher	$out3,$out3,v24
-	vncipher	$out4,$out4,v24
-
-	andi.		r0,r0,16
-	cmpwi		$taillen,0
-	vncipher	$out0,$out0,v25
-	vncipher	$out1,$out1,v25
-	vncipher	$out2,$out2,v25
-	vncipher	$out3,$out3,v25
-	vncipher	$out4,$out4,v25
-	 vxor		$twk0,$twk0,v31
-
-	sub		$inp,$inp,r0
-	vncipher	$out0,$out0,v26
-	vncipher	$out1,$out1,v26
-	vncipher	$out2,$out2,v26
-	vncipher	$out3,$out3,v26
-	vncipher	$out4,$out4,v26
-	 vxor		$in1,$twk1,v31
-
-	vncipher	$out0,$out0,v27
-	lvx_u		$in0,0,$inp
-	vncipher	$out1,$out1,v27
-	vncipher	$out2,$out2,v27
-	vncipher	$out3,$out3,v27
-	vncipher	$out4,$out4,v27
-	 vxor		$in2,$twk2,v31
-
-	addi		$key_,$sp,$FRAME+15	# rewind $key_
-	vncipher	$out0,$out0,v28
-	vncipher	$out1,$out1,v28
-	vncipher	$out2,$out2,v28
-	vncipher	$out3,$out3,v28
-	vncipher	$out4,$out4,v28
-	lvx		v24,$x00,$key_		# re-pre-load round[1]
-	 vxor		$in3,$twk3,v31
-
-	vncipher	$out0,$out0,v29
-	le?vperm	$in0,$in0,$in0,$leperm
-	vncipher	$out1,$out1,v29
-	vncipher	$out2,$out2,v29
-	vncipher	$out3,$out3,v29
-	vncipher	$out4,$out4,v29
-	lvx		v25,$x10,$key_		# re-pre-load round[2]
-	 vxor		$in4,$twk4,v31
-
-	vncipher	$out0,$out0,v30
-	vncipher	$out1,$out1,v30
-	vncipher	$out2,$out2,v30
-	vncipher	$out3,$out3,v30
-	vncipher	$out4,$out4,v30
-
-	vncipherlast	$out0,$out0,$twk0
-	vncipherlast	$out1,$out1,$in1
-	vncipherlast	$out2,$out2,$in2
-	vncipherlast	$out3,$out3,$in3
-	vncipherlast	$out4,$out4,$in4
-	mtctr		$rounds
-	blr
-        .long   	0
-        .byte   	0,12,0x14,0,0,0,0,0
-___
-}}	}}}
-
-my $consts=1;
-foreach(split("\n",$code)) {
-        s/\`([^\`]*)\`/eval($1)/geo;
-
-	# constants table endian-specific conversion
-	if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
-	    my $conv=$3;
-	    my @bytes=();
-
-	    # convert to endian-agnostic format
-	    if ($1 eq "long") {
-	      foreach (split(/,\s*/,$2)) {
-		my $l = /^0/?oct:int;
-		push @bytes,($l>>24)&0xff,($l>>16)&0xff,($l>>8)&0xff,$l&0xff;
-	      }
-	    } else {
-		@bytes = map(/^0/?oct:int,split(/,\s*/,$2));
-	    }
-
-	    # little-endian conversion
-	    if ($flavour =~ /le$/o) {
-		SWITCH: for($conv)  {
-		    /\?inv/ && do   { @bytes=map($_^0xf,@bytes); last; };
-		    /\?rev/ && do   { @bytes=reverse(@bytes);    last; };
-		}
-	    }
-
-	    #emit
-	    print ".byte\t",join(',',map (sprintf("0x%02x",$_),@bytes)),"\n";
-	    next;
-	}
-	$consts=0 if (m/Lconsts:/o);	# end of table
-
-	# instructions prefixed with '?' are endian-specific and need
-	# to be adjusted accordingly...
-	if ($flavour =~ /le$/o) {	# little-endian
-	    s/le\?//o		or
-	    s/be\?/#be#/o	or
-	    s/\?lvsr/lvsl/o	or
-	    s/\?lvsl/lvsr/o	or
-	    s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
-	    s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
-	    s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
-	} else {			# big-endian
-	    s/le\?/#le#/o	or
-	    s/be\?//o		or
-	    s/\?([a-z]+)/$1/o;
-	}
-
-        print $_,"\n";
-}
-
-close STDOUT;
diff --git a/arch/powerpc/crypto/vmx.c b/arch/powerpc/crypto/vmx.c
index 0b725e826388..7d2beb774f99 100644
--- a/arch/powerpc/crypto/vmx.c
+++ b/arch/powerpc/crypto/vmx.c
@@ -27,13 +27,9 @@ static int __init p8_init(void)
 	if (ret)
 		goto err;
 
-	ret = crypto_register_alg(&p8_aes_alg);
-	if (ret)
-		goto err_unregister_ghash;
-
 	ret = crypto_register_skcipher(&p8_aes_cbc_alg);
 	if (ret)
-		goto err_unregister_aes;
+		goto err_unregister_ghash;
 
 	ret = crypto_register_skcipher(&p8_aes_ctr_alg);
 	if (ret)
@@ -49,8 +45,6 @@ err_unregister_aes_ctr:
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 err_unregister_aes_cbc:
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-err_unregister_aes:
-	crypto_unregister_alg(&p8_aes_alg);
 err_unregister_ghash:
 	crypto_unregister_shash(&p8_ghash_alg);
 err:
@@ -62,7 +56,6 @@ static void __exit p8_exit(void)
 	crypto_unregister_skcipher(&p8_aes_xts_alg);
 	crypto_unregister_skcipher(&p8_aes_ctr_alg);
 	crypto_unregister_skcipher(&p8_aes_cbc_alg);
-	crypto_unregister_alg(&p8_aes_alg);
 	crypto_unregister_shash(&p8_ghash_alg);
 }
 
@@ -74,4 +67,3 @@ MODULE_DESCRIPTION("IBM VMX cryptographic acceleration instructions "
 		   "support on Power 8");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("1.0.0");
-MODULE_IMPORT_NS("CRYPTO_INTERNAL");
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 14c5acb935e9..22d4eaab15f3 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -6,11 +6,9 @@ config CRYPTO_AES_RISCV64
 	tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS"
 	depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
 		   RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
-	select CRYPTO_ALGAPI
 	select CRYPTO_LIB_AES
 	select CRYPTO_SKCIPHER
 	help
-	  Block cipher: AES cipher algorithms
 	  Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS
 
 	  Architecture: riscv64 using:
diff --git a/arch/riscv/crypto/aes-macros.S b/arch/riscv/crypto/aes-macros.S
index d1a258d04bc7..1384164621a5 100644
--- a/arch/riscv/crypto/aes-macros.S
+++ b/arch/riscv/crypto/aes-macros.S
@@ -51,8 +51,10 @@
 //   - If AES-256, loads round keys into v1-v15 and continues onwards.
 //
 // Also sets vl=4 and vtype=e32,m1,ta,ma.  Clobbers t0 and t1.
-.macro	aes_begin	keyp, label128, label192
+.macro	aes_begin	keyp, label128, label192, key_len
+.ifb \key_len
 	lwu		t0, 480(\keyp)	// t0 = key length in bytes
+.endif
 	li		t1, 24		// t1 = key length for AES-192
 	vsetivli	zero, 4, e32, m1, ta, ma
 	vle32.v		v1, (\keyp)
@@ -76,12 +78,20 @@
 	vle32.v		v10, (\keyp)
 	addi		\keyp, \keyp, 16
 	vle32.v		v11, (\keyp)
+.ifb \key_len
 	blt		t0, t1, \label128	// If AES-128, goto label128.
+.else
+	blt		\key_len, t1, \label128	// If AES-128, goto label128.
+.endif
 	addi		\keyp, \keyp, 16
 	vle32.v		v12, (\keyp)
 	addi		\keyp, \keyp, 16
 	vle32.v		v13, (\keyp)
+.ifb \key_len
 	beq		t0, t1, \label192	// If AES-192, goto label192.
+.else
+	beq		\key_len, t1, \label192	// If AES-192, goto label192.
+.endif
 	// Else, it's AES-256.
 	addi		\keyp, \keyp, 16
 	vle32.v		v14, (\keyp)
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index f814ee048555..8bbf7f348c23 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * AES using the RISC-V vector crypto extensions.  Includes the bare block
- * cipher and the ECB, CBC, CBC-CTS, CTR, and XTS modes.
+ * AES modes using the RISC-V vector crypto extensions
  *
  * Copyright (C) 2023 VRULL GmbH
  * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
@@ -15,7 +14,6 @@
 #include <asm/simd.h>
 #include <asm/vector.h>
 #include <crypto/aes.h>
-#include <crypto/internal/cipher.h>
 #include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
@@ -23,13 +21,6 @@
 #include <linux/linkage.h>
 #include <linux/module.h>
 
-asmlinkage void aes_encrypt_zvkned(const struct crypto_aes_ctx *key,
-				   const u8 in[AES_BLOCK_SIZE],
-				   u8 out[AES_BLOCK_SIZE]);
-asmlinkage void aes_decrypt_zvkned(const struct crypto_aes_ctx *key,
-				   const u8 in[AES_BLOCK_SIZE],
-				   u8 out[AES_BLOCK_SIZE]);
-
 asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
 				       const u8 *in, u8 *out, size_t len);
 asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key,
@@ -86,14 +77,6 @@ static int riscv64_aes_setkey(struct crypto_aes_ctx *ctx,
 	return aes_expandkey(ctx, key, keylen);
 }
 
-static int riscv64_aes_setkey_cipher(struct crypto_tfm *tfm,
-				     const u8 *key, unsigned int keylen)
-{
-	struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	return riscv64_aes_setkey(ctx, key, keylen);
-}
-
 static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
 				       const u8 *key, unsigned int keylen)
 {
@@ -102,34 +85,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
 	return riscv64_aes_setkey(ctx, key, keylen);
 }
 
-/* Bare AES, without a mode of operation */
-
-static void riscv64_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (crypto_simd_usable()) {
-		kernel_vector_begin();
-		aes_encrypt_zvkned(ctx, src, dst);
-		kernel_vector_end();
-	} else {
-		aes_encrypt(ctx, dst, src);
-	}
-}
-
-static void riscv64_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	const struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	if (crypto_simd_usable()) {
-		kernel_vector_begin();
-		aes_decrypt_zvkned(ctx, src, dst);
-		kernel_vector_end();
-	} else {
-		aes_decrypt(ctx, dst, src);
-	}
-}
-
 /* AES-ECB */
 
 static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc)
@@ -338,7 +293,7 @@ static int riscv64_aes_ctr_crypt(struct skcipher_request *req)
 
 struct riscv64_aes_xts_ctx {
 	struct crypto_aes_ctx ctx1;
-	struct crypto_aes_ctx ctx2;
+	struct aes_enckey tweak_key;
 };
 
 static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
@@ -348,7 +303,7 @@ static int riscv64_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
 
 	return xts_verify_key(tfm, key, keylen) ?:
 	       riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?:
-	       riscv64_aes_setkey(&ctx->ctx2, key + keylen / 2, keylen / 2);
+	       aes_prepareenckey(&ctx->tweak_key, key + keylen / 2, keylen / 2);
 }
 
 static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc)
@@ -366,9 +321,7 @@ static int riscv64_aes_xts_crypt(struct skcipher_request *req, bool enc)
 		return -EINVAL;
 
 	/* Encrypt the IV with the tweak key to get the first tweak. */
-	kernel_vector_begin();
-	aes_encrypt_zvkned(&ctx->ctx2, req->iv, req->iv);
-	kernel_vector_end();
+	aes_encrypt(&ctx->tweak_key, req->iv, req->iv);
 
 	err = skcipher_walk_virt(&walk, req, false);
 
@@ -456,23 +409,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
 
 /* Algorithm definitions */
 
-static struct crypto_alg riscv64_zvkned_aes_cipher_alg = {
-	.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize = AES_BLOCK_SIZE,
-	.cra_ctxsize = sizeof(struct crypto_aes_ctx),
-	.cra_priority = 300,
-	.cra_name = "aes",
-	.cra_driver_name = "aes-riscv64-zvkned",
-	.cra_cipher = {
-		.cia_min_keysize = AES_MIN_KEY_SIZE,
-		.cia_max_keysize = AES_MAX_KEY_SIZE,
-		.cia_setkey = riscv64_aes_setkey_cipher,
-		.cia_encrypt = riscv64_aes_encrypt,
-		.cia_decrypt = riscv64_aes_decrypt,
-	},
-	.cra_module = THIS_MODULE,
-};
-
 static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
 	{
 		.setkey = riscv64_aes_setkey_skcipher,
@@ -574,15 +510,11 @@ static int __init riscv64_aes_mod_init(void)
 
 	if (riscv_isa_extension_available(NULL, ZVKNED) &&
 	    riscv_vector_vlen() >= 128) {
-		err = crypto_register_alg(&riscv64_zvkned_aes_cipher_alg);
-		if (err)
-			return err;
-
 		err = crypto_register_skciphers(
 			riscv64_zvkned_aes_skcipher_algs,
 			ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
 		if (err)
-			goto unregister_zvkned_cipher_alg;
+			return err;
 
 		if (riscv_isa_extension_available(NULL, ZVKB)) {
 			err = crypto_register_skcipher(
@@ -607,8 +539,6 @@ unregister_zvkned_zvkb_skcipher_alg:
 unregister_zvkned_skcipher_algs:
 	crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
 				    ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
-unregister_zvkned_cipher_alg:
-	crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg);
 	return err;
 }
 
@@ -620,7 +550,6 @@ static void __exit riscv64_aes_mod_exit(void)
 		crypto_unregister_skcipher(&riscv64_zvkned_zvkb_aes_skcipher_alg);
 	crypto_unregister_skciphers(riscv64_zvkned_aes_skcipher_algs,
 				    ARRAY_SIZE(riscv64_zvkned_aes_skcipher_algs));
-	crypto_unregister_alg(&riscv64_zvkned_aes_cipher_alg);
 }
 
 module_init(riscv64_aes_mod_init);
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S
index 23d063f94ce6..d0fc4581a380 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned.S
+++ b/arch/riscv/crypto/aes-riscv64-zvkned.S
@@ -56,33 +56,6 @@
 #define LEN		a3
 #define IVP		a4
 
-.macro	__aes_crypt_zvkned	enc, keylen
-	vle32.v		v16, (INP)
-	aes_crypt	v16, \enc, \keylen
-	vse32.v		v16, (OUTP)
-	ret
-.endm
-
-.macro	aes_crypt_zvkned	enc
-	aes_begin	KEYP, 128f, 192f
-	__aes_crypt_zvkned	\enc, 256
-128:
-	__aes_crypt_zvkned	\enc, 128
-192:
-	__aes_crypt_zvkned	\enc, 192
-.endm
-
-// void aes_encrypt_zvkned(const struct crypto_aes_ctx *key,
-//			   const u8 in[16], u8 out[16]);
-SYM_FUNC_START(aes_encrypt_zvkned)
-	aes_crypt_zvkned	1
-SYM_FUNC_END(aes_encrypt_zvkned)
-
-// Same prototype and calling convention as the encryption function
-SYM_FUNC_START(aes_decrypt_zvkned)
-	aes_crypt_zvkned	0
-SYM_FUNC_END(aes_decrypt_zvkned)
-
 .macro	__aes_ecb_crypt	enc, keylen
 	srli		t0, LEN, 2
 	// t0 is the remaining length in 32-bit words.  It's a multiple of 4.
diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig
index 7a91d300c549..6996f3a3101a 100644
--- a/arch/s390/configs/debug_defconfig
+++ b/arch/s390/configs/debug_defconfig
@@ -771,7 +771,7 @@ CONFIG_CRYPTO_DH=m
 CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
-CONFIG_CRYPTO_AES_TI=m
+CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig
index 3bb2aa8ecd13..f7c99bbc3a6f 100644
--- a/arch/s390/configs/defconfig
+++ b/arch/s390/configs/defconfig
@@ -755,7 +755,7 @@ CONFIG_CRYPTO_DH=m
 CONFIG_CRYPTO_ECDH=m
 CONFIG_CRYPTO_ECDSA=m
 CONFIG_CRYPTO_ECRDSA=m
-CONFIG_CRYPTO_AES_TI=m
+CONFIG_CRYPTO_AES=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig
index f838ca055f6d..79a2d0034258 100644
--- a/arch/s390/crypto/Kconfig
+++ b/arch/s390/crypto/Kconfig
@@ -14,10 +14,8 @@ config CRYPTO_GHASH_S390
 
 config CRYPTO_AES_S390
 	tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM"
-	select CRYPTO_ALGAPI
 	select CRYPTO_SKCIPHER
 	help
-	  Block cipher: AES cipher algorithms (FIPS 197)
 	  AEAD cipher: AES with GCM
 	  Length-preserving ciphers: AES with ECB, CBC, XTS, and CTR modes
 
diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index d0a295435680..62edc66d5478 100644
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -20,7 +20,6 @@
 #include <crypto/algapi.h>
 #include <crypto/ghash.h>
 #include <crypto/internal/aead.h>
-#include <crypto/internal/cipher.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/scatterwalk.h>
 #include <linux/err.h>
@@ -45,7 +44,6 @@ struct s390_aes_ctx {
 	unsigned long fc;
 	union {
 		struct crypto_skcipher *skcipher;
-		struct crypto_cipher *cip;
 	} fallback;
 };
 
@@ -72,109 +70,6 @@ struct gcm_sg_walk {
 	unsigned int nbytes;
 };
 
-static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key,
-		unsigned int key_len)
-{
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-
-	sctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
-	sctx->fallback.cip->base.crt_flags |= (tfm->crt_flags &
-			CRYPTO_TFM_REQ_MASK);
-
-	return crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
-}
-
-static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
-		       unsigned int key_len)
-{
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-	unsigned long fc;
-
-	/* Pick the correct function code based on the key length */
-	fc = (key_len == 16) ? CPACF_KM_AES_128 :
-	     (key_len == 24) ? CPACF_KM_AES_192 :
-	     (key_len == 32) ? CPACF_KM_AES_256 : 0;
-
-	/* Check if the function code is available */
-	sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
-	if (!sctx->fc)
-		return setkey_fallback_cip(tfm, in_key, key_len);
-
-	sctx->key_len = key_len;
-	memcpy(sctx->key, in_key, key_len);
-	return 0;
-}
-
-static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-
-	if (unlikely(!sctx->fc)) {
-		crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
-		return;
-	}
-	cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE);
-}
-
-static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
-{
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-
-	if (unlikely(!sctx->fc)) {
-		crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
-		return;
-	}
-	cpacf_km(sctx->fc | CPACF_DECRYPT,
-		 &sctx->key, out, in, AES_BLOCK_SIZE);
-}
-
-static int fallback_init_cip(struct crypto_tfm *tfm)
-{
-	const char *name = tfm->__crt_alg->cra_name;
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-
-	sctx->fallback.cip = crypto_alloc_cipher(name, 0,
-						 CRYPTO_ALG_NEED_FALLBACK);
-
-	if (IS_ERR(sctx->fallback.cip)) {
-		pr_err("Allocating AES fallback algorithm %s failed\n",
-		       name);
-		return PTR_ERR(sctx->fallback.cip);
-	}
-
-	return 0;
-}
-
-static void fallback_exit_cip(struct crypto_tfm *tfm)
-{
-	struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
-
-	crypto_free_cipher(sctx->fallback.cip);
-	sctx->fallback.cip = NULL;
-}
-
-static struct crypto_alg aes_alg = {
-	.cra_name		=	"aes",
-	.cra_driver_name	=	"aes-s390",
-	.cra_priority		=	300,
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER |
-					CRYPTO_ALG_NEED_FALLBACK,
-	.cra_blocksize		=	AES_BLOCK_SIZE,
-	.cra_ctxsize		=	sizeof(struct s390_aes_ctx),
-	.cra_module		=	THIS_MODULE,
-	.cra_init               =       fallback_init_cip,
-	.cra_exit               =       fallback_exit_cip,
-	.cra_u			=	{
-		.cipher = {
-			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
-			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
-			.cia_setkey		=	aes_set_key,
-			.cia_encrypt		=	crypto_aes_encrypt,
-			.cia_decrypt		=	crypto_aes_decrypt,
-		}
-	}
-};
-
 static int setkey_fallback_skcipher(struct crypto_skcipher *tfm, const u8 *key,
 				    unsigned int len)
 {
@@ -1049,7 +944,6 @@ static struct aead_alg gcm_aes_aead = {
 	},
 };
 
-static struct crypto_alg *aes_s390_alg;
 static struct skcipher_alg *aes_s390_skcipher_algs[5];
 static int aes_s390_skciphers_num;
 static struct aead_alg *aes_s390_aead_alg;
@@ -1066,8 +960,6 @@ static int aes_s390_register_skcipher(struct skcipher_alg *alg)
 
 static void aes_s390_fini(void)
 {
-	if (aes_s390_alg)
-		crypto_unregister_alg(aes_s390_alg);
 	while (aes_s390_skciphers_num--)
 		crypto_unregister_skcipher(aes_s390_skcipher_algs[aes_s390_skciphers_num]);
 	if (ctrblk)
@@ -1090,10 +982,6 @@ static int __init aes_s390_init(void)
 	if (cpacf_test_func(&km_functions, CPACF_KM_AES_128) ||
 	    cpacf_test_func(&km_functions, CPACF_KM_AES_192) ||
 	    cpacf_test_func(&km_functions, CPACF_KM_AES_256)) {
-		ret = crypto_register_alg(&aes_alg);
-		if (ret)
-			goto out_err;
-		aes_s390_alg = &aes_alg;
 		ret = aes_s390_register_skcipher(&ecb_aes_alg);
 		if (ret)
 			goto out_err;
@@ -1156,4 +1044,3 @@ MODULE_ALIAS_CRYPTO("aes-all");
 
 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
 MODULE_LICENSE("GPL");
-MODULE_IMPORT_NS("CRYPTO_INTERNAL");
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index f755da979534..c1932ce46c7f 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -19,9 +19,9 @@ config CRYPTO_DES_SPARC64
 config CRYPTO_AES_SPARC64
 	tristate "Ciphers: AES, modes: ECB, CBC, CTR"
 	depends on SPARC64
+	select CRYPTO_LIB_AES
 	select CRYPTO_SKCIPHER
 	help
-	  Block ciphers: AES cipher algorithms (FIPS-197)
 	  Length-preseving ciphers: AES with ECB, CBC, and CTR modes
 
 	  Architecture: sparc64 using crypto instructions
diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile
index 7b4796842ddd..cdf9f4b3efbb 100644
--- a/arch/sparc/crypto/Makefile
+++ b/arch/sparc/crypto/Makefile
@@ -7,6 +7,6 @@ obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o
 obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o
 obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o
 
-aes-sparc64-y := aes_asm.o aes_glue.o
+aes-sparc64-y := aes_glue.o
 des-sparc64-y := des_asm.o des_glue.o
 camellia-sparc64-y := camellia_asm.o camellia_glue.o
diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S
deleted file mode 100644
index f291174a72a1..000000000000
--- a/arch/sparc/crypto/aes_asm.S
+++ /dev/null
@@ -1,1543 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#include <linux/linkage.h>
-#include <asm/opcodes.h>
-#include <asm/visasm.h>
-
-#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
-	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
-	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
-	AES_EROUND01(KEY_BASE +  4, T0, T1, I0) \
-	AES_EROUND23(KEY_BASE +  6, T0, T1, I1)
-
-#define ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
-	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
-	AES_EROUND01(KEY_BASE +  0, I2, I3, T2) \
-	AES_EROUND23(KEY_BASE +  2, I2, I3, T3) \
-	AES_EROUND01(KEY_BASE +  4, T0, T1, I0) \
-	AES_EROUND23(KEY_BASE +  6, T0, T1, I1) \
-	AES_EROUND01(KEY_BASE +  4, T2, T3, I2) \
-	AES_EROUND23(KEY_BASE +  6, T2, T3, I3)
-
-#define ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
-	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
-	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
-	AES_EROUND01_L(KEY_BASE +  4, T0, T1, I0) \
-	AES_EROUND23_L(KEY_BASE +  6, T0, T1, I1)
-
-#define ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	AES_EROUND01(KEY_BASE +  0, I0, I1, T0) \
-	AES_EROUND23(KEY_BASE +  2, I0, I1, T1) \
-	AES_EROUND01(KEY_BASE +  0, I2, I3, T2) \
-	AES_EROUND23(KEY_BASE +  2, I2, I3, T3) \
-	AES_EROUND01_L(KEY_BASE +  4, T0, T1, I0) \
-	AES_EROUND23_L(KEY_BASE +  6, T0, T1, I1) \
-	AES_EROUND01_L(KEY_BASE +  4, T2, T3, I2) \
-	AES_EROUND23_L(KEY_BASE +  6, T2, T3, I3)
-
-	/* 10 rounds */
-#define ENCRYPT_128(KEY_BASE, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
-
-#define ENCRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
-
-	/* 12 rounds */
-#define ENCRYPT_192(KEY_BASE, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
-
-#define ENCRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
-	ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
-
-	/* 14 rounds */
-#define ENCRYPT_256(KEY_BASE, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
-	ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
-
-#define ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
-	ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
-			     TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
-
-#define ENCRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, KEY_BASE + 48) \
-	ldd	[%o0 + 0xd0], %f56; \
-	ldd	[%o0 + 0xd8], %f58; \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, KEY_BASE +  0) \
-	ldd	[%o0 + 0xe0], %f60; \
-	ldd	[%o0 + 0xe8], %f62; \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE +  0) \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE +  0) \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE +  0) \
-	ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE +  0) \
-	AES_EROUND01(KEY_BASE +  48, I0, I1, KEY_BASE + 0) \
-	AES_EROUND23(KEY_BASE +  50, I0, I1, KEY_BASE + 2) \
-	AES_EROUND01(KEY_BASE +  48, I2, I3, KEY_BASE + 4) \
-	AES_EROUND23(KEY_BASE +  50, I2, I3, KEY_BASE + 6) \
-	AES_EROUND01_L(KEY_BASE +  52, KEY_BASE + 0, KEY_BASE + 2, I0) \
-	AES_EROUND23_L(KEY_BASE +  54, KEY_BASE + 0, KEY_BASE + 2, I1) \
-	ldd	[%o0 + 0x10], %f8; \
-	ldd	[%o0 + 0x18], %f10; \
-	AES_EROUND01_L(KEY_BASE +  52, KEY_BASE + 4, KEY_BASE + 6, I2) \
-	AES_EROUND23_L(KEY_BASE +  54, KEY_BASE + 4, KEY_BASE + 6, I3) \
-	ldd	[%o0 + 0x20], %f12; \
-	ldd	[%o0 + 0x28], %f14;
-
-#define DECRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \
-	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
-	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
-	AES_DROUND23(KEY_BASE +  4, T0, T1, I1) \
-	AES_DROUND01(KEY_BASE +  6, T0, T1, I0)
-
-#define DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
-	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
-	AES_DROUND23(KEY_BASE +  0, I2, I3, T3) \
-	AES_DROUND01(KEY_BASE +  2, I2, I3, T2) \
-	AES_DROUND23(KEY_BASE +  4, T0, T1, I1) \
-	AES_DROUND01(KEY_BASE +  6, T0, T1, I0) \
-	AES_DROUND23(KEY_BASE +  4, T2, T3, I3) \
-	AES_DROUND01(KEY_BASE +  6, T2, T3, I2)
-
-#define DECRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \
-	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
-	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
-	AES_DROUND23_L(KEY_BASE +  4, T0, T1, I1) \
-	AES_DROUND01_L(KEY_BASE +  6, T0, T1, I0)
-
-#define DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	AES_DROUND23(KEY_BASE +  0, I0, I1, T1) \
-	AES_DROUND01(KEY_BASE +  2, I0, I1, T0) \
-	AES_DROUND23(KEY_BASE +  0, I2, I3, T3) \
-	AES_DROUND01(KEY_BASE +  2, I2, I3, T2) \
-	AES_DROUND23_L(KEY_BASE +  4, T0, T1, I1) \
-	AES_DROUND01_L(KEY_BASE +  6, T0, T1, I0) \
-	AES_DROUND23_L(KEY_BASE +  4, T2, T3, I3) \
-	AES_DROUND01_L(KEY_BASE +  6, T2, T3, I2)
-
-	/* 10 rounds */
-#define DECRYPT_128(KEY_BASE, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1)
-
-#define DECRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3)
-
-	/* 12 rounds */
-#define DECRYPT_192(KEY_BASE, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1)
-
-#define DECRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \
-	DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3)
-
-	/* 14 rounds */
-#define DECRYPT_256(KEY_BASE, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  0, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE +  8, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \
-	DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1)
-
-#define DECRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \
-	DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \
-			     TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6)
-
-#define DECRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE +  0, I0, I1, I2, I3, KEY_BASE + 48) \
-	ldd	[%o0 + 0x18], %f56; \
-	ldd	[%o0 + 0x10], %f58; \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE +  8, I0, I1, I2, I3, KEY_BASE +  0) \
-	ldd	[%o0 + 0x08], %f60; \
-	ldd	[%o0 + 0x00], %f62; \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE +  0) \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE +  0) \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE +  0) \
-	DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE +  0) \
-	AES_DROUND23(KEY_BASE +  48, I0, I1, KEY_BASE + 2) \
-	AES_DROUND01(KEY_BASE +  50, I0, I1, KEY_BASE + 0) \
-	AES_DROUND23(KEY_BASE +  48, I2, I3, KEY_BASE + 6) \
-	AES_DROUND01(KEY_BASE +  50, I2, I3, KEY_BASE + 4) \
-	AES_DROUND23_L(KEY_BASE +  52, KEY_BASE + 0, KEY_BASE + 2, I1) \
-	AES_DROUND01_L(KEY_BASE +  54, KEY_BASE + 0, KEY_BASE + 2, I0) \
-	ldd	[%o0 + 0xd8], %f8; \
-	ldd	[%o0 + 0xd0], %f10; \
-	AES_DROUND23_L(KEY_BASE +  52, KEY_BASE + 4, KEY_BASE + 6, I3) \
-	AES_DROUND01_L(KEY_BASE +  54, KEY_BASE + 4, KEY_BASE + 6, I2) \
-	ldd	[%o0 + 0xc8], %f12; \
-	ldd	[%o0 + 0xc0], %f14;
-
-	.align	32
-ENTRY(aes_sparc64_key_expand)
-	/* %o0=input_key, %o1=output_key, %o2=key_len */
-	VISEntry
-	ld	[%o0 + 0x00], %f0
-	ld	[%o0 + 0x04], %f1
-	ld	[%o0 + 0x08], %f2
-	ld	[%o0 + 0x0c], %f3
-
-	std	%f0, [%o1 + 0x00]
-	std	%f2, [%o1 + 0x08]
-	add	%o1, 0x10, %o1
-
-	cmp	%o2, 24
-	bl	2f
-	 nop
-
-	be	1f
-	 nop
-
-	/* 256-bit key expansion */
-	ld	[%o0 + 0x10], %f4
-	ld	[%o0 + 0x14], %f5
-	ld	[%o0 + 0x18], %f6
-	ld	[%o0 + 0x1c], %f7
-
-	std	%f4, [%o1 + 0x00]
-	std	%f6, [%o1 + 0x08]
-	add	%o1, 0x10, %o1
-
-	AES_KEXPAND1(0, 6, 0x0, 8)
-	AES_KEXPAND2(2, 8, 10)
-	AES_KEXPAND0(4, 10, 12)
-	AES_KEXPAND2(6, 12, 14)
-	AES_KEXPAND1(8, 14, 0x1, 16)
-	AES_KEXPAND2(10, 16, 18)
-	AES_KEXPAND0(12, 18, 20)
-	AES_KEXPAND2(14, 20, 22)
-	AES_KEXPAND1(16, 22, 0x2, 24)
-	AES_KEXPAND2(18, 24, 26)
-	AES_KEXPAND0(20, 26, 28)
-	AES_KEXPAND2(22, 28, 30)
-	AES_KEXPAND1(24, 30, 0x3, 32)
-	AES_KEXPAND2(26, 32, 34)
-	AES_KEXPAND0(28, 34, 36)
-	AES_KEXPAND2(30, 36, 38)
-	AES_KEXPAND1(32, 38, 0x4, 40)
-	AES_KEXPAND2(34, 40, 42)
-	AES_KEXPAND0(36, 42, 44)
-	AES_KEXPAND2(38, 44, 46)
-	AES_KEXPAND1(40, 46, 0x5, 48)
-	AES_KEXPAND2(42, 48, 50)
-	AES_KEXPAND0(44, 50, 52)
-	AES_KEXPAND2(46, 52, 54)
-	AES_KEXPAND1(48, 54, 0x6, 56)
-	AES_KEXPAND2(50, 56, 58)
-
-	std	%f8, [%o1 + 0x00]
-	std	%f10, [%o1 + 0x08]
-	std	%f12, [%o1 + 0x10]
-	std	%f14, [%o1 + 0x18]
-	std	%f16, [%o1 + 0x20]
-	std	%f18, [%o1 + 0x28]
-	std	%f20, [%o1 + 0x30]
-	std	%f22, [%o1 + 0x38]
-	std	%f24, [%o1 + 0x40]
-	std	%f26, [%o1 + 0x48]
-	std	%f28, [%o1 + 0x50]
-	std	%f30, [%o1 + 0x58]
-	std	%f32, [%o1 + 0x60]
-	std	%f34, [%o1 + 0x68]
-	std	%f36, [%o1 + 0x70]
-	std	%f38, [%o1 + 0x78]
-	std	%f40, [%o1 + 0x80]
-	std	%f42, [%o1 + 0x88]
-	std	%f44, [%o1 + 0x90]
-	std	%f46, [%o1 + 0x98]
-	std	%f48, [%o1 + 0xa0]
-	std	%f50, [%o1 + 0xa8]
-	std	%f52, [%o1 + 0xb0]
-	std	%f54, [%o1 + 0xb8]
-	std	%f56, [%o1 + 0xc0]
-	ba,pt	%xcc, 80f
-	 std	%f58, [%o1 + 0xc8]
-
-1:	
-	/* 192-bit key expansion */
-	ld	[%o0 + 0x10], %f4
-	ld	[%o0 + 0x14], %f5
-
-	std	%f4, [%o1 + 0x00]
-	add	%o1, 0x08, %o1
-
-	AES_KEXPAND1(0, 4, 0x0, 6)
-	AES_KEXPAND2(2, 6, 8)
-	AES_KEXPAND2(4, 8, 10)
-	AES_KEXPAND1(6, 10, 0x1, 12)
-	AES_KEXPAND2(8, 12, 14)
-	AES_KEXPAND2(10, 14, 16)
-	AES_KEXPAND1(12, 16, 0x2, 18)
-	AES_KEXPAND2(14, 18, 20)
-	AES_KEXPAND2(16, 20, 22)
-	AES_KEXPAND1(18, 22, 0x3, 24)
-	AES_KEXPAND2(20, 24, 26)
-	AES_KEXPAND2(22, 26, 28)
-	AES_KEXPAND1(24, 28, 0x4, 30)
-	AES_KEXPAND2(26, 30, 32)
-	AES_KEXPAND2(28, 32, 34)
-	AES_KEXPAND1(30, 34, 0x5, 36)
-	AES_KEXPAND2(32, 36, 38)
-	AES_KEXPAND2(34, 38, 40)
-	AES_KEXPAND1(36, 40, 0x6, 42)
-	AES_KEXPAND2(38, 42, 44)
-	AES_KEXPAND2(40, 44, 46)
-	AES_KEXPAND1(42, 46, 0x7, 48)
-	AES_KEXPAND2(44, 48, 50)
-
-	std	%f6, [%o1 + 0x00]
-	std	%f8, [%o1 + 0x08]
-	std	%f10, [%o1 + 0x10]
-	std	%f12, [%o1 + 0x18]
-	std	%f14, [%o1 + 0x20]
-	std	%f16, [%o1 + 0x28]
-	std	%f18, [%o1 + 0x30]
-	std	%f20, [%o1 + 0x38]
-	std	%f22, [%o1 + 0x40]
-	std	%f24, [%o1 + 0x48]
-	std	%f26, [%o1 + 0x50]
-	std	%f28, [%o1 + 0x58]
-	std	%f30, [%o1 + 0x60]
-	std	%f32, [%o1 + 0x68]
-	std	%f34, [%o1 + 0x70]
-	std	%f36, [%o1 + 0x78]
-	std	%f38, [%o1 + 0x80]
-	std	%f40, [%o1 + 0x88]
-	std	%f42, [%o1 + 0x90]
-	std	%f44, [%o1 + 0x98]
-	std	%f46, [%o1 + 0xa0]
-	std	%f48, [%o1 + 0xa8]
-	ba,pt	%xcc, 80f
-	 std	%f50, [%o1 + 0xb0]
-
-2:
-	/* 128-bit key expansion */
-	AES_KEXPAND1(0, 2, 0x0, 4)
-	AES_KEXPAND2(2, 4, 6)
-	AES_KEXPAND1(4, 6, 0x1, 8)
-	AES_KEXPAND2(6, 8, 10)
-	AES_KEXPAND1(8, 10, 0x2, 12)
-	AES_KEXPAND2(10, 12, 14)
-	AES_KEXPAND1(12, 14, 0x3, 16)
-	AES_KEXPAND2(14, 16, 18)
-	AES_KEXPAND1(16, 18, 0x4, 20)
-	AES_KEXPAND2(18, 20, 22)
-	AES_KEXPAND1(20, 22, 0x5, 24)
-	AES_KEXPAND2(22, 24, 26)
-	AES_KEXPAND1(24, 26, 0x6, 28)
-	AES_KEXPAND2(26, 28, 30)
-	AES_KEXPAND1(28, 30, 0x7, 32)
-	AES_KEXPAND2(30, 32, 34)
-	AES_KEXPAND1(32, 34, 0x8, 36)
-	AES_KEXPAND2(34, 36, 38)
-	AES_KEXPAND1(36, 38, 0x9, 40)
-	AES_KEXPAND2(38, 40, 42)
-
-	std	%f4, [%o1 + 0x00]
-	std	%f6, [%o1 + 0x08]
-	std	%f8, [%o1 + 0x10]
-	std	%f10, [%o1 + 0x18]
-	std	%f12, [%o1 + 0x20]
-	std	%f14, [%o1 + 0x28]
-	std	%f16, [%o1 + 0x30]
-	std	%f18, [%o1 + 0x38]
-	std	%f20, [%o1 + 0x40]
-	std	%f22, [%o1 + 0x48]
-	std	%f24, [%o1 + 0x50]
-	std	%f26, [%o1 + 0x58]
-	std	%f28, [%o1 + 0x60]
-	std	%f30, [%o1 + 0x68]
-	std	%f32, [%o1 + 0x70]
-	std	%f34, [%o1 + 0x78]
-	std	%f36, [%o1 + 0x80]
-	std	%f38, [%o1 + 0x88]
-	std	%f40, [%o1 + 0x90]
-	std	%f42, [%o1 + 0x98]
-80:
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_key_expand)
-
-	.align		32
-ENTRY(aes_sparc64_encrypt_128)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-	ldd		[%o0 + 0x00], %f8
-	ldd		[%o0 + 0x08], %f10
-	ldd		[%o0 + 0x10], %f12
-	ldd		[%o0 + 0x18], %f14
-	ldd		[%o0 + 0x20], %f16
-	ldd		[%o0 + 0x28], %f18
-	ldd		[%o0 + 0x30], %f20
-	ldd		[%o0 + 0x38], %f22
-	ldd		[%o0 + 0x40], %f24
-	ldd		[%o0 + 0x48], %f26
-	ldd		[%o0 + 0x50], %f28
-	ldd		[%o0 + 0x58], %f30
-	ldd		[%o0 + 0x60], %f32
-	ldd		[%o0 + 0x68], %f34
-	ldd		[%o0 + 0x70], %f36
-	ldd		[%o0 + 0x78], %f38
-	ldd		[%o0 + 0x80], %f40
-	ldd		[%o0 + 0x88], %f42
-	ldd		[%o0 + 0x90], %f44
-	ldd		[%o0 + 0x98], %f46
-	ldd		[%o0 + 0xa0], %f48
-	ldd		[%o0 + 0xa8], %f50
-	fxor		%f8, %f4, %f4
-	fxor		%f10, %f6, %f6
-	ENCRYPT_128(12, 4, 6, 0, 2)
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_encrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_encrypt_192)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-
-	ldd		[%o0 + 0x00], %f8
-	ldd		[%o0 + 0x08], %f10
-
-	fxor		%f8, %f4, %f4
-	fxor		%f10, %f6, %f6
-
-	ldd		[%o0 + 0x10], %f8
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	add		%o0, 0x20, %o0
-
-	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
-
-	ldd		[%o0 + 0x10], %f12
-	ldd		[%o0 + 0x18], %f14
-	ldd		[%o0 + 0x20], %f16
-	ldd		[%o0 + 0x28], %f18
-	ldd		[%o0 + 0x30], %f20
-	ldd		[%o0 + 0x38], %f22
-	ldd		[%o0 + 0x40], %f24
-	ldd		[%o0 + 0x48], %f26
-	ldd		[%o0 + 0x50], %f28
-	ldd		[%o0 + 0x58], %f30
-	ldd		[%o0 + 0x60], %f32
-	ldd		[%o0 + 0x68], %f34
-	ldd		[%o0 + 0x70], %f36
-	ldd		[%o0 + 0x78], %f38
-	ldd		[%o0 + 0x80], %f40
-	ldd		[%o0 + 0x88], %f42
-	ldd		[%o0 + 0x90], %f44
-	ldd		[%o0 + 0x98], %f46
-	ldd		[%o0 + 0xa0], %f48
-	ldd		[%o0 + 0xa8], %f50
-
-
-	ENCRYPT_128(12, 4, 6, 0, 2)
-
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_encrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_encrypt_256)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-
-	ldd		[%o0 + 0x00], %f8
-	ldd		[%o0 + 0x08], %f10
-
-	fxor		%f8, %f4, %f4
-	fxor		%f10, %f6, %f6
-
-	ldd		[%o0 + 0x10], %f8
-
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	add		%o0, 0x20, %o0
-
-	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
-
-	ldd		[%o0 + 0x10], %f8
-
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	add		%o0, 0x20, %o0
-
-	ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2)
-
-	ldd		[%o0 + 0x10], %f12
-	ldd		[%o0 + 0x18], %f14
-	ldd		[%o0 + 0x20], %f16
-	ldd		[%o0 + 0x28], %f18
-	ldd		[%o0 + 0x30], %f20
-	ldd		[%o0 + 0x38], %f22
-	ldd		[%o0 + 0x40], %f24
-	ldd		[%o0 + 0x48], %f26
-	ldd		[%o0 + 0x50], %f28
-	ldd		[%o0 + 0x58], %f30
-	ldd		[%o0 + 0x60], %f32
-	ldd		[%o0 + 0x68], %f34
-	ldd		[%o0 + 0x70], %f36
-	ldd		[%o0 + 0x78], %f38
-	ldd		[%o0 + 0x80], %f40
-	ldd		[%o0 + 0x88], %f42
-	ldd		[%o0 + 0x90], %f44
-	ldd		[%o0 + 0x98], %f46
-	ldd		[%o0 + 0xa0], %f48
-	ldd		[%o0 + 0xa8], %f50
-
-	ENCRYPT_128(12, 4, 6, 0, 2)
-
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_encrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_decrypt_128)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-	ldd		[%o0 + 0xa0], %f8
-	ldd		[%o0 + 0xa8], %f10
-	ldd		[%o0 + 0x98], %f12
-	ldd		[%o0 + 0x90], %f14
-	ldd		[%o0 + 0x88], %f16
-	ldd		[%o0 + 0x80], %f18
-	ldd		[%o0 + 0x78], %f20
-	ldd		[%o0 + 0x70], %f22
-	ldd		[%o0 + 0x68], %f24
-	ldd		[%o0 + 0x60], %f26
-	ldd		[%o0 + 0x58], %f28
-	ldd		[%o0 + 0x50], %f30
-	ldd		[%o0 + 0x48], %f32
-	ldd		[%o0 + 0x40], %f34
-	ldd		[%o0 + 0x38], %f36
-	ldd		[%o0 + 0x30], %f38
-	ldd		[%o0 + 0x28], %f40
-	ldd		[%o0 + 0x20], %f42
-	ldd		[%o0 + 0x18], %f44
-	ldd		[%o0 + 0x10], %f46
-	ldd		[%o0 + 0x08], %f48
-	ldd		[%o0 + 0x00], %f50
-	fxor		%f8, %f4, %f4
-	fxor		%f10, %f6, %f6
-	DECRYPT_128(12, 4, 6, 0, 2)
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_decrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_decrypt_192)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-	ldd		[%o0 + 0xc0], %f8
-	ldd		[%o0 + 0xc8], %f10
-	ldd		[%o0 + 0xb8], %f12
-	ldd		[%o0 + 0xb0], %f14
-	ldd		[%o0 + 0xa8], %f16
-	ldd		[%o0 + 0xa0], %f18
-	fxor		%f8, %f4, %f4
-	fxor		%f10, %f6, %f6
-	ldd		[%o0 + 0x98], %f20
-	ldd		[%o0 + 0x90], %f22
-	ldd		[%o0 + 0x88], %f24
-	ldd		[%o0 + 0x80], %f26
-	DECRYPT_TWO_ROUNDS(12, 4, 6, 0, 2)
-	ldd		[%o0 + 0x78], %f28
-	ldd		[%o0 + 0x70], %f30
-	ldd		[%o0 + 0x68], %f32
-	ldd		[%o0 + 0x60], %f34
-	ldd		[%o0 + 0x58], %f36
-	ldd		[%o0 + 0x50], %f38
-	ldd		[%o0 + 0x48], %f40
-	ldd		[%o0 + 0x40], %f42
-	ldd		[%o0 + 0x38], %f44
-	ldd		[%o0 + 0x30], %f46
-	ldd		[%o0 + 0x28], %f48
-	ldd		[%o0 + 0x20], %f50
-	ldd		[%o0 + 0x18], %f52
-	ldd		[%o0 + 0x10], %f54
-	ldd		[%o0 + 0x08], %f56
-	ldd		[%o0 + 0x00], %f58
-	DECRYPT_128(20, 4, 6, 0, 2)
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_decrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_decrypt_256)
-	/* %o0=key, %o1=input, %o2=output */
-	VISEntry
-	ld		[%o1 + 0x00], %f4
-	ld		[%o1 + 0x04], %f5
-	ld		[%o1 + 0x08], %f6
-	ld		[%o1 + 0x0c], %f7
-	ldd		[%o0 + 0xe0], %f8
-	ldd		[%o0 + 0xe8], %f10
-	ldd		[%o0 + 0xd8], %f12
-	ldd		[%o0 + 0xd0], %f14
-	ldd		[%o0 + 0xc8], %f16
-	fxor		%f8, %f4, %f4
-	ldd		[%o0 + 0xc0], %f18
-	fxor		%f10, %f6, %f6
-	ldd		[%o0 + 0xb8], %f20
-	AES_DROUND23(12, 4, 6, 2)
-	ldd		[%o0 + 0xb0], %f22
-	AES_DROUND01(14, 4, 6, 0)
-	ldd		[%o0 + 0xa8], %f24
-	AES_DROUND23(16, 0, 2, 6)
-	ldd		[%o0 + 0xa0], %f26
-	AES_DROUND01(18, 0, 2, 4)
-	ldd		[%o0 + 0x98], %f12
-	AES_DROUND23(20, 4, 6, 2)
-	ldd		[%o0 + 0x90], %f14
-	AES_DROUND01(22, 4, 6, 0)
-	ldd		[%o0 + 0x88], %f16
-	AES_DROUND23(24, 0, 2, 6)
-	ldd		[%o0 + 0x80], %f18
-	AES_DROUND01(26, 0, 2, 4)
-	ldd		[%o0 + 0x78], %f20
-	AES_DROUND23(12, 4, 6, 2)
-	ldd		[%o0 + 0x70], %f22
-	AES_DROUND01(14, 4, 6, 0)
-	ldd		[%o0 + 0x68], %f24
-	AES_DROUND23(16, 0, 2, 6)
-	ldd		[%o0 + 0x60], %f26
-	AES_DROUND01(18, 0, 2, 4)
-	ldd		[%o0 + 0x58], %f28
-	AES_DROUND23(20, 4, 6, 2)
-	ldd		[%o0 + 0x50], %f30
-	AES_DROUND01(22, 4, 6, 0)
-	ldd		[%o0 + 0x48], %f32
-	AES_DROUND23(24, 0, 2, 6)
-	ldd		[%o0 + 0x40], %f34
-	AES_DROUND01(26, 0, 2, 4)
-	ldd		[%o0 + 0x38], %f36
-	AES_DROUND23(28, 4, 6, 2)
-	ldd		[%o0 + 0x30], %f38
-	AES_DROUND01(30, 4, 6, 0)
-	ldd		[%o0 + 0x28], %f40
-	AES_DROUND23(32, 0, 2, 6)
-	ldd		[%o0 + 0x20], %f42
-	AES_DROUND01(34, 0, 2, 4)
-	ldd		[%o0 + 0x18], %f44
-	AES_DROUND23(36, 4, 6, 2)
-	ldd		[%o0 + 0x10], %f46
-	AES_DROUND01(38, 4, 6, 0)
-	ldd		[%o0 + 0x08], %f48
-	AES_DROUND23(40, 0, 2, 6)
-	ldd		[%o0 + 0x00], %f50
-	AES_DROUND01(42, 0, 2, 4)
-	AES_DROUND23(44, 4, 6, 2)
-	AES_DROUND01(46, 4, 6, 0)
-	AES_DROUND23_L(48, 0, 2, 6)
-	AES_DROUND01_L(50, 0, 2, 4)
-	st		%f4, [%o2 + 0x00]
-	st		%f5, [%o2 + 0x04]
-	st		%f6, [%o2 + 0x08]
-	st		%f7, [%o2 + 0x0c]
-	retl
-	 VISExit
-ENDPROC(aes_sparc64_decrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_load_encrypt_keys_128)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0x10], %f8
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	ldd		[%o0 + 0x30], %f16
-	ldd		[%o0 + 0x38], %f18
-	ldd		[%o0 + 0x40], %f20
-	ldd		[%o0 + 0x48], %f22
-	ldd		[%o0 + 0x50], %f24
-	ldd		[%o0 + 0x58], %f26
-	ldd		[%o0 + 0x60], %f28
-	ldd		[%o0 + 0x68], %f30
-	ldd		[%o0 + 0x70], %f32
-	ldd		[%o0 + 0x78], %f34
-	ldd		[%o0 + 0x80], %f36
-	ldd		[%o0 + 0x88], %f38
-	ldd		[%o0 + 0x90], %f40
-	ldd		[%o0 + 0x98], %f42
-	ldd		[%o0 + 0xa0], %f44
-	retl
-	 ldd		[%o0 + 0xa8], %f46
-ENDPROC(aes_sparc64_load_encrypt_keys_128)
-
-	.align		32
-ENTRY(aes_sparc64_load_encrypt_keys_192)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0x10], %f8
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	ldd		[%o0 + 0x30], %f16
-	ldd		[%o0 + 0x38], %f18
-	ldd		[%o0 + 0x40], %f20
-	ldd		[%o0 + 0x48], %f22
-	ldd		[%o0 + 0x50], %f24
-	ldd		[%o0 + 0x58], %f26
-	ldd		[%o0 + 0x60], %f28
-	ldd		[%o0 + 0x68], %f30
-	ldd		[%o0 + 0x70], %f32
-	ldd		[%o0 + 0x78], %f34
-	ldd		[%o0 + 0x80], %f36
-	ldd		[%o0 + 0x88], %f38
-	ldd		[%o0 + 0x90], %f40
-	ldd		[%o0 + 0x98], %f42
-	ldd		[%o0 + 0xa0], %f44
-	ldd		[%o0 + 0xa8], %f46
-	ldd		[%o0 + 0xb0], %f48
-	ldd		[%o0 + 0xb8], %f50
-	ldd		[%o0 + 0xc0], %f52
-	retl
-	 ldd		[%o0 + 0xc8], %f54
-ENDPROC(aes_sparc64_load_encrypt_keys_192)
-
-	.align		32
-ENTRY(aes_sparc64_load_encrypt_keys_256)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0x10], %f8
-	ldd		[%o0 + 0x18], %f10
-	ldd		[%o0 + 0x20], %f12
-	ldd		[%o0 + 0x28], %f14
-	ldd		[%o0 + 0x30], %f16
-	ldd		[%o0 + 0x38], %f18
-	ldd		[%o0 + 0x40], %f20
-	ldd		[%o0 + 0x48], %f22
-	ldd		[%o0 + 0x50], %f24
-	ldd		[%o0 + 0x58], %f26
-	ldd		[%o0 + 0x60], %f28
-	ldd		[%o0 + 0x68], %f30
-	ldd		[%o0 + 0x70], %f32
-	ldd		[%o0 + 0x78], %f34
-	ldd		[%o0 + 0x80], %f36
-	ldd		[%o0 + 0x88], %f38
-	ldd		[%o0 + 0x90], %f40
-	ldd		[%o0 + 0x98], %f42
-	ldd		[%o0 + 0xa0], %f44
-	ldd		[%o0 + 0xa8], %f46
-	ldd		[%o0 + 0xb0], %f48
-	ldd		[%o0 + 0xb8], %f50
-	ldd		[%o0 + 0xc0], %f52
-	ldd		[%o0 + 0xc8], %f54
-	ldd		[%o0 + 0xd0], %f56
-	ldd		[%o0 + 0xd8], %f58
-	ldd		[%o0 + 0xe0], %f60
-	retl
-	 ldd		[%o0 + 0xe8], %f62
-ENDPROC(aes_sparc64_load_encrypt_keys_256)
-
-	.align		32
-ENTRY(aes_sparc64_load_decrypt_keys_128)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0x98], %f8
-	ldd		[%o0 + 0x90], %f10
-	ldd		[%o0 + 0x88], %f12
-	ldd		[%o0 + 0x80], %f14
-	ldd		[%o0 + 0x78], %f16
-	ldd		[%o0 + 0x70], %f18
-	ldd		[%o0 + 0x68], %f20
-	ldd		[%o0 + 0x60], %f22
-	ldd		[%o0 + 0x58], %f24
-	ldd		[%o0 + 0x50], %f26
-	ldd		[%o0 + 0x48], %f28
-	ldd		[%o0 + 0x40], %f30
-	ldd		[%o0 + 0x38], %f32
-	ldd		[%o0 + 0x30], %f34
-	ldd		[%o0 + 0x28], %f36
-	ldd		[%o0 + 0x20], %f38
-	ldd		[%o0 + 0x18], %f40
-	ldd		[%o0 + 0x10], %f42
-	ldd		[%o0 + 0x08], %f44
-	retl
-	 ldd		[%o0 + 0x00], %f46
-ENDPROC(aes_sparc64_load_decrypt_keys_128)
-
-	.align		32
-ENTRY(aes_sparc64_load_decrypt_keys_192)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0xb8], %f8
-	ldd		[%o0 + 0xb0], %f10
-	ldd		[%o0 + 0xa8], %f12
-	ldd		[%o0 + 0xa0], %f14
-	ldd		[%o0 + 0x98], %f16
-	ldd		[%o0 + 0x90], %f18
-	ldd		[%o0 + 0x88], %f20
-	ldd		[%o0 + 0x80], %f22
-	ldd		[%o0 + 0x78], %f24
-	ldd		[%o0 + 0x70], %f26
-	ldd		[%o0 + 0x68], %f28
-	ldd		[%o0 + 0x60], %f30
-	ldd		[%o0 + 0x58], %f32
-	ldd		[%o0 + 0x50], %f34
-	ldd		[%o0 + 0x48], %f36
-	ldd		[%o0 + 0x40], %f38
-	ldd		[%o0 + 0x38], %f40
-	ldd		[%o0 + 0x30], %f42
-	ldd		[%o0 + 0x28], %f44
-	ldd		[%o0 + 0x20], %f46
-	ldd		[%o0 + 0x18], %f48
-	ldd		[%o0 + 0x10], %f50
-	ldd		[%o0 + 0x08], %f52
-	retl
-	 ldd		[%o0 + 0x00], %f54
-ENDPROC(aes_sparc64_load_decrypt_keys_192)
-
-	.align		32
-ENTRY(aes_sparc64_load_decrypt_keys_256)
-	/* %o0=key */
-	VISEntry
-	ldd		[%o0 + 0xd8], %f8
-	ldd		[%o0 + 0xd0], %f10
-	ldd		[%o0 + 0xc8], %f12
-	ldd		[%o0 + 0xc0], %f14
-	ldd		[%o0 + 0xb8], %f16
-	ldd		[%o0 + 0xb0], %f18
-	ldd		[%o0 + 0xa8], %f20
-	ldd		[%o0 + 0xa0], %f22
-	ldd		[%o0 + 0x98], %f24
-	ldd		[%o0 + 0x90], %f26
-	ldd		[%o0 + 0x88], %f28
-	ldd		[%o0 + 0x80], %f30
-	ldd		[%o0 + 0x78], %f32
-	ldd		[%o0 + 0x70], %f34
-	ldd		[%o0 + 0x68], %f36
-	ldd		[%o0 + 0x60], %f38
-	ldd		[%o0 + 0x58], %f40
-	ldd		[%o0 + 0x50], %f42
-	ldd		[%o0 + 0x48], %f44
-	ldd		[%o0 + 0x40], %f46
-	ldd		[%o0 + 0x38], %f48
-	ldd		[%o0 + 0x30], %f50
-	ldd		[%o0 + 0x28], %f52
-	ldd		[%o0 + 0x20], %f54
-	ldd		[%o0 + 0x18], %f56
-	ldd		[%o0 + 0x10], %f58
-	ldd		[%o0 + 0x08], %f60
-	retl
-	 ldd		[%o0 + 0x00], %f62
-ENDPROC(aes_sparc64_load_decrypt_keys_256)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_encrypt_128)
-	/* %o0=key, %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 + 0x00], %g1
-	subcc		%o3, 0x10, %o3
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F60
-	MOVXTOD_G7_F62
-	ENCRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	ENCRYPT_128(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_encrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_encrypt_192)
-	/* %o0=key, %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 + 0x00], %g1
-	subcc		%o3, 0x10, %o3
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F60
-	MOVXTOD_G7_F62
-	ENCRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	ENCRYPT_192(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_encrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_encrypt_256)
-	/* %o0=key, %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 + 0x00], %g1
-	subcc		%o3, 0x10, %o3
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F0
-	MOVXTOD_G7_F2
-	ENCRYPT_256_2(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f0, [%o2 + 0x10]
-	std		%f2, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldd		[%o0 + 0xd0], %f56
-	ldd		[%o0 + 0xd8], %f58
-	ldd		[%o0 + 0xe0], %f60
-	ldd		[%o0 + 0xe8], %f62
-	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	ENCRYPT_256(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_encrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_decrypt_128)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 - 0x10], %g1
-	subcc		%o3, 0x10, %o3
-	be		10f
-	 ldx		[%o0 - 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F60
-	MOVXTOD_G7_F62
-	DECRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz,pt		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_128(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_decrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_decrypt_192)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 - 0x10], %g1
-	subcc		%o3, 0x10, %o3
-	be		10f
-	 ldx		[%o0 - 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F60
-	MOVXTOD_G7_F62
-	DECRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz,pt		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_192(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_decrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_ecb_decrypt_256)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
-	ldx		[%o0 - 0x10], %g1
-	subcc		%o3, 0x10, %o3
-	ldx		[%o0 - 0x08], %g2
-	be		10f
-	 sub		%o0, 0xf0, %o0
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	ldx		[%o1 + 0x10], %o4
-	ldx		[%o1 + 0x18], %o5
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	xor		%g1, %o4, %g3
-	xor		%g2, %o5, %g7
-	MOVXTOD_G3_F0
-	MOVXTOD_G7_F2
-	DECRYPT_256_2(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	std		%f0, [%o2 + 0x10]
-	std		%f2, [%o2 + 0x18]
-	sub		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz,pt		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldd		[%o0 + 0x18], %f56
-	ldd		[%o0 + 0x10], %f58
-	ldd		[%o0 + 0x08], %f60
-	ldd		[%o0 + 0x00], %f62
-	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_256(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	retl
-	 nop
-ENDPROC(aes_sparc64_ecb_decrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_encrypt_128)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldd		[%o4 + 0x00], %f4
-	ldd		[%o4 + 0x08], %f6
-	ldx		[%o0 + 0x00], %g1
-	ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F0
-	MOVXTOD_G7_F2
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	ENCRYPT_128(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	std		%f4, [%o4 + 0x00]
-	std		%f6, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_encrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_encrypt_192)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldd		[%o4 + 0x00], %f4
-	ldd		[%o4 + 0x08], %f6
-	ldx		[%o0 + 0x00], %g1
-	ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F0
-	MOVXTOD_G7_F2
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	ENCRYPT_192(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	std		%f4, [%o4 + 0x00]
-	std		%f6, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_encrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_encrypt_256)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldd		[%o4 + 0x00], %f4
-	ldd		[%o4 + 0x08], %f6
-	ldx		[%o0 + 0x00], %g1
-	ldx		[%o0 + 0x08], %g2
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F0
-	MOVXTOD_G7_F2
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	ENCRYPT_256(8, 4, 6, 0, 2)
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	std		%f4, [%o4 + 0x00]
-	std		%f6, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_encrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_decrypt_128)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
-	ldx		[%o0 - 0x10], %g1
-	ldx		[%o0 - 0x08], %g2
-	ldx		[%o4 + 0x00], %o0
-	ldx		[%o4 + 0x08], %o5
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_128(8, 4, 6, 0, 2)
-	MOVXTOD_O0_F0
-	MOVXTOD_O5_F2
-	xor		%g1, %g3, %o0
-	xor		%g2, %g7, %o5
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	stx		%o0, [%o4 + 0x00]
-	stx		%o5, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_decrypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_decrypt_192)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
-	ldx		[%o0 - 0x10], %g1
-	ldx		[%o0 - 0x08], %g2
-	ldx		[%o4 + 0x00], %o0
-	ldx		[%o4 + 0x08], %o5
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_192(8, 4, 6, 0, 2)
-	MOVXTOD_O0_F0
-	MOVXTOD_O5_F2
-	xor		%g1, %g3, %o0
-	xor		%g2, %g7, %o5
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	stx		%o0, [%o4 + 0x00]
-	stx		%o5, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_decrypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_cbc_decrypt_256)
-	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */
-	ldx		[%o0 - 0x10], %g1
-	ldx		[%o0 - 0x08], %g2
-	ldx		[%o4 + 0x00], %o0
-	ldx		[%o4 + 0x08], %o5
-1:	ldx		[%o1 + 0x00], %g3
-	ldx		[%o1 + 0x08], %g7
-	add		%o1, 0x10, %o1
-	xor		%g1, %g3, %g3
-	xor		%g2, %g7, %g7
-	MOVXTOD_G3_F4
-	MOVXTOD_G7_F6
-	DECRYPT_256(8, 4, 6, 0, 2)
-	MOVXTOD_O0_F0
-	MOVXTOD_O5_F2
-	xor		%g1, %g3, %o0
-	xor		%g2, %g7, %o5
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-	subcc		%o3, 0x10, %o3
-	bne,pt		%xcc, 1b
-	 add		%o2, 0x10, %o2
-	stx		%o0, [%o4 + 0x00]
-	stx		%o5, [%o4 + 0x08]
-	retl
-	 nop
-ENDPROC(aes_sparc64_cbc_decrypt_256)
-
-	.align		32
-ENTRY(aes_sparc64_ctr_crypt_128)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldx		[%o4 + 0x00], %g3
-	ldx		[%o4 + 0x08], %g7
-	subcc		%o3, 0x10, %o3
-	ldx		[%o0 + 0x00], %g1
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F4
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F6
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_128_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
-	ldd		[%o1 + 0x00], %f56
-	ldd		[%o1 + 0x08], %f58
-	ldd		[%o1 + 0x10], %f60
-	ldd		[%o1 + 0x18], %f62
-	fxor		%f56, %f0, %f56
-	fxor		%f58, %f2, %f58
-	fxor		%f60, %f4, %f60
-	fxor		%f62, %f6, %f62
-	std		%f56, [%o2 + 0x00]
-	std		%f58, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	subcc		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_128(8, 0, 2, 4, 6)
-	ldd		[%o1 + 0x00], %f4
-	ldd		[%o1 + 0x08], %f6
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	stx		%g3, [%o4 + 0x00]
-	retl
-	 stx		%g7, [%o4 + 0x08]
-ENDPROC(aes_sparc64_ctr_crypt_128)
-
-	.align		32
-ENTRY(aes_sparc64_ctr_crypt_192)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldx		[%o4 + 0x00], %g3
-	ldx		[%o4 + 0x08], %g7
-	subcc		%o3, 0x10, %o3
-	ldx		[%o0 + 0x00], %g1
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F4
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F6
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_192_2(8, 0, 2, 4, 6, 56, 58, 60, 62)
-	ldd		[%o1 + 0x00], %f56
-	ldd		[%o1 + 0x08], %f58
-	ldd		[%o1 + 0x10], %f60
-	ldd		[%o1 + 0x18], %f62
-	fxor		%f56, %f0, %f56
-	fxor		%f58, %f2, %f58
-	fxor		%f60, %f4, %f60
-	fxor		%f62, %f6, %f62
-	std		%f56, [%o2 + 0x00]
-	std		%f58, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	subcc		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_192(8, 0, 2, 4, 6)
-	ldd		[%o1 + 0x00], %f4
-	ldd		[%o1 + 0x08], %f6
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	stx		%g3, [%o4 + 0x00]
-	retl
-	 stx		%g7, [%o4 + 0x08]
-ENDPROC(aes_sparc64_ctr_crypt_192)
-
-	.align		32
-ENTRY(aes_sparc64_ctr_crypt_256)
-	/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
-	ldx		[%o4 + 0x00], %g3
-	ldx		[%o4 + 0x08], %g7
-	subcc		%o3, 0x10, %o3
-	ldx		[%o0 + 0x00], %g1
-	be		10f
-	 ldx		[%o0 + 0x08], %g2
-1:	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F4
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F6
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_256_2(8, 0, 2, 4, 6)
-	ldd		[%o1 + 0x00], %f56
-	ldd		[%o1 + 0x08], %f58
-	ldd		[%o1 + 0x10], %f60
-	ldd		[%o1 + 0x18], %f62
-	fxor		%f56, %f0, %f56
-	fxor		%f58, %f2, %f58
-	fxor		%f60, %f4, %f60
-	fxor		%f62, %f6, %f62
-	std		%f56, [%o2 + 0x00]
-	std		%f58, [%o2 + 0x08]
-	std		%f60, [%o2 + 0x10]
-	std		%f62, [%o2 + 0x18]
-	subcc		%o3, 0x20, %o3
-	add		%o1, 0x20, %o1
-	brgz		%o3, 1b
-	 add		%o2, 0x20, %o2
-	brlz,pt		%o3, 11f
-	 nop
-10:	ldd		[%o0 + 0xd0], %f56
-	ldd		[%o0 + 0xd8], %f58
-	ldd		[%o0 + 0xe0], %f60
-	ldd		[%o0 + 0xe8], %f62
-	xor		%g1, %g3, %o5
-	MOVXTOD_O5_F0
-	xor		%g2, %g7, %o5
-	MOVXTOD_O5_F2
-	add		%g7, 1, %g7
-	add		%g3, 1, %o5
-	movrz		%g7, %o5, %g3
-	ENCRYPT_256(8, 0, 2, 4, 6)
-	ldd		[%o1 + 0x00], %f4
-	ldd		[%o1 + 0x08], %f6
-	fxor		%f4, %f0, %f4
-	fxor		%f6, %f2, %f6
-	std		%f4, [%o2 + 0x00]
-	std		%f6, [%o2 + 0x08]
-11:	stx		%g3, [%o4 + 0x00]
-	retl
-	 stx		%g7, [%o4 + 0x08]
-ENDPROC(aes_sparc64_ctr_crypt_256)
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 359f22643b05..661561837415 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -32,8 +32,6 @@
 #include <asm/elf.h>
 
 struct aes_ops {
-	void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
-	void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
 	void (*load_encrypt_keys)(const u64 *key);
 	void (*load_decrypt_keys)(const u64 *key);
 	void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
@@ -55,79 +53,7 @@ struct crypto_sparc64_aes_ctx {
 	u32 expanded_key_length;
 };
 
-extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
-				    u32 *output);
-extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
-				    u32 *output);
-extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
-				    u32 *output);
-
-extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
-				    u32 *output);
-extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
-				    u32 *output);
-extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
-				    u32 *output);
-
-extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
-extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
-extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
-
-extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
-extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
-extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
-
-extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-
-extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len);
-
-extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
-					u64 *output, unsigned int len,
-					u64 *iv);
-
-extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
-				      u64 *output, unsigned int len,
-				      u64 *iv);
-extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
-				      u64 *output, unsigned int len,
-				      u64 *iv);
-extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
-				      u64 *output, unsigned int len,
-				      u64 *iv);
-
 static struct aes_ops aes128_ops = {
-	.encrypt		= aes_sparc64_encrypt_128,
-	.decrypt		= aes_sparc64_decrypt_128,
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_128,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_128,
 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_128,
@@ -138,8 +64,6 @@ static struct aes_ops aes128_ops = {
 };
 
 static struct aes_ops aes192_ops = {
-	.encrypt		= aes_sparc64_encrypt_192,
-	.decrypt		= aes_sparc64_decrypt_192,
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_192,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_192,
 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_192,
@@ -150,8 +74,6 @@ static struct aes_ops aes192_ops = {
 };
 
 static struct aes_ops aes256_ops = {
-	.encrypt		= aes_sparc64_encrypt_256,
-	.decrypt		= aes_sparc64_decrypt_256,
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_256,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_256,
 	.ecb_encrypt		= aes_sparc64_ecb_encrypt_256,
@@ -161,13 +83,10 @@ static struct aes_ops aes256_ops = {
 	.ctr_crypt		= aes_sparc64_ctr_crypt_256,
 };
 
-extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
-				   unsigned int key_len);
-
-static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
-		       unsigned int key_len)
+static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
+				unsigned int key_len)
 {
-	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+	struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
 
 	switch (key_len) {
 	case AES_KEYSIZE_128:
@@ -195,26 +114,6 @@ static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
-				unsigned int key_len)
-{
-	return aes_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
-}
-
-static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
-}
-
-static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
-	ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
-}
-
 static int ecb_encrypt(struct skcipher_request *req)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -358,26 +257,6 @@ static int ctr_crypt(struct skcipher_request *req)
 	return err;
 }
 
-static struct crypto_alg cipher_alg = {
-	.cra_name		= "aes",
-	.cra_driver_name	= "aes-sparc64",
-	.cra_priority		= SPARC_CR_OPCODE_PRIORITY,
-	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= sizeof(struct crypto_sparc64_aes_ctx),
-	.cra_alignmask		= 3,
-	.cra_module		= THIS_MODULE,
-	.cra_u	= {
-		.cipher	= {
-			.cia_min_keysize	= AES_MIN_KEY_SIZE,
-			.cia_max_keysize	= AES_MAX_KEY_SIZE,
-			.cia_setkey		= aes_set_key,
-			.cia_encrypt		= crypto_aes_encrypt,
-			.cia_decrypt		= crypto_aes_decrypt
-		}
-	}
-};
-
 static struct skcipher_alg skcipher_algs[] = {
 	{
 		.base.cra_name		= "ecb(aes)",
@@ -440,26 +319,17 @@ static bool __init sparc64_has_aes_opcode(void)
 
 static int __init aes_sparc64_mod_init(void)
 {
-	int err;
-
 	if (!sparc64_has_aes_opcode()) {
 		pr_info("sparc64 aes opcodes not available.\n");
 		return -ENODEV;
 	}
 	pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
-	err = crypto_register_alg(&cipher_alg);
-	if (err)
-		return err;
-	err = crypto_register_skciphers(skcipher_algs,
-					ARRAY_SIZE(skcipher_algs));
-	if (err)
-		crypto_unregister_alg(&cipher_alg);
-	return err;
+	return crypto_register_skciphers(skcipher_algs,
+					 ARRAY_SIZE(skcipher_algs));
 }
 
 static void __exit aes_sparc64_mod_fini(void)
 {
-	crypto_unregister_alg(&cipher_alg);
 	crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
 }
 
diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig
index 3fd2423d3cf8..7fb2319a0916 100644
--- a/arch/x86/crypto/Kconfig
+++ b/arch/x86/crypto/Kconfig
@@ -7,10 +7,8 @@ config CRYPTO_AES_NI_INTEL
 	select CRYPTO_AEAD
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_GF128MUL
-	select CRYPTO_ALGAPI
 	select CRYPTO_SKCIPHER
 	help
-	  Block cipher: AES cipher algorithms
 	  AEAD cipher: AES with GCM
 	  Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XCTR, XTS
 
@@ -333,26 +331,6 @@ config CRYPTO_AEGIS128_AESNI_SSE2
 	  - AES-NI (AES New Instructions)
 	  - SSE4.1 (Streaming SIMD Extensions 4.1)
 
-config CRYPTO_NHPOLY1305_SSE2
-	tristate "Hash functions: NHPoly1305 (SSE2)"
-	depends on 64BIT
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function for Adiantum
-
-	  Architecture: x86_64 using:
-	  - SSE2 (Streaming SIMD Extensions 2)
-
-config CRYPTO_NHPOLY1305_AVX2
-	tristate "Hash functions: NHPoly1305 (AVX2)"
-	depends on 64BIT
-	select CRYPTO_NHPOLY1305
-	help
-	  NHPoly1305 hash function for Adiantum
-
-	  Architecture: x86_64 using:
-	  - AVX2 (Advanced Vector Extensions 2)
-
 config CRYPTO_SM3_AVX_X86_64
 	tristate "Hash functions: SM3 (AVX)"
 	depends on 64BIT
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 5f2fb4f148fe..b21ad0978c52 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -53,11 +53,6 @@ aesni-intel-$(CONFIG_64BIT) += aes-ctr-avx-x86_64.o \
 obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
 
-obj-$(CONFIG_CRYPTO_NHPOLY1305_SSE2) += nhpoly1305-sse2.o
-nhpoly1305-sse2-y := nh-sse2-x86_64.o nhpoly1305-sse2-glue.o
-obj-$(CONFIG_CRYPTO_NHPOLY1305_AVX2) += nhpoly1305-avx2.o
-nhpoly1305-avx2-y := nh-avx2-x86_64.o nhpoly1305-avx2-glue.o
-
 obj-$(CONFIG_CRYPTO_SM3_AVX_X86_64) += sm3-avx-x86_64.o
 sm3-avx-x86_64-y := sm3-avx-asm_64.o sm3_avx_glue.o
 
diff --git a/arch/x86/crypto/aes-gcm-aesni-x86_64.S b/arch/x86/crypto/aes-gcm-aesni-x86_64.S
index 7c8a8a32bd3c..6b2abb76827e 100644
--- a/arch/x86/crypto/aes-gcm-aesni-x86_64.S
+++ b/arch/x86/crypto/aes-gcm-aesni-x86_64.S
@@ -143,10 +143,11 @@
 	.octa	0
 
 // Offsets in struct aes_gcm_key_aesni
-#define OFFSETOF_AESKEYLEN	480
-#define OFFSETOF_H_POWERS	496
-#define OFFSETOF_H_POWERS_XORED	624
-#define OFFSETOF_H_TIMES_X64	688
+#define OFFSETOF_AESKEYLEN	0
+#define OFFSETOF_AESROUNDKEYS	16
+#define OFFSETOF_H_POWERS	272
+#define OFFSETOF_H_POWERS_XORED	400
+#define OFFSETOF_H_TIMES_X64	464
 
 .text
 
@@ -505,9 +506,9 @@
 
 	// Encrypt an all-zeroes block to get the raw hash subkey.
 	movl		OFFSETOF_AESKEYLEN(KEY), %eax
-	lea		6*16(KEY,%rax,4), RNDKEYLAST_PTR
-	movdqa		(KEY), H_POW1  // Zero-th round key XOR all-zeroes block
-	lea		16(KEY), %rax
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR
+	movdqa		OFFSETOF_AESROUNDKEYS(KEY), H_POW1
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rax
 1:
 	aesenc		(%rax), H_POW1
 	add		$16, %rax
@@ -624,7 +625,7 @@
 // the zero-th AES round key.  Clobbers TMP0 and TMP1.
 .macro	_ctr_begin_8x
 	movq		.Lone(%rip), TMP0
-	movdqa		(KEY), TMP1		// zero-th round key
+	movdqa		OFFSETOF_AESROUNDKEYS(KEY), TMP1 // zero-th round key
 .irp i, 0,1,2,3,4,5,6,7
 	_vpshufb	BSWAP_MASK, LE_CTR, AESDATA\i
 	pxor		TMP1, AESDATA\i
@@ -726,7 +727,7 @@
 	movdqu		(LE_CTR_PTR), LE_CTR
 
 	movl		OFFSETOF_AESKEYLEN(KEY), AESKEYLEN
-	lea		6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
 
 	// If there are at least 8*16 bytes of data, then continue into the main
 	// loop, which processes 8*16 bytes of data per iteration.
@@ -745,7 +746,7 @@
 .if \enc
 	// Encrypt the first 8 plaintext blocks.
 	_ctr_begin_8x
-	lea		16(KEY), %rsi
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rsi
 	.p2align 4
 1:
 	movdqa		(%rsi), TMP0
@@ -767,7 +768,7 @@
 
 	// Generate the next set of 8 counter blocks and start encrypting them.
 	_ctr_begin_8x
-	lea		16(KEY), %rsi
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rsi
 
 	// Do a round of AES, and start the GHASH update of 8 ciphertext blocks
 	// by doing the unreduced multiplication for the first ciphertext block.
@@ -869,7 +870,7 @@
 	// Encrypt the next counter block.
 	_vpshufb	BSWAP_MASK, LE_CTR, TMP0
 	paddd		ONE, LE_CTR
-	pxor		(KEY), TMP0
+	pxor		OFFSETOF_AESROUNDKEYS(KEY), TMP0
 	lea		-6*16(RNDKEYLAST_PTR), %rsi	// Reduce code size
 	cmp		$24, AESKEYLEN
 	jl		128f	// AES-128?
@@ -926,8 +927,8 @@
 
 	// Encrypt a counter block for the last time.
 	pshufb		BSWAP_MASK, LE_CTR
-	pxor		(KEY), LE_CTR
-	lea		16(KEY), %rsi
+	pxor		OFFSETOF_AESROUNDKEYS(KEY), LE_CTR
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rsi
 1:
 	aesenc		(%rsi), LE_CTR
 	add		$16, %rsi
@@ -1038,12 +1039,12 @@
 
 	// Make %rax point to the 6th from last AES round key.  (Using signed
 	// byte offsets -7*16 through 6*16 decreases code size.)
-	lea		(KEY,AESKEYLEN64,4), %rax
+	lea		OFFSETOF_AESROUNDKEYS(KEY,AESKEYLEN64,4), %rax
 
 	// AES-encrypt the counter block and also multiply GHASH_ACC by H^1.
 	// Interleave the AES and GHASH instructions to improve performance.
 	pshufb		BSWAP_MASK, %xmm0
-	pxor		(KEY), %xmm0
+	pxor		OFFSETOF_AESROUNDKEYS(KEY), %xmm0
 	cmp		$24, AESKEYLEN
 	jl		128f	// AES-128?
 	je		192f	// AES-192?
diff --git a/arch/x86/crypto/aes-gcm-vaes-avx2.S b/arch/x86/crypto/aes-gcm-vaes-avx2.S
index 93c9504a488f..9cc387957fa9 100644
--- a/arch/x86/crypto/aes-gcm-vaes-avx2.S
+++ b/arch/x86/crypto/aes-gcm-vaes-avx2.S
@@ -122,8 +122,9 @@
 	.octa	2
 
 // Offsets in struct aes_gcm_key_vaes_avx2
-#define OFFSETOF_AESKEYLEN	480
-#define OFFSETOF_H_POWERS	512
+#define OFFSETOF_AESKEYLEN	0
+#define OFFSETOF_AESROUNDKEYS	16
+#define OFFSETOF_H_POWERS	288
 #define NUM_H_POWERS		8
 #define OFFSETOFEND_H_POWERS    (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16))
 #define OFFSETOF_H_POWERS_XORED	OFFSETOFEND_H_POWERS
@@ -240,9 +241,9 @@ SYM_FUNC_START(aes_gcm_precompute_vaes_avx2)
 
 	// Encrypt an all-zeroes block to get the raw hash subkey.
 	movl		OFFSETOF_AESKEYLEN(KEY), %eax
-	lea		6*16(KEY,%rax,4), RNDKEYLAST_PTR
-	vmovdqu		(KEY), H_CUR_XMM  // Zero-th round key XOR all-zeroes block
-	lea		16(KEY), %rax
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR
+	vmovdqu		OFFSETOF_AESROUNDKEYS(KEY), H_CUR_XMM
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rax
 1:
 	vaesenc		(%rax), H_CUR_XMM, H_CUR_XMM
 	add		$16, %rax
@@ -635,7 +636,7 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2)
 // the last AES round.  Clobbers %rax and TMP0.
 .macro	_aesenc_loop	vecs:vararg
 	_ctr_begin	\vecs
-	lea		16(KEY), %rax
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rax
 .Laesenc_loop\@:
 	vbroadcasti128	(%rax), TMP0
 	_vaesenc	TMP0, \vecs
@@ -768,8 +769,8 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2)
 	// Make RNDKEYLAST_PTR point to the last AES round key.  This is the
 	// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
 	// respectively.  Then load the zero-th and last round keys.
-	lea		6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
-	vbroadcasti128	(KEY), RNDKEY0
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
+	vbroadcasti128	OFFSETOF_AESROUNDKEYS(KEY), RNDKEY0
 	vbroadcasti128	(RNDKEYLAST_PTR), RNDKEYLAST
 
 	// Finish initializing LE_CTR by adding 1 to the second block.
@@ -1069,12 +1070,12 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2)
 .endif
 
 	// Make %rax point to the last AES round key for the chosen AES variant.
-	lea		6*16(KEY,AESKEYLEN64,4), %rax
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), %rax
 
 	// Start the AES encryption of the counter block by swapping the counter
 	// block to big-endian and XOR-ing it with the zero-th AES round key.
 	vpshufb		BSWAP_MASK, LE_CTR, %xmm0
-	vpxor		(KEY), %xmm0, %xmm0
+	vpxor		OFFSETOF_AESROUNDKEYS(KEY), %xmm0, %xmm0
 
 	// Complete the AES encryption and multiply GHASH_ACC by H^1.
 	// Interleave the AES and GHASH instructions to improve performance.
diff --git a/arch/x86/crypto/aes-gcm-vaes-avx512.S b/arch/x86/crypto/aes-gcm-vaes-avx512.S
index 06b71314d65c..516747db4659 100644
--- a/arch/x86/crypto/aes-gcm-vaes-avx512.S
+++ b/arch/x86/crypto/aes-gcm-vaes-avx512.S
@@ -86,10 +86,13 @@
 #define NUM_H_POWERS		16
 
 // Offset to AES key length (in bytes) in the key struct
-#define OFFSETOF_AESKEYLEN	480
+#define OFFSETOF_AESKEYLEN	0
+
+// Offset to AES round keys in the key struct
+#define OFFSETOF_AESROUNDKEYS	16
 
 // Offset to start of hash key powers array in the key struct
-#define OFFSETOF_H_POWERS	512
+#define OFFSETOF_H_POWERS	320
 
 // Offset to end of hash key powers array in the key struct.
 //
@@ -301,9 +304,9 @@ SYM_FUNC_START(aes_gcm_precompute_vaes_avx512)
 
 	// Encrypt an all-zeroes block to get the raw hash subkey.
 	movl		OFFSETOF_AESKEYLEN(KEY), %eax
-	lea		6*16(KEY,%rax,4), RNDKEYLAST_PTR
-	vmovdqu		(KEY), %xmm0  // Zero-th round key XOR all-zeroes block
-	add		$16, KEY
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR
+	vmovdqu		OFFSETOF_AESROUNDKEYS(KEY), %xmm0
+	add		$OFFSETOF_AESROUNDKEYS+16, KEY
 1:
 	vaesenc		(KEY), %xmm0, %xmm0
 	add		$16, KEY
@@ -790,8 +793,8 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512)
 	// Make RNDKEYLAST_PTR point to the last AES round key.  This is the
 	// round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256
 	// respectively.  Then load the zero-th and last round keys.
-	lea		6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
-	vbroadcasti32x4	(KEY), RNDKEY0
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR
+	vbroadcasti32x4	OFFSETOF_AESROUNDKEYS(KEY), RNDKEY0
 	vbroadcasti32x4	(RNDKEYLAST_PTR), RNDKEYLAST
 
 	// Finish initializing LE_CTR by adding [0, 1, ...] to its low words.
@@ -834,7 +837,7 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512)
 	// Encrypt the first 4 vectors of plaintext blocks.  Leave the resulting
 	// ciphertext in GHASHDATA[0-3] for GHASH.
 	_ctr_begin_4x
-	lea		16(KEY), %rax
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rax
 1:
 	vbroadcasti32x4	(%rax), RNDKEY
 	_vaesenc_4x	RNDKEY
@@ -957,7 +960,7 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512)
 	vpshufb		BSWAP_MASK, LE_CTR, %zmm0
 	vpaddd		LE_CTR_INC, LE_CTR, LE_CTR
 	vpxord		RNDKEY0, %zmm0, %zmm0
-	lea		16(KEY), %rax
+	lea		OFFSETOF_AESROUNDKEYS+16(KEY), %rax
 1:
 	vbroadcasti32x4	(%rax), RNDKEY
 	vaesenc		RNDKEY, %zmm0, %zmm0
@@ -1087,12 +1090,12 @@ SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512)
 .endif
 
 	// Make %rax point to the last AES round key for the chosen AES variant.
-	lea		6*16(KEY,AESKEYLEN64,4), %rax
+	lea		OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), %rax
 
 	// Start the AES encryption of the counter block by swapping the counter
 	// block to big-endian and XOR-ing it with the zero-th AES round key.
 	vpshufb		BSWAP_MASK, LE_CTR, %xmm0
-	vpxor		(KEY), %xmm0, %xmm0
+	vpxor		OFFSETOF_AESROUNDKEYS(KEY), %xmm0, %xmm0
 
 	// Complete the AES encryption and multiply GHASH_ACC by H^1.
 	// Interleave the AES and GHASH instructions to improve performance.
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S
index b37881bb9f15..6abe5e38a6d7 100644
--- a/arch/x86/crypto/aesni-intel_asm.S
+++ b/arch/x86/crypto/aesni-intel_asm.S
@@ -437,31 +437,6 @@ SYM_FUNC_START_LOCAL(_aesni_enc4)
 SYM_FUNC_END(_aesni_enc4)
 
 /*
- * void aesni_dec (const void *ctx, u8 *dst, const u8 *src)
- */
-SYM_FUNC_START(aesni_dec)
-	FRAME_BEGIN
-#ifndef __x86_64__
-	pushl KEYP
-	pushl KLEN
-	movl (FRAME_OFFSET+12)(%esp), KEYP	# ctx
-	movl (FRAME_OFFSET+16)(%esp), OUTP	# dst
-	movl (FRAME_OFFSET+20)(%esp), INP	# src
-#endif
-	mov 480(KEYP), KLEN		# key length
-	add $240, KEYP
-	movups (INP), STATE		# input
-	call _aesni_dec1
-	movups STATE, (OUTP)		#output
-#ifndef __x86_64__
-	popl KLEN
-	popl KEYP
-#endif
-	FRAME_END
-	RET
-SYM_FUNC_END(aesni_dec)
-
-/*
  * _aesni_dec1:		internal ABI
  * input:
  *	KEYP:		key struct pointer
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 48405e02d6e4..e6c38d1d8a92 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -60,7 +60,6 @@ static inline void *aes_align_addr(void *addr)
 asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
 			      unsigned int key_len);
 asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
-asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
 			      const u8 *in, unsigned int len);
 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
@@ -113,39 +112,6 @@ static int aes_set_key_common(struct crypto_aes_ctx *ctx,
 	return 0;
 }
 
-static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
-		       unsigned int key_len)
-{
-	return aes_set_key_common(aes_ctx(crypto_tfm_ctx(tfm)), in_key,
-				  key_len);
-}
-
-static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
-
-	if (!crypto_simd_usable()) {
-		aes_encrypt(ctx, dst, src);
-	} else {
-		kernel_fpu_begin();
-		aesni_enc(ctx, dst, src);
-		kernel_fpu_end();
-	}
-}
-
-static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
-
-	if (!crypto_simd_usable()) {
-		aes_decrypt(ctx, dst, src);
-	} else {
-		kernel_fpu_begin();
-		aesni_dec(ctx, dst, src);
-		kernel_fpu_end();
-	}
-}
-
 static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 			         unsigned int len)
 {
@@ -544,25 +510,6 @@ static int xts_decrypt_aesni(struct skcipher_request *req)
 	return xts_crypt(req, aesni_xts_encrypt_iv, aesni_xts_decrypt);
 }
 
-static struct crypto_alg aesni_cipher_alg = {
-	.cra_name		= "aes",
-	.cra_driver_name	= "aes-aesni",
-	.cra_priority		= 300,
-	.cra_flags		= CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		= AES_BLOCK_SIZE,
-	.cra_ctxsize		= CRYPTO_AES_CTX_SIZE,
-	.cra_module		= THIS_MODULE,
-	.cra_u	= {
-		.cipher	= {
-			.cia_min_keysize	= AES_MIN_KEY_SIZE,
-			.cia_max_keysize	= AES_MAX_KEY_SIZE,
-			.cia_setkey		= aes_set_key,
-			.cia_encrypt		= aesni_encrypt,
-			.cia_decrypt		= aesni_decrypt
-		}
-	}
-};
-
 static struct skcipher_alg aesni_skciphers[] = {
 	{
 		.base = {
@@ -833,7 +780,7 @@ DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
 /* The common part of the x86_64 AES-GCM key struct */
 struct aes_gcm_key {
 	/* Expanded AES key and the AES key length in bytes */
-	struct crypto_aes_ctx aes_key;
+	struct aes_enckey aes_key;
 
 	/* RFC4106 nonce (used only by the rfc4106 algorithms) */
 	u32 rfc4106_nonce;
@@ -842,11 +789,10 @@ struct aes_gcm_key {
 /* Key struct used by the AES-NI implementations of AES-GCM */
 struct aes_gcm_key_aesni {
 	/*
-	 * Common part of the key.  The assembly code requires 16-byte alignment
-	 * for the round keys; we get this by them being located at the start of
-	 * the struct and the whole struct being 16-byte aligned.
+	 * Common part of the key.  16-byte alignment is required by the
+	 * assembly code.
 	 */
-	struct aes_gcm_key base;
+	struct aes_gcm_key base __aligned(16);
 
 	/*
 	 * Powers of the hash key H^8 through H^1.  These are 128-bit values.
@@ -877,10 +823,9 @@ struct aes_gcm_key_aesni {
 struct aes_gcm_key_vaes_avx2 {
 	/*
 	 * Common part of the key.  The assembly code prefers 16-byte alignment
-	 * for the round keys; we get this by them being located at the start of
-	 * the struct and the whole struct being 32-byte aligned.
+	 * for this.
 	 */
-	struct aes_gcm_key base;
+	struct aes_gcm_key base __aligned(16);
 
 	/*
 	 * Powers of the hash key H^8 through H^1.  These are 128-bit values.
@@ -907,10 +852,9 @@ struct aes_gcm_key_vaes_avx2 {
 struct aes_gcm_key_vaes_avx512 {
 	/*
 	 * Common part of the key.  The assembly code prefers 16-byte alignment
-	 * for the round keys; we get this by them being located at the start of
-	 * the struct and the whole struct being 64-byte aligned.
+	 * for this.
 	 */
-	struct aes_gcm_key base;
+	struct aes_gcm_key base __aligned(16);
 
 	/*
 	 * Powers of the hash key H^16 through H^1.  These are 128-bit values.
@@ -1235,26 +1179,26 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
 	}
 
 	/* The assembly code assumes the following offsets. */
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_enc) != 0);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_length) != 480);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) != 512);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) != 640);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_enc) != 0);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_length) != 480);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) != 512);
-	BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, padding) != 768);
+	static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.len) == 0);
+	static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.k.rndkeys) == 16);
+	static_assert(offsetof(struct aes_gcm_key_aesni, h_powers) == 272);
+	static_assert(offsetof(struct aes_gcm_key_aesni, h_powers_xored) == 400);
+	static_assert(offsetof(struct aes_gcm_key_aesni, h_times_x64) == 464);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.len) == 0);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.k.rndkeys) == 16);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) == 288);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) == 416);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.len) == 0);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.k.rndkeys) == 16);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) == 320);
+	static_assert(offsetof(struct aes_gcm_key_vaes_avx512, padding) == 576);
+
+	err = aes_prepareenckey(&key->aes_key, raw_key, keylen);
+	if (err)
+		return err;
 
 	if (likely(crypto_simd_usable())) {
-		err = aes_check_keylen(keylen);
-		if (err)
-			return err;
 		kernel_fpu_begin();
-		aesni_set_key(&key->aes_key, raw_key, keylen);
 		aes_gcm_precompute(key, flags);
 		kernel_fpu_end();
 	} else {
@@ -1268,10 +1212,6 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
 		be128 h;
 		int i;
 
-		err = aes_expandkey(&key->aes_key, raw_key, keylen);
-		if (err)
-			return err;
-
 		/* Encrypt the all-zeroes block to get the hash key H^1 */
 		aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1);
 
@@ -1689,14 +1629,10 @@ static int __init aesni_init(void)
 	if (!x86_match_cpu(aesni_cpu_id))
 		return -ENODEV;
 
-	err = crypto_register_alg(&aesni_cipher_alg);
-	if (err)
-		return err;
-
 	err = crypto_register_skciphers(aesni_skciphers,
 					ARRAY_SIZE(aesni_skciphers));
 	if (err)
-		goto unregister_cipher;
+		return err;
 
 	err = crypto_register_aeads(aes_gcm_algs_aesni,
 				    ARRAY_SIZE(aes_gcm_algs_aesni));
@@ -1716,8 +1652,6 @@ unregister_avx:
 unregister_skciphers:
 	crypto_unregister_skciphers(aesni_skciphers,
 				    ARRAY_SIZE(aesni_skciphers));
-unregister_cipher:
-	crypto_unregister_alg(&aesni_cipher_alg);
 	return err;
 }
 
@@ -1727,7 +1661,6 @@ static void __exit aesni_exit(void)
 				ARRAY_SIZE(aes_gcm_algs_aesni));
 	crypto_unregister_skciphers(aesni_skciphers,
 				    ARRAY_SIZE(aesni_skciphers));
-	crypto_unregister_alg(&aesni_cipher_alg);
 	unregister_avx_algs();
 }
 
diff --git a/arch/x86/crypto/nh-avx2-x86_64.S b/arch/x86/crypto/nh-avx2-x86_64.S
deleted file mode 100644
index 791386d9a83a..000000000000
--- a/arch/x86/crypto/nh-avx2-x86_64.S
+++ /dev/null
@@ -1,159 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 AVX2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define		PASS0_SUMS	%ymm0
-#define		PASS1_SUMS	%ymm1
-#define		PASS2_SUMS	%ymm2
-#define		PASS3_SUMS	%ymm3
-#define		K0		%ymm4
-#define		K0_XMM		%xmm4
-#define		K1		%ymm5
-#define		K1_XMM		%xmm5
-#define		K2		%ymm6
-#define		K2_XMM		%xmm6
-#define		K3		%ymm7
-#define		K3_XMM		%xmm7
-#define		T0		%ymm8
-#define		T1		%ymm9
-#define		T2		%ymm10
-#define		T2_XMM		%xmm10
-#define		T3		%ymm11
-#define		T3_XMM		%xmm11
-#define		T4		%ymm12
-#define		T5		%ymm13
-#define		T6		%ymm14
-#define		T7		%ymm15
-#define		KEY		%rdi
-#define		MESSAGE		%rsi
-#define		MESSAGE_LEN	%rdx
-#define		HASH		%rcx
-
-.macro _nh_2xstride	k0, k1, k2, k3
-
-	// Add message words to key words
-	vpaddd		\k0, T3, T0
-	vpaddd		\k1, T3, T1
-	vpaddd		\k2, T3, T2
-	vpaddd		\k3, T3, T3
-
-	// Multiply 32x32 => 64 and accumulate
-	vpshufd		$0x10, T0, T4
-	vpshufd		$0x32, T0, T0
-	vpshufd		$0x10, T1, T5
-	vpshufd		$0x32, T1, T1
-	vpshufd		$0x10, T2, T6
-	vpshufd		$0x32, T2, T2
-	vpshufd		$0x10, T3, T7
-	vpshufd		$0x32, T3, T3
-	vpmuludq	T4, T0, T0
-	vpmuludq	T5, T1, T1
-	vpmuludq	T6, T2, T2
-	vpmuludq	T7, T3, T3
-	vpaddq		T0, PASS0_SUMS, PASS0_SUMS
-	vpaddq		T1, PASS1_SUMS, PASS1_SUMS
-	vpaddq		T2, PASS2_SUMS, PASS2_SUMS
-	vpaddq		T3, PASS3_SUMS, PASS3_SUMS
-.endm
-
-/*
- * void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_avx2)
-
-	vmovdqu		0x00(KEY), K0
-	vmovdqu		0x10(KEY), K1
-	add		$0x20, KEY
-	vpxor		PASS0_SUMS, PASS0_SUMS, PASS0_SUMS
-	vpxor		PASS1_SUMS, PASS1_SUMS, PASS1_SUMS
-	vpxor		PASS2_SUMS, PASS2_SUMS, PASS2_SUMS
-	vpxor		PASS3_SUMS, PASS3_SUMS, PASS3_SUMS
-
-	sub		$0x40, MESSAGE_LEN
-	jl		.Lloop4_done
-.Lloop4:
-	vmovdqu		(MESSAGE), T3
-	vmovdqu		0x00(KEY), K2
-	vmovdqu		0x10(KEY), K3
-	_nh_2xstride	K0, K1, K2, K3
-
-	vmovdqu		0x20(MESSAGE), T3
-	vmovdqu		0x20(KEY), K0
-	vmovdqu		0x30(KEY), K1
-	_nh_2xstride	K2, K3, K0, K1
-
-	add		$0x40, MESSAGE
-	add		$0x40, KEY
-	sub		$0x40, MESSAGE_LEN
-	jge		.Lloop4
-
-.Lloop4_done:
-	and		$0x3f, MESSAGE_LEN
-	jz		.Ldone
-
-	cmp		$0x20, MESSAGE_LEN
-	jl		.Llast
-
-	// 2 or 3 strides remain; do 2 more.
-	vmovdqu		(MESSAGE), T3
-	vmovdqu		0x00(KEY), K2
-	vmovdqu		0x10(KEY), K3
-	_nh_2xstride	K0, K1, K2, K3
-	add		$0x20, MESSAGE
-	add		$0x20, KEY
-	sub		$0x20, MESSAGE_LEN
-	jz		.Ldone
-	vmovdqa		K2, K0
-	vmovdqa		K3, K1
-.Llast:
-	// Last stride.  Zero the high 128 bits of the message and keys so they
-	// don't affect the result when processing them like 2 strides.
-	vmovdqu		(MESSAGE), T3_XMM
-	vmovdqa		K0_XMM, K0_XMM
-	vmovdqa		K1_XMM, K1_XMM
-	vmovdqu		0x00(KEY), K2_XMM
-	vmovdqu		0x10(KEY), K3_XMM
-	_nh_2xstride	K0, K1, K2, K3
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-
-	// PASS0_SUMS is (0A 0B 0C 0D)
-	// PASS1_SUMS is (1A 1B 1C 1D)
-	// PASS2_SUMS is (2A 2B 2C 2D)
-	// PASS3_SUMS is (3A 3B 3C 3D)
-	// We need the horizontal sums:
-	//     (0A + 0B + 0C + 0D,
-	//	1A + 1B + 1C + 1D,
-	//	2A + 2B + 2C + 2D,
-	//	3A + 3B + 3C + 3D)
-	//
-
-	vpunpcklqdq	PASS1_SUMS, PASS0_SUMS, T0	// T0 = (0A 1A 0C 1C)
-	vpunpckhqdq	PASS1_SUMS, PASS0_SUMS, T1	// T1 = (0B 1B 0D 1D)
-	vpunpcklqdq	PASS3_SUMS, PASS2_SUMS, T2	// T2 = (2A 3A 2C 3C)
-	vpunpckhqdq	PASS3_SUMS, PASS2_SUMS, T3	// T3 = (2B 3B 2D 3D)
-
-	vinserti128	$0x1, T2_XMM, T0, T4		// T4 = (0A 1A 2A 3A)
-	vinserti128	$0x1, T3_XMM, T1, T5		// T5 = (0B 1B 2B 3B)
-	vperm2i128	$0x31, T2, T0, T0		// T0 = (0C 1C 2C 3C)
-	vperm2i128	$0x31, T3, T1, T1		// T1 = (0D 1D 2D 3D)
-
-	vpaddq		T5, T4, T4
-	vpaddq		T1, T0, T0
-	vpaddq		T4, T0, T0
-	vmovdqu		T0, (HASH)
-	vzeroupper
-	RET
-SYM_FUNC_END(nh_avx2)
diff --git a/arch/x86/crypto/nh-sse2-x86_64.S b/arch/x86/crypto/nh-sse2-x86_64.S
deleted file mode 100644
index 75fb994b6d17..000000000000
--- a/arch/x86/crypto/nh-sse2-x86_64.S
+++ /dev/null
@@ -1,124 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * NH - ε-almost-universal hash function, x86_64 SSE2 accelerated
- *
- * Copyright 2018 Google LLC
- *
- * Author: Eric Biggers <ebiggers@google.com>
- */
-
-#include <linux/linkage.h>
-#include <linux/cfi_types.h>
-
-#define		PASS0_SUMS	%xmm0
-#define		PASS1_SUMS	%xmm1
-#define		PASS2_SUMS	%xmm2
-#define		PASS3_SUMS	%xmm3
-#define		K0		%xmm4
-#define		K1		%xmm5
-#define		K2		%xmm6
-#define		K3		%xmm7
-#define		T0		%xmm8
-#define		T1		%xmm9
-#define		T2		%xmm10
-#define		T3		%xmm11
-#define		T4		%xmm12
-#define		T5		%xmm13
-#define		T6		%xmm14
-#define		T7		%xmm15
-#define		KEY		%rdi
-#define		MESSAGE		%rsi
-#define		MESSAGE_LEN	%rdx
-#define		HASH		%rcx
-
-.macro _nh_stride	k0, k1, k2, k3, offset
-
-	// Load next message stride
-	movdqu		\offset(MESSAGE), T1
-
-	// Load next key stride
-	movdqu		\offset(KEY), \k3
-
-	// Add message words to key words
-	movdqa		T1, T2
-	movdqa		T1, T3
-	paddd		T1, \k0    // reuse k0 to avoid a move
-	paddd		\k1, T1
-	paddd		\k2, T2
-	paddd		\k3, T3
-
-	// Multiply 32x32 => 64 and accumulate
-	pshufd		$0x10, \k0, T4
-	pshufd		$0x32, \k0, \k0
-	pshufd		$0x10, T1, T5
-	pshufd		$0x32, T1, T1
-	pshufd		$0x10, T2, T6
-	pshufd		$0x32, T2, T2
-	pshufd		$0x10, T3, T7
-	pshufd		$0x32, T3, T3
-	pmuludq		T4, \k0
-	pmuludq		T5, T1
-	pmuludq		T6, T2
-	pmuludq		T7, T3
-	paddq		\k0, PASS0_SUMS
-	paddq		T1, PASS1_SUMS
-	paddq		T2, PASS2_SUMS
-	paddq		T3, PASS3_SUMS
-.endm
-
-/*
- * void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
- *		__le64 hash[NH_NUM_PASSES])
- *
- * It's guaranteed that message_len % 16 == 0.
- */
-SYM_TYPED_FUNC_START(nh_sse2)
-
-	movdqu		0x00(KEY), K0
-	movdqu		0x10(KEY), K1
-	movdqu		0x20(KEY), K2
-	add		$0x30, KEY
-	pxor		PASS0_SUMS, PASS0_SUMS
-	pxor		PASS1_SUMS, PASS1_SUMS
-	pxor		PASS2_SUMS, PASS2_SUMS
-	pxor		PASS3_SUMS, PASS3_SUMS
-
-	sub		$0x40, MESSAGE_LEN
-	jl		.Lloop4_done
-.Lloop4:
-	_nh_stride	K0, K1, K2, K3, 0x00
-	_nh_stride	K1, K2, K3, K0, 0x10
-	_nh_stride	K2, K3, K0, K1, 0x20
-	_nh_stride	K3, K0, K1, K2, 0x30
-	add		$0x40, KEY
-	add		$0x40, MESSAGE
-	sub		$0x40, MESSAGE_LEN
-	jge		.Lloop4
-
-.Lloop4_done:
-	and		$0x3f, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K0, K1, K2, K3, 0x00
-
-	sub		$0x10, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K1, K2, K3, K0, 0x10
-
-	sub		$0x10, MESSAGE_LEN
-	jz		.Ldone
-	_nh_stride	K2, K3, K0, K1, 0x20
-
-.Ldone:
-	// Sum the accumulators for each pass, then store the sums to 'hash'
-	movdqa		PASS0_SUMS, T0
-	movdqa		PASS2_SUMS, T1
-	punpcklqdq	PASS1_SUMS, T0		// => (PASS0_SUM_A PASS1_SUM_A)
-	punpcklqdq	PASS3_SUMS, T1		// => (PASS2_SUM_A PASS3_SUM_A)
-	punpckhqdq	PASS1_SUMS, PASS0_SUMS	// => (PASS0_SUM_B PASS1_SUM_B)
-	punpckhqdq	PASS3_SUMS, PASS2_SUMS	// => (PASS2_SUM_B PASS3_SUM_B)
-	paddq		PASS0_SUMS, T0
-	paddq		PASS2_SUMS, T1
-	movdqu		T0, 0x00(HASH)
-	movdqu		T1, 0x10(HASH)
-	RET
-SYM_FUNC_END(nh_sse2)
diff --git a/arch/x86/crypto/nhpoly1305-avx2-glue.c b/arch/x86/crypto/nhpoly1305-avx2-glue.c
deleted file mode 100644
index c3a872f4d6a7..000000000000
--- a/arch/x86/crypto/nhpoly1305-avx2-glue.c
+++ /dev/null
@@ -1,81 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (AVX2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_avx2_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_fpu_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_avx2);
-		kernel_fpu_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_avx2_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_avx2_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-avx2",
-	.base.cra_priority	= 300,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_avx2_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_avx2_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!boot_cpu_has(X86_FEATURE_AVX2) ||
-	    !boot_cpu_has(X86_FEATURE_OSXSAVE))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (AVX2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-avx2");
diff --git a/arch/x86/crypto/nhpoly1305-sse2-glue.c b/arch/x86/crypto/nhpoly1305-sse2-glue.c
deleted file mode 100644
index a268a8439a5c..000000000000
--- a/arch/x86/crypto/nhpoly1305-sse2-glue.c
+++ /dev/null
@@ -1,80 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * NHPoly1305 - ε-almost-∆-universal hash function for Adiantum
- * (SSE2 accelerated version)
- *
- * Copyright 2018 Google LLC
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/nhpoly1305.h>
-#include <linux/module.h>
-#include <linux/sizes.h>
-#include <asm/simd.h>
-
-asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
-			__le64 hash[NH_NUM_PASSES]);
-
-static int nhpoly1305_sse2_update(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen)
-{
-	if (srclen < 64 || !crypto_simd_usable())
-		return crypto_nhpoly1305_update(desc, src, srclen);
-
-	do {
-		unsigned int n = min_t(unsigned int, srclen, SZ_4K);
-
-		kernel_fpu_begin();
-		crypto_nhpoly1305_update_helper(desc, src, n, nh_sse2);
-		kernel_fpu_end();
-		src += n;
-		srclen -= n;
-	} while (srclen);
-	return 0;
-}
-
-static int nhpoly1305_sse2_digest(struct shash_desc *desc,
-				  const u8 *src, unsigned int srclen, u8 *out)
-{
-	return crypto_nhpoly1305_init(desc) ?:
-	       nhpoly1305_sse2_update(desc, src, srclen) ?:
-	       crypto_nhpoly1305_final(desc, out);
-}
-
-static struct shash_alg nhpoly1305_alg = {
-	.base.cra_name		= "nhpoly1305",
-	.base.cra_driver_name	= "nhpoly1305-sse2",
-	.base.cra_priority	= 200,
-	.base.cra_ctxsize	= sizeof(struct nhpoly1305_key),
-	.base.cra_module	= THIS_MODULE,
-	.digestsize		= POLY1305_DIGEST_SIZE,
-	.init			= crypto_nhpoly1305_init,
-	.update			= nhpoly1305_sse2_update,
-	.final			= crypto_nhpoly1305_final,
-	.digest			= nhpoly1305_sse2_digest,
-	.setkey			= crypto_nhpoly1305_setkey,
-	.descsize		= sizeof(struct nhpoly1305_state),
-};
-
-static int __init nhpoly1305_mod_init(void)
-{
-	if (!boot_cpu_has(X86_FEATURE_XMM2))
-		return -ENODEV;
-
-	return crypto_register_shash(&nhpoly1305_alg);
-}
-
-static void __exit nhpoly1305_mod_exit(void)
-{
-	crypto_unregister_shash(&nhpoly1305_alg);
-}
-
-module_init(nhpoly1305_mod_init);
-module_exit(nhpoly1305_mod_exit);
-
-MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function (SSE2-accelerated)");
-MODULE_LICENSE("GPL v2");
-MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
-MODULE_ALIAS_CRYPTO("nhpoly1305");
-MODULE_ALIAS_CRYPTO("nhpoly1305-sse2");