lib/crypto: s390/ghash: Migrate optimized code into library

Remove the "ghash-s390" crypto_shash algorithm, and replace it with an
implementation of ghash_blocks_arch() for the GHASH library.

This makes the GHASH library be optimized with CPACF.  It also greatly
reduces the amount of s390-specific glue code that is needed, and it
fixes the issue where this GHASH optimization was disabled by default.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260319061723.1140720-14-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>

2 files changed, 55 insertions, 0 deletions

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 027802e0de33..a39e7707e9ee 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -124,6 +124,7 @@ config CRYPTO_LIB_GF128HASH_ARCH
 	default y if PPC64 && VSX
 	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
 		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
 	default y if X86_64
 
 config CRYPTO_LIB_MD5
diff --git a/lib/crypto/s390/gf128hash.h b/lib/crypto/s390/gf128hash.h
new file mode 100644
index 000000000000..1e46ce4bca40
--- /dev/null
+++ b/lib/crypto/s390/gf128hash.h
@@ -0,0 +1,54 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * GHASH optimized using the CP Assist for Cryptographic Functions (CPACF)
+ *
+ * Copyright 2026 Google LLC
+ */
+#include <asm/cpacf.h>
+#include <linux/cpufeature.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_ghash);
+
+#define ghash_preparekey_arch ghash_preparekey_arch
+static void ghash_preparekey_arch(struct ghash_key *key,
+				  const u8 raw_key[GHASH_BLOCK_SIZE])
+{
+	/* Save key in POLYVAL format for fallback */
+	ghash_key_to_polyval(raw_key, &key->h);
+
+	/* Save key in GHASH format for CPACF_KIMD_GHASH */
+	memcpy(key->h_raw, raw_key, GHASH_BLOCK_SIZE);
+}
+
+#define ghash_blocks_arch ghash_blocks_arch
+static void ghash_blocks_arch(struct polyval_elem *acc,
+			      const struct ghash_key *key,
+			      const u8 *data, size_t nblocks)
+{
+	if (static_branch_likely(&have_cpacf_ghash)) {
+		/*
+		 * CPACF_KIMD_GHASH requires the accumulator and key in a single
+		 * buffer, each using the GHASH convention.
+		 */
+		u8 ctx[2][GHASH_BLOCK_SIZE] __aligned(8);
+
+		polyval_acc_to_ghash(acc, ctx[0]);
+		memcpy(ctx[1], key->h_raw, GHASH_BLOCK_SIZE);
+
+		cpacf_kimd(CPACF_KIMD_GHASH, ctx, data,
+			   nblocks * GHASH_BLOCK_SIZE);
+
+		ghash_acc_to_polyval(ctx[0], acc);
+		memzero_explicit(ctx, sizeof(ctx));
+	} else {
+		ghash_blocks_generic(acc, &key->h, data, nblocks);
+	}
+}
+
+#define gf128hash_mod_init_arch gf128hash_mod_init_arch
+static void gf128hash_mod_init_arch(void)
+{
+	if (cpu_have_feature(S390_CPU_FEATURE_MSA) &&
+	    cpacf_query_func(CPACF_KIMD, CPACF_KIMD_GHASH))
+		static_branch_enable(&have_cpacf_ghash);
+}