From 7137cbf2b5c9feb6302d6da116eab2047c5f05d2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Feb 2026 13:34:48 -0800 Subject: crypto: aes - Add cmac, xcbc, and cbcmac algorithms using library Update the "aes" module to implement "cmac(aes)", "xcbc(aes)", and "cbcmac(aes)" algorithms using the corresponding library functions, and register these with the crypto_shash API. Each algorithm is included only if the corresponding existing kconfig option is enabled. This allows the architecture-optimized implementations of these algorithms to continue to be accessible via the crypto_shash API once they are migrated into the library. For "xcbc(aes)", I also fixed the bug where AES key lengths other than 128 bits were allowed, so that this bug didn't have to be implemented in the library. The AES-XCBC-MAC specification (RFC 3566) is clear that key lengths other than 128 bits MUST NOT be supported. AES-XCBC-MAC derives a 128-bit subkey internally, so the nonstandard support for longer AES keys didn't really work: AES-128 was still used internally. In the unlikely event that someone is actually relying on the broken and nonstandard support for longer AES-XCBC-MAC keys, we can fairly easily reintroduce it. But it seems unnecessary: the only user of "xcbc(aes)" seems to be IPsec, which uses 128-bit keys with it. Reviewed-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260218213501.136844-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 + crypto/aes.c | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- crypto/testmgr.c | 10 +-- 3 files changed, 189 insertions(+), 6 deletions(-) (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index b4bb85e8e226..c977d40f906d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -361,6 +361,8 @@ config CRYPTO_AES tristate "AES (Advanced Encryption Standard)" select CRYPTO_ALGAPI select CRYPTO_LIB_AES + select CRYPTO_LIB_AES_CBC_MACS if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM + select CRYPTO_HASH if CRYPTO_CMAC || CRYPTO_XCBC || CRYPTO_CCM help AES cipher algorithms (Rijndael)(FIPS-197, ISO/IEC 18033-3) diff --git a/crypto/aes.c b/crypto/aes.c index ae8385df0ce5..6bf23eb0503f 100644 --- a/crypto/aes.c +++ b/crypto/aes.c @@ -5,8 +5,10 @@ * Copyright 2026 Google LLC */ +#include #include #include +#include #include static_assert(__alignof__(struct aes_key) <= CRYPTO_MINALIGN); @@ -33,6 +35,98 @@ static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) aes_decrypt(key, out, in); } +static_assert(__alignof__(struct aes_cmac_key) <= CRYPTO_MINALIGN); +#define AES_CMAC_KEY(tfm) ((struct aes_cmac_key *)crypto_shash_ctx(tfm)) +#define AES_CMAC_CTX(desc) ((struct aes_cmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_cmac_preparekey(AES_CMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_xcbc_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + if (key_len != AES_KEYSIZE_128) + return -EINVAL; + aes_xcbcmac_preparekey(AES_CMAC_KEY(tfm), in_key); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_init(struct shash_desc *desc) +{ + aes_cmac_init(AES_CMAC_CTX(desc), AES_CMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cmac_update(AES_CMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cmac_final(AES_CMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cmac(AES_CMAC_KEY(desc->tfm), data, len, out); + return 0; +} + +static_assert(__alignof__(struct aes_enckey) <= CRYPTO_MINALIGN); +#define AES_CBCMAC_KEY(tfm) ((struct aes_enckey *)crypto_shash_ctx(tfm)) +#define AES_CBCMAC_CTX(desc) ((struct aes_cbcmac_ctx *)shash_desc_ctx(desc)) + +static int __maybe_unused crypto_aes_cbcmac_setkey(struct crypto_shash *tfm, + const u8 *in_key, + unsigned int key_len) +{ + return aes_prepareenckey(AES_CBCMAC_KEY(tfm), in_key, key_len); +} + +static int __maybe_unused crypto_aes_cbcmac_init(struct shash_desc *desc) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_update(struct shash_desc *desc, + const u8 *data, + unsigned int len) +{ + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_final(struct shash_desc *desc, + u8 *out) +{ + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + +static int __maybe_unused crypto_aes_cbcmac_digest(struct shash_desc *desc, + const u8 *data, + unsigned int len, u8 *out) +{ + aes_cbcmac_init(AES_CBCMAC_CTX(desc), AES_CBCMAC_KEY(desc->tfm)); + aes_cbcmac_update(AES_CBCMAC_CTX(desc), data, len); + aes_cbcmac_final(AES_CBCMAC_CTX(desc), out); + return 0; +} + static struct crypto_alg alg = { .cra_name = "aes", .cra_driver_name = "aes-lib", @@ -48,19 +142,106 @@ static struct crypto_alg alg = { .cia_decrypt = crypto_aes_decrypt } } }; +static struct shash_alg mac_algs[] = { +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) + { + .base.cra_name = "cmac(aes)", + .base.cra_driver_name = "cmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cmac_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) + { + /* + * Note that the only difference between xcbc(aes) and cmac(aes) + * is the preparekey function. + */ + .base.cra_name = "xcbc(aes)", + .base.cra_driver_name = "xcbc-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_cmac_key), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_xcbc_setkey, + .init = crypto_aes_cmac_init, + .update = crypto_aes_cmac_update, + .final = crypto_aes_cmac_final, + .digest = crypto_aes_cmac_digest, + .descsize = sizeof(struct aes_cmac_ctx), + }, +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) + { + .base.cra_name = "cbcmac(aes)", + .base.cra_driver_name = "cbcmac-aes-lib", + .base.cra_priority = 300, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct aes_enckey), + .base.cra_module = THIS_MODULE, + .digestsize = AES_BLOCK_SIZE, + .setkey = crypto_aes_cbcmac_setkey, + .init = crypto_aes_cbcmac_init, + .update = crypto_aes_cbcmac_update, + .final = crypto_aes_cbcmac_final, + .digest = crypto_aes_cbcmac_digest, + .descsize = sizeof(struct aes_cbcmac_ctx), + }, +#endif +}; + static int __init crypto_aes_mod_init(void) { - return crypto_register_alg(&alg); + int err = crypto_register_alg(&alg); + + if (err) + return err; + + if (ARRAY_SIZE(mac_algs) > 0) { + err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs)); + if (err) + goto err_unregister_alg; + } /* Else, CONFIG_CRYPTO_HASH might not be enabled. */ + return 0; + +err_unregister_alg: + crypto_unregister_alg(&alg); + return err; } module_init(crypto_aes_mod_init); static void __exit crypto_aes_mod_exit(void) { + if (ARRAY_SIZE(mac_algs) > 0) + crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs)); crypto_unregister_alg(&alg); } module_exit(crypto_aes_mod_exit); MODULE_DESCRIPTION("Crypto API support for AES block cipher"); +MODULE_IMPORT_NS("CRYPTO_INTERNAL"); MODULE_LICENSE("GPL"); MODULE_ALIAS_CRYPTO("aes"); MODULE_ALIAS_CRYPTO("aes-lib"); +#if IS_ENABLED(CONFIG_CRYPTO_CMAC) +MODULE_ALIAS_CRYPTO("cmac(aes)"); +MODULE_ALIAS_CRYPTO("cmac-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_XCBC) +MODULE_ALIAS_CRYPTO("xcbc(aes)"); +MODULE_ALIAS_CRYPTO("xcbc-aes-lib"); +#endif +#if IS_ENABLED(CONFIG_CRYPTO_CCM) +MODULE_ALIAS_CRYPTO("cbcmac(aes)"); +MODULE_ALIAS_CRYPTO("cbcmac-aes-lib"); +#endif diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 4985411dedae..fec950f1628b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4388,7 +4388,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif .alg = "cbcmac(aes)", - .generic_driver = "cbcmac(aes-lib)", + .generic_driver = "cbcmac-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_cbcmac_tv_template) @@ -4401,7 +4401,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ccm(aes)", - .generic_driver = "ccm_base(ctr(aes-lib),cbcmac(aes-lib))", + .generic_driver = "ccm_base(ctr(aes-lib),cbcmac-aes-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4429,7 +4429,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, }, { .alg = "cmac(aes)", - .generic_driver = "cmac(aes-lib)", + .generic_driver = "cmac-aes-lib", .fips_allowed = 1, .test = alg_test_hash, .suite = { @@ -5326,7 +5326,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4309(ccm(aes))", - .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac(aes-lib)))", + .generic_driver = "rfc4309(ccm_base(ctr(aes-lib),cbcmac-aes-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5515,7 +5515,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "xcbc(aes)", - .generic_driver = "xcbc(aes-lib)", + .generic_driver = "xcbc-aes-lib", .test = alg_test_hash, .suite = { .hash = __VECS(aes_xcbc128_tv_template) -- cgit v1.2.3 From ce260754bb435aea18e6a1a1ce3759249013f5a4 Mon Sep 17 00:00:00 2001 From: David Howells Date: Wed, 25 Feb 2026 17:00:05 -0800 Subject: crypto: jitterentropy - Use SHA-3 library Make the jitterentropy RNG use the SHA-3 library API instead of crypto_shash. This ends up being quite a bit simpler, as various dynamic allocations and error checks become unnecessary. Signed-off-by: David Howells Co-developed-by: Eric Biggers Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260226010005.43528-1-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/jitterentropy-kcapi.c | 114 +++++++++++-------------------------------- crypto/jitterentropy.c | 25 +++++----- crypto/jitterentropy.h | 19 +++++--- 4 files changed, 52 insertions(+), 108 deletions(-) (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index c977d40f906d..b8608ef6823b 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1177,8 +1177,8 @@ endif # if CRYPTO_DRBG_MENU config CRYPTO_JITTERENTROPY tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)" + select CRYPTO_LIB_SHA3 select CRYPTO_RNG - select CRYPTO_SHA3 help CPU Jitter RNG (Random Number Generator) from the Jitterentropy library diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 7c880cf34c52..4ad729357441 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -37,7 +37,6 @@ * DAMAGE. */ -#include #include #include #include @@ -48,8 +47,6 @@ #include "jitterentropy.h" -#define JENT_CONDITIONING_HASH "sha3-256" - /*************************************************************************** * Helper function ***************************************************************************/ @@ -101,22 +98,14 @@ void jent_get_nstime(__u64 *out) jent_raw_hires_entropy_store(tmp); } -int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck) +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; - SHASH_DESC_ON_STACK(desc, hash_state_desc->tfm); + struct sha3_ctx tmp_state; /* zeroized by sha3_final() */ u8 intermediary[SHA3_256_DIGEST_SIZE]; __u64 j = 0; - int ret; - - desc->tfm = hash_state_desc->tfm; - if (sizeof(intermediary) != crypto_shash_digestsize(desc->tfm)) { - pr_warn_ratelimited("Unexpected digest size\n"); - return -EINVAL; - } kmsan_unpoison_memory(intermediary, sizeof(intermediary)); /* @@ -130,24 +119,20 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, * * Note, it does not matter which or how much data you inject, we are * interested in one Keccack1600 compression operation performed with - * the crypto_shash_final. + * the sha3_final. */ for (j = 0; j < hash_loop_cnt; j++) { - ret = crypto_shash_init(desc) ?: - crypto_shash_update(desc, intermediary, - sizeof(intermediary)) ?: - crypto_shash_finup(desc, addtl, addtl_len, intermediary); - if (ret) - goto err; + sha3_256_init(&tmp_state); + sha3_update(&tmp_state, intermediary, sizeof(intermediary)); + sha3_update(&tmp_state, addtl, addtl_len); + sha3_final(&tmp_state, intermediary); } /* * Inject the data from the previous loop into the pool. This data is * not considered to contain any entropy, but it stirs the pool a bit. */ - ret = crypto_shash_update(hash_state_desc, intermediary, sizeof(intermediary)); - if (ret) - goto err; + sha3_update(hash_state, intermediary, sizeof(intermediary)); /* * Insert the time stamp into the hash context representing the pool. @@ -162,30 +147,24 @@ int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, time = 0; } - ret = crypto_shash_update(hash_state_desc, (u8 *)&time, sizeof(__u64)); - -err: - shash_desc_zero(desc); + sha3_update(hash_state, (u8 *)&time, sizeof(__u64)); memzero_explicit(intermediary, sizeof(intermediary)); - - return ret; } -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len) { - struct shash_desc *hash_state_desc = (struct shash_desc *)hash_state; u8 jent_block[SHA3_256_DIGEST_SIZE]; + /* Obtain data from entropy pool and re-initialize it */ - int ret = crypto_shash_final(hash_state_desc, jent_block) ?: - crypto_shash_init(hash_state_desc) ?: - crypto_shash_update(hash_state_desc, jent_block, - sizeof(jent_block)); + sha3_final(hash_state, jent_block); + sha3_256_init(hash_state); + sha3_update(hash_state, jent_block, sizeof(jent_block)); - if (!ret && dst_len) + if (dst_len) memcpy(dst, jent_block, dst_len); memzero_explicit(jent_block, sizeof(jent_block)); - return ret; } /*************************************************************************** @@ -195,8 +174,7 @@ int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len) struct jitterentropy { spinlock_t jent_lock; struct rand_data *entropy_collector; - struct crypto_shash *tfm; - struct shash_desc *sdesc; + struct sha3_ctx hash_state; }; static void jent_kcapi_cleanup(struct crypto_tfm *tfm) @@ -205,15 +183,7 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) spin_lock(&rng->jent_lock); - if (rng->sdesc) { - shash_desc_zero(rng->sdesc); - kfree(rng->sdesc); - } - rng->sdesc = NULL; - - if (rng->tfm) - crypto_free_shash(rng->tfm); - rng->tfm = NULL; + memzero_explicit(&rng->hash_state, sizeof(rng->hash_state)); if (rng->entropy_collector) jent_entropy_collector_free(rng->entropy_collector); @@ -224,34 +194,15 @@ static void jent_kcapi_cleanup(struct crypto_tfm *tfm) static int jent_kcapi_init(struct crypto_tfm *tfm) { struct jitterentropy *rng = crypto_tfm_ctx(tfm); - struct crypto_shash *hash; - struct shash_desc *sdesc; - int size, ret = 0; + int ret = 0; spin_lock_init(&rng->jent_lock); /* Use SHA3-256 as conditioner */ - hash = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(hash)) { - pr_err("Cannot allocate conditioning digest\n"); - return PTR_ERR(hash); - } - rng->tfm = hash; - - size = sizeof(struct shash_desc) + crypto_shash_descsize(hash); - sdesc = kmalloc(size, GFP_KERNEL); - if (!sdesc) { - ret = -ENOMEM; - goto err; - } - - sdesc->tfm = hash; - crypto_shash_init(sdesc); - rng->sdesc = sdesc; + sha3_256_init(&rng->hash_state); - rng->entropy_collector = - jent_entropy_collector_alloc(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, - sdesc); + rng->entropy_collector = jent_entropy_collector_alloc( + CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &rng->hash_state); if (!rng->entropy_collector) { ret = -ENOMEM; goto err; @@ -326,23 +277,16 @@ static struct rng_alg jent_alg = { static int __init jent_mod_init(void) { - SHASH_DESC_ON_STACK(desc, tfm); - struct crypto_shash *tfm; + struct sha3_ctx hash_state; int ret = 0; jent_testing_init(); - tfm = crypto_alloc_shash(JENT_CONDITIONING_HASH, 0, 0); - if (IS_ERR(tfm)) { - jent_testing_exit(); - return PTR_ERR(tfm); - } + sha3_256_init(&hash_state); - desc->tfm = tfm; - crypto_shash_init(desc); - ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, desc, NULL); - shash_desc_zero(desc); - crypto_free_shash(tfm); + ret = jent_entropy_init(CONFIG_CRYPTO_JITTERENTROPY_OSR, 0, &hash_state, + NULL); + memzero_explicit(&hash_state, sizeof(hash_state)); if (ret) { /* Handle permanent health test error */ if (fips_enabled) diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index 3f93cdc9a7af..d5832caa8ab3 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -68,7 +68,7 @@ struct rand_data { * of the RNG are marked as SENSITIVE. A user must not * access that information while the RNG executes its loops to * calculate the next random value. */ - void *hash_state; /* SENSITIVE hash state entropy pool */ + struct sha3_ctx *hash_state; /* SENSITIVE hash state entropy pool */ __u64 prev_time; /* SENSITIVE Previous time stamp */ __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ @@ -417,10 +417,9 @@ static __u64 jent_loop_shuffle(unsigned int bits, unsigned int min) * time [in] time stamp to be injected * stuck [in] Is the time stamp identified as stuck? * - * Output: - * updated hash context in the entropy collector or error code + * Output: updated hash context in the entropy collector */ -static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) +static void jent_condition_data(struct rand_data *ec, __u64 time, int stuck) { #define SHA3_HASH_LOOP (1<<3) struct { @@ -435,8 +434,8 @@ static int jent_condition_data(struct rand_data *ec, __u64 time, int stuck) ec->apt_base }; - return jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), - SHA3_HASH_LOOP, stuck); + jent_hash_time(ec->hash_state, time, (u8 *)&addtl, sizeof(addtl), + SHA3_HASH_LOOP, stuck); } /* @@ -538,8 +537,7 @@ static int jent_measure_jitter(struct rand_data *ec, __u64 *ret_current_delta) stuck = jent_stuck(ec, current_delta); /* Now call the next noise sources which also injects the data */ - if (jent_condition_data(ec, current_delta, stuck)) - stuck = 1; + jent_condition_data(ec, current_delta, stuck); /* return the raw entropy value */ if (ret_current_delta) @@ -597,7 +595,7 @@ static void jent_gen_entropy(struct rand_data *ec) * @return 0 when request is fulfilled or an error * * The following error codes can occur: - * -1 entropy_collector is NULL or the generation failed + * -1 entropy_collector is NULL * -2 Intermittent health failure * -3 Permanent health failure */ @@ -640,8 +638,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, } tocopy = min(DATA_SIZE_BITS / 8, len); - if (jent_read_random_block(ec->hash_state, p, tocopy)) - return -1; + jent_read_random_block(ec->hash_state, p, tocopy); len -= tocopy; p += tocopy; @@ -656,7 +653,7 @@ int jent_read_entropy(struct rand_data *ec, unsigned char *data, struct rand_data *jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, - void *hash_state) + struct sha3_ctx *hash_state) { struct rand_data *entropy_collector; @@ -704,8 +701,8 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector); } -int jent_entropy_init(unsigned int osr, unsigned int flags, void *hash_state, - struct rand_data *p_ec) +int jent_entropy_init(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state, struct rand_data *p_ec) { /* * If caller provides an allocated ec, reuse it which implies that the diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index 4c5dbf2a8d8f..5bb15cb33000 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -1,24 +1,27 @@ // SPDX-License-Identifier: GPL-2.0-or-later +struct sha3_ctx; extern void *jent_kvzalloc(unsigned int len); extern void jent_kvzfree(void *ptr, unsigned int len); extern void *jent_zalloc(unsigned int len); extern void jent_zfree(void *ptr); extern void jent_get_nstime(__u64 *out); -extern int jent_hash_time(void *hash_state, __u64 time, u8 *addtl, - unsigned int addtl_len, __u64 hash_loop_cnt, - unsigned int stuck); -int jent_read_random_block(void *hash_state, char *dst, unsigned int dst_len); +void jent_hash_time(struct sha3_ctx *hash_state, __u64 time, u8 *addtl, + unsigned int addtl_len, __u64 hash_loop_cnt, + unsigned int stuck); +void jent_read_random_block(struct sha3_ctx *hash_state, char *dst, + unsigned int dst_len); struct rand_data; extern int jent_entropy_init(unsigned int osr, unsigned int flags, - void *hash_state, struct rand_data *p_ec); + struct sha3_ctx *hash_state, + struct rand_data *p_ec); extern int jent_read_entropy(struct rand_data *ec, unsigned char *data, unsigned int len); -extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, - unsigned int flags, - void *hash_state); +extern struct rand_data * +jent_entropy_collector_alloc(unsigned int osr, unsigned int flags, + struct sha3_ctx *hash_state); extern void jent_entropy_collector_free(struct rand_data *entropy_collector); #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE -- cgit v1.2.3 From 61f66c5216a961784b12307be60a25204525605c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:02 -0700 Subject: lib/crypto: gf128hash: Rename polyval module to gf128hash Currently, the standalone GHASH code is coupled with crypto_shash. This has resulted in unnecessary complexity and overhead, as well as the code being unavailable to library code such as the AES-GCM library. Like was done with POLYVAL, it needs to find a new home in lib/crypto/. GHASH and POLYVAL are closely related and can each be implemented in terms of each other. Optimized code for one can be reused with the other. But also since GHASH tends to be difficult to implement directly due to its unnatural bit order, most modern GHASH implementations (including the existing arm, arm64, powerpc, and x86 optimized GHASH code, and the new generic GHASH code I'll be adding) actually reinterpret the GHASH computation as an equivalent POLYVAL computation, pre and post-processing the inputs and outputs to map to/from POLYVAL. Given this close relationship, it makes sense to group the GHASH and POLYVAL code together in the same module. This gives us a wide range of options for implementing them, reusing code between the two and properly utilizing whatever instructions each architecture provides. Thus, GHASH support will be added to the library module that is currently called "polyval". Rename it to an appropriate name: "gf128hash". Rename files, options, functions, etc. where appropriate to reflect the upcoming sharing with GHASH. (Note: polyval_kunit is not renamed, as ghash_kunit will be added alongside it instead.) Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/hctr2.c | 2 +- include/crypto/gf128hash.h | 190 ++++++++++++++++++++++++ include/crypto/polyval.h | 190 ------------------------ lib/crypto/Kconfig | 24 +-- lib/crypto/Makefile | 20 +-- lib/crypto/arm64/gf128hash.h | 80 ++++++++++ lib/crypto/arm64/polyval.h | 80 ---------- lib/crypto/gf128hash.c | 307 +++++++++++++++++++++++++++++++++++++++ lib/crypto/polyval.c | 307 --------------------------------------- lib/crypto/tests/Kconfig | 4 +- lib/crypto/tests/polyval_kunit.c | 2 +- lib/crypto/x86/gf128hash.h | 83 +++++++++++ lib/crypto/x86/polyval.h | 83 ----------- 14 files changed, 687 insertions(+), 687 deletions(-) create mode 100644 include/crypto/gf128hash.h delete mode 100644 include/crypto/polyval.h create mode 100644 lib/crypto/arm64/gf128hash.h delete mode 100644 lib/crypto/arm64/polyval.h create mode 100644 lib/crypto/gf128hash.c delete mode 100644 lib/crypto/polyval.c create mode 100644 lib/crypto/x86/gf128hash.h delete mode 100644 lib/crypto/x86/polyval.h (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index b8608ef6823b..5627b3691561 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -686,7 +686,7 @@ config CRYPTO_ECB config CRYPTO_HCTR2 tristate "HCTR2" select CRYPTO_XCTR - select CRYPTO_LIB_POLYVAL + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help HCTR2 length-preserving encryption mode diff --git a/crypto/hctr2.c b/crypto/hctr2.c index f4cd6c29b4d3..ad5edf9366ac 100644 --- a/crypto/hctr2.c +++ b/crypto/hctr2.c @@ -16,9 +16,9 @@ * (https://eprint.iacr.org/2021/1441.pdf) */ +#include #include #include -#include #include #include diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h new file mode 100644 index 000000000000..5ffa86f5c13f --- /dev/null +++ b/include/crypto/gf128hash.h @@ -0,0 +1,190 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GF(2^128) polynomial hashing: GHASH and POLYVAL + * + * Copyright 2025 Google LLC + */ + +#ifndef _CRYPTO_GF128HASH_H +#define _CRYPTO_GF128HASH_H + +#include +#include + +#define POLYVAL_BLOCK_SIZE 16 +#define POLYVAL_DIGEST_SIZE 16 + +/** + * struct polyval_elem - An element of the POLYVAL finite field + * @bytes: View of the element as a byte array (unioned with @lo and @hi) + * @lo: The low 64 terms of the element's polynomial + * @hi: The high 64 terms of the element's polynomial + * + * This represents an element of the finite field GF(2^128), using the POLYVAL + * convention: little-endian byte order and natural bit order. + */ +struct polyval_elem { + union { + u8 bytes[POLYVAL_BLOCK_SIZE]; + struct { + __le64 lo; + __le64 hi; + }; + }; +}; + +/** + * struct polyval_key - Prepared key for POLYVAL + * + * This may contain just the raw key H, or it may contain precomputed key + * powers, depending on the platform's POLYVAL implementation. Use + * polyval_preparekey() to initialize this. + * + * By H^i we mean H^(i-1) * H * x^-128, with base case H^1 = H. I.e. the + * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. + */ +struct polyval_key { +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#ifdef CONFIG_ARM64 + /** @h_powers: Powers of the hash key H^8 through H^1 */ + struct polyval_elem h_powers[8]; +#elif defined(CONFIG_X86) + /** @h_powers: Powers of the hash key H^8 through H^1 */ + struct polyval_elem h_powers[8]; +#else +#error "Unhandled arch" +#endif +#else /* CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ + /** @h: The hash key H */ + struct polyval_elem h; +#endif /* !CONFIG_CRYPTO_LIB_GF128HASH_ARCH */ +}; + +/** + * struct polyval_ctx - Context for computing a POLYVAL value + * @key: Pointer to the prepared POLYVAL key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator + * @partial: Number of data bytes processed so far modulo POLYVAL_BLOCK_SIZE + */ +struct polyval_ctx { + const struct polyval_key *key; + struct polyval_elem acc; + size_t partial; +}; + +/** + * polyval_preparekey() - Prepare a POLYVAL key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a POLYVAL key structure from a raw key. This may be a simple + * copy, or it may involve precomputing powers of the key, depending on the + * platform's POLYVAL implementation. + * + * Context: Any context. + */ +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +void polyval_preparekey(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]); + +#else +static inline void polyval_preparekey(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + /* Just a simple copy, so inline it. */ + memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); +} +#endif + +/** + * polyval_init() - Initialize a POLYVAL context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void polyval_init(struct polyval_ctx *ctx, + const struct polyval_key *key) +{ + *ctx = (struct polyval_ctx){ .key = key }; +} + +/** + * polyval_import_blkaligned() - Import a POLYVAL accumulator value + * @ctx: The context to initialize + * @key: The key to import. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + * @acc: The accumulator value to import. + * + * This imports an accumulator that was saved by polyval_export_blkaligned(). + * The same key must be used. + */ +static inline void +polyval_import_blkaligned(struct polyval_ctx *ctx, + const struct polyval_key *key, + const struct polyval_elem *acc) +{ + *ctx = (struct polyval_ctx){ .key = key, .acc = *acc }; +} + +/** + * polyval_export_blkaligned() - Export a POLYVAL accumulator value + * @ctx: The context to export the accumulator value from + * @acc: (output) The exported accumulator value + * + * This exports the accumulator from a POLYVAL context. The number of data + * bytes processed so far must be a multiple of POLYVAL_BLOCK_SIZE. + */ +static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, + struct polyval_elem *acc) +{ + *acc = ctx->acc; +} + +/** + * polyval_update() - Update a POLYVAL context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); + +/** + * polyval_final() - Finish computing a POLYVAL value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of POLYVAL_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); + +/** + * polyval() - Compute a POLYVAL value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void polyval(const struct polyval_key *key, + const u8 *data, size_t len, + u8 out[POLYVAL_BLOCK_SIZE]) +{ + struct polyval_ctx ctx; + + polyval_init(&ctx, key); + polyval_update(&ctx, data, len); + polyval_final(&ctx, out); +} + +#endif /* _CRYPTO_GF128HASH_H */ diff --git a/include/crypto/polyval.h b/include/crypto/polyval.h deleted file mode 100644 index b28b8ef11353..000000000000 --- a/include/crypto/polyval.h +++ /dev/null @@ -1,190 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library API - * - * Copyright 2025 Google LLC - */ - -#ifndef _CRYPTO_POLYVAL_H -#define _CRYPTO_POLYVAL_H - -#include -#include - -#define POLYVAL_BLOCK_SIZE 16 -#define POLYVAL_DIGEST_SIZE 16 - -/** - * struct polyval_elem - An element of the POLYVAL finite field - * @bytes: View of the element as a byte array (unioned with @lo and @hi) - * @lo: The low 64 terms of the element's polynomial - * @hi: The high 64 terms of the element's polynomial - * - * This represents an element of the finite field GF(2^128), using the POLYVAL - * convention: little-endian byte order and natural bit order. - */ -struct polyval_elem { - union { - u8 bytes[POLYVAL_BLOCK_SIZE]; - struct { - __le64 lo; - __le64 hi; - }; - }; -}; - -/** - * struct polyval_key - Prepared key for POLYVAL - * - * This may contain just the raw key H, or it may contain precomputed key - * powers, depending on the platform's POLYVAL implementation. Use - * polyval_preparekey() to initialize this. - * - * By H^i we mean H^(i-1) * H * x^-128, with base case H^1 = H. I.e. the - * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. - */ -struct polyval_key { -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#ifdef CONFIG_ARM64 - /** @h_powers: Powers of the hash key H^8 through H^1 */ - struct polyval_elem h_powers[8]; -#elif defined(CONFIG_X86) - /** @h_powers: Powers of the hash key H^8 through H^1 */ - struct polyval_elem h_powers[8]; -#else -#error "Unhandled arch" -#endif -#else /* CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ - /** @h: The hash key H */ - struct polyval_elem h; -#endif /* !CONFIG_CRYPTO_LIB_POLYVAL_ARCH */ -}; - -/** - * struct polyval_ctx - Context for computing a POLYVAL value - * @key: Pointer to the prepared POLYVAL key. The user of the API is - * responsible for ensuring that the key lives as long as the context. - * @acc: The accumulator - * @partial: Number of data bytes processed so far modulo POLYVAL_BLOCK_SIZE - */ -struct polyval_ctx { - const struct polyval_key *key; - struct polyval_elem acc; - size_t partial; -}; - -/** - * polyval_preparekey() - Prepare a POLYVAL key - * @key: (output) The key structure to initialize - * @raw_key: The raw hash key - * - * Initialize a POLYVAL key structure from a raw key. This may be a simple - * copy, or it may involve precomputing powers of the key, depending on the - * platform's POLYVAL implementation. - * - * Context: Any context. - */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -void polyval_preparekey(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]); - -#else -static inline void polyval_preparekey(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - /* Just a simple copy, so inline it. */ - memcpy(key->h.bytes, raw_key, POLYVAL_BLOCK_SIZE); -} -#endif - -/** - * polyval_init() - Initialize a POLYVAL context for a new message - * @ctx: The context to initialize - * @key: The key to use. Note that a pointer to the key is saved in the - * context, so the key must live at least as long as the context. - */ -static inline void polyval_init(struct polyval_ctx *ctx, - const struct polyval_key *key) -{ - *ctx = (struct polyval_ctx){ .key = key }; -} - -/** - * polyval_import_blkaligned() - Import a POLYVAL accumulator value - * @ctx: The context to initialize - * @key: The key to import. Note that a pointer to the key is saved in the - * context, so the key must live at least as long as the context. - * @acc: The accumulator value to import. - * - * This imports an accumulator that was saved by polyval_export_blkaligned(). - * The same key must be used. - */ -static inline void -polyval_import_blkaligned(struct polyval_ctx *ctx, - const struct polyval_key *key, - const struct polyval_elem *acc) -{ - *ctx = (struct polyval_ctx){ .key = key, .acc = *acc }; -} - -/** - * polyval_export_blkaligned() - Export a POLYVAL accumulator value - * @ctx: The context to export the accumulator value from - * @acc: (output) The exported accumulator value - * - * This exports the accumulator from a POLYVAL context. The number of data - * bytes processed so far must be a multiple of POLYVAL_BLOCK_SIZE. - */ -static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, - struct polyval_elem *acc) -{ - *acc = ctx->acc; -} - -/** - * polyval_update() - Update a POLYVAL context with message data - * @ctx: The context to update; must have been initialized - * @data: The message data - * @len: The data length in bytes. Doesn't need to be block-aligned. - * - * This can be called any number of times. - * - * Context: Any context. - */ -void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); - -/** - * polyval_final() - Finish computing a POLYVAL value - * @ctx: The context to finalize - * @out: The output value - * - * If the total data length isn't a multiple of POLYVAL_BLOCK_SIZE, then the - * final block is automatically zero-padded. - * - * After finishing, this zeroizes @ctx. So the caller does not need to do it. - * - * Context: Any context. - */ -void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); - -/** - * polyval() - Compute a POLYVAL value - * @key: The prepared key - * @data: The message data - * @len: The data length in bytes. Doesn't need to be block-aligned. - * @out: The output value - * - * Context: Any context. - */ -static inline void polyval(const struct polyval_key *key, - const u8 *data, size_t len, - u8 out[POLYVAL_BLOCK_SIZE]) -{ - struct polyval_ctx ctx; - - polyval_init(&ctx, key); - polyval_update(&ctx, data, len); - polyval_final(&ctx, out); -} - -#endif /* _CRYPTO_POLYVAL_H */ diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 4910fe20e42a..98cedd95c2a5 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -110,6 +110,18 @@ config CRYPTO_LIB_CURVE25519_GENERIC config CRYPTO_LIB_DES tristate +config CRYPTO_LIB_GF128HASH + tristate + help + The GHASH and POLYVAL library functions. Select this if your module + uses any of the functions from . + +config CRYPTO_LIB_GF128HASH_ARCH + bool + depends on CRYPTO_LIB_GF128HASH && !UML + default y if ARM64 + default y if X86_64 + config CRYPTO_LIB_MD5 tristate help @@ -178,18 +190,6 @@ config CRYPTO_LIB_POLY1305_RSIZE default 9 if ARM || ARM64 default 1 -config CRYPTO_LIB_POLYVAL - tristate - help - The POLYVAL library functions. Select this if your module uses any of - the functions from . - -config CRYPTO_LIB_POLYVAL_ARCH - bool - depends on CRYPTO_LIB_POLYVAL && !UML - default y if ARM64 - default y if X86_64 - config CRYPTO_LIB_CHACHA20POLY1305 tristate select CRYPTO_LIB_CHACHA diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index a961615c8c7f..fc30622123d2 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -154,6 +154,16 @@ libdes-y := des.o ################################################################################ +obj-$(CONFIG_CRYPTO_LIB_GF128HASH) += libgf128hash.o +libgf128hash-y := gf128hash.o +ifeq ($(CONFIG_CRYPTO_LIB_GF128HASH_ARCH),y) +CFLAGS_gf128hash.o += -I$(src)/$(SRCARCH) +libgf128hash-$(CONFIG_ARM64) += arm64/polyval-ce-core.o +libgf128hash-$(CONFIG_X86) += x86/polyval-pclmul-avx.o +endif + +################################################################################ + obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o libmd5-y := md5.o ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y) @@ -251,16 +261,6 @@ clean-files += arm/poly1305-core.S \ ################################################################################ -obj-$(CONFIG_CRYPTO_LIB_POLYVAL) += libpolyval.o -libpolyval-y := polyval.o -ifeq ($(CONFIG_CRYPTO_LIB_POLYVAL_ARCH),y) -CFLAGS_polyval.o += -I$(src)/$(SRCARCH) -libpolyval-$(CONFIG_ARM64) += arm64/polyval-ce-core.o -libpolyval-$(CONFIG_X86) += x86/polyval-pclmul-avx.o -endif - -################################################################################ - obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o libsha1-y := sha1.o ifeq ($(CONFIG_CRYPTO_LIB_SHA1_ARCH),y) diff --git a/lib/crypto/arm64/gf128hash.h b/lib/crypto/arm64/gf128hash.h new file mode 100644 index 000000000000..c1012007adcf --- /dev/null +++ b/lib/crypto/arm64/gf128hash.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * POLYVAL library functions, arm64 optimized + * + * Copyright 2025 Google LLC + */ +#include +#include + +#define NUM_H_POWERS 8 + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); + +asmlinkage void polyval_mul_pmull(struct polyval_elem *a, + const struct polyval_elem *b); +asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks); + +static void polyval_preparekey_arch(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); + memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_pmull( + &key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } + } else { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_generic(&key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } +} + +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + if (static_branch_likely(&have_pmull) && may_use_simd()) { + scoped_ksimd() + polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); + } else { + polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + } +} + +static void polyval_blocks_arch(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pmull) && may_use_simd()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / POLYVAL_BLOCK_SIZE); + + scoped_ksimd() + polyval_blocks_pmull(acc, key, data, n); + data += n * POLYVAL_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], + data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (cpu_have_named_feature(PMULL)) + static_branch_enable(&have_pmull); +} diff --git a/lib/crypto/arm64/polyval.h b/lib/crypto/arm64/polyval.h deleted file mode 100644 index a39763395e9b..000000000000 --- a/lib/crypto/arm64/polyval.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library functions, arm64 optimized - * - * Copyright 2025 Google LLC - */ -#include -#include - -#define NUM_H_POWERS 8 - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull); - -asmlinkage void polyval_mul_pmull(struct polyval_elem *a, - const struct polyval_elem *b); -asmlinkage void polyval_blocks_pmull(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks); - -static void polyval_preparekey_arch(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); - memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_pmull( - &key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } - } else { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_generic(&key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } -} - -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - scoped_ksimd() - polyval_mul_pmull(acc, &key->h_powers[NUM_H_POWERS - 1]); - } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); - } -} - -static void polyval_blocks_arch(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_pmull) && may_use_simd()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - scoped_ksimd() - polyval_blocks_pmull(acc, key, data, n); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); - } else { - polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], - data, nblocks); - } -} - -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) -{ - if (cpu_have_named_feature(PMULL)) - static_branch_enable(&have_pmull); -} diff --git a/lib/crypto/gf128hash.c b/lib/crypto/gf128hash.c new file mode 100644 index 000000000000..8bb848bf26b7 --- /dev/null +++ b/lib/crypto/gf128hash.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * GF(2^128) polynomial hashing: GHASH and POLYVAL + * + * Copyright 2025 Google LLC + */ + +#include +#include +#include +#include +#include + +/* + * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL + * interprets the message as the coefficients of a polynomial in GF(2^128) and + * evaluates that polynomial at a secret point. POLYVAL has a simple + * mathematical relationship with GHASH, but it uses a better field convention + * which makes it easier and faster to implement. + * + * POLYVAL is not a cryptographic hash function, and it should be used only by + * algorithms that are specifically designed to use it. + * + * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated + * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) + * + * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" + * (https://eprint.iacr.org/2021/1441.pdf). + * + * This file provides a library API for POLYVAL. This API can delegate to + * either a generic implementation or an architecture-optimized implementation. + * + * For the generic implementation, we don't use the traditional table approach + * to GF(2^128) multiplication. That approach is not constant-time and requires + * a lot of memory. Instead, we use a different approach which emulates + * carryless multiplication using standard multiplications by spreading the data + * bits apart using "holes". This allows the carries to spill harmlessly. This + * approach is borrowed from BoringSSL, which in turn credits BearSSL's + * documentation (https://bearssl.org/constanttime.html#ghash-for-gcm) for the + * "holes" trick and a presentation by Shay Gueron + * (https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf) for the + * 256-bit => 128-bit reduction algorithm. + */ + +#ifdef CONFIG_ARCH_SUPPORTS_INT128 + +/* Do a 64 x 64 => 128 bit carryless multiplication. */ +static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) +{ + /* + * With 64-bit multiplicands and one term every 4 bits, there would be + * up to 64 / 4 = 16 one bits per column when each multiplication is + * written out as a series of additions in the schoolbook manner. + * Unfortunately, that doesn't work since the value 16 is 1 too large to + * fit in 4 bits. Carries would sometimes overflow into the next term. + * + * Using one term every 5 bits would work. However, that would cost + * 5 x 5 = 25 multiplications instead of 4 x 4 = 16. + * + * Instead, mask off 4 bits from one multiplicand, giving a max of 15 + * one bits per column. Then handle those 4 bits separately. + */ + u64 a0 = a & 0x1111111111111110; + u64 a1 = a & 0x2222222222222220; + u64 a2 = a & 0x4444444444444440; + u64 a3 = a & 0x8888888888888880; + + u64 b0 = b & 0x1111111111111111; + u64 b1 = b & 0x2222222222222222; + u64 b2 = b & 0x4444444444444444; + u64 b3 = b & 0x8888888888888888; + + /* Multiply the high 60 bits of @a by @b. */ + u128 c0 = (a0 * (u128)b0) ^ (a1 * (u128)b3) ^ + (a2 * (u128)b2) ^ (a3 * (u128)b1); + u128 c1 = (a0 * (u128)b1) ^ (a1 * (u128)b0) ^ + (a2 * (u128)b3) ^ (a3 * (u128)b2); + u128 c2 = (a0 * (u128)b2) ^ (a1 * (u128)b1) ^ + (a2 * (u128)b0) ^ (a3 * (u128)b3); + u128 c3 = (a0 * (u128)b3) ^ (a1 * (u128)b2) ^ + (a2 * (u128)b1) ^ (a3 * (u128)b0); + + /* Multiply the low 4 bits of @a by @b. */ + u64 e0 = -(a & 1) & b; + u64 e1 = -((a >> 1) & 1) & b; + u64 e2 = -((a >> 2) & 1) & b; + u64 e3 = -((a >> 3) & 1) & b; + u64 extra_lo = e0 ^ (e1 << 1) ^ (e2 << 2) ^ (e3 << 3); + u64 extra_hi = (e1 >> 63) ^ (e2 >> 62) ^ (e3 >> 61); + + /* Add all the intermediate products together. */ + *out_lo = (((u64)c0) & 0x1111111111111111) ^ + (((u64)c1) & 0x2222222222222222) ^ + (((u64)c2) & 0x4444444444444444) ^ + (((u64)c3) & 0x8888888888888888) ^ extra_lo; + *out_hi = (((u64)(c0 >> 64)) & 0x1111111111111111) ^ + (((u64)(c1 >> 64)) & 0x2222222222222222) ^ + (((u64)(c2 >> 64)) & 0x4444444444444444) ^ + (((u64)(c3 >> 64)) & 0x8888888888888888) ^ extra_hi; +} + +#else /* CONFIG_ARCH_SUPPORTS_INT128 */ + +/* Do a 32 x 32 => 64 bit carryless multiplication. */ +static u64 clmul32(u32 a, u32 b) +{ + /* + * With 32-bit multiplicands and one term every 4 bits, there are up to + * 32 / 4 = 8 one bits per column when each multiplication is written + * out as a series of additions in the schoolbook manner. The value 8 + * fits in 4 bits, so the carries don't overflow into the next term. + */ + u32 a0 = a & 0x11111111; + u32 a1 = a & 0x22222222; + u32 a2 = a & 0x44444444; + u32 a3 = a & 0x88888888; + + u32 b0 = b & 0x11111111; + u32 b1 = b & 0x22222222; + u32 b2 = b & 0x44444444; + u32 b3 = b & 0x88888888; + + u64 c0 = (a0 * (u64)b0) ^ (a1 * (u64)b3) ^ + (a2 * (u64)b2) ^ (a3 * (u64)b1); + u64 c1 = (a0 * (u64)b1) ^ (a1 * (u64)b0) ^ + (a2 * (u64)b3) ^ (a3 * (u64)b2); + u64 c2 = (a0 * (u64)b2) ^ (a1 * (u64)b1) ^ + (a2 * (u64)b0) ^ (a3 * (u64)b3); + u64 c3 = (a0 * (u64)b3) ^ (a1 * (u64)b2) ^ + (a2 * (u64)b1) ^ (a3 * (u64)b0); + + /* Add all the intermediate products together. */ + return (c0 & 0x1111111111111111) ^ + (c1 & 0x2222222222222222) ^ + (c2 & 0x4444444444444444) ^ + (c3 & 0x8888888888888888); +} + +/* Do a 64 x 64 => 128 bit carryless multiplication. */ +static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) +{ + u32 a_lo = (u32)a; + u32 a_hi = a >> 32; + u32 b_lo = (u32)b; + u32 b_hi = b >> 32; + + /* Karatsuba multiplication */ + u64 lo = clmul32(a_lo, b_lo); + u64 hi = clmul32(a_hi, b_hi); + u64 mi = clmul32(a_lo ^ a_hi, b_lo ^ b_hi) ^ lo ^ hi; + + *out_lo = lo ^ (mi << 32); + *out_hi = hi ^ (mi >> 32); +} +#endif /* !CONFIG_ARCH_SUPPORTS_INT128 */ + +/* Compute @a = @a * @b * x^-128 in the POLYVAL field. */ +static void __maybe_unused +polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) +{ + u64 c0, c1, c2, c3, mi0, mi1; + + /* + * Carryless-multiply @a by @b using Karatsuba multiplication. Store + * the 256-bit product in @c0 (low) through @c3 (high). + */ + clmul64(le64_to_cpu(a->lo), le64_to_cpu(b->lo), &c0, &c1); + clmul64(le64_to_cpu(a->hi), le64_to_cpu(b->hi), &c2, &c3); + clmul64(le64_to_cpu(a->lo ^ a->hi), le64_to_cpu(b->lo ^ b->hi), + &mi0, &mi1); + mi0 ^= c0 ^ c2; + mi1 ^= c1 ^ c3; + c1 ^= mi0; + c2 ^= mi1; + + /* + * Cancel out the low 128 bits of the product by adding multiples of + * G(x) = x^128 + x^127 + x^126 + x^121 + 1. Do this in two steps, each + * of which cancels out 64 bits. Note that we break G(x) into three + * parts: 1, x^64 * (x^63 + x^62 + x^57), and x^128 * 1. + */ + + /* + * First, add G(x) times c0 as follows: + * + * (c0, c1, c2) = (0, + * c1 + (c0 * (x^63 + x^62 + x^57) mod x^64), + * c2 + c0 + floor((c0 * (x^63 + x^62 + x^57)) / x^64)) + */ + c1 ^= (c0 << 63) ^ (c0 << 62) ^ (c0 << 57); + c2 ^= c0 ^ (c0 >> 1) ^ (c0 >> 2) ^ (c0 >> 7); + + /* + * Second, add G(x) times the new c1: + * + * (c1, c2, c3) = (0, + * c2 + (c1 * (x^63 + x^62 + x^57) mod x^64), + * c3 + c1 + floor((c1 * (x^63 + x^62 + x^57)) / x^64)) + */ + c2 ^= (c1 << 63) ^ (c1 << 62) ^ (c1 << 57); + c3 ^= c1 ^ (c1 >> 1) ^ (c1 >> 2) ^ (c1 >> 7); + + /* Return (c2, c3). This implicitly multiplies by x^-128. */ + a->lo = cpu_to_le64(c2); + a->hi = cpu_to_le64(c3); +} + +static void __maybe_unused +polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, + const u8 *data, size_t nblocks) +{ + do { + acc->lo ^= get_unaligned((__le64 *)data); + acc->hi ^= get_unaligned((__le64 *)(data + 8)); + polyval_mul_generic(acc, key); + data += POLYVAL_BLOCK_SIZE; + } while (--nblocks); +} + +/* Include the arch-optimized implementation of POLYVAL, if one is available. */ +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH +#include "gf128hash.h" /* $(SRCARCH)/gf128hash.h */ +void polyval_preparekey(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + polyval_preparekey_arch(key, raw_key); +} +EXPORT_SYMBOL_GPL(polyval_preparekey); +#endif /* Else, polyval_preparekey() is an inline function. */ + +/* + * polyval_mul_generic() and polyval_blocks_generic() take the key as a + * polyval_elem rather than a polyval_key, so that arch-optimized + * implementations with a different key format can use it as a fallback (if they + * have H^1 stored somewhere in their struct). Thus, the following dispatch + * code is needed to pass the appropriate key argument. + */ + +static void polyval_mul(struct polyval_ctx *ctx) +{ +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH + polyval_mul_arch(&ctx->acc, ctx->key); +#else + polyval_mul_generic(&ctx->acc, &ctx->key->h); +#endif +} + +static void polyval_blocks(struct polyval_ctx *ctx, + const u8 *data, size_t nblocks) +{ +#ifdef CONFIG_CRYPTO_LIB_GF128HASH_ARCH + polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); +#else + polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); +#endif +} + +void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len) +{ + if (unlikely(ctx->partial)) { + size_t n = min(len, POLYVAL_BLOCK_SIZE - ctx->partial); + + len -= n; + while (n--) + ctx->acc.bytes[ctx->partial++] ^= *data++; + if (ctx->partial < POLYVAL_BLOCK_SIZE) + return; + polyval_mul(ctx); + } + if (len >= POLYVAL_BLOCK_SIZE) { + size_t nblocks = len / POLYVAL_BLOCK_SIZE; + + polyval_blocks(ctx, data, nblocks); + data += len & ~(POLYVAL_BLOCK_SIZE - 1); + len &= POLYVAL_BLOCK_SIZE - 1; + } + for (size_t i = 0; i < len; i++) + ctx->acc.bytes[i] ^= data[i]; + ctx->partial = len; +} +EXPORT_SYMBOL_GPL(polyval_update); + +void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) +{ + if (unlikely(ctx->partial)) + polyval_mul(ctx); + memcpy(out, &ctx->acc, POLYVAL_BLOCK_SIZE); + memzero_explicit(ctx, sizeof(*ctx)); +} +EXPORT_SYMBOL_GPL(polyval_final); + +#ifdef gf128hash_mod_init_arch +static int __init gf128hash_mod_init(void) +{ + gf128hash_mod_init_arch(); + return 0; +} +subsys_initcall(gf128hash_mod_init); + +static void __exit gf128hash_mod_exit(void) +{ +} +module_exit(gf128hash_mod_exit); +#endif + +MODULE_DESCRIPTION("GF(2^128) polynomial hashing: GHASH and POLYVAL"); +MODULE_LICENSE("GPL"); diff --git a/lib/crypto/polyval.c b/lib/crypto/polyval.c deleted file mode 100644 index 5796275f574a..000000000000 --- a/lib/crypto/polyval.c +++ /dev/null @@ -1,307 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * POLYVAL library functions - * - * Copyright 2025 Google LLC - */ - -#include -#include -#include -#include -#include - -/* - * POLYVAL is an almost-XOR-universal hash function. Similar to GHASH, POLYVAL - * interprets the message as the coefficients of a polynomial in GF(2^128) and - * evaluates that polynomial at a secret point. POLYVAL has a simple - * mathematical relationship with GHASH, but it uses a better field convention - * which makes it easier and faster to implement. - * - * POLYVAL is not a cryptographic hash function, and it should be used only by - * algorithms that are specifically designed to use it. - * - * POLYVAL is specified by "AES-GCM-SIV: Nonce Misuse-Resistant Authenticated - * Encryption" (https://datatracker.ietf.org/doc/html/rfc8452) - * - * POLYVAL is also used by HCTR2. See "Length-preserving encryption with HCTR2" - * (https://eprint.iacr.org/2021/1441.pdf). - * - * This file provides a library API for POLYVAL. This API can delegate to - * either a generic implementation or an architecture-optimized implementation. - * - * For the generic implementation, we don't use the traditional table approach - * to GF(2^128) multiplication. That approach is not constant-time and requires - * a lot of memory. Instead, we use a different approach which emulates - * carryless multiplication using standard multiplications by spreading the data - * bits apart using "holes". This allows the carries to spill harmlessly. This - * approach is borrowed from BoringSSL, which in turn credits BearSSL's - * documentation (https://bearssl.org/constanttime.html#ghash-for-gcm) for the - * "holes" trick and a presentation by Shay Gueron - * (https://crypto.stanford.edu/RealWorldCrypto/slides/gueron.pdf) for the - * 256-bit => 128-bit reduction algorithm. - */ - -#ifdef CONFIG_ARCH_SUPPORTS_INT128 - -/* Do a 64 x 64 => 128 bit carryless multiplication. */ -static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) -{ - /* - * With 64-bit multiplicands and one term every 4 bits, there would be - * up to 64 / 4 = 16 one bits per column when each multiplication is - * written out as a series of additions in the schoolbook manner. - * Unfortunately, that doesn't work since the value 16 is 1 too large to - * fit in 4 bits. Carries would sometimes overflow into the next term. - * - * Using one term every 5 bits would work. However, that would cost - * 5 x 5 = 25 multiplications instead of 4 x 4 = 16. - * - * Instead, mask off 4 bits from one multiplicand, giving a max of 15 - * one bits per column. Then handle those 4 bits separately. - */ - u64 a0 = a & 0x1111111111111110; - u64 a1 = a & 0x2222222222222220; - u64 a2 = a & 0x4444444444444440; - u64 a3 = a & 0x8888888888888880; - - u64 b0 = b & 0x1111111111111111; - u64 b1 = b & 0x2222222222222222; - u64 b2 = b & 0x4444444444444444; - u64 b3 = b & 0x8888888888888888; - - /* Multiply the high 60 bits of @a by @b. */ - u128 c0 = (a0 * (u128)b0) ^ (a1 * (u128)b3) ^ - (a2 * (u128)b2) ^ (a3 * (u128)b1); - u128 c1 = (a0 * (u128)b1) ^ (a1 * (u128)b0) ^ - (a2 * (u128)b3) ^ (a3 * (u128)b2); - u128 c2 = (a0 * (u128)b2) ^ (a1 * (u128)b1) ^ - (a2 * (u128)b0) ^ (a3 * (u128)b3); - u128 c3 = (a0 * (u128)b3) ^ (a1 * (u128)b2) ^ - (a2 * (u128)b1) ^ (a3 * (u128)b0); - - /* Multiply the low 4 bits of @a by @b. */ - u64 e0 = -(a & 1) & b; - u64 e1 = -((a >> 1) & 1) & b; - u64 e2 = -((a >> 2) & 1) & b; - u64 e3 = -((a >> 3) & 1) & b; - u64 extra_lo = e0 ^ (e1 << 1) ^ (e2 << 2) ^ (e3 << 3); - u64 extra_hi = (e1 >> 63) ^ (e2 >> 62) ^ (e3 >> 61); - - /* Add all the intermediate products together. */ - *out_lo = (((u64)c0) & 0x1111111111111111) ^ - (((u64)c1) & 0x2222222222222222) ^ - (((u64)c2) & 0x4444444444444444) ^ - (((u64)c3) & 0x8888888888888888) ^ extra_lo; - *out_hi = (((u64)(c0 >> 64)) & 0x1111111111111111) ^ - (((u64)(c1 >> 64)) & 0x2222222222222222) ^ - (((u64)(c2 >> 64)) & 0x4444444444444444) ^ - (((u64)(c3 >> 64)) & 0x8888888888888888) ^ extra_hi; -} - -#else /* CONFIG_ARCH_SUPPORTS_INT128 */ - -/* Do a 32 x 32 => 64 bit carryless multiplication. */ -static u64 clmul32(u32 a, u32 b) -{ - /* - * With 32-bit multiplicands and one term every 4 bits, there are up to - * 32 / 4 = 8 one bits per column when each multiplication is written - * out as a series of additions in the schoolbook manner. The value 8 - * fits in 4 bits, so the carries don't overflow into the next term. - */ - u32 a0 = a & 0x11111111; - u32 a1 = a & 0x22222222; - u32 a2 = a & 0x44444444; - u32 a3 = a & 0x88888888; - - u32 b0 = b & 0x11111111; - u32 b1 = b & 0x22222222; - u32 b2 = b & 0x44444444; - u32 b3 = b & 0x88888888; - - u64 c0 = (a0 * (u64)b0) ^ (a1 * (u64)b3) ^ - (a2 * (u64)b2) ^ (a3 * (u64)b1); - u64 c1 = (a0 * (u64)b1) ^ (a1 * (u64)b0) ^ - (a2 * (u64)b3) ^ (a3 * (u64)b2); - u64 c2 = (a0 * (u64)b2) ^ (a1 * (u64)b1) ^ - (a2 * (u64)b0) ^ (a3 * (u64)b3); - u64 c3 = (a0 * (u64)b3) ^ (a1 * (u64)b2) ^ - (a2 * (u64)b1) ^ (a3 * (u64)b0); - - /* Add all the intermediate products together. */ - return (c0 & 0x1111111111111111) ^ - (c1 & 0x2222222222222222) ^ - (c2 & 0x4444444444444444) ^ - (c3 & 0x8888888888888888); -} - -/* Do a 64 x 64 => 128 bit carryless multiplication. */ -static void clmul64(u64 a, u64 b, u64 *out_lo, u64 *out_hi) -{ - u32 a_lo = (u32)a; - u32 a_hi = a >> 32; - u32 b_lo = (u32)b; - u32 b_hi = b >> 32; - - /* Karatsuba multiplication */ - u64 lo = clmul32(a_lo, b_lo); - u64 hi = clmul32(a_hi, b_hi); - u64 mi = clmul32(a_lo ^ a_hi, b_lo ^ b_hi) ^ lo ^ hi; - - *out_lo = lo ^ (mi << 32); - *out_hi = hi ^ (mi >> 32); -} -#endif /* !CONFIG_ARCH_SUPPORTS_INT128 */ - -/* Compute @a = @a * @b * x^-128 in the POLYVAL field. */ -static void __maybe_unused -polyval_mul_generic(struct polyval_elem *a, const struct polyval_elem *b) -{ - u64 c0, c1, c2, c3, mi0, mi1; - - /* - * Carryless-multiply @a by @b using Karatsuba multiplication. Store - * the 256-bit product in @c0 (low) through @c3 (high). - */ - clmul64(le64_to_cpu(a->lo), le64_to_cpu(b->lo), &c0, &c1); - clmul64(le64_to_cpu(a->hi), le64_to_cpu(b->hi), &c2, &c3); - clmul64(le64_to_cpu(a->lo ^ a->hi), le64_to_cpu(b->lo ^ b->hi), - &mi0, &mi1); - mi0 ^= c0 ^ c2; - mi1 ^= c1 ^ c3; - c1 ^= mi0; - c2 ^= mi1; - - /* - * Cancel out the low 128 bits of the product by adding multiples of - * G(x) = x^128 + x^127 + x^126 + x^121 + 1. Do this in two steps, each - * of which cancels out 64 bits. Note that we break G(x) into three - * parts: 1, x^64 * (x^63 + x^62 + x^57), and x^128 * 1. - */ - - /* - * First, add G(x) times c0 as follows: - * - * (c0, c1, c2) = (0, - * c1 + (c0 * (x^63 + x^62 + x^57) mod x^64), - * c2 + c0 + floor((c0 * (x^63 + x^62 + x^57)) / x^64)) - */ - c1 ^= (c0 << 63) ^ (c0 << 62) ^ (c0 << 57); - c2 ^= c0 ^ (c0 >> 1) ^ (c0 >> 2) ^ (c0 >> 7); - - /* - * Second, add G(x) times the new c1: - * - * (c1, c2, c3) = (0, - * c2 + (c1 * (x^63 + x^62 + x^57) mod x^64), - * c3 + c1 + floor((c1 * (x^63 + x^62 + x^57)) / x^64)) - */ - c2 ^= (c1 << 63) ^ (c1 << 62) ^ (c1 << 57); - c3 ^= c1 ^ (c1 >> 1) ^ (c1 >> 2) ^ (c1 >> 7); - - /* Return (c2, c3). This implicitly multiplies by x^-128. */ - a->lo = cpu_to_le64(c2); - a->hi = cpu_to_le64(c3); -} - -static void __maybe_unused -polyval_blocks_generic(struct polyval_elem *acc, const struct polyval_elem *key, - const u8 *data, size_t nblocks) -{ - do { - acc->lo ^= get_unaligned((__le64 *)data); - acc->hi ^= get_unaligned((__le64 *)(data + 8)); - polyval_mul_generic(acc, key); - data += POLYVAL_BLOCK_SIZE; - } while (--nblocks); -} - -/* Include the arch-optimized implementation of POLYVAL, if one is available. */ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH -#include "polyval.h" /* $(SRCARCH)/polyval.h */ -void polyval_preparekey(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - polyval_preparekey_arch(key, raw_key); -} -EXPORT_SYMBOL_GPL(polyval_preparekey); -#endif /* Else, polyval_preparekey() is an inline function. */ - -/* - * polyval_mul_generic() and polyval_blocks_generic() take the key as a - * polyval_elem rather than a polyval_key, so that arch-optimized - * implementations with a different key format can use it as a fallback (if they - * have H^1 stored somewhere in their struct). Thus, the following dispatch - * code is needed to pass the appropriate key argument. - */ - -static void polyval_mul(struct polyval_ctx *ctx) -{ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH - polyval_mul_arch(&ctx->acc, ctx->key); -#else - polyval_mul_generic(&ctx->acc, &ctx->key->h); -#endif -} - -static void polyval_blocks(struct polyval_ctx *ctx, - const u8 *data, size_t nblocks) -{ -#ifdef CONFIG_CRYPTO_LIB_POLYVAL_ARCH - polyval_blocks_arch(&ctx->acc, ctx->key, data, nblocks); -#else - polyval_blocks_generic(&ctx->acc, &ctx->key->h, data, nblocks); -#endif -} - -void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len) -{ - if (unlikely(ctx->partial)) { - size_t n = min(len, POLYVAL_BLOCK_SIZE - ctx->partial); - - len -= n; - while (n--) - ctx->acc.bytes[ctx->partial++] ^= *data++; - if (ctx->partial < POLYVAL_BLOCK_SIZE) - return; - polyval_mul(ctx); - } - if (len >= POLYVAL_BLOCK_SIZE) { - size_t nblocks = len / POLYVAL_BLOCK_SIZE; - - polyval_blocks(ctx, data, nblocks); - data += len & ~(POLYVAL_BLOCK_SIZE - 1); - len &= POLYVAL_BLOCK_SIZE - 1; - } - for (size_t i = 0; i < len; i++) - ctx->acc.bytes[i] ^= data[i]; - ctx->partial = len; -} -EXPORT_SYMBOL_GPL(polyval_update); - -void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]) -{ - if (unlikely(ctx->partial)) - polyval_mul(ctx); - memcpy(out, &ctx->acc, POLYVAL_BLOCK_SIZE); - memzero_explicit(ctx, sizeof(*ctx)); -} -EXPORT_SYMBOL_GPL(polyval_final); - -#ifdef polyval_mod_init_arch -static int __init polyval_mod_init(void) -{ - polyval_mod_init_arch(); - return 0; -} -subsys_initcall(polyval_mod_init); - -static void __exit polyval_mod_exit(void) -{ -} -module_exit(polyval_mod_exit); -#endif - -MODULE_DESCRIPTION("POLYVAL almost-XOR-universal hash function"); -MODULE_LICENSE("GPL"); diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig index 42e1770e1883..aa627b6b9855 100644 --- a/lib/crypto/tests/Kconfig +++ b/lib/crypto/tests/Kconfig @@ -69,7 +69,7 @@ config CRYPTO_LIB_POLY1305_KUNIT_TEST config CRYPTO_LIB_POLYVAL_KUNIT_TEST tristate "KUnit tests for POLYVAL" if !KUNIT_ALL_TESTS - depends on KUNIT && CRYPTO_LIB_POLYVAL + depends on KUNIT && CRYPTO_LIB_GF128HASH default KUNIT_ALL_TESTS select CRYPTO_LIB_BENCHMARK_VISIBLE help @@ -122,11 +122,11 @@ config CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT select CRYPTO_LIB_AES_CBC_MACS select CRYPTO_LIB_BLAKE2B select CRYPTO_LIB_CURVE25519 + select CRYPTO_LIB_GF128HASH select CRYPTO_LIB_MD5 select CRYPTO_LIB_MLDSA select CRYPTO_LIB_NH select CRYPTO_LIB_POLY1305 - select CRYPTO_LIB_POLYVAL select CRYPTO_LIB_SHA1 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 diff --git a/lib/crypto/tests/polyval_kunit.c b/lib/crypto/tests/polyval_kunit.c index f47f41a39a41..d1f53a690ab8 100644 --- a/lib/crypto/tests/polyval_kunit.c +++ b/lib/crypto/tests/polyval_kunit.c @@ -2,7 +2,7 @@ /* * Copyright 2025 Google LLC */ -#include +#include #include "polyval-testvecs.h" /* diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h new file mode 100644 index 000000000000..fe506cf6431b --- /dev/null +++ b/lib/crypto/x86/gf128hash.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * POLYVAL library functions, x86_64 optimized + * + * Copyright 2025 Google LLC + */ +#include +#include + +#define NUM_H_POWERS 8 + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); + +asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, + const struct polyval_elem *b); +asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks); + +static void polyval_preparekey_arch(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]) +{ + static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); + memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); + if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + kernel_fpu_begin(); + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_pclmul_avx( + &key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + kernel_fpu_end(); + } else { + for (int i = NUM_H_POWERS - 2; i >= 0; i--) { + key->h_powers[i] = key->h_powers[i + 1]; + polyval_mul_generic(&key->h_powers[i], + &key->h_powers[NUM_H_POWERS - 1]); + } + } +} + +static void polyval_mul_arch(struct polyval_elem *acc, + const struct polyval_key *key) +{ + if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + kernel_fpu_begin(); + polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); + kernel_fpu_end(); + } else { + polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); + } +} + +static void polyval_blocks_arch(struct polyval_elem *acc, + const struct polyval_key *key, + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { + do { + /* Allow rescheduling every 4 KiB. */ + size_t n = min_t(size_t, nblocks, + 4096 / POLYVAL_BLOCK_SIZE); + + kernel_fpu_begin(); + polyval_blocks_pclmul_avx(acc, key, data, n); + kernel_fpu_end(); + data += n * POLYVAL_BLOCK_SIZE; + nblocks -= n; + } while (nblocks); + } else { + polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], + data, nblocks); + } +} + +#define gf128hash_mod_init_arch gf128hash_mod_init_arch +static void gf128hash_mod_init_arch(void) +{ + if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && + boot_cpu_has(X86_FEATURE_AVX)) + static_branch_enable(&have_pclmul_avx); +} diff --git a/lib/crypto/x86/polyval.h b/lib/crypto/x86/polyval.h deleted file mode 100644 index ef8797521420..000000000000 --- a/lib/crypto/x86/polyval.h +++ /dev/null @@ -1,83 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * POLYVAL library functions, x86_64 optimized - * - * Copyright 2025 Google LLC - */ -#include -#include - -#define NUM_H_POWERS 8 - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pclmul_avx); - -asmlinkage void polyval_mul_pclmul_avx(struct polyval_elem *a, - const struct polyval_elem *b); -asmlinkage void polyval_blocks_pclmul_avx(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks); - -static void polyval_preparekey_arch(struct polyval_key *key, - const u8 raw_key[POLYVAL_BLOCK_SIZE]) -{ - static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS); - memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE); - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { - kernel_fpu_begin(); - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_pclmul_avx( - &key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - kernel_fpu_end(); - } else { - for (int i = NUM_H_POWERS - 2; i >= 0; i--) { - key->h_powers[i] = key->h_powers[i + 1]; - polyval_mul_generic(&key->h_powers[i], - &key->h_powers[NUM_H_POWERS - 1]); - } - } -} - -static void polyval_mul_arch(struct polyval_elem *acc, - const struct polyval_key *key) -{ - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { - kernel_fpu_begin(); - polyval_mul_pclmul_avx(acc, &key->h_powers[NUM_H_POWERS - 1]); - kernel_fpu_end(); - } else { - polyval_mul_generic(acc, &key->h_powers[NUM_H_POWERS - 1]); - } -} - -static void polyval_blocks_arch(struct polyval_elem *acc, - const struct polyval_key *key, - const u8 *data, size_t nblocks) -{ - if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) { - do { - /* Allow rescheduling every 4 KiB. */ - size_t n = min_t(size_t, nblocks, - 4096 / POLYVAL_BLOCK_SIZE); - - kernel_fpu_begin(); - polyval_blocks_pclmul_avx(acc, key, data, n); - kernel_fpu_end(); - data += n * POLYVAL_BLOCK_SIZE; - nblocks -= n; - } while (nblocks); - } else { - polyval_blocks_generic(acc, &key->h_powers[NUM_H_POWERS - 1], - data, nblocks); - } -} - -#define polyval_mod_init_arch polyval_mod_init_arch -static void polyval_mod_init_arch(void) -{ - if (boot_cpu_has(X86_FEATURE_PCLMULQDQ) && - boot_cpu_has(X86_FEATURE_AVX)) - static_branch_enable(&have_pclmul_avx); -} -- cgit v1.2.3 From 9f4e9553a1f40841ebce9ab749896e9312b1701b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:16 -0700 Subject: crypto: gcm - Use GHASH library instead of crypto_ahash Make the "gcm" template access GHASH using the library API instead of crypto_ahash. This is much simpler and more efficient, especially given that all GHASH implementations are synchronous and CPU-based anyway. Note that this allows "ghash" to be removed from the crypto_ahash (and crypto_shash) API, which a later commit will do. This mirrors the similar cleanup that was done with POLYVAL. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-16-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 2 +- crypto/gcm.c | 413 +++++++---------------------------- crypto/testmgr.c | 10 +- drivers/crypto/starfive/jh7110-aes.c | 2 +- 4 files changed, 85 insertions(+), 342 deletions(-) (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index 5627b3691561..13ccf5ac2f1a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -794,7 +794,7 @@ config CRYPTO_GCM tristate "GCM (Galois/Counter Mode) and GMAC (GCM MAC)" select CRYPTO_CTR select CRYPTO_AEAD - select CRYPTO_GHASH + select CRYPTO_LIB_GF128HASH select CRYPTO_MANAGER help GCM (Galois/Counter Mode) authenticated encryption mode and GMAC diff --git a/crypto/gcm.c b/crypto/gcm.c index e1e878d37410..5f16b237b3c5 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -5,13 +5,11 @@ * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen */ -#include #include #include -#include #include #include -#include +#include #include #include #include @@ -20,12 +18,11 @@ struct gcm_instance_ctx { struct crypto_skcipher_spawn ctr; - struct crypto_ahash_spawn ghash; }; struct crypto_gcm_ctx { struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; + struct ghash_key ghash; }; struct crypto_rfc4106_ctx { @@ -52,31 +49,15 @@ struct crypto_rfc4543_req_ctx { struct aead_request subreq; }; -struct crypto_gcm_ghash_ctx { - unsigned int cryptlen; - struct scatterlist *src; - int (*complete)(struct aead_request *req, u32 flags); -}; - struct crypto_gcm_req_priv_ctx { u8 iv[16]; u8 auth_tag[16]; u8 iauth_tag[16]; struct scatterlist src[3]; struct scatterlist dst[3]; - struct scatterlist sg; - struct crypto_gcm_ghash_ctx ghash_ctx; - union { - struct ahash_request ahreq; - struct skcipher_request skreq; - } u; + struct skcipher_request skreq; /* Must be last */ }; -static struct { - u8 buf[16]; - struct scatterlist sg; -} *gcm_zeroes; - static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -89,10 +70,9 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); - struct crypto_ahash *ghash = ctx->ghash; struct crypto_skcipher *ctr = ctx->ctr; struct { - be128 hash; + u8 h[GHASH_BLOCK_SIZE]; u8 iv[16]; struct crypto_wait wait; @@ -115,14 +95,14 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, return -ENOMEM; crypto_init_wait(&data->wait); - sg_init_one(data->sg, &data->hash, sizeof(data->hash)); + sg_init_one(data->sg, data->h, sizeof(data->h)); skcipher_request_set_tfm(&data->req, ctr); skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &data->wait); skcipher_request_set_crypt(&data->req, data->sg, data->sg, - sizeof(data->hash), data->iv); + sizeof(data->h), data->iv); err = crypto_wait_req(crypto_skcipher_encrypt(&data->req), &data->wait); @@ -130,10 +110,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); - crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & - CRYPTO_TFM_REQ_MASK); - err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); + ghash_preparekey(&ctx->ghash, data->h); out: kfree_sensitive(data); return err; @@ -176,7 +153,7 @@ static void crypto_gcm_init_crypt(struct aead_request *req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; dst = req->src == req->dst ? pctx->src : pctx->dst; @@ -187,244 +164,65 @@ static void crypto_gcm_init_crypt(struct aead_request *req, pctx->iv); } -static inline unsigned int gcm_remain(unsigned int len) -{ - len &= 0xfU; - return len ? 16 - len : 0; -} - -static void gcm_hash_len_done(void *data, int err); - -static int gcm_hash_update(struct aead_request *req, - crypto_completion_t compl, - struct scatterlist *src, - unsigned int len, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - - ahash_request_set_callback(ahreq, flags, compl, req); - ahash_request_set_crypt(ahreq, src, NULL, len); - - return crypto_ahash_update(ahreq); -} - -static int gcm_hash_remain(struct aead_request *req, - unsigned int remain, - crypto_completion_t compl, u32 flags) -{ - return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags); -} - -static int gcm_hash_len(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - be128 lengths; - - lengths.a = cpu_to_be64(req->assoclen * 8); - lengths.b = cpu_to_be64(gctx->cryptlen * 8); - memcpy(pctx->iauth_tag, &lengths, 16); - sg_init_one(&pctx->sg, pctx->iauth_tag, 16); - ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req); - ahash_request_set_crypt(ahreq, &pctx->sg, - pctx->iauth_tag, sizeof(lengths)); - - return crypto_ahash_finup(ahreq); -} - -static int gcm_hash_len_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - return gctx->complete(req, flags); -} - -static void gcm_hash_len_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_len_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags) -{ - return gcm_hash_len(req, flags) ?: - gcm_hash_len_continue(req, flags); -} - -static void gcm_hash_crypt_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int remain; - - remain = gcm_remain(gctx->cryptlen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_crypt_remain_done, flags) ?: - gcm_hash_crypt_remain_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_crypt_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_crypt_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - if (gctx->cryptlen) - return gcm_hash_update(req, gcm_hash_crypt_done, - gctx->src, gctx->cryptlen, flags) ?: - gcm_hash_crypt_continue(req, flags); - - return gcm_hash_crypt_remain_continue(req, flags); -} - -static void gcm_hash_assoc_remain_done(void *data, int err) -{ - struct aead_request *req = data; - - if (err) - goto out; - - err = gcm_hash_assoc_remain_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags) +static void ghash_update_sg_and_pad(struct ghash_ctx *ghash, + struct scatterlist *sg, unsigned int len) { - unsigned int remain; + static const u8 zeroes[GHASH_BLOCK_SIZE]; - remain = gcm_remain(req->assoclen); - if (remain) - return gcm_hash_remain(req, remain, - gcm_hash_assoc_remain_done, flags) ?: - gcm_hash_assoc_remain_continue(req, flags); + if (len) { + unsigned int pad_len = -len % GHASH_BLOCK_SIZE; + struct scatter_walk walk; - return gcm_hash_assoc_remain_continue(req, flags); -} + scatterwalk_start(&walk, sg); + do { + unsigned int n = scatterwalk_next(&walk, len); -static void gcm_hash_assoc_done(void *data, int err) -{ - struct aead_request *req = data; + ghash_update(ghash, walk.addr, n); + scatterwalk_done_src(&walk, n); + len -= n; + } while (len); - if (err) - goto out; - - err = gcm_hash_assoc_continue(req, 0); - if (err == -EINPROGRESS) - return; - -out: - aead_request_complete(req, err); -} - -static int gcm_hash_init_continue(struct aead_request *req, u32 flags) -{ - if (req->assoclen) - return gcm_hash_update(req, gcm_hash_assoc_done, - req->src, req->assoclen, flags) ?: - gcm_hash_assoc_continue(req, flags); - - return gcm_hash_assoc_remain_continue(req, flags); + if (pad_len) + ghash_update(ghash, zeroes, pad_len); + } } -static void gcm_hash_init_done(void *data, int err) +static void gcm_hash(struct aead_request *req, struct scatterlist *ctext, + unsigned int datalen, u8 out[GHASH_BLOCK_SIZE]) { - struct aead_request *req = data; - - if (err) - goto out; + const struct crypto_gcm_ctx *ctx = + crypto_aead_ctx(crypto_aead_reqtfm(req)); + __be64 lengths[2] = { + cpu_to_be64(8 * (u64)req->assoclen), + cpu_to_be64(8 * (u64)datalen), + }; + struct ghash_ctx ghash; - err = gcm_hash_init_continue(req, 0); - if (err == -EINPROGRESS) - return; + ghash_init(&ghash, &ctx->ghash); -out: - aead_request_complete(req, err); -} + /* Associated data, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, req->src, req->assoclen); -static int gcm_hash(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ahash_request *ahreq = &pctx->u.ahreq; - struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + /* Ciphertext, then zero-padding to the next 16-byte boundary */ + ghash_update_sg_and_pad(&ghash, ctext, datalen); - ahash_request_set_tfm(ahreq, ctx->ghash); + /* Lengths block */ + ghash_update(&ghash, (const u8 *)lengths, sizeof(lengths)); - ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req); - return crypto_ahash_init(ahreq) ?: - gcm_hash_init_continue(req, flags); + ghash_final(&ghash, out); } -static int gcm_enc_copy_hash(struct aead_request *req, u32 flags) +static int gcm_add_auth_tag(struct aead_request *req) { - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); - u8 *auth_tag = pctx->auth_tag; - - crypto_xor(auth_tag, pctx->iauth_tag, 16); - scatterwalk_map_and_copy(auth_tag, req->dst, - req->assoclen + req->cryptlen, - crypto_aead_authsize(aead), 1); - return 0; -} - -static int gcm_encrypt_continue(struct aead_request *req, u32 flags) -{ struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst); - gctx->cryptlen = req->cryptlen; - gctx->complete = gcm_enc_copy_hash; - - return gcm_hash(req, flags); + gcm_hash(req, sg_next(req->src == req->dst ? pctx->src : pctx->dst), + req->cryptlen, pctx->iauth_tag); + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + memcpy_to_sglist(req->dst, req->assoclen + req->cryptlen, + pctx->auth_tag, crypto_aead_authsize(aead)); + return 0; } static void gcm_encrypt_done(void *data, int err) @@ -434,9 +232,7 @@ static void gcm_encrypt_done(void *data, int err) if (err) goto out; - err = gcm_encrypt_continue(req, 0); - if (err == -EINPROGRESS) - return; + err = gcm_add_auth_tag(req); out: aead_request_complete(req, err); @@ -445,15 +241,14 @@ out: static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; + struct skcipher_request *skreq = &pctx->skreq; u32 flags = aead_request_flags(req); crypto_gcm_init_common(req); crypto_gcm_init_crypt(req, req->cryptlen); skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req); - return crypto_skcipher_encrypt(skreq) ?: - gcm_encrypt_continue(req, flags); + return crypto_skcipher_encrypt(skreq) ?: gcm_add_auth_tag(req); } static int crypto_gcm_verify(struct aead_request *req) @@ -481,35 +276,21 @@ static void gcm_decrypt_done(void *data, int err) aead_request_complete(req, err); } -static int gcm_dec_hash_continue(struct aead_request *req, u32 flags) -{ - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct skcipher_request *skreq = &pctx->u.skreq; - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - - crypto_gcm_init_crypt(req, gctx->cryptlen); - skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req); - return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); -} - static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; - unsigned int authsize = crypto_aead_authsize(aead); - unsigned int cryptlen = req->cryptlen; - u32 flags = aead_request_flags(req); - - cryptlen -= authsize; + struct skcipher_request *skreq = &pctx->skreq; + unsigned int datalen = req->cryptlen - crypto_aead_authsize(aead); crypto_gcm_init_common(req); - gctx->src = sg_next(pctx->src); - gctx->cryptlen = cryptlen; - gctx->complete = gcm_dec_hash_continue; + gcm_hash(req, sg_next(pctx->src), datalen, pctx->iauth_tag); - return gcm_hash(req, flags); + crypto_gcm_init_crypt(req, datalen); + skcipher_request_set_callback(skreq, aead_request_flags(req), + gcm_decrypt_done, req); + return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req); } static int crypto_gcm_init_tfm(struct crypto_aead *tfm) @@ -518,43 +299,26 @@ static int crypto_gcm_init_tfm(struct crypto_aead *tfm) struct gcm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_skcipher *ctr; - struct crypto_ahash *ghash; unsigned long align; - int err; - - ghash = crypto_spawn_ahash(&ictx->ghash); - if (IS_ERR(ghash)) - return PTR_ERR(ghash); ctr = crypto_spawn_skcipher(&ictx->ctr); - err = PTR_ERR(ctr); if (IS_ERR(ctr)) - goto err_free_hash; + return PTR_ERR(ctr); ctx->ctr = ctr; - ctx->ghash = ghash; align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize(tfm, - align + offsetof(struct crypto_gcm_req_priv_ctx, u) + - max(sizeof(struct skcipher_request) + - crypto_skcipher_reqsize(ctr), - sizeof(struct ahash_request) + - crypto_ahash_reqsize(ghash))); - + align + sizeof(struct crypto_gcm_req_priv_ctx) + + crypto_skcipher_reqsize(ctr)); return 0; - -err_free_hash: - crypto_free_ahash(ghash); - return err; } static void crypto_gcm_exit_tfm(struct crypto_aead *tfm) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm); - crypto_free_ahash(ctx->ghash); crypto_free_skcipher(ctx->ctr); } @@ -563,20 +327,16 @@ static void crypto_gcm_free(struct aead_instance *inst) struct gcm_instance_ctx *ctx = aead_instance_ctx(inst); crypto_drop_skcipher(&ctx->ctr); - crypto_drop_ahash(&ctx->ghash); kfree(inst); } static int crypto_gcm_create_common(struct crypto_template *tmpl, - struct rtattr **tb, - const char *ctr_name, - const char *ghash_name) + struct rtattr **tb, const char *ctr_name) { struct skcipher_alg_common *ctr; u32 mask; struct aead_instance *inst; struct gcm_instance_ctx *ctx; - struct hash_alg_common *ghash; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); @@ -588,17 +348,6 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, return -ENOMEM; ctx = aead_instance_ctx(inst); - err = crypto_grab_ahash(&ctx->ghash, aead_crypto_instance(inst), - ghash_name, 0, mask); - if (err) - goto err_free_inst; - ghash = crypto_spawn_ahash_alg(&ctx->ghash); - - err = -EINVAL; - if (strcmp(ghash->base.cra_name, "ghash") != 0 || - ghash->digestsize != 16) - goto err_free_inst; - err = crypto_grab_skcipher(&ctx->ctr, aead_crypto_instance(inst), ctr_name, 0, mask); if (err) @@ -617,13 +366,11 @@ static int crypto_gcm_create_common(struct crypto_template *tmpl, goto err_free_inst; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s,%s)", ctr->base.cra_driver_name, - ghash->base.cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) + "gcm_base(%s,ghash-lib)", + ctr->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; - inst->alg.base.cra_priority = (ghash->base.cra_priority + - ctr->base.cra_priority) / 2; + inst->alg.base.cra_priority = ctr->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx); @@ -660,7 +407,7 @@ static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; - return crypto_gcm_create_common(tmpl, tb, ctr_name, "ghash"); + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_gcm_base_create(struct crypto_template *tmpl, @@ -677,7 +424,16 @@ static int crypto_gcm_base_create(struct crypto_template *tmpl, if (IS_ERR(ghash_name)) return PTR_ERR(ghash_name); - return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name); + /* + * Originally this parameter allowed requesting a specific + * implementation of GHASH. This is no longer supported. Now the best + * implementation of GHASH is just always used. + */ + if (strcmp(ghash_name, "ghash") != 0 && + strcmp(ghash_name, "ghash-lib") != 0) + return -EINVAL; + + return crypto_gcm_create_common(tmpl, tb, ctr_name); } static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key, @@ -1096,25 +852,12 @@ static struct crypto_template crypto_gcm_tmpls[] = { static int __init crypto_gcm_module_init(void) { - int err; - - gcm_zeroes = kzalloc_obj(*gcm_zeroes); - if (!gcm_zeroes) - return -ENOMEM; - - sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf)); - - err = crypto_register_templates(crypto_gcm_tmpls, - ARRAY_SIZE(crypto_gcm_tmpls)); - if (err) - kfree(gcm_zeroes); - - return err; + return crypto_register_templates(crypto_gcm_tmpls, + ARRAY_SIZE(crypto_gcm_tmpls)); } static void __exit crypto_gcm_module_exit(void) { - kfree(gcm_zeroes); crypto_unregister_templates(crypto_gcm_tmpls, ARRAY_SIZE(crypto_gcm_tmpls)); } diff --git a/crypto/testmgr.c b/crypto/testmgr.c index fec950f1628b..0b0ad358e091 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4965,7 +4965,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { #endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */ .alg = "gcm(aes)", - .generic_driver = "gcm_base(ctr(aes-lib),ghash-generic)", + .generic_driver = "gcm_base(ctr(aes-lib),ghash-lib)", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -4973,14 +4973,14 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "gcm(aria)", - .generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(aria-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(aria_gcm_tv_template) } }, { .alg = "gcm(sm4)", - .generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)", + .generic_driver = "gcm_base(ctr(sm4-generic),ghash-lib)", .test = alg_test_aead, .suite = { .aead = __VECS(sm4_gcm_tv_template) @@ -5314,7 +5314,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4106(gcm(aes))", - .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4106(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .fips_allowed = 1, .suite = { @@ -5338,7 +5338,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "rfc4543(gcm(aes))", - .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-generic))", + .generic_driver = "rfc4543(gcm_base(ctr(aes-lib),ghash-lib))", .test = alg_test_aead, .suite = { .aead = { diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 2e2d97d17e6c..a0713aa21250 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -1008,7 +1008,7 @@ static int starfive_aes_ccm_init_tfm(struct crypto_aead *tfm) static int starfive_aes_gcm_init_tfm(struct crypto_aead *tfm) { - return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-generic)"); + return starfive_aes_aead_init_tfm(tfm, "gcm_base(ctr(aes-lib),ghash-lib)"); } static struct skcipher_engine_alg skcipher_algs[] = { -- cgit v1.2.3 From 662a05a245078e7d03e75895403c851967dc8384 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 18 Mar 2026 23:17:17 -0700 Subject: crypto: ghash - Remove ghash from crypto_shash API Now that there are no users of the "ghash" crypto_shash algorithm, remove it. GHASH remains supported via the library API. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260319061723.1140720-17-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Kconfig | 7 --- crypto/Makefile | 1 - crypto/ghash-generic.c | 162 ------------------------------------------------- crypto/tcrypt.c | 9 --- crypto/testmgr.c | 6 -- crypto/testmgr.h | 109 --------------------------------- 6 files changed, 294 deletions(-) delete mode 100644 crypto/ghash-generic.c (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index 13ccf5ac2f1a..efb482ea192d 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -888,13 +888,6 @@ config CRYPTO_CMAC CMAC (Cipher-based Message Authentication Code) authentication mode (NIST SP800-38B and IETF RFC4493) -config CRYPTO_GHASH - tristate "GHASH" - select CRYPTO_HASH - select CRYPTO_LIB_GF128MUL - help - GCM GHASH function (NIST SP800-38D) - config CRYPTO_HMAC tristate "HMAC (Keyed-Hash MAC)" select CRYPTO_HASH diff --git a/crypto/Makefile b/crypto/Makefile index 04e269117589..17f4fca9b9e5 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -171,7 +171,6 @@ UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE) += jitterentropy-testing.o obj-$(CONFIG_CRYPTO_BENCHMARK) += tcrypt.o -obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c deleted file mode 100644 index e5803c249c12..000000000000 --- a/crypto/ghash-generic.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * GHASH: hash function for GCM (Galois/Counter Mode). - * - * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying - */ - -/* - * GHASH is a keyed hash function used in GCM authentication tag generation. - * - * The original GCM paper [1] presents GHASH as a function GHASH(H, A, C) which - * takes a 16-byte hash key H, additional authenticated data A, and a ciphertext - * C. It formats A and C into a single byte string X, interprets X as a - * polynomial over GF(2^128), and evaluates this polynomial at the point H. - * - * However, the NIST standard for GCM [2] presents GHASH as GHASH(H, X) where X - * is the already-formatted byte string containing both A and C. - * - * "ghash" in the Linux crypto API uses the 'X' (pre-formatted) convention, - * since the API supports only a single data stream per hash. Thus, the - * formatting of 'A' and 'C' is done in the "gcm" template, not in "ghash". - * - * The reason "ghash" is separate from "gcm" is to allow "gcm" to use an - * accelerated "ghash" when a standalone accelerated "gcm(aes)" is unavailable. - * It is generally inappropriate to use "ghash" for other purposes, since it is - * an "ε-almost-XOR-universal hash function", not a cryptographic hash function. - * It can only be used securely in crypto modes specially designed to use it. - * - * [1] The Galois/Counter Mode of Operation (GCM) - * (http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.694.695&rep=rep1&type=pdf) - * [2] Recommendation for Block Cipher Modes of Operation: Galois/Counter Mode (GCM) and GMAC - * (https://csrc.nist.gov/publications/detail/sp/800-38d/final) - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -static int ghash_init(struct shash_desc *desc) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - - memset(dctx, 0, sizeof(*dctx)); - - return 0; -} - -static int ghash_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(tfm); - be128 k; - - if (keylen != GHASH_BLOCK_SIZE) - return -EINVAL; - - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); - - BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); - memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ - ctx->gf128 = gf128mul_init_4k_lle(&k); - memzero_explicit(&k, GHASH_BLOCK_SIZE); - - if (!ctx->gf128) - return -ENOMEM; - - return 0; -} - -static int ghash_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - u8 *dst = dctx->buffer; - - do { - crypto_xor(dst, src, GHASH_BLOCK_SIZE); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - src += GHASH_BLOCK_SIZE; - srclen -= GHASH_BLOCK_SIZE; - } while (srclen >= GHASH_BLOCK_SIZE); - - return srclen; -} - -static void ghash_flush(struct shash_desc *desc, const u8 *src, - unsigned int len) -{ - struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *dst = dctx->buffer; - - if (len) { - crypto_xor(dst, src, len); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } -} - -static int ghash_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *dst) -{ - struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); - u8 *buf = dctx->buffer; - - ghash_flush(desc, src, len); - memcpy(dst, buf, GHASH_BLOCK_SIZE); - - return 0; -} - -static void ghash_exit_tfm(struct crypto_tfm *tfm) -{ - struct ghash_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->gf128) - gf128mul_free_4k(ctx->gf128); -} - -static struct shash_alg ghash_alg = { - .digestsize = GHASH_DIGEST_SIZE, - .init = ghash_init, - .update = ghash_update, - .finup = ghash_finup, - .setkey = ghash_setkey, - .descsize = sizeof(struct ghash_desc_ctx), - .base = { - .cra_name = "ghash", - .cra_driver_name = "ghash-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, - .cra_blocksize = GHASH_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct ghash_ctx), - .cra_module = THIS_MODULE, - .cra_exit = ghash_exit_tfm, - }, -}; - -static int __init ghash_mod_init(void) -{ - return crypto_register_shash(&ghash_alg); -} - -static void __exit ghash_mod_exit(void) -{ - crypto_unregister_shash(&ghash_alg); -} - -module_init(ghash_mod_init); -module_exit(ghash_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("GHASH hash function"); -MODULE_ALIAS_CRYPTO("ghash"); -MODULE_ALIAS_CRYPTO("ghash-generic"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index aded37546137..1773f5f71351 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1650,10 +1650,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret = min(ret, tcrypt_test("rfc4309(ccm(aes))")); break; - case 46: - ret = min(ret, tcrypt_test("ghash")); - break; - case 48: ret = min(ret, tcrypt_test("sha3-224")); break; @@ -2251,11 +2247,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("blake2b-512", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; - case 318: - klen = 16; - test_hash_speed("ghash", sec, generic_hash_speed_template); - if (mode > 300 && mode < 400) break; - fallthrough; case 319: test_hash_speed("crc32c", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 0b0ad358e091..dd01f86dd6fe 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4985,12 +4985,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .aead = __VECS(sm4_gcm_tv_template) } - }, { - .alg = "ghash", - .test = alg_test_hash, - .suite = { - .hash = __VECS(ghash_tv_template) - } }, { .alg = "hctr2(aes)", .generic_driver = "hctr2_base(xctr(aes-lib),polyval-lib)", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 1c69c11c0cdb..a3274abacfde 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -6183,115 +6183,6 @@ static const struct hash_testvec wp256_tv_template[] = { }, }; -static const struct hash_testvec ghash_tv_template[] = -{ - { - .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" - "\xff\xca\xff\x95\xf8\x30\xf0\x61", - .ksize = 16, - .plaintext = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .psize = 16, - .digest = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - }, { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, - .plaintext = "what do ya want for nothing?", - .psize = 28, - .digest = "\x3e\x1f\x5c\x4d\x65\xf0\xef\xce" - "\x0d\x61\x06\x27\x66\x51\xd5\xe2", - }, { - .key = "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa" - "\xaa\xaa\xaa\xaa\xaa\xaa\xaa\xaa", - .ksize = 16, - .plaintext = "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd" - "\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd\xdd", - .psize = 50, - .digest = "\xfb\x49\x8a\x36\xe1\x96\xe1\x96" - "\xe1\x96\xe1\x96\xe1\x96\xe1\x96", - }, { - .key = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" - "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", - .ksize = 16, - .plaintext = "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd" - "\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd\xcd", - .psize = 50, - .digest = "\x2b\x5c\x0c\x7f\x52\xd1\x60\xc2" - "\x49\xed\x6e\x32\x7a\xa9\xbe\x08", - }, { - .key = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0" - "\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", - .ksize = 16, - .plaintext = "Test With Truncation", - .psize = 20, - .digest = "\xf8\x94\x87\x2a\x4b\x63\x99\x28" - "\x23\xf7\x93\xf7\x19\xf5\x96\xd9", - }, { - .key = "\x0a\x1b\x2c\x3d\x4e\x5f\x64\x71" - "\x82\x93\xa4\xb5\xc6\xd7\xe8\xf9", - .ksize = 16, - .plaintext = "\x56\x6f\x72\x20\x6c\x61\x75\x74" - "\x65\x72\x20\x4c\x61\x75\x73\x63" - "\x68\x65\x6e\x20\x75\x6e\x64\x20" - "\x53\x74\x61\x75\x6e\x65\x6e\x20" - "\x73\x65\x69\x20\x73\x74\x69\x6c" - "\x6c\x2c\x0a\x64\x75\x20\x6d\x65" - "\x69\x6e\x20\x74\x69\x65\x66\x74" - "\x69\x65\x66\x65\x73\x20\x4c\x65" - "\x62\x65\x6e\x3b\x0a\x64\x61\x73" - "\x73\x20\x64\x75\x20\x77\x65\x69" - "\xc3\x9f\x74\x20\x77\x61\x73\x20" - "\x64\x65\x72\x20\x57\x69\x6e\x64" - "\x20\x64\x69\x72\x20\x77\x69\x6c" - "\x6c\x2c\x0a\x65\x68\x20\x6e\x6f" - "\x63\x68\x20\x64\x69\x65\x20\x42" - "\x69\x72\x6b\x65\x6e\x20\x62\x65" - "\x62\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x77\x65\x6e\x6e\x20\x64" - "\x69\x72\x20\x65\x69\x6e\x6d\x61" - "\x6c\x20\x64\x61\x73\x20\x53\x63" - "\x68\x77\x65\x69\x67\x65\x6e\x20" - "\x73\x70\x72\x61\x63\x68\x2c\x0a" - "\x6c\x61\x73\x73\x20\x64\x65\x69" - "\x6e\x65\x20\x53\x69\x6e\x6e\x65" - "\x20\x62\x65\x73\x69\x65\x67\x65" - "\x6e\x2e\x0a\x4a\x65\x64\x65\x6d" - "\x20\x48\x61\x75\x63\x68\x65\x20" - "\x67\x69\x62\x74\x20\x64\x69\x63" - "\x68\x2c\x20\x67\x69\x62\x20\x6e" - "\x61\x63\x68\x2c\x0a\x65\x72\x20" - "\x77\x69\x72\x64\x20\x64\x69\x63" - "\x68\x20\x6c\x69\x65\x62\x65\x6e" - "\x20\x75\x6e\x64\x20\x77\x69\x65" - "\x67\x65\x6e\x2e\x0a\x0a\x55\x6e" - "\x64\x20\x64\x61\x6e\x6e\x20\x6d" - "\x65\x69\x6e\x65\x20\x53\x65\x65" - "\x6c\x65\x20\x73\x65\x69\x74\x20" - "\x77\x65\x69\x74\x2c\x20\x73\x65" - "\x69\x20\x77\x65\x69\x74\x2c\x0a" - "\x64\x61\x73\x73\x20\x64\x69\x72" - "\x20\x64\x61\x73\x20\x4c\x65\x62" - "\x65\x6e\x20\x67\x65\x6c\x69\x6e" - "\x67\x65\x2c\x0a\x62\x72\x65\x69" - "\x74\x65\x20\x64\x69\x63\x68\x20" - "\x77\x69\x65\x20\x65\x69\x6e\x20" - "\x46\x65\x69\x65\x72\x6b\x6c\x65" - "\x69\x64\x0a\xc3\xbc\x62\x65\x72" - "\x20\x64\x69\x65\x20\x73\x69\x6e" - "\x6e\x65\x6e\x64\x65\x6e\x20\x44" - "\x69\x6e\x67\x65\x2e\x2e\x2e\x0a", - .psize = 400, - .digest = "\xad\xb1\xc1\xe9\x56\x70\x31\x1d" - "\xbb\x5b\xdf\x5e\x70\x72\x1a\x57", - }, -}; - /* * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). -- cgit v1.2.3 From 77e4ca814c2824d7aa0c4170678bfbc6e3caa556 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:25 -0700 Subject: crypto: sm3 - Remove sm3_zero_message_hash and SM3_T[1-2] Remove these, since they are unused. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/sm3_generic.c | 8 -------- include/crypto/sm3.h | 5 ----- 2 files changed, 13 deletions(-) (limited to 'crypto') diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index 7529139fcc96..0c606f526347 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -14,14 +14,6 @@ #include #include -const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE] = { - 0x1A, 0xB2, 0x1D, 0x83, 0x55, 0xCF, 0xA1, 0x7F, - 0x8e, 0x61, 0x19, 0x48, 0x31, 0xE8, 0x1A, 0x8F, - 0x22, 0xBE, 0xC8, 0xC7, 0x28, 0xFE, 0xFB, 0x74, - 0x7E, 0xD0, 0x35, 0xEB, 0x50, 0x82, 0xAA, 0x2B -}; -EXPORT_SYMBOL_GPL(sm3_zero_message_hash); - static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index c09f6bf4c0bf..918d318795ef 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -16,9 +16,6 @@ #define SM3_BLOCK_SIZE 64 #define SM3_STATE_SIZE 40 -#define SM3_T1 0x79CC4519 -#define SM3_T2 0x7A879D8A - #define SM3_IVA 0x7380166f #define SM3_IVB 0x4914b2b9 #define SM3_IVC 0x172442d7 @@ -28,8 +25,6 @@ #define SM3_IVG 0xe38dee4d #define SM3_IVH 0xb0fb0e4e -extern const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE]; - struct sm3_state { u32 state[SM3_DIGEST_SIZE / 4]; u64 count; -- cgit v1.2.3 From 6dc7fce91041ec8d2f5e6fd589ee2962898d9f44 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:26 -0700 Subject: crypto: sm3 - Rename CRYPTO_SM3_GENERIC to CRYPTO_SM3 The kconfig options for generic crypto API modules have traditionally *not* had a "_GENERIC" suffix. Also, the "_GENERIC" suffix will make even less sense once the architecture-optimized SM3 code is moved into lib/crypto/ and the "sm3" crypto_shash is reimplemented on top of that. Thus, rename CRYPTO_SM3_GENERIC to CRYPTO_SM3. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- arch/loongarch/configs/loongson32_defconfig | 2 +- arch/loongarch/configs/loongson64_defconfig | 2 +- arch/m68k/configs/amiga_defconfig | 2 +- arch/m68k/configs/apollo_defconfig | 2 +- arch/m68k/configs/atari_defconfig | 2 +- arch/m68k/configs/bvme6000_defconfig | 2 +- arch/m68k/configs/hp300_defconfig | 2 +- arch/m68k/configs/mac_defconfig | 2 +- arch/m68k/configs/multi_defconfig | 2 +- arch/m68k/configs/mvme147_defconfig | 2 +- arch/m68k/configs/mvme16x_defconfig | 2 +- arch/m68k/configs/q40_defconfig | 2 +- arch/m68k/configs/sun3_defconfig | 2 +- arch/m68k/configs/sun3x_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- crypto/Kconfig | 2 +- crypto/Makefile | 2 +- drivers/crypto/Kconfig | 2 +- drivers/crypto/starfive/Kconfig | 2 +- security/integrity/ima/Kconfig | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) (limited to 'crypto') diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig index 276b1577e0be..7abbb21f4f8f 100644 --- a/arch/loongarch/configs/loongson32_defconfig +++ b/arch/loongarch/configs/loongson32_defconfig @@ -1080,7 +1080,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig index a14db1a95e7e..51ccd18ecdae 100644 --- a/arch/loongarch/configs/loongson64_defconfig +++ b/arch/loongarch/configs/loongson64_defconfig @@ -1113,7 +1113,7 @@ CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_DEFLATE=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 31d16cba9879..03a8c192a7a3 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -581,7 +581,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index c0c419ec9a9e..0aee1939ac7a 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -538,7 +538,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 2b7547ecc4c4..756256770afc 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -558,7 +558,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index 0b63787cff0d..8cfb75bb0add 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 308836b60bba..b2f5c9749e9b 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -540,7 +540,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 97e108c0d24f..c4fddaaa6a86 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -557,7 +557,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 7e9f83af9af4..926f12bc3d1d 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -644,7 +644,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 2fe33271d249..e507012dbbc1 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -530,7 +530,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 4308daaa7f74..6195cedd914b 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -531,7 +531,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 36eb29ec54ee..9087bd9e3c35 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -547,7 +547,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 524a89fa6953..25115bda7c8a 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index f4fbc65c52d9..15a086634ba5 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -528,7 +528,7 @@ CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA1=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_LZO=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index aa862d4fcc68..f8fcc29adbd3 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -797,7 +797,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 74f943307c46..3c6ccc0de018 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -781,7 +781,7 @@ CONFIG_CRYPTO_MD5=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_XCBC=m CONFIG_CRYPTO_CRC32=m diff --git a/crypto/Kconfig b/crypto/Kconfig index efb482ea192d..9919fe0db7f6 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -974,7 +974,7 @@ config CRYPTO_SHA3 help SHA-3 secure hash algorithms (FIPS 202, ISO/IEC 10118-3) -config CRYPTO_SM3_GENERIC +config CRYPTO_SM3 tristate "SM3 (ShangMi 3)" select CRYPTO_HASH select CRYPTO_LIB_SM3 diff --git a/crypto/Makefile b/crypto/Makefile index 17f4fca9b9e5..3fcbf0cd522d 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -83,7 +83,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3.o -obj-$(CONFIG_CRYPTO_SM3_GENERIC) += sm3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 8d3b5d2890f8..9960100e6066 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -843,7 +843,7 @@ config CRYPTO_DEV_CCREE select CRYPTO_CTR select CRYPTO_XTS select CRYPTO_SM4_GENERIC - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 help Say 'Y' to enable a driver for the REE interface of the Arm TrustZone CryptoCell family of processors. Currently the diff --git a/drivers/crypto/starfive/Kconfig b/drivers/crypto/starfive/Kconfig index 0fe389e9f932..11518ca3eea1 100644 --- a/drivers/crypto/starfive/Kconfig +++ b/drivers/crypto/starfive/Kconfig @@ -10,7 +10,7 @@ config CRYPTO_DEV_JH7110 select CRYPTO_HMAC select CRYPTO_SHA256 select CRYPTO_SHA512 - select CRYPTO_SM3_GENERIC + select CRYPTO_SM3 select CRYPTO_RSA select CRYPTO_AES select CRYPTO_CCM diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index 976e75f9b9ba..862fbee2b174 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -111,7 +111,7 @@ choice config IMA_DEFAULT_HASH_SM3 bool "SM3" - depends on CRYPTO_SM3_GENERIC=y + depends on CRYPTO_SM3=y endchoice config IMA_DEFAULT_HASH -- cgit v1.2.3 From ed065bd06ebe8d92d1647d230a14b9c035ad5b30 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 20 Mar 2026 21:09:29 -0700 Subject: crypto: sm3 - Replace with wrapper around library Reimplement the "sm3" crypto_shash on top of the SM3 library, closely mirroring the other hash algorithms (e.g. SHA-*). The result, after later commits migrate the architecture-optimized SM3 code into the library as well, is that crypto/sm3.c will be the single point of integration between crypto_shash and the actual SM3 implementations, simplifying the code. Note: to see the diff from crypto/sm3_generic.c to crypto/sm3.c, view this commit with 'git show -M10'. Acked-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20260321040935.410034-7-ebiggers@kernel.org Signed-off-by: Eric Biggers --- crypto/Makefile | 2 +- crypto/sm3.c | 89 +++++++++++++++++++++++++++++++++++ crypto/sm3_generic.c | 64 ------------------------- crypto/testmgr.c | 2 + drivers/crypto/starfive/jh7110-hash.c | 4 +- 5 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 crypto/sm3.c delete mode 100644 crypto/sm3_generic.c (limited to 'crypto') diff --git a/crypto/Makefile b/crypto/Makefile index 3fcbf0cd522d..68646175f6ab 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -83,7 +83,7 @@ obj-$(CONFIG_CRYPTO_SHA1) += sha1.o obj-$(CONFIG_CRYPTO_SHA256) += sha256.o obj-$(CONFIG_CRYPTO_SHA512) += sha512.o obj-$(CONFIG_CRYPTO_SHA3) += sha3.o -obj-$(CONFIG_CRYPTO_SM3) += sm3_generic.o +obj-$(CONFIG_CRYPTO_SM3) += sm3.o obj-$(CONFIG_CRYPTO_STREEBOG) += streebog_generic.o obj-$(CONFIG_CRYPTO_WP512) += wp512.o CFLAGS_wp512.o := $(call cc-option,-fno-schedule-insns) # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=79149 diff --git a/crypto/sm3.c b/crypto/sm3.c new file mode 100644 index 000000000000..05111a99b851 --- /dev/null +++ b/crypto/sm3.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and + * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 + * + * Copyright (C) 2017 ARM Limited or its affiliates. + * Written by Gilad Ben-Yossef + * Copyright (C) 2021 Tianjia Zhang + * Copyright 2026 Google LLC + */ + +#include +#include +#include +#include + +#define SM3_CTX(desc) ((struct sm3_ctx *)shash_desc_ctx(desc)) + +static int crypto_sm3_init(struct shash_desc *desc) +{ + sm3_init(SM3_CTX(desc)); + return 0; +} + +static int crypto_sm3_update(struct shash_desc *desc, + const u8 *data, unsigned int len) +{ + sm3_update(SM3_CTX(desc), data, len); + return 0; +} + +static int crypto_sm3_final(struct shash_desc *desc, u8 *out) +{ + sm3_final(SM3_CTX(desc), out); + return 0; +} + +static int crypto_sm3_digest(struct shash_desc *desc, + const u8 *data, unsigned int len, u8 *out) +{ + sm3(data, len, out); + return 0; +} + +static int crypto_sm3_export_core(struct shash_desc *desc, void *out) +{ + memcpy(out, SM3_CTX(desc), sizeof(struct sm3_ctx)); + return 0; +} + +static int crypto_sm3_import_core(struct shash_desc *desc, const void *in) +{ + memcpy(SM3_CTX(desc), in, sizeof(struct sm3_ctx)); + return 0; +} + +static struct shash_alg sm3_alg = { + .base.cra_name = "sm3", + .base.cra_driver_name = "sm3-lib", + .base.cra_priority = 300, + .base.cra_blocksize = SM3_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = SM3_DIGEST_SIZE, + .init = crypto_sm3_init, + .update = crypto_sm3_update, + .final = crypto_sm3_final, + .digest = crypto_sm3_digest, + .export_core = crypto_sm3_export_core, + .import_core = crypto_sm3_import_core, + .descsize = sizeof(struct sm3_ctx), +}; + +static int __init crypto_sm3_mod_init(void) +{ + return crypto_register_shash(&sm3_alg); +} +module_init(crypto_sm3_mod_init); + +static void __exit crypto_sm3_mod_exit(void) +{ + crypto_unregister_shash(&sm3_alg); +} +module_exit(crypto_sm3_mod_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Crypto API support for SM3"); + +MODULE_ALIAS_CRYPTO("sm3"); +MODULE_ALIAS_CRYPTO("sm3-lib"); diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c deleted file mode 100644 index 0c606f526347..000000000000 --- a/crypto/sm3_generic.c +++ /dev/null @@ -1,64 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * SM3 secure hash, as specified by OSCCA GM/T 0004-2012 SM3 and - * described at https://tools.ietf.org/html/draft-shen-sm3-hash-01 - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef - * Copyright (C) 2021 Tianjia Zhang - */ - -#include -#include -#include -#include -#include - -static int crypto_sm3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sm3_base_do_update_blocks(desc, data, len, sm3_block_generic); -} - -static int crypto_sm3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash) -{ - sm3_base_do_finup(desc, data, len, sm3_block_generic); - return sm3_base_finish(desc, hash); -} - -static struct shash_alg sm3_alg = { - .digestsize = SM3_DIGEST_SIZE, - .init = sm3_base_init, - .update = crypto_sm3_update, - .finup = crypto_sm3_finup, - .descsize = SM3_STATE_SIZE, - .base = { - .cra_name = "sm3", - .cra_driver_name = "sm3-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SM3_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sm3_generic_mod_init(void) -{ - return crypto_register_shash(&sm3_alg); -} - -static void __exit sm3_generic_mod_fini(void) -{ - crypto_unregister_shash(&sm3_alg); -} - -module_init(sm3_generic_mod_init); -module_exit(sm3_generic_mod_fini); - -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("SM3 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sm3"); -MODULE_ALIAS_CRYPTO("sm3-generic"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index dd01f86dd6fe..60b6e4379aa6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5079,6 +5079,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "hmac(sm3)", + .generic_driver = "hmac(sm3-lib)", .test = alg_test_hash, .suite = { .hash = __VECS(hmac_sm3_tv_template) @@ -5446,6 +5447,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "sm3", + .generic_driver = "sm3-lib", .test = alg_test_hash, .suite = { .hash = __VECS(sm3_tv_template) diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index 54b7af4a7aee..742038a5201a 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -520,7 +520,7 @@ static int starfive_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "sm3-generic", + return starfive_hash_init_tfm(hash, "sm3-lib", STARFIVE_HASH_SM3, 0); } @@ -550,7 +550,7 @@ static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) { - return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", + return starfive_hash_init_tfm(hash, "hmac(sm3-lib)", STARFIVE_HASH_SM3, 1); } -- cgit v1.2.3