From 1fe244c5916ad63ab7b706a0de713cc16200f47a Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Fri, 17 Jan 2025 15:42:22 +0100 Subject: crypto: skcipher - use str_yes_no() helper in crypto_skcipher_show() Remove hard-coded strings by using the str_yes_no() helper function. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- crypto/skcipher.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index a9eb2dcf2898..e3751cc88b76 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "skcipher.h" @@ -612,7 +613,7 @@ static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "type : skcipher\n"); seq_printf(m, "async : %s\n", - alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no"); + str_yes_no(alg->cra_flags & CRYPTO_ALG_ASYNC)); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); seq_printf(m, "min keysize : %u\n", skcipher->min_keysize); seq_printf(m, "max keysize : %u\n", skcipher->max_keysize); -- cgit v1.2.3 From f4144b6bb74cc358054041e7b062bc9354c59e6c Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sun, 2 Feb 2025 20:00:51 +0100 Subject: crypto: sig - Prepare for algorithms with variable signature size The callers of crypto_sig_sign() assume that the signature size is always equivalent to the key size. This happens to be true for RSA, which is currently the only algorithm implementing the ->sign() callback. But it is false e.g. for X9.62 encoded ECDSA signatures because they have variable length. Prepare for addition of a ->sign() callback to such algorithms by letting the callback return the signature size (or a negative integer on error). When testing the ->sign() callback in test_sig_one(), use crypto_sig_maxsize() instead of crypto_sig_keysize() to verify that the test vector's signature does not exceed an algorithm's maximum signature size. There has been a relatively recent effort to upstream ECDSA signature generation support which may benefit from this change: https://lore.kernel.org/linux-crypto/20220908200036.2034-1-ignat@cloudflare.com/ However the main motivation for this commit is to reduce the number of crypto_sig_keysize() callers: This function is about to be changed to return the size in bits instead of bytes and that will require amending most callers to divide the return value by 8. Signed-off-by: Lukas Wunner Reviewed-by: Stefan Berger Cc: Ignat Korchagin Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/public_key.c | 9 ++------- crypto/rsassa-pkcs1.c | 2 +- crypto/testmgr.c | 7 ++++--- include/crypto/sig.h | 5 +++-- 4 files changed, 10 insertions(+), 13 deletions(-) (limited to 'crypto') diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index bbd07a9022e6..bf165d321440 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -267,7 +267,6 @@ static int software_key_eds_op(struct kernel_pkey_params *params, struct crypto_sig *sig; char *key, *ptr; bool issig; - int ksz; int ret; pr_devel("==>%s()\n", __func__); @@ -302,8 +301,6 @@ static int software_key_eds_op(struct kernel_pkey_params *params, ret = crypto_sig_set_pubkey(sig, key, pkey->keylen); if (ret) goto error_free_tfm; - - ksz = crypto_sig_keysize(sig); } else { tfm = crypto_alloc_akcipher(alg_name, 0, 0); if (IS_ERR(tfm)) { @@ -317,8 +314,6 @@ static int software_key_eds_op(struct kernel_pkey_params *params, ret = crypto_akcipher_set_pub_key(tfm, key, pkey->keylen); if (ret) goto error_free_tfm; - - ksz = crypto_akcipher_maxsize(tfm); } ret = -EINVAL; @@ -347,8 +342,8 @@ static int software_key_eds_op(struct kernel_pkey_params *params, BUG(); } - if (ret == 0) - ret = ksz; + if (!issig && ret == 0) + ret = crypto_akcipher_maxsize(tfm); error_free_tfm: if (issig) diff --git a/crypto/rsassa-pkcs1.c b/crypto/rsassa-pkcs1.c index f68ffd338f48..d01ac75635e0 100644 --- a/crypto/rsassa-pkcs1.c +++ b/crypto/rsassa-pkcs1.c @@ -210,7 +210,7 @@ static int rsassa_pkcs1_sign(struct crypto_sig *tfm, memset(dst, 0, pad_len); } - return 0; + return ctx->key_size; } static int rsassa_pkcs1_verify(struct crypto_sig *tfm, diff --git a/crypto/testmgr.c b/crypto/testmgr.c index e61490ba4095..b69877db3f33 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4328,7 +4328,7 @@ static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs) if (vecs->public_key_vec) return 0; - sig_size = crypto_sig_keysize(tfm); + sig_size = crypto_sig_maxsize(tfm); if (sig_size < vecs->c_size) { pr_err("alg: sig: invalid maxsize %u\n", sig_size); return -EINVAL; @@ -4340,13 +4340,14 @@ static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs) /* Run asymmetric signature generation */ err = crypto_sig_sign(tfm, vecs->m, vecs->m_size, sig, sig_size); - if (err) { + if (err < 0) { pr_err("alg: sig: sign test failed: err %d\n", err); return err; } /* Verify that generated signature equals cooked signature */ - if (memcmp(sig, vecs->c, vecs->c_size) || + if (err != vecs->c_size || + memcmp(sig, vecs->c, vecs->c_size) || memchr_inv(sig + vecs->c_size, 0, sig_size - vecs->c_size)) { pr_err("alg: sig: sign test failed: invalid output\n"); hexdump(sig, sig_size); diff --git a/include/crypto/sig.h b/include/crypto/sig.h index cff41ad93824..11024708c069 100644 --- a/include/crypto/sig.h +++ b/include/crypto/sig.h @@ -23,7 +23,8 @@ struct crypto_sig { * struct sig_alg - generic public key signature algorithm * * @sign: Function performs a sign operation as defined by public key - * algorithm. Optional. + * algorithm. On success, the signature size is returned. + * Optional. * @verify: Function performs a complete verify operation as defined by * public key algorithm, returning verification status. Optional. * @set_pub_key: Function invokes the algorithm specific set public key @@ -186,7 +187,7 @@ static inline unsigned int crypto_sig_maxsize(struct crypto_sig *tfm) * @dst: destination obuffer * @dlen: destination length * - * Return: zero on success; error code in case of error + * Return: signature size on success; error code in case of error */ static inline int crypto_sig_sign(struct crypto_sig *tfm, const void *src, unsigned int slen, -- cgit v1.2.3 From b16510a530d1e6ab9683f04f8fb34f2e0f538275 Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Sun, 2 Feb 2025 20:00:52 +0100 Subject: crypto: ecdsa - Harden against integer overflows in DIV_ROUND_UP() Herbert notes that DIV_ROUND_UP() may overflow unnecessarily if an ecdsa implementation's ->key_size() callback returns an unusually large value. Herbert instead suggests (for a division by 8): X / 8 + !!(X & 7) Based on this formula, introduce a generic DIV_ROUND_UP_POW2() macro and use it in lieu of DIV_ROUND_UP() for ->key_size() return values. Additionally, use the macro in ecc_digits_from_bytes(), whose "nbytes" parameter is a ->key_size() return value in some instances, or a user-specified ASN.1 length in the case of ecdsa_get_signature_rs(). Link: https://lore.kernel.org/r/Z3iElsILmoSu6FuC@gondor.apana.org.au/ Signed-off-by: Lukas Wunner Signed-off-by: Lukas Wunner Signed-off-by: Herbert Xu --- crypto/ecc.c | 2 +- crypto/ecdsa-p1363.c | 2 +- crypto/ecdsa-x962.c | 4 ++-- include/linux/math.h | 12 ++++++++++++ 4 files changed, 16 insertions(+), 4 deletions(-) (limited to 'crypto') diff --git a/crypto/ecc.c b/crypto/ecc.c index 50ad2d4ed672..6cf9a945fc6c 100644 --- a/crypto/ecc.c +++ b/crypto/ecc.c @@ -71,7 +71,7 @@ EXPORT_SYMBOL(ecc_get_curve); void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, u64 *out, unsigned int ndigits) { - int diff = ndigits - DIV_ROUND_UP(nbytes, sizeof(u64)); + int diff = ndigits - DIV_ROUND_UP_POW2(nbytes, sizeof(u64)); unsigned int o = nbytes & 7; __be64 msd = 0; diff --git a/crypto/ecdsa-p1363.c b/crypto/ecdsa-p1363.c index eaae7214d69b..4454f1f8f33f 100644 --- a/crypto/ecdsa-p1363.c +++ b/crypto/ecdsa-p1363.c @@ -22,7 +22,7 @@ static int ecdsa_p1363_verify(struct crypto_sig *tfm, { struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); unsigned int keylen = crypto_sig_keysize(ctx->child); - unsigned int ndigits = DIV_ROUND_UP(keylen, sizeof(u64)); + unsigned int ndigits = DIV_ROUND_UP_POW2(keylen, sizeof(u64)); struct ecdsa_raw_sig sig; if (slen != 2 * keylen) diff --git a/crypto/ecdsa-x962.c b/crypto/ecdsa-x962.c index 6a77c13e192b..90a04f4b9a2f 100644 --- a/crypto/ecdsa-x962.c +++ b/crypto/ecdsa-x962.c @@ -81,8 +81,8 @@ static int ecdsa_x962_verify(struct crypto_sig *tfm, struct ecdsa_x962_signature_ctx sig_ctx; int err; - sig_ctx.ndigits = DIV_ROUND_UP(crypto_sig_keysize(ctx->child), - sizeof(u64)); + sig_ctx.ndigits = DIV_ROUND_UP_POW2(crypto_sig_keysize(ctx->child), + sizeof(u64)); err = asn1_ber_decoder(&ecdsasignature_decoder, &sig_ctx, src, slen); if (err < 0) diff --git a/include/linux/math.h b/include/linux/math.h index f5f18dc3616b..0198c92cbe3e 100644 --- a/include/linux/math.h +++ b/include/linux/math.h @@ -34,6 +34,18 @@ */ #define round_down(x, y) ((x) & ~__round_mask(x, y)) +/** + * DIV_ROUND_UP_POW2 - divide and round up + * @n: numerator + * @d: denominator (must be a power of 2) + * + * Divides @n by @d and rounds up to next multiple of @d (which must be a power + * of 2). Avoids integer overflows that may occur with __KERNEL_DIV_ROUND_UP(). + * Performance is roughly equivalent to __KERNEL_DIV_ROUND_UP(). + */ +#define DIV_ROUND_UP_POW2(n, d) \ + ((n) / (d) + !!((n) & ((d) - 1))) + #define DIV_ROUND_UP __KERNEL_DIV_ROUND_UP #define DIV_ROUND_DOWN_ULL(ll, d) \ -- cgit v1.2.3 From 77cb2f63ad6c546267b2fcb428cf9fceedef279a Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 10 Feb 2025 11:04:48 +0100 Subject: crypto: ahash - use str_yes_no() helper in crypto_ahash_show() Remove hard-coded strings by using the str_yes_no() helper function. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- crypto/ahash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index b08b89ec26ec..923c68a39ddd 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "hash.h" @@ -536,8 +537,8 @@ static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) { seq_printf(m, "type : ahash\n"); - seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? - "yes" : "no"); + seq_printf(m, "async : %s\n", + str_yes_no(alg->cra_flags & CRYPTO_ALG_ASYNC)); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); seq_printf(m, "digestsize : %u\n", __crypto_hash_alg_common(alg)->digestsize); -- cgit v1.2.3 From 844c683d1f00cd5f950dd09aefd53a522f686bc5 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Tue, 11 Feb 2025 10:52:54 +0100 Subject: crypto: aead - use str_yes_no() helper in crypto_aead_show() Remove hard-coded strings by using the str_yes_no() helper function. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- crypto/aead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/aead.c b/crypto/aead.c index cade532413bf..12f5b42171af 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "internal.h" @@ -156,8 +157,8 @@ static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) struct aead_alg *aead = container_of(alg, struct aead_alg, base); seq_printf(m, "type : aead\n"); - seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? - "yes" : "no"); + seq_printf(m, "async : %s\n", + str_yes_no(alg->cra_flags & CRYPTO_ALG_ASYNC)); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); seq_printf(m, "ivsize : %u\n", aead->ivsize); seq_printf(m, "maxauthsize : %u\n", aead->maxauthsize); -- cgit v1.2.3 From dcc47a028c24e793ce6d6efebfef1a1e92f80297 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 12 Feb 2025 14:10:07 +0800 Subject: crypto: null - Use spin lock instead of mutex As the null algorithm may be freed in softirq context through af_alg, use spin locks instead of mutexes to protect the default null algorithm. Reported-by: syzbot+b3e02953598f447d4d2a@syzkaller.appspotmail.com Signed-off-by: Herbert Xu --- crypto/crypto_null.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) (limited to 'crypto') diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 5b84b0f7cc17..337867028653 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -17,10 +17,10 @@ #include #include #include -#include +#include #include -static DEFINE_MUTEX(crypto_default_null_skcipher_lock); +static DEFINE_SPINLOCK(crypto_default_null_skcipher_lock); static struct crypto_sync_skcipher *crypto_default_null_skcipher; static int crypto_default_null_skcipher_refcnt; @@ -152,23 +152,32 @@ MODULE_ALIAS_CRYPTO("cipher_null"); struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void) { + struct crypto_sync_skcipher *ntfm = NULL; struct crypto_sync_skcipher *tfm; - mutex_lock(&crypto_default_null_skcipher_lock); + spin_lock_bh(&crypto_default_null_skcipher_lock); tfm = crypto_default_null_skcipher; if (!tfm) { - tfm = crypto_alloc_sync_skcipher("ecb(cipher_null)", 0, 0); - if (IS_ERR(tfm)) - goto unlock; - - crypto_default_null_skcipher = tfm; + spin_unlock_bh(&crypto_default_null_skcipher_lock); + + ntfm = crypto_alloc_sync_skcipher("ecb(cipher_null)", 0, 0); + if (IS_ERR(ntfm)) + return ntfm; + + spin_lock_bh(&crypto_default_null_skcipher_lock); + tfm = crypto_default_null_skcipher; + if (!tfm) { + tfm = ntfm; + ntfm = NULL; + crypto_default_null_skcipher = tfm; + } } crypto_default_null_skcipher_refcnt++; + spin_unlock_bh(&crypto_default_null_skcipher_lock); -unlock: - mutex_unlock(&crypto_default_null_skcipher_lock); + crypto_free_sync_skcipher(ntfm); return tfm; } @@ -176,12 +185,16 @@ EXPORT_SYMBOL_GPL(crypto_get_default_null_skcipher); void crypto_put_default_null_skcipher(void) { - mutex_lock(&crypto_default_null_skcipher_lock); + struct crypto_sync_skcipher *tfm = NULL; + + spin_lock_bh(&crypto_default_null_skcipher_lock); if (!--crypto_default_null_skcipher_refcnt) { - crypto_free_sync_skcipher(crypto_default_null_skcipher); + tfm = crypto_default_null_skcipher; crypto_default_null_skcipher = NULL; } - mutex_unlock(&crypto_default_null_skcipher_lock); + spin_unlock_bh(&crypto_default_null_skcipher_lock); + + crypto_free_sync_skcipher(tfm); } EXPORT_SYMBOL_GPL(crypto_put_default_null_skcipher); -- cgit v1.2.3 From 7505436e2925d89a13706a295a6734d6cabb4b43 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 14 Feb 2025 10:31:25 +0800 Subject: crypto: api - Fix larval relookup type and mask When the lookup is retried after instance construction, it uses the type and mask from the larval, which may not match the values used by the caller. For example, if the caller is requesting for a !NEEDS_FALLBACK algorithm, it may end up getting an algorithm that needs fallbacks. Fix this by making the caller supply the type/mask and using that for the lookup. Reported-by: Coiby Xu Fixes: 96ad59552059 ("crypto: api - Remove instance larval fulfilment") Signed-off-by: Herbert Xu --- crypto/api.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'crypto') diff --git a/crypto/api.c b/crypto/api.c index bfd177a4313a..c2c4eb14ef95 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -36,7 +36,8 @@ EXPORT_SYMBOL_GPL(crypto_chain); DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished); #endif -static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg); +static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg, + u32 type, u32 mask); static struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, u32 mask); @@ -145,7 +146,7 @@ static struct crypto_alg *crypto_larval_add(const char *name, u32 type, if (alg != &larval->alg) { kfree(larval); if (crypto_is_larval(alg)) - alg = crypto_larval_wait(alg); + alg = crypto_larval_wait(alg, type, mask); } return alg; @@ -197,7 +198,8 @@ static void crypto_start_test(struct crypto_larval *larval) crypto_schedule_test(larval); } -static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg) +static struct crypto_alg *crypto_larval_wait(struct crypto_alg *alg, + u32 type, u32 mask) { struct crypto_larval *larval; long time_left; @@ -219,12 +221,7 @@ again: crypto_larval_kill(larval); alg = ERR_PTR(-ETIMEDOUT); } else if (!alg) { - u32 type; - u32 mask; - alg = &larval->alg; - type = alg->cra_flags & ~(CRYPTO_ALG_LARVAL | CRYPTO_ALG_DEAD); - mask = larval->mask; alg = crypto_alg_lookup(alg->cra_name, type, mask) ?: ERR_PTR(-EAGAIN); } else if (IS_ERR(alg)) @@ -304,7 +301,7 @@ static struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, } if (!IS_ERR_OR_NULL(alg) && crypto_is_larval(alg)) - alg = crypto_larval_wait(alg); + alg = crypto_larval_wait(alg, type, mask); else if (alg) ; else if (!(mask & CRYPTO_ALG_TESTED)) @@ -352,7 +349,7 @@ struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask) ok = crypto_probing_notify(CRYPTO_MSG_ALG_REQUEST, larval); if (ok == NOTIFY_STOP) - alg = crypto_larval_wait(larval); + alg = crypto_larval_wait(larval, type, mask); else { crypto_mod_put(larval); alg = ERR_PTR(-ENOENT); -- cgit v1.2.3 From ee509efc74ddbc59bb5d6fd6e050f9ef25f74bff Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Feb 2025 08:57:51 +0800 Subject: crypto: skcipher - Zap type in crypto_alloc_sync_skcipher The type needs to be zeroed as otherwise the user could use it to allocate an asynchronous sync skcipher. Signed-off-by: Herbert Xu --- crypto/skcipher.c | 1 + 1 file changed, 1 insertion(+) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index e3751cc88b76..cdf0f11c7eaa 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -682,6 +682,7 @@ struct crypto_sync_skcipher *crypto_alloc_sync_skcipher( /* Only sync algorithms allowed. */ mask |= CRYPTO_ALG_ASYNC | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE; + type &= ~(CRYPTO_ALG_ASYNC | CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE); tfm = crypto_alloc_tfm(alg_name, &crypto_skcipher_type, type, mask); -- cgit v1.2.3 From 075db21426b17609e2374274751b761e35d39664 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Feb 2025 11:07:12 +0800 Subject: crypto: ahash - Only save callback and data in ahash_save_req As unaligned operations are supported by the underlying algorithm, ahash_save_req and ahash_restore_req can be greatly simplified to only preserve the callback and data. Signed-off-by: Herbert Xu --- crypto/ahash.c | 97 +++++++++++++++++++-------------------------------- include/crypto/hash.h | 3 -- 2 files changed, 35 insertions(+), 65 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index 923c68a39ddd..e54e06dd76f5 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -41,6 +41,12 @@ struct crypto_hash_walk { struct scatterlist *sg; }; +struct ahash_save_req_state { + struct ahash_request *req; + crypto_completion_t compl; + void *data; +}; + static int hash_walk_next(struct crypto_hash_walk *walk) { unsigned int offset = walk->offset; @@ -279,67 +285,34 @@ int crypto_ahash_init(struct ahash_request *req) } EXPORT_SYMBOL_GPL(crypto_ahash_init); -static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt, - bool has_state) +static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - unsigned int ds = crypto_ahash_digestsize(tfm); - struct ahash_request *subreq; - unsigned int subreq_size; - unsigned int reqsize; - u8 *result; + struct ahash_save_req_state *state; gfp_t gfp; u32 flags; - subreq_size = sizeof(*subreq); - reqsize = crypto_ahash_reqsize(tfm); - reqsize = ALIGN(reqsize, crypto_tfm_ctx_alignment()); - subreq_size += reqsize; - subreq_size += ds; - flags = ahash_request_flags(req); gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - subreq = kmalloc(subreq_size, gfp); - if (!subreq) + state = kmalloc(sizeof(*state), gfp); + if (!state) return -ENOMEM; - ahash_request_set_tfm(subreq, tfm); - ahash_request_set_callback(subreq, flags, cplt, req); - - result = (u8 *)(subreq + 1) + reqsize; - - ahash_request_set_crypt(subreq, req->src, result, req->nbytes); - - if (has_state) { - void *state; - - state = kmalloc(crypto_ahash_statesize(tfm), gfp); - if (!state) { - kfree(subreq); - return -ENOMEM; - } - - crypto_ahash_export(req, state); - crypto_ahash_import(subreq, state); - kfree_sensitive(state); - } - - req->priv = subreq; + state->compl = req->base.complete; + state->data = req->base.data; + req->base.complete = cplt; + req->base.data = state; + state->req = req; return 0; } -static void ahash_restore_req(struct ahash_request *req, int err) +static void ahash_restore_req(struct ahash_request *req) { - struct ahash_request *subreq = req->priv; - - if (!err) - memcpy(req->result, subreq->result, - crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); + struct ahash_save_req_state *state = req->base.data; - req->priv = NULL; - - kfree_sensitive(subreq); + req->base.complete = state->compl; + req->base.data = state->data; + kfree(state); } int crypto_ahash_update(struct ahash_request *req) @@ -391,51 +364,51 @@ EXPORT_SYMBOL_GPL(crypto_ahash_digest); static void ahash_def_finup_done2(void *data, int err) { - struct ahash_request *areq = data; + struct ahash_save_req_state *state = data; + struct ahash_request *areq = state->req; if (err == -EINPROGRESS) return; - ahash_restore_req(areq, err); - + ahash_restore_req(areq); ahash_request_complete(areq, err); } static int ahash_def_finup_finish1(struct ahash_request *req, int err) { - struct ahash_request *subreq = req->priv; - if (err) goto out; - subreq->base.complete = ahash_def_finup_done2; + req->base.complete = ahash_def_finup_done2; - err = crypto_ahash_alg(crypto_ahash_reqtfm(req))->final(subreq); + err = crypto_ahash_alg(crypto_ahash_reqtfm(req))->final(req); if (err == -EINPROGRESS || err == -EBUSY) return err; out: - ahash_restore_req(req, err); + ahash_restore_req(req); return err; } static void ahash_def_finup_done1(void *data, int err) { - struct ahash_request *areq = data; - struct ahash_request *subreq; + struct ahash_save_req_state *state0 = data; + struct ahash_save_req_state state; + struct ahash_request *areq; + state = *state0; + areq = state.req; if (err == -EINPROGRESS) goto out; - subreq = areq->priv; - subreq->base.flags &= CRYPTO_TFM_REQ_MAY_BACKLOG; + areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = ahash_def_finup_finish1(areq, err); if (err == -EINPROGRESS || err == -EBUSY) return; out: - ahash_request_complete(areq, err); + state.compl(state.data, err); } static int ahash_def_finup(struct ahash_request *req) @@ -443,11 +416,11 @@ static int ahash_def_finup(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); int err; - err = ahash_save_req(req, ahash_def_finup_done1, true); + err = ahash_save_req(req, ahash_def_finup_done1); if (err) return err; - err = crypto_ahash_alg(tfm)->update(req->priv); + err = crypto_ahash_alg(tfm)->update(req); if (err == -EINPROGRESS || err == -EBUSY) return err; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 2d5ea9f9ff43..9c1f8ca59a77 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -55,9 +55,6 @@ struct ahash_request { struct scatterlist *src; u8 *result; - /* This field may only be used by the ahash API code. */ - void *priv; - void *__ctx[] CRYPTO_MINALIGN_ATTR; }; -- cgit v1.2.3 From f2ffe5a9183d22eec718edac03e8bfcedf4dee70 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Feb 2025 11:07:17 +0800 Subject: crypto: hash - Add request chaining API This adds request chaining to the ahash interface. Request chaining allows multiple requests to be submitted in one shot. An algorithm can elect to receive chained requests by setting the flag CRYPTO_ALG_REQ_CHAIN. If this bit is not set, the API will break up chained requests and submit them one-by-one. A new err field is added to struct crypto_async_request to record the return value for each individual request. Signed-off-by: Herbert Xu --- crypto/ahash.c | 261 +++++++++++++++++++++++++++++++++++++---- crypto/algapi.c | 2 +- include/crypto/algapi.h | 11 ++ include/crypto/hash.h | 28 +++-- include/crypto/internal/hash.h | 10 ++ include/linux/crypto.h | 24 ++++ 6 files changed, 299 insertions(+), 37 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index e54e06dd76f5..e6bdb5ae2dac 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -42,11 +42,19 @@ struct crypto_hash_walk { }; struct ahash_save_req_state { - struct ahash_request *req; + struct list_head head; + struct ahash_request *req0; + struct ahash_request *cur; + int (*op)(struct ahash_request *req); crypto_completion_t compl; void *data; }; +static void ahash_reqchain_done(void *data, int err); +static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); +static void ahash_restore_req(struct ahash_request *req); +static int ahash_def_finup(struct ahash_request *req); + static int hash_walk_next(struct crypto_hash_walk *walk) { unsigned int offset = walk->offset; @@ -273,24 +281,145 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); +static int ahash_reqchain_finish(struct ahash_save_req_state *state, + int err, u32 mask) +{ + struct ahash_request *req0 = state->req0; + struct ahash_request *req = state->cur; + struct ahash_request *n; + + req->base.err = err; + + if (req != req0) + list_add_tail(&req->base.list, &req0->base.list); + + list_for_each_entry_safe(req, n, &state->head, base.list) { + list_del_init(&req->base.list); + + req->base.flags &= mask; + req->base.complete = ahash_reqchain_done; + req->base.data = state; + state->cur = req; + err = state->op(req); + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + err = -EBUSY; + goto out; + } + + if (err == -EBUSY) + goto out; + + req->base.err = err; + list_add_tail(&req->base.list, &req0->base.list); + } + + ahash_restore_req(req0); + +out: + return err; +} + +static void ahash_reqchain_done(void *data, int err) +{ + struct ahash_save_req_state *state = data; + crypto_completion_t compl = state->compl; + + data = state->data; + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + return; + goto notify; + } + + err = ahash_reqchain_finish(state, err, CRYPTO_TFM_REQ_MAY_BACKLOG); + if (err == -EBUSY) + return; + +notify: + compl(data, err); +} + +static int ahash_do_req_chain(struct ahash_request *req, + int (*op)(struct ahash_request *req)) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ahash_save_req_state *state; + struct ahash_save_req_state state0; + int err; + + if (!ahash_request_chained(req) || crypto_ahash_req_chain(tfm)) + return op(req); + + state = &state0; + + if (ahash_is_async(tfm)) { + err = ahash_save_req(req, ahash_reqchain_done); + if (err) { + struct ahash_request *r2; + + req->base.err = err; + list_for_each_entry(r2, &req->base.list, base.list) + r2->base.err = err; + + return err; + } + + state = req->base.data; + } + + state->op = op; + state->cur = req; + INIT_LIST_HEAD(&state->head); + list_splice_init(&req->base.list, &state->head); + + err = op(req); + if (err == -EBUSY || err == -EINPROGRESS) + return -EBUSY; + + return ahash_reqchain_finish(state, err, ~0); +} + int crypto_ahash_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) - return crypto_shash_init(prepare_shash_desc(req, tfm)); + if (likely(tfm->using_shash)) { + struct ahash_request *r2; + int err; + + err = crypto_shash_init(prepare_shash_desc(req, tfm)); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + struct shash_desc *desc; + + desc = prepare_shash_desc(r2, tfm); + r2->base.err = crypto_shash_init(desc); + } + + return err; + } + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_ahash_alg(tfm)->init(req); + + return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); } EXPORT_SYMBOL_GPL(crypto_ahash_init); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct ahash_save_req_state *state; gfp_t gfp; u32 flags; + if (!ahash_is_async(tfm)) + return 0; + flags = ahash_request_flags(req); gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; state = kmalloc(sizeof(*state), gfp); @@ -301,14 +430,20 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) state->data = req->base.data; req->base.complete = cplt; req->base.data = state; - state->req = req; + state->req0 = req; return 0; } static void ahash_restore_req(struct ahash_request *req) { - struct ahash_save_req_state *state = req->base.data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ahash_save_req_state *state; + + if (!ahash_is_async(tfm)) + return; + + state = req->base.data; req->base.complete = state->compl; req->base.data = state->data; @@ -319,10 +454,24 @@ int crypto_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) - return shash_ahash_update(req, ahash_request_ctx(req)); + if (likely(tfm->using_shash)) { + struct ahash_request *r2; + int err; + + err = shash_ahash_update(req, ahash_request_ctx(req)); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + struct shash_desc *desc; - return crypto_ahash_alg(tfm)->update(req); + desc = ahash_request_ctx(r2); + r2->base.err = shash_ahash_update(r2, desc); + } + + return err; + } + + return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); } EXPORT_SYMBOL_GPL(crypto_ahash_update); @@ -330,10 +479,24 @@ int crypto_ahash_final(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) - return crypto_shash_final(ahash_request_ctx(req), req->result); + if (likely(tfm->using_shash)) { + struct ahash_request *r2; + int err; + + err = crypto_shash_final(ahash_request_ctx(req), req->result); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + struct shash_desc *desc; - return crypto_ahash_alg(tfm)->final(req); + desc = ahash_request_ctx(r2); + r2->base.err = crypto_shash_final(desc, r2->result); + } + + return err; + } + + return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); } EXPORT_SYMBOL_GPL(crypto_ahash_final); @@ -341,10 +504,27 @@ int crypto_ahash_finup(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) - return shash_ahash_finup(req, ahash_request_ctx(req)); + if (likely(tfm->using_shash)) { + struct ahash_request *r2; + int err; + + err = shash_ahash_finup(req, ahash_request_ctx(req)); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + struct shash_desc *desc; + + desc = ahash_request_ctx(r2); + r2->base.err = shash_ahash_finup(r2, desc); + } + + return err; + } - return crypto_ahash_alg(tfm)->finup(req); + if (!crypto_ahash_alg(tfm)->finup) + return ahash_def_finup(req); + + return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); @@ -352,20 +532,34 @@ int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) - return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + if (likely(tfm->using_shash)) { + struct ahash_request *r2; + int err; + + err = shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) { + struct shash_desc *desc; + + desc = prepare_shash_desc(r2, tfm); + r2->base.err = shash_ahash_digest(r2, desc); + } + + return err; + } if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return crypto_ahash_alg(tfm)->digest(req); + return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); static void ahash_def_finup_done2(void *data, int err) { struct ahash_save_req_state *state = data; - struct ahash_request *areq = state->req; + struct ahash_request *areq = state->req0; if (err == -EINPROGRESS) return; @@ -376,12 +570,15 @@ static void ahash_def_finup_done2(void *data, int err) static int ahash_def_finup_finish1(struct ahash_request *req, int err) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + if (err) goto out; - req->base.complete = ahash_def_finup_done2; + if (ahash_is_async(tfm)) + req->base.complete = ahash_def_finup_done2; - err = crypto_ahash_alg(crypto_ahash_reqtfm(req))->final(req); + err = crypto_ahash_final(req); if (err == -EINPROGRESS || err == -EBUSY) return err; @@ -397,7 +594,7 @@ static void ahash_def_finup_done1(void *data, int err) struct ahash_request *areq; state = *state0; - areq = state.req; + areq = state.req0; if (err == -EINPROGRESS) goto out; @@ -413,14 +610,13 @@ out: static int ahash_def_finup(struct ahash_request *req) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); int err; err = ahash_save_req(req, ahash_def_finup_done1); if (err) return err; - err = crypto_ahash_alg(tfm)->update(req); + err = crypto_ahash_update(req); if (err == -EINPROGRESS || err == -EBUSY) return err; @@ -635,8 +831,6 @@ static int ahash_prepare_alg(struct ahash_alg *alg) base->cra_type = &crypto_ahash_type; base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; - if (!alg->finup) - alg->finup = ahash_def_finup; if (!alg->setkey) alg->setkey = ahash_nosetkey; @@ -707,5 +901,20 @@ int ahash_register_instance(struct crypto_template *tmpl, } EXPORT_SYMBOL_GPL(ahash_register_instance); +void ahash_request_free(struct ahash_request *req) +{ + struct ahash_request *tmp; + struct ahash_request *r2; + + if (unlikely(!req)) + return; + + list_for_each_entry_safe(r2, tmp, &req->base.list, base.list) + kfree_sensitive(r2); + + kfree_sensitive(req); +} +EXPORT_SYMBOL_GPL(ahash_request_free); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/algapi.c b/crypto/algapi.c index 5318c214debb..e7a9a2ada2cf 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -955,7 +955,7 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) queue->backlog = queue->backlog->next; request = queue->list.next; - list_del(request); + list_del_init(request); return list_entry(request, struct crypto_async_request, list); } diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 156de41ca760..11065978d360 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -271,4 +272,14 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } +static inline bool crypto_request_chained(struct crypto_async_request *req) +{ + return !list_empty(&req->list); +} + +static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) +{ + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; +} + #endif /* _CRYPTO_ALGAPI_H */ diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 9c1f8ca59a77..0a6f744ce4a1 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -572,16 +572,7 @@ static inline struct ahash_request *ahash_request_alloc_noprof( * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ -static inline void ahash_request_free(struct ahash_request *req) -{ - kfree_sensitive(req); -} - -static inline void ahash_request_zero(struct ahash_request *req) -{ - memzero_explicit(req, sizeof(*req) + - crypto_ahash_reqsize(crypto_ahash_reqtfm(req))); -} +void ahash_request_free(struct ahash_request *req); static inline struct ahash_request *ahash_request_cast( struct crypto_async_request *req) @@ -622,6 +613,7 @@ static inline void ahash_request_set_callback(struct ahash_request *req, req->base.complete = compl; req->base.data = data; req->base.flags = flags; + crypto_reqchain_init(&req->base); } /** @@ -646,6 +638,12 @@ static inline void ahash_request_set_crypt(struct ahash_request *req, req->result = result; } +static inline void ahash_request_chain(struct ahash_request *req, + struct ahash_request *head) +{ + crypto_request_chain(&req->base, &head->base); +} + /** * DOC: Synchronous Message Digest API * @@ -947,4 +945,14 @@ static inline void shash_desc_zero(struct shash_desc *desc) sizeof(*desc) + crypto_shash_descsize(desc->tfm)); } +static inline int ahash_request_err(struct ahash_request *req) +{ + return req->base.err; +} + +static inline bool ahash_is_async(struct crypto_ahash *tfm) +{ + return crypto_tfm_is_async(&tfm->base); +} + #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 84da3424decc..36425ecd2c37 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -247,5 +247,15 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) return container_of(tfm, struct crypto_shash, base); } +static inline bool ahash_request_chained(struct ahash_request *req) +{ + return crypto_request_chained(&req->base); +} + +static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) +{ + return crypto_tfm_req_chain(&tfm->base); +} + #endif /* _CRYPTO_INTERNAL_HASH_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index b164da5e129e..1d2a6c515d58 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -13,6 +13,8 @@ #define _LINUX_CRYPTO_H #include +#include +#include #include #include #include @@ -124,6 +126,9 @@ */ #define CRYPTO_ALG_FIPS_INTERNAL 0x00020000 +/* Set if the algorithm supports request chains. */ +#define CRYPTO_ALG_REQ_CHAIN 0x00040000 + /* * Transform masks and values (for crt_flags). */ @@ -174,6 +179,7 @@ struct crypto_async_request { struct crypto_tfm *tfm; u32 flags; + int err; }; /** @@ -540,5 +546,23 @@ int crypto_comp_decompress(struct crypto_comp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); +static inline void crypto_reqchain_init(struct crypto_async_request *req) +{ + req->err = -EINPROGRESS; + INIT_LIST_HEAD(&req->list); +} + +static inline void crypto_request_chain(struct crypto_async_request *req, + struct crypto_async_request *head) +{ + req->err = -EINPROGRESS; + list_add_tail(&req->list, &head->list); +} + +static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) +{ + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; +} + #endif /* _LINUX_CRYPTO_H */ -- cgit v1.2.3 From c664f034172705a75f3f8a0c409b9bf95b633093 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Feb 2025 11:07:19 +0800 Subject: crypto: tcrypt - Restore multibuffer ahash tests This patch is a revert of commit 388ac25efc8ce3bf9768ce7bf24268d6fac285d5. As multibuffer ahash is coming back in the form of request chaining, restore the multibuffer ahash tests using the new interface. Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 231 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) (limited to 'crypto') diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index e1a74cb2cfbe..f618f61c5615 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -716,6 +716,207 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) return crypto_wait_req(ret, wait); } +struct test_mb_ahash_data { + struct scatterlist sg[XBUFSIZE]; + char result[64]; + struct ahash_request *req; + struct crypto_wait wait; + char *xbuf[XBUFSIZE]; +}; + +static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb, + int *rc) +{ + int i, err; + + /* Fire up a bunch of concurrent requests */ + err = crypto_ahash_digest(data[0].req); + + /* Wait for all requests to finish */ + err = crypto_wait_req(err, &data[0].wait); + if (num_mb < 2) + return err; + + for (i = 0; i < num_mb; i++) { + rc[i] = ahash_request_err(data[i].req); + if (rc[i]) { + pr_info("concurrent request %d error %d\n", i, rc[i]); + err = rc[i]; + } + } + + return err; +} + +static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, + int secs, u32 num_mb) +{ + unsigned long start, end; + int bcount; + int ret = 0; + int *rc; + + rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); + if (!rc) + return -ENOMEM; + + for (start = jiffies, end = start + secs * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = do_mult_ahash_op(data, num_mb, rc); + if (ret) + goto out; + } + + pr_cont("%d operations in %d seconds (%llu bytes)\n", + bcount * num_mb, secs, (u64)bcount * blen * num_mb); + +out: + kfree(rc); + return ret; +} + +static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen, + u32 num_mb) +{ + unsigned long cycles = 0; + int ret = 0; + int i; + int *rc; + + rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); + if (!rc) + return -ENOMEM; + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = do_mult_ahash_op(data, num_mb, rc); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + ret = do_mult_ahash_op(data, num_mb, rc); + end = get_cycles(); + + if (ret) + goto out; + + cycles += end - start; + } + + pr_cont("1 operation in %lu cycles (%d bytes)\n", + (cycles + 4) / (8 * num_mb), blen); + +out: + kfree(rc); + return ret; +} + +static void test_mb_ahash_speed(const char *algo, unsigned int secs, + struct hash_speed *speed, u32 num_mb) +{ + struct test_mb_ahash_data *data; + struct crypto_ahash *tfm; + unsigned int i, j, k; + int ret; + + data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); + if (!data) + return; + + tfm = crypto_alloc_ahash(algo, 0, 0); + if (IS_ERR(tfm)) { + pr_err("failed to load transform for %s: %ld\n", + algo, PTR_ERR(tfm)); + goto free_data; + } + + for (i = 0; i < num_mb; ++i) { + if (testmgr_alloc_buf(data[i].xbuf)) + goto out; + + crypto_init_wait(&data[i].wait); + + data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!data[i].req) { + pr_err("alg: hash: Failed to allocate request for %s\n", + algo); + goto out; + } + + + if (i) { + ahash_request_set_callback(data[i].req, 0, NULL, NULL); + ahash_request_chain(data[i].req, data[0].req); + } else + ahash_request_set_callback(data[0].req, 0, + crypto_req_done, + &data[0].wait); + + sg_init_table(data[i].sg, XBUFSIZE); + for (j = 0; j < XBUFSIZE; j++) { + sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE); + memset(data[i].xbuf[j], 0xff, PAGE_SIZE); + } + } + + pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, + get_driver_name(crypto_ahash, tfm)); + + for (i = 0; speed[i].blen != 0; i++) { + /* For some reason this only tests digests. */ + if (speed[i].blen != speed[i].plen) + continue; + + if (speed[i].blen > XBUFSIZE * PAGE_SIZE) { + pr_err("template (%u) too big for tvmem (%lu)\n", + speed[i].blen, XBUFSIZE * PAGE_SIZE); + goto out; + } + + if (klen) + crypto_ahash_setkey(tfm, tvmem[0], klen); + + for (k = 0; k < num_mb; k++) + ahash_request_set_crypt(data[k].req, data[k].sg, + data[k].result, speed[i].blen); + + pr_info("test%3u " + "(%5u byte blocks,%5u bytes per update,%4u updates): ", + i, speed[i].blen, speed[i].plen, + speed[i].blen / speed[i].plen); + + if (secs) { + ret = test_mb_ahash_jiffies(data, speed[i].blen, secs, + num_mb); + cond_resched(); + } else { + ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb); + } + + + if (ret) { + pr_err("At least one hashing failed ret=%d\n", ret); + break; + } + } + +out: + ahash_request_free(data[0].req); + + for (k = 0; k < num_mb; ++k) + testmgr_free_buf(data[k].xbuf); + + crypto_free_ahash(tfm); + +free_data: + kfree(data); +} + static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, char *out, int secs) { @@ -2391,6 +2592,36 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("sm3", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; + case 450: + test_mb_ahash_speed("sha1", sec, generic_hash_speed_template, + num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; + case 451: + test_mb_ahash_speed("sha256", sec, generic_hash_speed_template, + num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; + case 452: + test_mb_ahash_speed("sha512", sec, generic_hash_speed_template, + num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; + case 453: + test_mb_ahash_speed("sm3", sec, generic_hash_speed_template, + num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; + case 454: + test_mb_ahash_speed("streebog256", sec, + generic_hash_speed_template, num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; + case 455: + test_mb_ahash_speed("streebog512", sec, + generic_hash_speed_template, num_mb); + if (mode > 400 && mode < 500) break; + fallthrough; case 499: break; -- cgit v1.2.3 From 439963cdc3aa447dfd3b5abc05fcd25cef4d22dc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Feb 2025 11:07:22 +0800 Subject: crypto: ahash - Add virtual address support This patch adds virtual address support to ahash. Virtual addresses were previously only supported through shash. The user may choose to use virtual addresses with ahash by calling ahash_request_set_virt instead of ahash_request_set_crypt. The API will take care of translating this to an SG list if necessary, unless the algorithm declares that it supports chaining. Therefore in order for an ahash algorithm to support chaining, it must also support virtual addresses directly. Signed-off-by: Herbert Xu --- crypto/ahash.c | 282 ++++++++++++++++++++++++++++++++++++----- include/crypto/hash.h | 38 +++++- include/crypto/internal/hash.h | 5 + include/linux/crypto.h | 2 +- 4 files changed, 293 insertions(+), 34 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index e6bdb5ae2dac..208aa4c8d725 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -29,7 +29,7 @@ #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e struct crypto_hash_walk { - char *data; + const char *data; unsigned int offset; unsigned int flags; @@ -48,11 +48,17 @@ struct ahash_save_req_state { int (*op)(struct ahash_request *req); crypto_completion_t compl; void *data; + struct scatterlist sg; + const u8 *src; + u8 *page; + unsigned int offset; + unsigned int nbytes; }; static void ahash_reqchain_done(void *data, int err); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); -static void ahash_restore_req(struct ahash_request *req); +static void ahash_restore_req(struct ahash_save_req_state *state); +static void ahash_def_finup_done1(void *data, int err); static int ahash_def_finup(struct ahash_request *req); static int hash_walk_next(struct crypto_hash_walk *walk) @@ -88,20 +94,29 @@ static int crypto_hash_walk_first(struct ahash_request *req, struct crypto_hash_walk *walk) { walk->total = req->nbytes; + walk->entrylen = 0; - if (!walk->total) { - walk->entrylen = 0; + if (!walk->total) return 0; + + walk->flags = req->base.flags; + + if (ahash_request_isvirt(req)) { + walk->data = req->svirt; + walk->total = 0; + return req->nbytes; } walk->sg = req->src; - walk->flags = req->base.flags; return hash_walk_new_entry(walk); } static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) { + if ((walk->flags & CRYPTO_AHASH_REQ_VIRT)) + return err; + walk->data -= walk->offset; kunmap_local(walk->data); @@ -188,6 +203,10 @@ int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) unsigned int offset; int err; + if (ahash_request_isvirt(req)) + return crypto_shash_digest(desc, req->svirt, nbytes, + req->result); + if (nbytes && (sg = req->src, offset = sg->offset, nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { @@ -281,18 +300,82 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); +static bool ahash_request_hasvirt(struct ahash_request *req) +{ + struct ahash_request *r2; + + if (ahash_request_isvirt(req)) + return true; + + list_for_each_entry(r2, &req->base.list, base.list) + if (ahash_request_isvirt(r2)) + return true; + + return false; +} + +static int ahash_reqchain_virt(struct ahash_save_req_state *state, + int err, u32 mask) +{ + struct ahash_request *req = state->cur; + + for (;;) { + unsigned len = state->nbytes; + + req->base.err = err; + + if (!state->offset) + break; + + if (state->offset == len || err) { + u8 *result = req->result; + + ahash_request_set_virt(req, state->src, result, len); + state->offset = 0; + break; + } + + len -= state->offset; + + len = min(PAGE_SIZE, len); + memcpy(state->page, state->src + state->offset, len); + state->offset += len; + req->nbytes = len; + + err = state->op(req); + if (err == -EINPROGRESS) { + if (!list_empty(&state->head) || + state->offset < state->nbytes) + err = -EBUSY; + break; + } + + if (err == -EBUSY) + break; + } + + return err; +} + static int ahash_reqchain_finish(struct ahash_save_req_state *state, int err, u32 mask) { struct ahash_request *req0 = state->req0; struct ahash_request *req = state->cur; + struct crypto_ahash *tfm; struct ahash_request *n; + bool update; - req->base.err = err; + err = ahash_reqchain_virt(state, err, mask); + if (err == -EINPROGRESS || err == -EBUSY) + goto out; if (req != req0) list_add_tail(&req->base.list, &req0->base.list); + tfm = crypto_ahash_reqtfm(req); + update = state->op == crypto_ahash_alg(tfm)->update; + list_for_each_entry_safe(req, n, &state->head, base.list) { list_del_init(&req->base.list); @@ -300,10 +383,27 @@ static int ahash_reqchain_finish(struct ahash_save_req_state *state, req->base.complete = ahash_reqchain_done; req->base.data = state; state->cur = req; + + if (update && ahash_request_isvirt(req) && req->nbytes) { + unsigned len = req->nbytes; + u8 *result = req->result; + + state->src = req->svirt; + state->nbytes = len; + + len = min(PAGE_SIZE, len); + + memcpy(state->page, req->svirt, len); + state->offset = len; + + ahash_request_set_crypt(req, &state->sg, result, len); + } + err = state->op(req); if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) + if (!list_empty(&state->head) || + state->offset < state->nbytes) err = -EBUSY; goto out; } @@ -311,11 +411,14 @@ static int ahash_reqchain_finish(struct ahash_save_req_state *state, if (err == -EBUSY) goto out; - req->base.err = err; + err = ahash_reqchain_virt(state, err, mask); + if (err == -EINPROGRESS || err == -EBUSY) + goto out; + list_add_tail(&req->base.list, &req0->base.list); } - ahash_restore_req(req0); + ahash_restore_req(state); out: return err; @@ -329,7 +432,7 @@ static void ahash_reqchain_done(void *data, int err) data = state->data; if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) + if (!list_empty(&state->head) || state->offset < state->nbytes) return; goto notify; } @@ -346,40 +449,84 @@ static int ahash_do_req_chain(struct ahash_request *req, int (*op)(struct ahash_request *req)) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + bool update = op == crypto_ahash_alg(tfm)->update; struct ahash_save_req_state *state; struct ahash_save_req_state state0; + struct ahash_request *r2; + u8 *page = NULL; int err; - if (!ahash_request_chained(req) || crypto_ahash_req_chain(tfm)) + if (crypto_ahash_req_chain(tfm) || + (!ahash_request_chained(req) && + (!update || !ahash_request_isvirt(req)))) return op(req); - state = &state0; + if (update && ahash_request_hasvirt(req)) { + gfp_t gfp; + u32 flags; + + flags = ahash_request_flags(req); + gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + page = (void *)__get_free_page(gfp); + err = -ENOMEM; + if (!page) + goto out_set_chain; + } + state = &state0; if (ahash_is_async(tfm)) { err = ahash_save_req(req, ahash_reqchain_done); - if (err) { - struct ahash_request *r2; - - req->base.err = err; - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = err; - - return err; - } + if (err) + goto out_free_page; state = req->base.data; } state->op = op; state->cur = req; + state->page = page; + state->offset = 0; + state->nbytes = 0; INIT_LIST_HEAD(&state->head); list_splice_init(&req->base.list, &state->head); + if (page) + sg_init_one(&state->sg, page, PAGE_SIZE); + + if (update && ahash_request_isvirt(req) && req->nbytes) { + unsigned len = req->nbytes; + u8 *result = req->result; + + state->src = req->svirt; + state->nbytes = len; + + len = min(PAGE_SIZE, len); + + memcpy(page, req->svirt, len); + state->offset = len; + + ahash_request_set_crypt(req, &state->sg, result, len); + } + err = op(req); if (err == -EBUSY || err == -EINPROGRESS) return -EBUSY; return ahash_reqchain_finish(state, err, ~0); + +out_free_page: + if (page) { + memset(page, 0, PAGE_SIZE); + free_page((unsigned long)page); + } + +out_set_chain: + req->base.err = err; + list_for_each_entry(r2, &req->base.list, base.list) + r2->base.err = err; + + return err; } int crypto_ahash_init(struct ahash_request *req) @@ -431,15 +578,19 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) req->base.complete = cplt; req->base.data = state; state->req0 = req; + state->page = NULL; return 0; } -static void ahash_restore_req(struct ahash_request *req) +static void ahash_restore_req(struct ahash_save_req_state *state) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_save_req_state *state; + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; + free_page((unsigned long)state->page); + + tfm = crypto_ahash_reqtfm(req); if (!ahash_is_async(tfm)) return; @@ -521,13 +672,74 @@ int crypto_ahash_finup(struct ahash_request *req) return err; } - if (!crypto_ahash_alg(tfm)->finup) + if (!crypto_ahash_alg(tfm)->finup || + (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req))) return ahash_def_finup(req); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); +static int ahash_def_digest_finish(struct ahash_save_req_state *state, int err) +{ + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; + + if (err) + goto out; + + tfm = crypto_ahash_reqtfm(req); + if (ahash_is_async(tfm)) + req->base.complete = ahash_def_finup_done1; + + err = crypto_ahash_update(req); + if (err == -EINPROGRESS || err == -EBUSY) + return err; + +out: + ahash_restore_req(state); + return err; +} + +static void ahash_def_digest_done(void *data, int err) +{ + struct ahash_save_req_state *state0 = data; + struct ahash_save_req_state state; + struct ahash_request *areq; + + state = *state0; + areq = state.req0; + if (err == -EINPROGRESS) + goto out; + + areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + err = ahash_def_digest_finish(state0, err); + if (err == -EINPROGRESS || err == -EBUSY) + return; + +out: + state.compl(state.data, err); +} + +static int ahash_def_digest(struct ahash_request *req) +{ + struct ahash_save_req_state *state; + int err; + + err = ahash_save_req(req, ahash_def_digest_done); + if (err) + return err; + + state = req->base.data; + + err = crypto_ahash_init(req); + if (err == -EINPROGRESS || err == -EBUSY) + return err; + + return ahash_def_digest_finish(state, err); +} + int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); @@ -549,6 +761,9 @@ int crypto_ahash_digest(struct ahash_request *req) return err; } + if (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req)) + return ahash_def_digest(req); + if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; @@ -564,17 +779,19 @@ static void ahash_def_finup_done2(void *data, int err) if (err == -EINPROGRESS) return; - ahash_restore_req(areq); + ahash_restore_req(state); ahash_request_complete(areq, err); } -static int ahash_def_finup_finish1(struct ahash_request *req, int err) +static int ahash_def_finup_finish1(struct ahash_save_req_state *state, int err) { - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; if (err) goto out; + tfm = crypto_ahash_reqtfm(req); if (ahash_is_async(tfm)) req->base.complete = ahash_def_finup_done2; @@ -583,7 +800,7 @@ static int ahash_def_finup_finish1(struct ahash_request *req, int err) return err; out: - ahash_restore_req(req); + ahash_restore_req(state); return err; } @@ -600,7 +817,7 @@ static void ahash_def_finup_done1(void *data, int err) areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - err = ahash_def_finup_finish1(areq, err); + err = ahash_def_finup_finish1(state0, err); if (err == -EINPROGRESS || err == -EBUSY) return; @@ -610,17 +827,20 @@ out: static int ahash_def_finup(struct ahash_request *req) { + struct ahash_save_req_state *state; int err; err = ahash_save_req(req, ahash_def_finup_done1); if (err) return err; + state = req->base.data; + err = crypto_ahash_update(req); if (err == -EINPROGRESS || err == -EBUSY) return err; - return ahash_def_finup_finish1(req, err); + return ahash_def_finup_finish1(state, err); } int crypto_ahash_export(struct ahash_request *req, void *out) diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 0a6f744ce4a1..4e87e39679cb 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -12,6 +12,9 @@ #include #include +/* Set this bit for virtual address instead of SG list. */ +#define CRYPTO_AHASH_REQ_VIRT 0x00000001 + struct crypto_ahash; /** @@ -52,7 +55,10 @@ struct ahash_request { struct crypto_async_request base; unsigned int nbytes; - struct scatterlist *src; + union { + struct scatterlist *src; + const u8 *svirt; + }; u8 *result; void *__ctx[] CRYPTO_MINALIGN_ATTR; @@ -610,9 +616,13 @@ static inline void ahash_request_set_callback(struct ahash_request *req, crypto_completion_t compl, void *data) { + u32 keep = CRYPTO_AHASH_REQ_VIRT; + req->base.complete = compl; req->base.data = data; - req->base.flags = flags; + flags &= ~keep; + req->base.flags &= keep; + req->base.flags |= flags; crypto_reqchain_init(&req->base); } @@ -636,6 +646,30 @@ static inline void ahash_request_set_crypt(struct ahash_request *req, req->src = src; req->nbytes = nbytes; req->result = result; + req->base.flags &= ~CRYPTO_AHASH_REQ_VIRT; +} + +/** + * ahash_request_set_virt() - set virtual address data buffers + * @req: ahash_request handle to be updated + * @src: source virtual address + * @result: buffer that is filled with the message digest -- the caller must + * ensure that the buffer has sufficient space by, for example, calling + * crypto_ahash_digestsize() + * @nbytes: number of bytes to process from the source virtual address + * + * By using this call, the caller references the source virtual address. + * The source virtual address points to the data the message digest is to + * be calculated for. + */ +static inline void ahash_request_set_virt(struct ahash_request *req, + const u8 *src, u8 *result, + unsigned int nbytes) +{ + req->svirt = src; + req->nbytes = nbytes; + req->result = result; + req->base.flags |= CRYPTO_AHASH_REQ_VIRT; } static inline void ahash_request_chain(struct ahash_request *req, diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 36425ecd2c37..485e22cf517e 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -252,6 +252,11 @@ static inline bool ahash_request_chained(struct ahash_request *req) return crypto_request_chained(&req->base); } +static inline bool ahash_request_isvirt(struct ahash_request *req) +{ + return req->base.flags & CRYPTO_AHASH_REQ_VIRT; +} + static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm) { return crypto_tfm_req_chain(&tfm->base); diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 1d2a6c515d58..61ac11226638 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -126,7 +126,7 @@ */ #define CRYPTO_ALG_FIPS_INTERNAL 0x00020000 -/* Set if the algorithm supports request chains. */ +/* Set if the algorithm supports request chains and virtual addresses. */ #define CRYPTO_ALG_REQ_CHAIN 0x00040000 /* -- cgit v1.2.3 From 9e01aaa1033d6e40f8d7cf4f20931a61ce9e3f04 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Feb 2025 11:07:24 +0800 Subject: crypto: ahash - Set default reqsize from ahash_alg Add a reqsize field to struct ahash_alg and use it to set the default reqsize so that algorithms with a static reqsize are not forced to create an init_tfm function. Signed-off-by: Herbert Xu --- crypto/ahash.c | 4 ++++ include/crypto/hash.h | 3 +++ 2 files changed, 7 insertions(+) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index 208aa4c8d725..9c26175c21a8 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -879,6 +879,7 @@ static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) struct ahash_alg *alg = crypto_ahash_alg(hash); crypto_ahash_set_statesize(hash, alg->halg.statesize); + crypto_ahash_set_reqsize(hash, alg->reqsize); if (tfm->__crt_alg->cra_type == &crypto_shash_type) return crypto_init_ahash_using_shash(tfm); @@ -1044,6 +1045,9 @@ static int ahash_prepare_alg(struct ahash_alg *alg) if (alg->halg.statesize == 0) return -EINVAL; + if (alg->reqsize && alg->reqsize < alg->halg.statesize) + return -EINVAL; + err = hash_prepare_alg(&alg->halg); if (err) return err; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 4e87e39679cb..2aa83ee0ec98 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -135,6 +135,7 @@ struct ahash_request { * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. + * @reqsize: Size of the request context. * @halg: see struct hash_alg_common */ struct ahash_alg { @@ -151,6 +152,8 @@ struct ahash_alg { void (*exit_tfm)(struct crypto_ahash *tfm); int (*clone_tfm)(struct crypto_ahash *dst, struct crypto_ahash *src); + unsigned int reqsize; + struct hash_alg_common halg; }; -- cgit v1.2.3 From f307c87ea06c64b87fcd3221a682cd713cde51e9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 17 Feb 2025 13:55:55 +0100 Subject: crypto: bpf - Add MODULE_DESCRIPTION for skcipher All modules should have a description, building with extra warnings enabled prints this outfor the for bpf_crypto_skcipher module: WARNING: modpost: missing MODULE_DESCRIPTION() in crypto/bpf_crypto_skcipher.o Add a description line. Fixes: fda4f71282b2 ("bpf: crypto: add skcipher to bpf crypto") Signed-off-by: Arnd Bergmann Reviewed-by: Vadim Fedorenko Signed-off-by: Herbert Xu --- crypto/bpf_crypto_skcipher.c | 1 + 1 file changed, 1 insertion(+) (limited to 'crypto') diff --git a/crypto/bpf_crypto_skcipher.c b/crypto/bpf_crypto_skcipher.c index b5e657415770..a88798d3e8c8 100644 --- a/crypto/bpf_crypto_skcipher.c +++ b/crypto/bpf_crypto_skcipher.c @@ -80,3 +80,4 @@ static void __exit bpf_crypto_skcipher_exit(void) module_init(bpf_crypto_skcipher_init); module_exit(bpf_crypto_skcipher_exit); MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Symmetric key cipher support for BPF"); -- cgit v1.2.3 From e21d01a2a3f56ee422cd155bf06c5e572523fcc1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:24 -0800 Subject: crypto: scatterwalk - add new functions for skipping data Add scatterwalk_skip() to skip the given number of bytes in a scatter_walk. Previously support for skipping was provided through scatterwalk_copychunks(..., 2) followed by scatterwalk_done(), which was confusing and less efficient. Also add scatterwalk_start_at_pos() which starts a scatter_walk at the given position, equivalent to scatterwalk_start() + scatterwalk_skip(). This addresses another common need in a more streamlined way. Later patches will convert various users to use these functions. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 15 +++++++++++++++ include/crypto/scatterwalk.h | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) (limited to 'crypto') diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 16f6ba896fb6..af436ad02e3f 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -15,6 +15,21 @@ #include #include +void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) +{ + struct scatterlist *sg = walk->sg; + + nbytes += walk->offset - sg->offset; + + while (nbytes > sg->length) { + nbytes -= sg->length; + sg = sg_next(sg); + } + walk->sg = sg; + walk->offset = sg->offset + nbytes; +} +EXPORT_SYMBOL_GPL(scatterwalk_skip); + static inline void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) { void *src = out ? buf : sgdata; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 924efbaefe67..5c7765f601e0 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -33,6 +33,22 @@ static inline void scatterwalk_start(struct scatter_walk *walk, walk->offset = sg->offset; } +/* + * This is equivalent to scatterwalk_start(walk, sg) followed by + * scatterwalk_skip(walk, pos). + */ +static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, + struct scatterlist *sg, + unsigned int pos) +{ + while (pos > sg->length) { + pos -= sg->length; + sg = sg_next(sg); + } + walk->sg = sg; + walk->offset = sg->offset + pos; +} + static inline unsigned int scatterwalk_pagelen(struct scatter_walk *walk) { unsigned int len = walk->sg->offset + walk->sg->length - walk->offset; @@ -92,6 +108,8 @@ static inline void scatterwalk_done(struct scatter_walk *walk, int out, scatterwalk_pagedone(walk, out, more); } +void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes); + void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out); -- cgit v1.2.3 From bb699e724f3a6cc5c016dad0724e7ed12bc7278b Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:26 -0800 Subject: crypto: scatterwalk - add new functions for copying data Add memcpy_from_sglist() and memcpy_to_sglist() which are more readable versions of scatterwalk_map_and_copy() with the 'out' argument 0 and 1 respectively. They follow the same argument order as memcpy_from_page() and memcpy_to_page() from . Note that in the case of memcpy_from_sglist(), this also happens to be the same argument order that scatterwalk_map_and_copy() uses. The new code is also faster, mainly because it builds the scatter_walk directly without creating a temporary scatterlist. E.g., a 20% performance improvement is seen for copying the AES-GCM auth tag. Make scatterwalk_map_and_copy() be a wrapper around memcpy_from_sglist() and memcpy_to_sglist(). Callers of scatterwalk_map_and_copy() should be updated to call memcpy_from_sglist() or memcpy_to_sglist() directly, but there are a lot of them so they aren't all being updated right away. Also add functions memcpy_from_scatterwalk() and memcpy_to_scatterwalk() which are similar but operate on a scatter_walk instead of a scatterlist. These will replace scatterwalk_copychunks() with the 'out' argument 0 and 1 respectively. Their behavior differs slightly from scatterwalk_copychunks() in that they automatically take care of flushing the dcache when needed, making them easier to use. scatterwalk_copychunks() itself is left unchanged for now. It will be removed after its callers are updated to use other functions instead. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 59 +++++++++++++++++++++++++++++++++++++------- include/crypto/scatterwalk.h | 24 ++++++++++++++++-- 2 files changed, 72 insertions(+), 11 deletions(-) (limited to 'crypto') diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index af436ad02e3f..2e7a532152d6 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -67,22 +67,63 @@ void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, } EXPORT_SYMBOL_GPL(scatterwalk_copychunks); -void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out) +inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, + unsigned int nbytes) +{ + do { + const void *src_addr; + unsigned int to_copy; + + src_addr = scatterwalk_next(walk, nbytes, &to_copy); + memcpy(buf, src_addr, to_copy); + scatterwalk_done_src(walk, src_addr, to_copy); + buf += to_copy; + nbytes -= to_copy; + } while (nbytes); +} +EXPORT_SYMBOL_GPL(memcpy_from_scatterwalk); + +inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, + unsigned int nbytes) +{ + do { + void *dst_addr; + unsigned int to_copy; + + dst_addr = scatterwalk_next(walk, nbytes, &to_copy); + memcpy(dst_addr, buf, to_copy); + scatterwalk_done_dst(walk, dst_addr, to_copy); + buf += to_copy; + nbytes -= to_copy; + } while (nbytes); +} +EXPORT_SYMBOL_GPL(memcpy_to_scatterwalk); + +void memcpy_from_sglist(void *buf, struct scatterlist *sg, + unsigned int start, unsigned int nbytes) { struct scatter_walk walk; - struct scatterlist tmp[2]; - if (!nbytes) + if (unlikely(nbytes == 0)) /* in case sg == NULL */ return; - sg = scatterwalk_ffwd(tmp, sg, start); + scatterwalk_start_at_pos(&walk, sg, start); + memcpy_from_scatterwalk(buf, &walk, nbytes); +} +EXPORT_SYMBOL_GPL(memcpy_from_sglist); + +void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, + const void *buf, unsigned int nbytes) +{ + struct scatter_walk walk; + + if (unlikely(nbytes == 0)) /* in case sg == NULL */ + return; - scatterwalk_start(&walk, sg); - scatterwalk_copychunks(buf, &walk, nbytes, out); - scatterwalk_done(&walk, out, 0); + scatterwalk_start_at_pos(&walk, sg, start); + memcpy_to_scatterwalk(&walk, buf, nbytes); } -EXPORT_SYMBOL_GPL(scatterwalk_map_and_copy); +EXPORT_SYMBOL_GPL(memcpy_to_sglist); struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 8e83c43016c9..1689ecd7ddaf 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -170,8 +170,28 @@ void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes); void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out); -void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out); +void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, + unsigned int nbytes); + +void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, + unsigned int nbytes); + +void memcpy_from_sglist(void *buf, struct scatterlist *sg, + unsigned int start, unsigned int nbytes); + +void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, + const void *buf, unsigned int nbytes); + +/* In new code, please use memcpy_{from,to}_sglist() directly instead. */ +static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, + unsigned int start, + unsigned int nbytes, int out) +{ + if (out) + memcpy_to_sglist(sg, start, buf, nbytes); + else + memcpy_from_sglist(buf, sg, start, nbytes); +} struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, -- cgit v1.2.3 From cb25dbb60542c56a68089b57f9edd994a6b5bbf4 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:28 -0800 Subject: crypto: skcipher - use scatterwalk_start_at_pos() In skcipher_walk_aead_common(), use scatterwalk_start_at_pos() instead of a sequence of scatterwalk_start(), scatterwalk_copychunks(..., 2), and scatterwalk_done(). This is simpler and faster. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/skcipher.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index cdf0f11c7eaa..59e33f63f845 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -363,14 +363,8 @@ static int skcipher_walk_aead_common(struct skcipher_walk *walk, if (unlikely(!walk->total)) return 0; - scatterwalk_start(&walk->in, req->src); - scatterwalk_start(&walk->out, req->dst); - - scatterwalk_copychunks(NULL, &walk->in, req->assoclen, 2); - scatterwalk_copychunks(NULL, &walk->out, req->assoclen, 2); - - scatterwalk_done(&walk->in, 0, walk->total); - scatterwalk_done(&walk->out, 0, walk->total); + scatterwalk_start_at_pos(&walk->in, req->src, req->assoclen); + scatterwalk_start_at_pos(&walk->out, req->dst, req->assoclen); /* * Accessing 'alg' directly generates better code than using the -- cgit v1.2.3 From c89edd931a10cf9d6bf29dc645324ce976cc0570 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:29 -0800 Subject: crypto: aegis - use the new scatterwalk functions Use scatterwalk_next() which consolidates scatterwalk_clamp() and scatterwalk_map(), and use scatterwalk_done_src() which consolidates scatterwalk_unmap(), scatterwalk_advance(), and scatterwalk_done(). Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/aegis128-core.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'crypto') diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 6cbff298722b..15d64d836356 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -284,10 +284,10 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, scatterwalk_start(&walk, sg_src); while (assoclen != 0) { - unsigned int size = scatterwalk_clamp(&walk, assoclen); + unsigned int size; + const u8 *mapped = scatterwalk_next(&walk, assoclen, &size); unsigned int left = size; - void *mapped = scatterwalk_map(&walk); - const u8 *src = (const u8 *)mapped; + const u8 *src = mapped; if (pos + size >= AEGIS_BLOCK_SIZE) { if (pos > 0) { @@ -308,9 +308,7 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, pos += left; assoclen -= size; - scatterwalk_unmap(mapped); - scatterwalk_advance(&walk, size); - scatterwalk_done(&walk, 0, assoclen); + scatterwalk_done_src(&walk, mapped, size); } if (pos > 0) { -- cgit v1.2.3 From 95dbd711b1d81278d703e05eb5288eac758430c1 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:39 -0800 Subject: crypto: skcipher - use the new scatterwalk functions Convert skcipher_walk to use the new scatterwalk functions. This includes a few changes to exactly where the different parts of the iteration happen. For example the dcache flush that previously happened in scatterwalk_done() now happens in scatterwalk_dst_done() or in memcpy_to_scatterwalk(). Advancing to the next sg entry now happens just-in-time in scatterwalk_clamp() instead of in scatterwalk_done(). Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/skcipher.c | 51 +++++++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 59e33f63f845..e9b803fe2441 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -49,16 +49,6 @@ static inline void skcipher_map_dst(struct skcipher_walk *walk) walk->dst.virt.addr = scatterwalk_map(&walk->out); } -static inline void skcipher_unmap_src(struct skcipher_walk *walk) -{ - scatterwalk_unmap(walk->src.virt.addr); -} - -static inline void skcipher_unmap_dst(struct skcipher_walk *walk) -{ - scatterwalk_unmap(walk->dst.virt.addr); -} - static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) { return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -70,14 +60,6 @@ static inline struct skcipher_alg *__crypto_skcipher_alg( return container_of(alg, struct skcipher_alg, base); } -static int skcipher_done_slow(struct skcipher_walk *walk, unsigned int bsize) -{ - u8 *addr = PTR_ALIGN(walk->buffer, walk->alignmask + 1); - - scatterwalk_copychunks(addr, &walk->out, bsize, 1); - return 0; -} - /** * skcipher_walk_done() - finish one step of a skcipher_walk * @walk: the skcipher_walk @@ -112,15 +94,14 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) if (likely(!(walk->flags & (SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | SKCIPHER_WALK_DIFF)))) { -unmap_src: - skcipher_unmap_src(walk); + scatterwalk_advance(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_DIFF) { - skcipher_unmap_dst(walk); - goto unmap_src; + scatterwalk_unmap(walk->src.virt.addr); + scatterwalk_advance(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_COPY) { + scatterwalk_advance(&walk->in, n); skcipher_map_dst(walk); memcpy(walk->dst.virt.addr, walk->page, n); - skcipher_unmap_dst(walk); } else { /* SKCIPHER_WALK_SLOW */ if (res > 0) { /* @@ -131,21 +112,23 @@ unmap_src: */ res = -EINVAL; total = 0; - } else - n = skcipher_done_slow(walk, n); + } else { + u8 *buf = PTR_ALIGN(walk->buffer, walk->alignmask + 1); + + memcpy_to_scatterwalk(&walk->out, buf, n); + } + goto dst_done; } + scatterwalk_done_dst(&walk->out, walk->dst.virt.addr, n); +dst_done: + if (res > 0) res = 0; walk->total = total; walk->nbytes = 0; - scatterwalk_advance(&walk->in, n); - scatterwalk_advance(&walk->out, n); - scatterwalk_done(&walk->in, 0, total); - scatterwalk_done(&walk->out, 1, total); - if (total) { if (walk->flags & SKCIPHER_WALK_SLEEP) cond_resched(); @@ -192,7 +175,7 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1); walk->src.virt.addr = walk->dst.virt.addr; - scatterwalk_copychunks(walk->src.virt.addr, &walk->in, bsize, 0); + memcpy_from_scatterwalk(walk->src.virt.addr, &walk->in, bsize); walk->nbytes = bsize; walk->flags |= SKCIPHER_WALK_SLOW; @@ -206,7 +189,11 @@ static int skcipher_next_copy(struct skcipher_walk *walk) skcipher_map_src(walk); memcpy(tmp, walk->src.virt.addr, walk->nbytes); - skcipher_unmap_src(walk); + scatterwalk_unmap(walk->src.virt.addr); + /* + * walk->in is advanced later when the number of bytes actually + * processed (which might be less than walk->nbytes) is known. + */ walk->src.virt.addr = tmp; walk->dst.virt.addr = tmp; -- cgit v1.2.3 From fa94e45436c15421284c5cd24d497675b1f46433 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:40 -0800 Subject: crypto: scatterwalk - remove obsolete functions Remove various functions that are no longer used. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 37 ------------------------------------- include/crypto/scatterwalk.h | 25 ------------------------- 2 files changed, 62 deletions(-) (limited to 'crypto') diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 2e7a532152d6..87c080f565d4 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -30,43 +30,6 @@ void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes) } EXPORT_SYMBOL_GPL(scatterwalk_skip); -static inline void memcpy_dir(void *buf, void *sgdata, size_t nbytes, int out) -{ - void *src = out ? buf : sgdata; - void *dst = out ? sgdata : buf; - - memcpy(dst, src, nbytes); -} - -void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, - size_t nbytes, int out) -{ - for (;;) { - unsigned int len_this_page = scatterwalk_pagelen(walk); - u8 *vaddr; - - if (len_this_page > nbytes) - len_this_page = nbytes; - - if (out != 2) { - vaddr = scatterwalk_map(walk); - memcpy_dir(buf, vaddr, len_this_page, out); - scatterwalk_unmap(vaddr); - } - - scatterwalk_advance(walk, len_this_page); - - if (nbytes == len_this_page) - break; - - buf += len_this_page; - nbytes -= len_this_page; - - scatterwalk_pagedone(walk, out & 1, 1); - } -} -EXPORT_SYMBOL_GPL(scatterwalk_copychunks); - inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, unsigned int nbytes) { diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index f6262d05a3c7..ac03fdf88b2a 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -115,28 +115,6 @@ static inline void *scatterwalk_next(struct scatter_walk *walk, return scatterwalk_map(walk); } -static inline void scatterwalk_pagedone(struct scatter_walk *walk, int out, - unsigned int more) -{ - if (out) { - struct page *page; - - page = sg_page(walk->sg) + ((walk->offset - 1) >> PAGE_SHIFT); - flush_dcache_page(page); - } - - if (more && walk->offset >= walk->sg->offset + walk->sg->length) - scatterwalk_start(walk, sg_next(walk->sg)); -} - -static inline void scatterwalk_done(struct scatter_walk *walk, int out, - int more) -{ - if (!more || walk->offset >= walk->sg->offset + walk->sg->length || - !(walk->offset & (PAGE_SIZE - 1))) - scatterwalk_pagedone(walk, out, more); -} - static inline void scatterwalk_advance(struct scatter_walk *walk, unsigned int nbytes) { @@ -184,9 +162,6 @@ static inline void scatterwalk_done_dst(struct scatter_walk *walk, void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes); -void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, - size_t nbytes, int out); - void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, unsigned int nbytes); -- cgit v1.2.3 From 641938d3bba64287f199431fafd917bc7b7c9d1a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 19 Feb 2025 10:23:41 -0800 Subject: crypto: scatterwalk - don't split at page boundaries when !HIGHMEM When !HIGHMEM, the kmap_local_page() in the scatterlist walker does not actually map anything, and the address it returns is just the address from the kernel's direct map, where each sg entry's data is virtually contiguous. To improve performance, stop unnecessarily clamping data segments to page boundaries in this case. For now, still limit segments to PAGE_SIZE. This is needed to prevent preemption from being disabled for too long when SIMD is used, and to support the alignmask case which still uses a page-sized bounce buffer. Even so, this change still helps a lot in cases where messages cross a page boundary. For example, testing IPsec with AES-GCM on x86_64, the messages are 1424 bytes which is less than PAGE_SIZE, but on the Rx side over a third cross a page boundary. These ended up being processed in three parts, with the middle part going through skcipher_next_slow which uses a 16-byte bounce buffer. That was causing a significant amount of overhead which unnecessarily reduced the performance benefit of the new x86_64 AES-GCM assembly code. This change solves the problem; all these messages now get passed to the assembly code in one part. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/skcipher.c | 4 +-- include/crypto/scatterwalk.h | 79 ++++++++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 24 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index e9b803fe2441..b0e1f3c12cde 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -206,8 +206,8 @@ static int skcipher_next_fast(struct skcipher_walk *walk) diff = offset_in_page(walk->in.offset) - offset_in_page(walk->out.offset); - diff |= (u8 *)scatterwalk_page(&walk->in) - - (u8 *)scatterwalk_page(&walk->out); + diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - + (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); skcipher_map_src(walk); walk->dst.virt.addr = walk->src.virt.addr; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index ac03fdf88b2a..3024adbdd443 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -49,24 +49,35 @@ static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, walk->offset = sg->offset + pos; } -static inline unsigned int scatterwalk_pagelen(struct scatter_walk *walk) -{ - unsigned int len = walk->sg->offset + walk->sg->length - walk->offset; - unsigned int len_this_page = offset_in_page(~walk->offset) + 1; - return len_this_page > len ? len : len_this_page; -} - static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, unsigned int nbytes) { + unsigned int len_this_sg; + unsigned int limit; + if (walk->offset >= walk->sg->offset + walk->sg->length) scatterwalk_start(walk, sg_next(walk->sg)); - return min(nbytes, scatterwalk_pagelen(walk)); -} + len_this_sg = walk->sg->offset + walk->sg->length - walk->offset; -static inline struct page *scatterwalk_page(struct scatter_walk *walk) -{ - return sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); + /* + * HIGHMEM case: the page may have to be mapped into memory. To avoid + * the complexity of having to map multiple pages at once per sg entry, + * clamp the returned length to not cross a page boundary. + * + * !HIGHMEM case: no mapping is needed; all pages of the sg entry are + * already mapped contiguously in the kernel's direct map. For improved + * performance, allow the walker to return data segments that cross a + * page boundary. Do still cap the length to PAGE_SIZE, since some + * users rely on that to avoid disabling preemption for too long when + * using SIMD. It's also needed for when skcipher_walk uses a bounce + * page due to the data not being aligned to the algorithm's alignmask. + */ + if (IS_ENABLED(CONFIG_HIGHMEM)) + limit = PAGE_SIZE - offset_in_page(walk->offset); + else + limit = PAGE_SIZE; + + return min3(nbytes, len_this_sg, limit); } /* @@ -86,15 +97,23 @@ static inline void scatterwalk_get_sglist(struct scatter_walk *walk, scatterwalk_crypto_chain(sg_out, sg_next(walk->sg), 2); } -static inline void scatterwalk_unmap(void *vaddr) -{ - kunmap_local(vaddr); -} - static inline void *scatterwalk_map(struct scatter_walk *walk) { - return kmap_local_page(scatterwalk_page(walk)) + - offset_in_page(walk->offset); + struct page *base_page = sg_page(walk->sg); + + if (IS_ENABLED(CONFIG_HIGHMEM)) + return kmap_local_page(base_page + (walk->offset >> PAGE_SHIFT)) + + offset_in_page(walk->offset); + /* + * When !HIGHMEM we allow the walker to return segments that span a page + * boundary; see scatterwalk_clamp(). To make it clear that in this + * case we're working in the linear buffer of the whole sg entry in the + * kernel's direct map rather than within the mapped buffer of a single + * page, compute the address as an offset from the page_address() of the + * first page of the sg entry. Either way the result is the address in + * the direct map, but this makes it clearer what is really going on. + */ + return page_address(base_page) + walk->offset; } /** @@ -115,6 +134,12 @@ static inline void *scatterwalk_next(struct scatter_walk *walk, return scatterwalk_map(walk); } +static inline void scatterwalk_unmap(const void *vaddr) +{ + if (IS_ENABLED(CONFIG_HIGHMEM)) + kunmap_local(vaddr); +} + static inline void scatterwalk_advance(struct scatter_walk *walk, unsigned int nbytes) { @@ -133,7 +158,7 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, static inline void scatterwalk_done_src(struct scatter_walk *walk, const void *vaddr, unsigned int nbytes) { - scatterwalk_unmap((void *)vaddr); + scatterwalk_unmap(vaddr); scatterwalk_advance(walk, nbytes); } @@ -154,9 +179,19 @@ static inline void scatterwalk_done_dst(struct scatter_walk *walk, * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just * relying on flush_dcache_page() being a no-op when not implemented, * since otherwise the BUG_ON in sg_page() does not get optimized out. + * This also avoids having to consider whether the loop would get + * reliably optimized out or not. */ - if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) - flush_dcache_page(scatterwalk_page(walk)); + if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) { + struct page *base_page, *start_page, *end_page, *page; + + base_page = sg_page(walk->sg); + start_page = base_page + (walk->offset >> PAGE_SHIFT); + end_page = base_page + ((walk->offset + nbytes + + PAGE_SIZE - 1) >> PAGE_SHIFT); + for (page = start_page; page < end_page; page++) + flush_dcache_page(page); + } scatterwalk_advance(walk, nbytes); } -- cgit v1.2.3 From f79d2d2852facc72b91a78e5c423722c7dc53d72 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 23 Feb 2025 14:27:51 +0800 Subject: crypto: skcipher - Use restrict rather than hand-rolling accesses Rather than accessing 'alg' directly to avoid the aliasing issue which leads to unnecessary reloads, use the __restrict keyword to explicitly tell the compiler that there is no aliasing. This generates equivalent if not superior code on x86 with gcc 12. Note that in skcipher_walk_virt the alg assignment is moved after might_sleep_if because that function is a compiler barrier and forces a reload. Signed-off-by: Herbert Xu --- crypto/skcipher.c | 51 +++++++++++++++++--------------------- include/crypto/internal/skcipher.h | 14 ++++++----- 2 files changed, 31 insertions(+), 34 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index b0e1f3c12cde..53123d3685d5 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -293,14 +293,16 @@ static int skcipher_walk_first(struct skcipher_walk *walk) return skcipher_walk_next(walk); } -int skcipher_walk_virt(struct skcipher_walk *walk, - struct skcipher_request *req, bool atomic) +int skcipher_walk_virt(struct skcipher_walk *__restrict walk, + struct skcipher_request *__restrict req, bool atomic) { - const struct skcipher_alg *alg = - crypto_skcipher_alg(crypto_skcipher_reqtfm(req)); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct skcipher_alg *alg; might_sleep_if(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); + alg = crypto_skcipher_alg(tfm); + walk->total = req->cryptlen; walk->nbytes = 0; walk->iv = req->iv; @@ -316,14 +318,9 @@ int skcipher_walk_virt(struct skcipher_walk *walk, scatterwalk_start(&walk->in, req->src); scatterwalk_start(&walk->out, req->dst); - /* - * Accessing 'alg' directly generates better code than using the - * crypto_skcipher_blocksize() and similar helper functions here, as it - * prevents the algorithm pointer from being repeatedly reloaded. - */ - walk->blocksize = alg->base.cra_blocksize; - walk->ivsize = alg->co.ivsize; - walk->alignmask = alg->base.cra_alignmask; + walk->blocksize = crypto_skcipher_blocksize(tfm); + walk->ivsize = crypto_skcipher_ivsize(tfm); + walk->alignmask = crypto_skcipher_alignmask(tfm); if (alg->co.base.cra_type != &crypto_skcipher_type) walk->stride = alg->co.chunksize; @@ -334,10 +331,11 @@ int skcipher_walk_virt(struct skcipher_walk *walk, } EXPORT_SYMBOL_GPL(skcipher_walk_virt); -static int skcipher_walk_aead_common(struct skcipher_walk *walk, - struct aead_request *req, bool atomic) +static int skcipher_walk_aead_common(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic) { - const struct aead_alg *alg = crypto_aead_alg(crypto_aead_reqtfm(req)); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); walk->nbytes = 0; walk->iv = req->iv; @@ -353,21 +351,17 @@ static int skcipher_walk_aead_common(struct skcipher_walk *walk, scatterwalk_start_at_pos(&walk->in, req->src, req->assoclen); scatterwalk_start_at_pos(&walk->out, req->dst, req->assoclen); - /* - * Accessing 'alg' directly generates better code than using the - * crypto_aead_blocksize() and similar helper functions here, as it - * prevents the algorithm pointer from being repeatedly reloaded. - */ - walk->blocksize = alg->base.cra_blocksize; - walk->stride = alg->chunksize; - walk->ivsize = alg->ivsize; - walk->alignmask = alg->base.cra_alignmask; + walk->blocksize = crypto_aead_blocksize(tfm); + walk->stride = crypto_aead_chunksize(tfm); + walk->ivsize = crypto_aead_ivsize(tfm); + walk->alignmask = crypto_aead_alignmask(tfm); return skcipher_walk_first(walk); } -int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic) +int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic) { walk->total = req->cryptlen; @@ -375,8 +369,9 @@ int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, } EXPORT_SYMBOL_GPL(skcipher_walk_aead_encrypt); -int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic) +int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index 4f49621d3eb6..d6ae7a86fed2 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -197,13 +197,15 @@ int lskcipher_register_instance(struct crypto_template *tmpl, struct lskcipher_instance *inst); int skcipher_walk_done(struct skcipher_walk *walk, int res); -int skcipher_walk_virt(struct skcipher_walk *walk, - struct skcipher_request *req, +int skcipher_walk_virt(struct skcipher_walk *__restrict walk, + struct skcipher_request *__restrict req, bool atomic); -int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic); -int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic); +int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic); +int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic); static inline void skcipher_walk_abort(struct skcipher_walk *walk) { -- cgit v1.2.3 From 17ec3e71ba797cdb62164fea9532c81b60f47167 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 27 Feb 2025 15:48:39 +0800 Subject: crypto: lib/Kconfig - Hide arch options from user The ARCH_MAY_HAVE patch missed arm64, mips and s390. But it may also lead to arch options being enabled but ineffective because of modular/built-in conflicts. As the primary user of all these options wireguard is selecting the arch options anyway, make the same selections at the lib/crypto option level and hide the arch options from the user. Instead of selecting them centrally from lib/crypto, simply set the default of each arch option as suggested by Eric Biggers. Change the Crypto API generic algorithms to select the top-level lib/crypto options instead of the generic one as otherwise there is no way to enable the arch options (Eric Biggers). Introduce a set of INTERNAL options to work around dependency cycles on the CONFIG_CRYPTO symbol. Fixes: 1047e21aecdf ("crypto: lib/Kconfig - Fix lib built-in failure when arch is modular") Reported-by: kernel test robot Reported-by: Arnd Bergmann Closes: https://lore.kernel.org/oe-kbuild-all/202502232152.JC84YDLp-lkp@intel.com/ Signed-off-by: Herbert Xu --- arch/arm/crypto/Kconfig | 16 ++++++++++------ arch/arm64/crypto/Kconfig | 6 ++++-- arch/mips/crypto/Kconfig | 7 +++++-- arch/powerpc/crypto/Kconfig | 11 +++++++---- arch/riscv/crypto/Kconfig | 1 - arch/s390/crypto/Kconfig | 3 ++- arch/x86/crypto/Kconfig | 17 +++++++++++------ crypto/Kconfig | 6 +++--- lib/crypto/Kconfig | 41 +++++++++++++++++++++-------------------- 9 files changed, 63 insertions(+), 45 deletions(-) (limited to 'crypto') diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig index 47d9cc59f254..23e4ea067ddb 100644 --- a/arch/arm/crypto/Kconfig +++ b/arch/arm/crypto/Kconfig @@ -3,10 +3,12 @@ menu "Accelerated Cryptographic Algorithms for CPU (arm)" config CRYPTO_CURVE25519_NEON - tristate "Public key crypto: Curve25519 (NEON)" + tristate depends on KERNEL_MODE_NEON + select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_CURVE25519 + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + default CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 algorithm @@ -45,9 +47,10 @@ config CRYPTO_NHPOLY1305_NEON - NEON (Advanced SIMD) extensions config CRYPTO_POLY1305_ARM - tristate "Hash functions: Poly1305 (NEON)" + tristate select CRYPTO_HASH - select CRYPTO_ARCH_MAY_HAVE_LIB_POLY1305 + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + default CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) @@ -212,9 +215,10 @@ config CRYPTO_AES_ARM_CE - ARMv8 Crypto Extensions config CRYPTO_CHACHA20_NEON - tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (NEON)" + tristate select CRYPTO_SKCIPHER - select CRYPTO_ARCH_MAY_HAVE_LIB_CHACHA + select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 5636ab83f22a..3418c8d3c78d 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -26,10 +26,11 @@ config CRYPTO_NHPOLY1305_NEON - NEON (Advanced SIMD) extensions config CRYPTO_POLY1305_NEON - tristate "Hash functions: Poly1305 (NEON)" + tristate depends on KERNEL_MODE_NEON select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 + default CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) @@ -186,11 +187,12 @@ config CRYPTO_AES_ARM64_NEON_BLK - NEON (Advanced SIMD) extensions config CRYPTO_CHACHA20_NEON - tristate "Ciphers: ChaCha (NEON)" + tristate depends on KERNEL_MODE_NEON select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms diff --git a/arch/mips/crypto/Kconfig b/arch/mips/crypto/Kconfig index 7decd40c4e20..545fc0e12422 100644 --- a/arch/mips/crypto/Kconfig +++ b/arch/mips/crypto/Kconfig @@ -3,9 +3,11 @@ menu "Accelerated Cryptographic Algorithms for CPU (mips)" config CRYPTO_POLY1305_MIPS - tristate "Hash functions: Poly1305" + tristate depends on MIPS + select CRYPTO_HASH select CRYPTO_ARCH_HAVE_LIB_POLY1305 + default CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) @@ -52,10 +54,11 @@ config CRYPTO_SHA512_OCTEON Architecture: mips OCTEON using crypto instructions, when available config CRYPTO_CHACHA_MIPS - tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (MIPS32r2)" + tristate depends on CPU_MIPS32_R2 select CRYPTO_SKCIPHER select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index e453cb0c82d2..370db8192ce6 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -3,10 +3,12 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" config CRYPTO_CURVE25519_PPC64 - tristate "Public key crypto: Curve25519 (PowerPC64)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN + select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_CURVE25519 + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + default CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 algorithm @@ -91,11 +93,12 @@ config CRYPTO_AES_GCM_P10 later CPU. This module supports stitched acceleration for AES/GCM. config CRYPTO_CHACHA20_P10 - tristate "Ciphers: ChaCha20, XChacha20, XChacha12 (P10 or later)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN && VSX select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_CHACHA + select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig index ad58dad9a580..c67095a3d669 100644 --- a/arch/riscv/crypto/Kconfig +++ b/arch/riscv/crypto/Kconfig @@ -22,7 +22,6 @@ config CRYPTO_CHACHA_RISCV64 tristate "Ciphers: ChaCha" depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC help Length-preserving ciphers: ChaCha20 stream cipher algorithm diff --git a/arch/s390/crypto/Kconfig b/arch/s390/crypto/Kconfig index b760232537f1..8c4db8b64fa2 100644 --- a/arch/s390/crypto/Kconfig +++ b/arch/s390/crypto/Kconfig @@ -108,11 +108,12 @@ config CRYPTO_DES_S390 As of z196 the CTR mode is hardware accelerated. config CRYPTO_CHACHA_S390 - tristate "Ciphers: ChaCha20" + tristate depends on S390 select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving cipher: ChaCha20 stream cipher (RFC 7539) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index c189dad0969b..3d948f10c94c 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -3,10 +3,12 @@ menu "Accelerated Cryptographic Algorithms for CPU (x86)" config CRYPTO_CURVE25519_X86 - tristate "Public key crypto: Curve25519 (ADX)" + tristate depends on X86 && 64BIT + select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_CURVE25519 + select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + default CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 algorithm @@ -348,11 +350,12 @@ config CRYPTO_ARIA_GFNI_AVX512_X86_64 Processes 64 blocks in parallel. config CRYPTO_CHACHA20_X86_64 - tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)" + tristate depends on X86 && 64BIT select CRYPTO_SKCIPHER select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_CHACHA + select CRYPTO_ARCH_HAVE_LIB_CHACHA + default CRYPTO_LIB_CHACHA_INTERNAL help Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms @@ -417,10 +420,12 @@ config CRYPTO_POLYVAL_CLMUL_NI - CLMUL-NI (carry-less multiplication new instructions) config CRYPTO_POLY1305_X86_64 - tristate "Hash functions: Poly1305 (SSE2/AVX2)" + tristate depends on X86 && 64BIT + select CRYPTO_HASH select CRYPTO_LIB_POLY1305_GENERIC - select CRYPTO_ARCH_MAY_HAVE_LIB_POLY1305 + select CRYPTO_ARCH_HAVE_LIB_POLY1305 + default CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) diff --git a/crypto/Kconfig b/crypto/Kconfig index 74ae5f52b784..b7771d7bd3b3 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -317,7 +317,7 @@ config CRYPTO_ECRDSA config CRYPTO_CURVE25519 tristate "Curve25519" select CRYPTO_KPP - select CRYPTO_LIB_CURVE25519_GENERIC + select CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 elliptic curve (RFC7748) @@ -615,7 +615,7 @@ config CRYPTO_ARC4 config CRYPTO_CHACHA20 tristate "ChaCha" - select CRYPTO_LIB_CHACHA_GENERIC + select CRYPTO_LIB_CHACHA_INTERNAL select CRYPTO_SKCIPHER help The ChaCha20, XChaCha20, and XChaCha12 stream cipher algorithms @@ -936,7 +936,7 @@ config CRYPTO_POLYVAL config CRYPTO_POLY1305 tristate "Poly1305" select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC + select CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index c542ef1d64d0..b09e78da959a 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -48,11 +48,6 @@ config CRYPTO_ARCH_HAVE_LIB_CHACHA accelerated implementation of the ChaCha library interface, either builtin or as a module. -config CRYPTO_ARCH_MAY_HAVE_LIB_CHACHA - tristate - select CRYPTO_ARCH_HAVE_LIB_CHACHA if CRYPTO_LIB_CHACHA=m - select CRYPTO_ARCH_HAVE_LIB_CHACHA if CRYPTO_ARCH_MAY_HAVE_LIB_CHACHA=y - config CRYPTO_LIB_CHACHA_GENERIC tristate select CRYPTO_LIB_UTILS @@ -63,9 +58,14 @@ config CRYPTO_LIB_CHACHA_GENERIC implementation is enabled, this implementation serves the users of CRYPTO_LIB_CHACHA. +config CRYPTO_LIB_CHACHA_INTERNAL + tristate + select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n + config CRYPTO_LIB_CHACHA tristate "ChaCha library interface" - select CRYPTO_LIB_CHACHA_GENERIC if CRYPTO_ARCH_HAVE_LIB_CHACHA=n + select CRYPTO + select CRYPTO_LIB_CHACHA_INTERNAL help Enable the ChaCha library interface. This interface may be fulfilled by either the generic implementation or an arch-specific one, if one @@ -78,13 +78,9 @@ config CRYPTO_ARCH_HAVE_LIB_CURVE25519 accelerated implementation of the Curve25519 library interface, either builtin or as a module. -config CRYPTO_ARCH_MAY_HAVE_LIB_CURVE25519 - tristate - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 if CRYPTO_LIB_CURVE25519=m - select CRYPTO_ARCH_HAVE_LIB_CURVE25519 if CRYPTO_ARCH_MAY_HAVE_LIB_CURVE25519=y - config CRYPTO_LIB_CURVE25519_GENERIC tristate + select CRYPTO_LIB_UTILS help This symbol can be depended upon by arch implementations of the Curve25519 library interface that require the generic code as a @@ -92,10 +88,14 @@ config CRYPTO_LIB_CURVE25519_GENERIC implementation is enabled, this implementation serves the users of CRYPTO_LIB_CURVE25519. +config CRYPTO_LIB_CURVE25519_INTERNAL + tristate + select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n + config CRYPTO_LIB_CURVE25519 tristate "Curve25519 scalar multiplication library" - select CRYPTO_LIB_CURVE25519_GENERIC if CRYPTO_ARCH_HAVE_LIB_CURVE25519=n - select CRYPTO_LIB_UTILS + select CRYPTO + select CRYPTO_LIB_CURVE25519_INTERNAL help Enable the Curve25519 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific @@ -118,11 +118,6 @@ config CRYPTO_ARCH_HAVE_LIB_POLY1305 accelerated implementation of the Poly1305 library interface, either builtin or as a module. -config CRYPTO_ARCH_MAY_HAVE_LIB_POLY1305 - tristate - select CRYPTO_ARCH_HAVE_LIB_POLY1305 if CRYPTO_LIB_POLY1305=m - select CRYPTO_ARCH_HAVE_LIB_POLY1305 if CRYPTO_ARCH_MAY_HAVE_LIB_POLY1305=y - config CRYPTO_LIB_POLY1305_GENERIC tristate help @@ -132,9 +127,14 @@ config CRYPTO_LIB_POLY1305_GENERIC implementation is enabled, this implementation serves the users of CRYPTO_LIB_POLY1305. +config CRYPTO_LIB_POLY1305_INTERNAL + tristate + select CRYPTO_LIB_POLY1305_GENERIC if CRYPTO_ARCH_HAVE_LIB_POLY1305=n + config CRYPTO_LIB_POLY1305 tristate "Poly1305 library interface" - select CRYPTO_LIB_POLY1305_GENERIC if CRYPTO_ARCH_HAVE_LIB_POLY1305=n + select CRYPTO + select CRYPTO_LIB_POLY1305_INTERNAL help Enable the Poly1305 library interface. This interface may be fulfilled by either the generic implementation or an arch-specific one, if one @@ -142,9 +142,10 @@ config CRYPTO_LIB_POLY1305 config CRYPTO_LIB_CHACHA20POLY1305 tristate "ChaCha20-Poly1305 AEAD support (8-byte nonce library version)" - depends on CRYPTO + select CRYPTO select CRYPTO_LIB_CHACHA select CRYPTO_LIB_POLY1305 + select CRYPTO_LIB_UTILS select CRYPTO_ALGAPI config CRYPTO_LIB_SHA1 -- cgit v1.2.3 From d1775a177f7f38156d541c8a3e3c91eaa6e69699 Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 17 Jan 2025 11:46:23 +0000 Subject: crypto: Add 'krb5enc' hash and cipher AEAD algorithm Add an AEAD template that does hash-then-cipher (unlike authenc that does cipher-then-hash). This is required for a number of Kerberos 5 encoding types. [!] Note that the net/sunrpc/auth_gss/ implementation gets a pair of ciphers, one non-CTS and one CTS, using the former to do all the aligned blocks and the latter to do the last two blocks if they aren't also aligned. It may be necessary to do this here too for performance reasons - but there are considerations both ways: (1) firstly, there is an optimised assembly version of cts(cbc(aes)) on x86_64 that should be used instead of having two ciphers; (2) secondly, none of the hardware offload drivers seem to offer CTS support (Intel QAT does not, for instance). However, I don't know if it's possible to query the crypto API to find out whether there's an optimised CTS algorithm available. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/Kconfig | 12 ++ crypto/Makefile | 1 + crypto/krb5enc.c | 504 +++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/authenc.h | 2 + 4 files changed, 519 insertions(+) create mode 100644 crypto/krb5enc.c (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index b7771d7bd3b3..15a6730d3ea1 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -228,6 +228,18 @@ config CRYPTO_AUTHENC This is required for IPSec ESP (XFRM_ESP). +config CRYPTO_KRB5ENC + tristate "Kerberos 5 combined hash+cipher support" + select CRYPTO_AEAD + select CRYPTO_SKCIPHER + select CRYPTO_MANAGER + select CRYPTO_HASH + select CRYPTO_NULL + help + Combined hash and cipher support for Kerberos 5 RFC3961 simplified + profile. This is required for Kerberos 5-style encryption, used by + sunrpc/NFS and rxrpc/AFS. + config CRYPTO_TEST tristate "Testing module" depends on m || EXPERT diff --git a/crypto/Makefile b/crypto/Makefile index f67e853c4690..20c8e3ee9835 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -159,6 +159,7 @@ obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_generic.o CFLAGS_crct10dif_generic.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o +obj-$(CONFIG_CRYPTO_KRB5ENC) += krb5enc.o obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o obj-$(CONFIG_CRYPTO_LZ4) += lz4.o obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o diff --git a/crypto/krb5enc.c b/crypto/krb5enc.c new file mode 100644 index 000000000000..d07769bf149e --- /dev/null +++ b/crypto/krb5enc.c @@ -0,0 +1,504 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * AEAD wrapper for Kerberos 5 RFC3961 simplified profile. + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * Derived from authenc: + * Copyright (c) 2007-2015 Herbert Xu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct krb5enc_instance_ctx { + struct crypto_ahash_spawn auth; + struct crypto_skcipher_spawn enc; + unsigned int reqoff; +}; + +struct krb5enc_ctx { + struct crypto_ahash *auth; + struct crypto_skcipher *enc; +}; + +struct krb5enc_request_ctx { + struct scatterlist src[2]; + struct scatterlist dst[2]; + char tail[]; +}; + +static void krb5enc_request_complete(struct aead_request *req, int err) +{ + if (err != -EINPROGRESS) + aead_request_complete(req, err); +} + +/** + * crypto_krb5enc_extractkeys - Extract Ke and Ki keys from the key blob. + * @keys: Where to put the key sizes and pointers + * @key: Encoded key material + * @keylen: Amount of key material + * + * Decode the key blob we're given. It starts with an rtattr that indicates + * the format and the length. Format CRYPTO_AUTHENC_KEYA_PARAM is: + * + * rtattr || __be32 enckeylen || authkey || enckey + * + * Note that the rtattr is in cpu-endian form, unlike enckeylen. This must be + * handled correctly in static testmgr data. + */ +int crypto_krb5enc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key, + unsigned int keylen) +{ + struct rtattr *rta = (struct rtattr *)key; + struct crypto_authenc_key_param *param; + + if (!RTA_OK(rta, keylen)) + return -EINVAL; + if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) + return -EINVAL; + + /* + * RTA_OK() didn't align the rtattr's payload when validating that it + * fits in the buffer. Yet, the keys should start on the next 4-byte + * aligned boundary. To avoid confusion, require that the rtattr + * payload be exactly the param struct, which has a 4-byte aligned size. + */ + if (RTA_PAYLOAD(rta) != sizeof(*param)) + return -EINVAL; + BUILD_BUG_ON(sizeof(*param) % RTA_ALIGNTO); + + param = RTA_DATA(rta); + keys->enckeylen = be32_to_cpu(param->enckeylen); + + key += rta->rta_len; + keylen -= rta->rta_len; + + if (keylen < keys->enckeylen) + return -EINVAL; + + keys->authkeylen = keylen - keys->enckeylen; + keys->authkey = key; + keys->enckey = key + keys->authkeylen; + return 0; +} +EXPORT_SYMBOL(crypto_krb5enc_extractkeys); + +static int krb5enc_setkey(struct crypto_aead *krb5enc, const u8 *key, + unsigned int keylen) +{ + struct crypto_authenc_keys keys; + struct krb5enc_ctx *ctx = crypto_aead_ctx(krb5enc); + struct crypto_skcipher *enc = ctx->enc; + struct crypto_ahash *auth = ctx->auth; + unsigned int flags = crypto_aead_get_flags(krb5enc); + int err = -EINVAL; + + if (crypto_krb5enc_extractkeys(&keys, key, keylen) != 0) + goto out; + + crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(auth, flags & CRYPTO_TFM_REQ_MASK); + err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen); + if (err) + goto out; + + crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(enc, flags & CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen); +out: + memzero_explicit(&keys, sizeof(keys)); + return err; +} + +static void krb5enc_encrypt_done(void *data, int err) +{ + struct aead_request *req = data; + + krb5enc_request_complete(req, err); +} + +/* + * Start the encryption of the plaintext. We skip over the associated data as + * that only gets included in the hash. + */ +static int krb5enc_dispatch_encrypt(struct aead_request *req, + unsigned int flags) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_ctx *ctx = crypto_aead_ctx(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct crypto_skcipher *enc = ctx->enc; + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ictx->reqoff); + struct scatterlist *src, *dst; + + src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); + if (req->src == req->dst) + dst = src; + else + dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); + + skcipher_request_set_tfm(skreq, enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + krb5enc_encrypt_done, req); + skcipher_request_set_crypt(skreq, src, dst, req->cryptlen, req->iv); + + return crypto_skcipher_encrypt(skreq); +} + +/* + * Insert the hash into the checksum field in the destination buffer directly + * after the encrypted region. + */ +static void krb5enc_insert_checksum(struct aead_request *req, u8 *hash) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + + scatterwalk_map_and_copy(hash, req->dst, + req->assoclen + req->cryptlen, + crypto_aead_authsize(krb5enc), 1); +} + +/* + * Upon completion of an asynchronous digest, transfer the hash to the checksum + * field. + */ +static void krb5enc_encrypt_ahash_done(void *data, int err) +{ + struct aead_request *req = data; + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + + if (err) + return krb5enc_request_complete(req, err); + + krb5enc_insert_checksum(req, ahreq->result); + + err = krb5enc_dispatch_encrypt(req, 0); + if (err != -EINPROGRESS) + aead_request_complete(req, err); +} + +/* + * Start the digest of the plaintext for encryption. In theory, this could be + * run in parallel with the encryption, provided the src and dst buffers don't + * overlap. + */ +static int krb5enc_dispatch_encrypt_hash(struct aead_request *req) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_ctx *ctx = crypto_aead_ctx(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_ahash *auth = ctx->auth; + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + u8 *hash = areq_ctx->tail; + int err; + + ahash_request_set_callback(ahreq, aead_request_flags(req), + krb5enc_encrypt_ahash_done, req); + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, req->src, hash, req->assoclen + req->cryptlen); + + err = crypto_ahash_digest(ahreq); + if (err) + return err; + + krb5enc_insert_checksum(req, hash); + return 0; +} + +/* + * Process an encryption operation. We can perform the cipher and the hash in + * parallel, provided the src and dst buffers are separate. + */ +static int krb5enc_encrypt(struct aead_request *req) +{ + int err; + + err = krb5enc_dispatch_encrypt_hash(req); + if (err < 0) + return err; + + return krb5enc_dispatch_encrypt(req, aead_request_flags(req)); +} + +static int krb5enc_verify_hash(struct aead_request *req) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + unsigned int authsize = crypto_aead_authsize(krb5enc); + u8 *calc_hash = areq_ctx->tail; + u8 *msg_hash = areq_ctx->tail + authsize; + + scatterwalk_map_and_copy(msg_hash, req->src, ahreq->nbytes, authsize, 0); + + if (crypto_memneq(msg_hash, calc_hash, authsize)) + return -EBADMSG; + return 0; +} + +static void krb5enc_decrypt_hash_done(void *data, int err) +{ + struct aead_request *req = data; + + if (err) + return krb5enc_request_complete(req, err); + + err = krb5enc_verify_hash(req); + krb5enc_request_complete(req, err); +} + +/* + * Dispatch the hashing of the plaintext after we've done the decryption. + */ +static int krb5enc_dispatch_decrypt_hash(struct aead_request *req) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_ctx *ctx = crypto_aead_ctx(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ictx->reqoff); + struct crypto_ahash *auth = ctx->auth; + unsigned int authsize = crypto_aead_authsize(krb5enc); + u8 *hash = areq_ctx->tail; + int err; + + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, req->dst, hash, + req->assoclen + req->cryptlen - authsize); + ahash_request_set_callback(ahreq, aead_request_flags(req), + krb5enc_decrypt_hash_done, req); + + err = crypto_ahash_digest(ahreq); + if (err < 0) + return err; + + return krb5enc_verify_hash(req); +} + +/* + * Dispatch the decryption of the ciphertext. + */ +static int krb5enc_dispatch_decrypt(struct aead_request *req) +{ + struct crypto_aead *krb5enc = crypto_aead_reqtfm(req); + struct aead_instance *inst = aead_alg_instance(krb5enc); + struct krb5enc_ctx *ctx = crypto_aead_ctx(krb5enc); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_request_ctx *areq_ctx = aead_request_ctx(req); + struct skcipher_request *skreq = (void *)(areq_ctx->tail + + ictx->reqoff); + unsigned int authsize = crypto_aead_authsize(krb5enc); + struct scatterlist *src, *dst; + + src = scatterwalk_ffwd(areq_ctx->src, req->src, req->assoclen); + dst = src; + + if (req->src != req->dst) + dst = scatterwalk_ffwd(areq_ctx->dst, req->dst, req->assoclen); + + skcipher_request_set_tfm(skreq, ctx->enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(skreq, src, dst, + req->cryptlen - authsize, req->iv); + + return crypto_skcipher_decrypt(skreq); +} + +static int krb5enc_decrypt(struct aead_request *req) +{ + int err; + + err = krb5enc_dispatch_decrypt(req); + if (err < 0) + return err; + + return krb5enc_dispatch_decrypt_hash(req); +} + +static int krb5enc_init_tfm(struct crypto_aead *tfm) +{ + struct aead_instance *inst = aead_alg_instance(tfm); + struct krb5enc_instance_ctx *ictx = aead_instance_ctx(inst); + struct krb5enc_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_ahash *auth; + struct crypto_skcipher *enc; + int err; + + auth = crypto_spawn_ahash(&ictx->auth); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + enc = crypto_spawn_skcipher(&ictx->enc); + err = PTR_ERR(enc); + if (IS_ERR(enc)) + goto err_free_ahash; + + ctx->auth = auth; + ctx->enc = enc; + + crypto_aead_set_reqsize( + tfm, + sizeof(struct krb5enc_request_ctx) + + ictx->reqoff + /* Space for two checksums */ + umax(sizeof(struct ahash_request) + crypto_ahash_reqsize(auth), + sizeof(struct skcipher_request) + crypto_skcipher_reqsize(enc))); + + return 0; + +err_free_ahash: + crypto_free_ahash(auth); + return err; +} + +static void krb5enc_exit_tfm(struct crypto_aead *tfm) +{ + struct krb5enc_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_ahash(ctx->auth); + crypto_free_skcipher(ctx->enc); +} + +static void krb5enc_free(struct aead_instance *inst) +{ + struct krb5enc_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->enc); + crypto_drop_ahash(&ctx->auth); + kfree(inst); +} + +/* + * Create an instance of a template for a specific hash and cipher pair. + */ +static int krb5enc_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct krb5enc_instance_ctx *ictx; + struct skcipher_alg_common *enc; + struct hash_alg_common *auth; + struct aead_instance *inst; + struct crypto_alg *auth_base; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); + if (err) { + pr_err("attr_type failed\n"); + return err; + } + + inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); + if (!inst) + return -ENOMEM; + ictx = aead_instance_ctx(inst); + + err = crypto_grab_ahash(&ictx->auth, aead_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) { + pr_err("grab ahash failed\n"); + goto err_free_inst; + } + auth = crypto_spawn_ahash_alg(&ictx->auth); + auth_base = &auth->base; + + err = crypto_grab_skcipher(&ictx->enc, aead_crypto_instance(inst), + crypto_attr_alg_name(tb[2]), 0, mask); + if (err) { + pr_err("grab skcipher failed\n"); + goto err_free_inst; + } + enc = crypto_spawn_skcipher_alg_common(&ictx->enc); + + ictx->reqoff = 2 * auth->digestsize; + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "krb5enc(%s,%s)", auth_base->cra_name, + enc->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "krb5enc(%s,%s)", auth_base->cra_driver_name, + enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + inst->alg.base.cra_priority = enc->base.cra_priority * 10 + + auth_base->cra_priority; + inst->alg.base.cra_blocksize = enc->base.cra_blocksize; + inst->alg.base.cra_alignmask = enc->base.cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct krb5enc_ctx); + + inst->alg.ivsize = enc->ivsize; + inst->alg.chunksize = enc->chunksize; + inst->alg.maxauthsize = auth->digestsize; + + inst->alg.init = krb5enc_init_tfm; + inst->alg.exit = krb5enc_exit_tfm; + + inst->alg.setkey = krb5enc_setkey; + inst->alg.encrypt = krb5enc_encrypt; + inst->alg.decrypt = krb5enc_decrypt; + + inst->free = krb5enc_free; + + err = aead_register_instance(tmpl, inst); + if (err) { + pr_err("ref failed\n"); + goto err_free_inst; + } + + return 0; + +err_free_inst: + krb5enc_free(inst); + return err; +} + +static struct crypto_template crypto_krb5enc_tmpl = { + .name = "krb5enc", + .create = krb5enc_create, + .module = THIS_MODULE, +}; + +static int __init crypto_krb5enc_module_init(void) +{ + return crypto_register_template(&crypto_krb5enc_tmpl); +} + +static void __exit crypto_krb5enc_module_exit(void) +{ + crypto_unregister_template(&crypto_krb5enc_tmpl); +} + +subsys_initcall(crypto_krb5enc_module_init); +module_exit(crypto_krb5enc_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Simple AEAD wrapper for Kerberos 5 RFC3961"); +MODULE_ALIAS_CRYPTO("krb5enc"); diff --git a/include/crypto/authenc.h b/include/crypto/authenc.h index 5f92a986083c..15a9caa2354a 100644 --- a/include/crypto/authenc.h +++ b/include/crypto/authenc.h @@ -28,5 +28,7 @@ struct crypto_authenc_keys { int crypto_authenc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key, unsigned int keylen); +int crypto_krb5enc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key, + unsigned int keylen); #endif /* _CRYPTO_AUTHENC_H */ -- cgit v1.2.3 From 1b80b6f446ed262077f5212ad213e8ead2cdecc3 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 9 Jan 2025 20:13:55 +0000 Subject: crypto/krb5: Test manager data Add Kerberos crypto tests to the test manager database. This covers: camellia128-cts-cmac samples from RFC6803 camellia256-cts-cmac samples from RFC6803 aes128-cts-hmac-sha256-128 samples from RFC8009 aes256-cts-hmac-sha384-192 samples from RFC8009 but not: aes128-cts-hmac-sha1-96 aes256-cts-hmac-sha1-96 as the test samples in RFC3962 don't seem to be suitable. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/testmgr.c | 16 +++ crypto/testmgr.h | 351 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 367 insertions(+) (limited to 'crypto') diff --git a/crypto/testmgr.c b/crypto/testmgr.c index b69877db3f33..cbce769b16ef 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4505,6 +4505,12 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "authenc(hmac(sha256),ctr(aes))", .test = alg_test_null, .fips_allowed = 1, + }, { + .alg = "authenc(hmac(sha256),cts(cbc(aes)))", + .test = alg_test_aead, + .suite = { + .aead = __VECS(krb5_test_aes128_cts_hmac_sha256_128) + } }, { .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))", .test = alg_test_null, @@ -4525,6 +4531,12 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "authenc(hmac(sha384),ctr(aes))", .test = alg_test_null, .fips_allowed = 1, + }, { + .alg = "authenc(hmac(sha384),cts(cbc(aes)))", + .test = alg_test_aead, + .suite = { + .aead = __VECS(krb5_test_aes256_cts_hmac_sha384_192) + } }, { .alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))", .test = alg_test_null, @@ -5398,6 +5410,10 @@ static const struct alg_test_desc alg_test_descs[] = { .alg = "jitterentropy_rng", .fips_allowed = 1, .test = alg_test_null, + }, { + .alg = "krb5enc(cmac(camellia),cts(cbc(camellia)))", + .test = alg_test_aead, + .suite.aead = __VECS(krb5_test_camellia_cts_cmac) }, { .alg = "lrw(aes)", .generic_driver = "lrw(ecb(aes-generic))", diff --git a/crypto/testmgr.h b/crypto/testmgr.h index d754ab997186..c384b2f7ec5d 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -38894,4 +38894,355 @@ static const struct cipher_testvec aes_hctr2_tv_template[] = { }; +#ifdef __LITTLE_ENDIAN +#define AUTHENC_KEY_HEADER(enckeylen) \ + "\x08\x00\x01\x00" /* LE rtattr */ \ + enckeylen /* crypto_authenc_key_param */ +#else +#define AUTHENC_KEY_HEADER(enckeylen) \ + "\x00\x08\x00\x01" /* BE rtattr */ \ + enckeylen /* crypto_authenc_key_param */ +#endif + +static const struct aead_testvec krb5_test_aes128_cts_hmac_sha256_128[] = { + /* rfc8009 Appendix A */ + { + /* "enc no plain" */ + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x9F\xDA\x0E\x56\xAB\x2D\x85\xE1\x56\x9A\x68\x86\x96\xC2\x6A\x6C" // Ki + "\x9B\x19\x7D\xD1\xE8\xC5\x60\x9D\x6E\x67\xC3\xE3\x7C\x62\xC7\x2E", // Ke + .klen = 4 + 4 + 16 + 16, + .ptext = + "\x7E\x58\x95\xEA\xF2\x67\x24\x35\xBA\xD8\x17\xF5\x45\xA3\x71\x48" // Confounder + "", // Plain + .plen = 16 + 0, + .ctext = + "\xEF\x85\xFB\x89\x0B\xB8\x47\x2F\x4D\xAB\x20\x39\x4D\xCA\x78\x1D" + "\xAD\x87\x7E\xDA\x39\xD5\x0C\x87\x0C\x0D\x5A\x0A\x8E\x48\xC7\x18", + .clen = 16 + 0 + 16, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", // IV + .alen = 16, + }, { + /* "enc plainblock" */ + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x9F\xDA\x0E\x56\xAB\x2D\x85\xE1\x56\x9A\x68\x86\x96\xC2\x6A\x6C" // Ki + "\x9B\x19\x7D\xD1\xE8\xC5\x60\x9D\x6E\x67\xC3\xE3\x7C\x62\xC7\x2E", // Ke + .klen = 4 + 4 + 16 + 16, + .ptext = + "\xA7\xA4\xE2\x9A\x47\x28\xCE\x10\x66\x4F\xB6\x4E\x49\xAD\x3F\xAC" // Confounder + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F" + "\x10\x11\x12\x13\x14", // Plain + .plen = 16 + 21, + .ctext = + "\x72\x0F\x73\xB1\x8D\x98\x59\xCD\x6C\xCB\x43\x46\x11\x5C\xD3\x36" + "\xC7\x0F\x58\xED\xC0\xC4\x43\x7C\x55\x73\x54\x4C\x31\xC8\x13\xBC" + "\xE1\xE6\xD0\x72\xC1\x86\xB3\x9A\x41\x3C\x2F\x92\xCA\x9B\x83\x34" + "\xA2\x87\xFF\xCB\xFC", + .clen = 16 + 21 + 16, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", // IV + .alen = 16, + }, +}; + +static const struct aead_testvec krb5_test_aes256_cts_hmac_sha384_192[] = { + /* rfc8009 Appendix A */ + { + /* "enc no plain" */ + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x69\xB1\x65\x14\xE3\xCD\x8E\x56\xB8\x20\x10\xD5\xC7\x30\x12\xB6" + "\x22\xC4\xD0\x0F\xFC\x23\xED\x1F" // Ki + "\x56\xAB\x22\xBE\xE6\x3D\x82\xD7\xBC\x52\x27\xF6\x77\x3F\x8E\xA7" + "\xA5\xEB\x1C\x82\x51\x60\xC3\x83\x12\x98\x0C\x44\x2E\x5C\x7E\x49", // Ke + .klen = 4 + 4 + 32 + 24, + .ptext = + "\xF7\x64\xE9\xFA\x15\xC2\x76\x47\x8B\x2C\x7D\x0C\x4E\x5F\x58\xE4" // Confounder + "", // Plain + .plen = 16 + 0, + .ctext = + "\x41\xF5\x3F\xA5\xBF\xE7\x02\x6D\x91\xFA\xF9\xBE\x95\x91\x95\xA0" + "\x58\x70\x72\x73\xA9\x6A\x40\xF0\xA0\x19\x60\x62\x1A\xC6\x12\x74" + "\x8B\x9B\xBF\xBE\x7E\xB4\xCE\x3C", + .clen = 16 + 0 + 24, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", // IV + .alen = 16, + }, { + /* "enc plainblock" */ + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x69\xB1\x65\x14\xE3\xCD\x8E\x56\xB8\x20\x10\xD5\xC7\x30\x12\xB6" + "\x22\xC4\xD0\x0F\xFC\x23\xED\x1F" // Ki + "\x56\xAB\x22\xBE\xE6\x3D\x82\xD7\xBC\x52\x27\xF6\x77\x3F\x8E\xA7" + "\xA5\xEB\x1C\x82\x51\x60\xC3\x83\x12\x98\x0C\x44\x2E\x5C\x7E\x49", // Ke + .klen = 4 + 4 + 32 + 24, + .ptext = + "\x76\x3E\x65\x36\x7E\x86\x4F\x02\xF5\x51\x53\xC7\xE3\xB5\x8A\xF1" // Confounder + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F" + "\x10\x11\x12\x13\x14", // Plain + .plen = 16 + 21, + .ctext = + "\x40\x01\x3E\x2D\xF5\x8E\x87\x51\x95\x7D\x28\x78\xBC\xD2\xD6\xFE" + "\x10\x1C\xCF\xD5\x56\xCB\x1E\xAE\x79\xDB\x3C\x3E\xE8\x64\x29\xF2" + "\xB2\xA6\x02\xAC\x86\xFE\xF6\xEC\xB6\x47\xD6\x29\x5F\xAE\x07\x7A" + "\x1F\xEB\x51\x75\x08\xD2\xC1\x6B\x41\x92\xE0\x1F\x62", + .clen = 16 + 21 + 24, + .assoc = "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", // IV + .alen = 16, + }, +}; + +static const struct aead_testvec krb5_test_camellia_cts_cmac[] = { + /* rfc6803 sec 10 */ + { + // "enc no plain" + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x45\xeb\x66\xe2\xef\xa8\x77\x8f\x7d\xf1\x46\x54\x53\x05\x98\x06" // Ki + "\xe9\x9b\x82\xb3\x6c\x4a\xe8\xea\x19\xe9\x5d\xfa\x9e\xde\x88\x2c", // Ke + .klen = 4 + 4 + 16 * 2, + .ptext = + "\xB6\x98\x22\xA1\x9A\x6B\x09\xC0\xEB\xC8\x55\x7D\x1F\x1B\x6C\x0A" // Confounder + "", // Plain + .plen = 16 + 0, + .ctext = + "\xC4\x66\xF1\x87\x10\x69\x92\x1E\xDB\x7C\x6F\xDE\x24\x4A\x52\xDB" + "\x0B\xA1\x0E\xDC\x19\x7B\xDB\x80\x06\x65\x8C\xA3\xCC\xCE\x6E\xB8", + .clen = 16 + 0 + 16, + }, { + // "enc 1 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x13\x5f\xe7\x11\x6f\x53\xc2\xaa\x36\x12\xb7\xea\xe0\xf2\x84\xaa" // Ki + "\xa7\xed\xcd\x53\x97\xea\x6d\x12\xb0\xaf\xf4\xcb\x8d\xaa\x57\xad", // Ke + .klen = 4 + 4 + 16 * 2, + .ptext = + "\x6F\x2F\xC3\xC2\xA1\x66\xFD\x88\x98\x96\x7A\x83\xDE\x95\x96\xD9" // Confounder + "1", // Plain + .plen = 16 + 1, + .ctext = + "\x84\x2D\x21\xFD\x95\x03\x11\xC0\xDD\x46\x4A\x3F\x4B\xE8\xD6\xDA" + "\x88\xA5\x6D\x55\x9C\x9B\x47\xD3\xF9\xA8\x50\x67\xAF\x66\x15\x59" + "\xB8", + .clen = 16 + 1 + 16, + }, { + // "enc 9 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x10\x2c\x34\xd0\x75\x74\x9f\x77\x8a\x15\xca\xd1\xe9\x7d\xa9\x86" // Ki + "\xdd\xe4\x2e\xca\x7c\xd9\x86\x3f\xc3\xce\x89\xcb\xc9\x43\x62\xd7", // Ke + .klen = 4 + 4 + 16 * 2, + .ptext = + "\xA5\xB4\xA7\x1E\x07\x7A\xEE\xF9\x3C\x87\x63\xC1\x8F\xDB\x1F\x10" // Confounder + "9 bytesss", // Plain + .plen = 16 + 9, + .ctext = + "\x61\x9F\xF0\x72\xE3\x62\x86\xFF\x0A\x28\xDE\xB3\xA3\x52\xEC\x0D" + "\x0E\xDF\x5C\x51\x60\xD6\x63\xC9\x01\x75\x8C\xCF\x9D\x1E\xD3\x3D" + "\x71\xDB\x8F\x23\xAA\xBF\x83\x48\xA0", + .clen = 16 + 9 + 16, + }, { + // "enc 13 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\xb8\xc4\x38\xcc\x1a\x00\x60\xfc\x91\x3a\x8e\x07\x16\x96\xbd\x08" // Ki + "\xc3\x11\x3a\x25\x85\x90\xb9\xae\xbf\x72\x1b\x1a\xf6\xb0\xcb\xf8", // Ke + .klen = 4 + 4 + 16 * 2, + .ptext = + "\x19\xFE\xE4\x0D\x81\x0C\x52\x4B\x5B\x22\xF0\x18\x74\xC6\x93\xDA" // Confounder + "13 bytes byte", // Plain + .plen = 16 + 13, + .ctext = + "\xB8\xEC\xA3\x16\x7A\xE6\x31\x55\x12\xE5\x9F\x98\xA7\xC5\x00\x20" + "\x5E\x5F\x63\xFF\x3B\xB3\x89\xAF\x1C\x41\xA2\x1D\x64\x0D\x86\x15" + "\xC9\xED\x3F\xBE\xB0\x5A\xB6\xAC\xB6\x76\x89\xB5\xEA", + .clen = 16 + 13 + 16, + }, { + // "enc 30 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x10") + "\x18\xaf\x19\xb0\x23\x74\x44\xfd\x75\x04\xad\x7d\xbd\x48\xad\xd3" // Ki + "\x8b\x07\xee\xd3\x01\x49\x91\x6a\xa2\x0d\xb3\xf5\xce\xd8\xaf\xad", // Ke + .klen = 4 + 4 + 16 * 2, + .ptext = + "\xCA\x7A\x7A\xB4\xBE\x19\x2D\xAB\xD6\x03\x50\x6D\xB1\x9C\x39\xE2" // Confounder + "30 bytes bytes bytes bytes byt", // Plain + .plen = 16 + 30, + .ctext = + "\xA2\x6A\x39\x05\xA4\xFF\xD5\x81\x6B\x7B\x1E\x27\x38\x0D\x08\x09" + "\x0C\x8E\xC1\xF3\x04\x49\x6E\x1A\xBD\xCD\x2B\xDC\xD1\xDF\xFC\x66" + "\x09\x89\xE1\x17\xA7\x13\xDD\xBB\x57\xA4\x14\x6C\x15\x87\xCB\xA4" + "\x35\x66\x65\x59\x1D\x22\x40\x28\x2F\x58\x42\xB1\x05\xA5", + .clen = 16 + 30 + 16, + }, { + // "enc no plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\xa2\xb8\x33\xe9\x43\xbb\x10\xee\x53\xb4\xa1\x9b\xc2\xbb\xc7\xe1" + "\x9b\x87\xad\x5d\xe9\x21\x22\xa4\x33\x8b\xe6\xf7\x32\xfd\x8a\x0e" // Ki + "\x6c\xcb\x3f\x25\xd8\xae\x57\xf4\xe8\xf6\xca\x47\x4b\xdd\xef\xf1" + "\x16\xce\x13\x1b\x3f\x71\x01\x2e\x75\x6d\x6b\x1e\x3f\x70\xa7\xf1", // Ke + .klen = 4 + 4 + 32 * 2, + .ptext = + "\x3C\xBB\xD2\xB4\x59\x17\x94\x10\x67\xF9\x65\x99\xBB\x98\x92\x6C" // Confounder + "", // Plain + .plen = 16 + 0, + .ctext = + "\x03\x88\x6D\x03\x31\x0B\x47\xA6\xD8\xF0\x6D\x7B\x94\xD1\xDD\x83" + "\x7E\xCC\xE3\x15\xEF\x65\x2A\xFF\x62\x08\x59\xD9\x4A\x25\x92\x66", + .clen = 16 + 0 + 16, + }, { + // "enc 1 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x84\x61\x4b\xfa\x98\xf1\x74\x8a\xa4\xaf\x99\x2b\x8c\x26\x28\x0d" + "\xc8\x98\x73\x29\xdf\x77\x5c\x1d\xb0\x4a\x43\xf1\x21\xaa\x86\x65" // Ki + "\xe9\x31\x73\xaa\x01\xeb\x3c\x24\x62\x31\xda\xfc\x78\x02\xee\x32" + "\xaf\x24\x85\x1d\x8c\x73\x87\xd1\x8c\xb9\xb2\xc5\xb7\xf5\x70\xb8", // Ke + .klen = 4 + 4 + 32 * 2, + .ptext = + "\xDE\xF4\x87\xFC\xEB\xE6\xDE\x63\x46\xD4\xDA\x45\x21\xBB\xA2\xD2" // Confounder + "1", // Plain + .plen = 16 + 1, + .ctext = + "\x2C\x9C\x15\x70\x13\x3C\x99\xBF\x6A\x34\xBC\x1B\x02\x12\x00\x2F" + "\xD1\x94\x33\x87\x49\xDB\x41\x35\x49\x7A\x34\x7C\xFC\xD9\xD1\x8A" + "\x12", + .clen = 16 + 1 + 16, + }, { + // "enc 9 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x47\xb9\xf5\xba\xd7\x63\x00\x58\x2a\x54\x45\xfa\x0c\x1b\x29\xc3" + "\xaa\x83\xec\x63\xb9\x0b\x4a\xb0\x08\x48\xc1\x85\x67\x4f\x44\xa7" // Ki + "\xcd\xa2\xd3\x9a\x9b\x24\x3f\xfe\xb5\x6e\x8d\x5f\x4b\xd5\x28\x74" + "\x1e\xcb\x52\x0c\x62\x12\x3f\xb0\x40\xb8\x41\x8b\x15\xc7\xd7\x0c", // Ke + .klen = 4 + 4 + 32 * 2, + .ptext = + "\xAD\x4F\xF9\x04\xD3\x4E\x55\x53\x84\xB1\x41\x00\xFC\x46\x5F\x88" // Confounder + "9 bytesss", // Plain + .plen = 16 + 9, + .ctext = + "\x9C\x6D\xE7\x5F\x81\x2D\xE7\xED\x0D\x28\xB2\x96\x35\x57\xA1\x15" + "\x64\x09\x98\x27\x5B\x0A\xF5\x15\x27\x09\x91\x3F\xF5\x2A\x2A\x9C" + "\x8E\x63\xB8\x72\xF9\x2E\x64\xC8\x39", + .clen = 16 + 9 + 16, + }, { + // "enc 13 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x15\x2f\x8c\x9d\xc9\x85\x79\x6e\xb1\x94\xed\x14\xc5\x9e\xac\xdd" + "\x41\x8a\x33\x32\x36\xb7\x8f\xaf\xa7\xc7\x9b\x04\xe0\xac\xe7\xbf" // Ki + "\xcd\x8a\x10\xe2\x79\xda\xdd\xb6\x90\x1e\xc3\x0b\xdf\x98\x73\x25" + "\x0f\x6e\xfc\x6a\x77\x36\x7d\x74\xdc\x3e\xe7\xf7\x4b\xc7\x77\x4e", // Ke + .klen = 4 + 4 + 32 * 2, + .ptext = + "\xCF\x9B\xCA\x6D\xF1\x14\x4E\x0C\x0A\xF9\xB8\xF3\x4C\x90\xD5\x14" // Confounder + "13 bytes byte", + .plen = 16 + 13, + .ctext = + "\xEE\xEC\x85\xA9\x81\x3C\xDC\x53\x67\x72\xAB\x9B\x42\xDE\xFC\x57" + "\x06\xF7\x26\xE9\x75\xDD\xE0\x5A\x87\xEB\x54\x06\xEA\x32\x4C\xA1" + "\x85\xC9\x98\x6B\x42\xAA\xBE\x79\x4B\x84\x82\x1B\xEE", + .clen = 16 + 13 + 16, + }, { + // "enc 30 plain", + .key = + AUTHENC_KEY_HEADER("\x00\x00\x00\x20") + "\x04\x8d\xeb\xf7\xb1\x2c\x09\x32\xe8\xb2\x96\x99\x6c\x23\xf8\xb7" + "\x9d\x59\xb9\x7e\xa1\x19\xfc\x0c\x15\x6b\xf7\x88\xdc\x8c\x85\xe8" // Ki + "\x1d\x51\x47\xf3\x4b\xb0\x01\xa0\x4a\x68\xa7\x13\x46\xe7\x65\x4e" + "\x02\x23\xa6\x0d\x90\xbc\x2b\x79\xb4\xd8\x79\x56\xd4\x7c\xd4\x2a", // Ke + .klen = 4 + 4 + 32 * 2, + .ptext = + "\x64\x4D\xEF\x38\xDA\x35\x00\x72\x75\x87\x8D\x21\x68\x55\xE2\x28" // Confounder + "30 bytes bytes bytes bytes byt", // Plain + .plen = 16 + 30, + .ctext = + "\x0E\x44\x68\x09\x85\x85\x5F\x2D\x1F\x18\x12\x52\x9C\xA8\x3B\xFD" + "\x8E\x34\x9D\xE6\xFD\x9A\xDA\x0B\xAA\xA0\x48\xD6\x8E\x26\x5F\xEB" + "\xF3\x4A\xD1\x25\x5A\x34\x49\x99\xAD\x37\x14\x68\x87\xA6\xC6\x84" + "\x57\x31\xAC\x7F\x46\x37\x6A\x05\x04\xCD\x06\x57\x14\x74", + .clen = 16 + 30 + 16, + }, +}; + #endif /* _CRYPTO_TESTMGR_H */ -- cgit v1.2.3 From 3936f02bf2d3308a7359dd37dd96cd60603d8170 Mon Sep 17 00:00:00 2001 From: David Howells Date: Tue, 10 Nov 2020 17:00:54 +0000 Subject: crypto/krb5: Implement Kerberos crypto core Provide core structures, an encoding-type registry and basic module and config bits for a generic Kerberos crypto library. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/Kconfig | 1 + crypto/Makefile | 2 + crypto/krb5/Kconfig | 14 +++++++ crypto/krb5/Makefile | 9 ++++ crypto/krb5/internal.h | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ crypto/krb5/krb5_api.c | 42 +++++++++++++++++++ include/crypto/krb5.h | 54 ++++++++++++++++++++++++ 7 files changed, 234 insertions(+) create mode 100644 crypto/krb5/Kconfig create mode 100644 crypto/krb5/Makefile create mode 100644 crypto/krb5/internal.h create mode 100644 crypto/krb5/krb5_api.c (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index 15a6730d3ea1..f0119d92decc 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -1472,5 +1472,6 @@ endif source "drivers/crypto/Kconfig" source "crypto/asymmetric_keys/Kconfig" source "certs/Kconfig" +source "crypto/krb5/Kconfig" endif # if CRYPTO diff --git a/crypto/Makefile b/crypto/Makefile index 20c8e3ee9835..d1e422249af6 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -213,3 +213,5 @@ obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o # Key derivation function # obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o + +obj-$(CONFIG_CRYPTO_KRB5) += krb5/ diff --git a/crypto/krb5/Kconfig b/crypto/krb5/Kconfig new file mode 100644 index 000000000000..079873618abf --- /dev/null +++ b/crypto/krb5/Kconfig @@ -0,0 +1,14 @@ +config CRYPTO_KRB5 + tristate "Kerberos 5 crypto" + select CRYPTO_MANAGER + select CRYPTO_KRB5ENC + select CRYPTO_AUTHENC + select CRYPTO_SKCIPHER + select CRYPTO_HASH_INFO + select CRYPTO_SHA1 + select CRYPTO_CBC + select CRYPTO_CTS + select CRYPTO_AES + help + Provide a library for provision of Kerberos-5-based crypto. This is + intended for network filesystems to use. diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile new file mode 100644 index 000000000000..c450d0754772 --- /dev/null +++ b/crypto/krb5/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for asymmetric cryptographic keys +# + +krb5-y += \ + krb5_api.o + +obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h new file mode 100644 index 000000000000..3ede858be4f7 --- /dev/null +++ b/crypto/krb5/internal.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Kerberos5 crypto internals + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include + +/* + * Profile used for key derivation and encryption. + */ +struct krb5_crypto_profile { + /* Pseudo-random function */ + int (*calc_PRF)(const struct krb5_enctype *krb5, + const struct krb5_buffer *protocol_key, + const struct krb5_buffer *octet_string, + struct krb5_buffer *result, + gfp_t gfp); + + /* Checksum key derivation */ + int (*calc_Kc)(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + const struct krb5_buffer *usage_constant, + struct krb5_buffer *Kc, + gfp_t gfp); + + /* Encryption key derivation */ + int (*calc_Ke)(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + const struct krb5_buffer *usage_constant, + struct krb5_buffer *Ke, + gfp_t gfp); + + /* Integrity key derivation */ + int (*calc_Ki)(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + const struct krb5_buffer *usage_constant, + struct krb5_buffer *Ki, + gfp_t gfp); + + /* Derive the keys needed for an encryption AEAD object. */ + int (*derive_encrypt_keys)(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp); + + /* Directly load the keys needed for an encryption AEAD object. */ + int (*load_encrypt_keys)(const struct krb5_enctype *krb5, + const struct krb5_buffer *Ke, + const struct krb5_buffer *Ki, + struct krb5_buffer *setkey, + gfp_t gfp); + + /* Derive the key needed for a checksum hash object. */ + int (*derive_checksum_key)(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp); + + /* Directly load the keys needed for a checksum hash object. */ + int (*load_checksum_key)(const struct krb5_enctype *krb5, + const struct krb5_buffer *Kc, + struct krb5_buffer *setkey, + gfp_t gfp); + + /* Encrypt data in-place, inserting confounder and checksum. */ + ssize_t (*encrypt)(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded); + + /* Decrypt data in-place, removing confounder and checksum */ + int (*decrypt)(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); + + /* Generate a MIC on part of a packet, inserting the checksum */ + ssize_t (*get_mic)(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len); + + /* Verify the MIC on a piece of data, removing the checksum */ + int (*verify_mic)(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); +}; + +/* + * Crypto size/alignment rounding convenience macros. + */ +#define crypto_roundup(X) ((unsigned int)round_up((X), CRYPTO_MINALIGN)) + +#define krb5_aead_size(TFM) \ + crypto_roundup(sizeof(struct aead_request) + crypto_aead_reqsize(TFM)) +#define krb5_aead_ivsize(TFM) \ + crypto_roundup(crypto_aead_ivsize(TFM)) +#define krb5_shash_size(TFM) \ + crypto_roundup(sizeof(struct shash_desc) + crypto_shash_descsize(TFM)) +#define krb5_digest_size(TFM) \ + crypto_roundup(crypto_shash_digestsize(TFM)) +#define round16(x) (((x) + 15) & ~15) diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c new file mode 100644 index 000000000000..5c1cd5d07fc3 --- /dev/null +++ b/crypto/krb5/krb5_api.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Kerberos 5 crypto library. + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include "internal.h" + +MODULE_DESCRIPTION("Kerberos 5 crypto"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +static const struct krb5_enctype *const krb5_supported_enctypes[] = { +}; + +/** + * crypto_krb5_find_enctype - Find the handler for a Kerberos5 encryption type + * @enctype: The standard Kerberos encryption type number + * + * Look up a Kerberos encryption type by number. If successful, returns a + * pointer to the type tables; returns NULL otherwise. + */ +const struct krb5_enctype *crypto_krb5_find_enctype(u32 enctype) +{ + const struct krb5_enctype *krb5; + size_t i; + + for (i = 0; i < ARRAY_SIZE(krb5_supported_enctypes); i++) { + krb5 = krb5_supported_enctypes[i]; + if (krb5->etype == enctype) + return krb5; + } + + return NULL; +} +EXPORT_SYMBOL(crypto_krb5_find_enctype); diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index 44a6342471d7..8fa6715ab35b 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -8,6 +8,12 @@ #ifndef _CRYPTO_KRB5_H #define _CRYPTO_KRB5_H +#include +#include + +struct crypto_shash; +struct scatterlist; + /* * Per Kerberos v5 protocol spec crypto types from the wire. These get mapped * to linux kernel crypto routines. @@ -48,4 +54,52 @@ #define KEY_USAGE_SEED_ENCRYPTION (0xAA) #define KEY_USAGE_SEED_INTEGRITY (0x55) +/* + * Mode of operation. + */ +enum krb5_crypto_mode { + KRB5_CHECKSUM_MODE, /* Checksum only */ + KRB5_ENCRYPT_MODE, /* Fully encrypted, possibly with integrity checksum */ +}; + +struct krb5_buffer { + unsigned int len; + void *data; +}; + +/* + * Kerberos encoding type definition. + */ +struct krb5_enctype { + int etype; /* Encryption (key) type */ + int ctype; /* Checksum type */ + const char *name; /* "Friendly" name */ + const char *encrypt_name; /* Crypto encrypt+checksum name */ + const char *cksum_name; /* Crypto checksum name */ + const char *hash_name; /* Crypto hash name */ + const char *derivation_enc; /* Cipher used in key derivation */ + u16 block_len; /* Length of encryption block */ + u16 conf_len; /* Length of confounder (normally == block_len) */ + u16 cksum_len; /* Length of checksum */ + u16 key_bytes; /* Length of raw key, in bytes */ + u16 key_len; /* Length of final key, in bytes */ + u16 hash_len; /* Length of hash in bytes */ + u16 prf_len; /* Length of PRF() result in bytes */ + u16 Kc_len; /* Length of Kc in bytes */ + u16 Ke_len; /* Length of Ke in bytes */ + u16 Ki_len; /* Length of Ki in bytes */ + bool keyed_cksum; /* T if a keyed cksum */ + + const struct krb5_crypto_profile *profile; + + int (*random_to_key)(const struct krb5_enctype *krb5, + const struct krb5_buffer *in, + struct krb5_buffer *out); /* complete key generation */ +}; + +/* + * krb5_api.c + */ +const struct krb5_enctype *crypto_krb5_find_enctype(u32 enctype); + #endif /* _CRYPTO_KRB5_H */ -- cgit v1.2.3 From 025ac491f4eeb48c03353719f0de20a6db36b826 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 19 Nov 2020 09:46:48 +0000 Subject: crypto/krb5: Add an API to query the layout of the crypto section Provide some functions to allow the called to find out about the layout of the crypto section: (1) Calculate, for a given size of data, how big a buffer will be required to hold it and where the data will be within it. (2) Calculate, for an amount of buffer, what's the maximum size of data that will fit therein, and where it will start. (3) Determine where the data will be in a received message. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/krb5_api.c | 108 +++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 9 +++++ 2 files changed, 117 insertions(+) (limited to 'crypto') diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 5c1cd5d07fc3..f6d1bc813daa 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -40,3 +40,111 @@ const struct krb5_enctype *crypto_krb5_find_enctype(u32 enctype) return NULL; } EXPORT_SYMBOL(crypto_krb5_find_enctype); + +/** + * crypto_krb5_how_much_buffer - Work out how much buffer is required for an amount of data + * @krb5: The encoding to use. + * @mode: The mode in which to operated (checksum/encrypt) + * @data_size: How much data we want to allow for + * @_offset: Where to place the offset into the buffer + * + * Calculate how much buffer space is required to wrap a given amount of data. + * This allows for a confounder, padding and checksum as appropriate. The + * amount of buffer required is returned and the offset into the buffer at + * which the data will start is placed in *_offset. + */ +size_t crypto_krb5_how_much_buffer(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t data_size, size_t *_offset) +{ + switch (mode) { + case KRB5_CHECKSUM_MODE: + *_offset = krb5->cksum_len; + return krb5->cksum_len + data_size; + + case KRB5_ENCRYPT_MODE: + *_offset = krb5->conf_len; + return krb5->conf_len + data_size + krb5->cksum_len; + + default: + WARN_ON(1); + *_offset = 0; + return 0; + } +} +EXPORT_SYMBOL(crypto_krb5_how_much_buffer); + +/** + * crypto_krb5_how_much_data - Work out how much data can fit in an amount of buffer + * @krb5: The encoding to use. + * @mode: The mode in which to operated (checksum/encrypt) + * @_buffer_size: How much buffer we want to allow for (may be reduced) + * @_offset: Where to place the offset into the buffer + * + * Calculate how much data can be fitted into given amount of buffer. This + * allows for a confounder, padding and checksum as appropriate. The amount of + * data that will fit is returned, the amount of buffer required is shrunk to + * allow for alignment and the offset into the buffer at which the data will + * start is placed in *_offset. + */ +size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_buffer_size, size_t *_offset) +{ + size_t buffer_size = *_buffer_size, data_size; + + switch (mode) { + case KRB5_CHECKSUM_MODE: + if (WARN_ON(buffer_size < krb5->cksum_len + 1)) + goto bad; + *_offset = krb5->cksum_len; + return buffer_size - krb5->cksum_len; + + case KRB5_ENCRYPT_MODE: + if (WARN_ON(buffer_size < krb5->conf_len + 1 + krb5->cksum_len)) + goto bad; + data_size = buffer_size - krb5->cksum_len; + *_offset = krb5->conf_len; + return data_size - krb5->conf_len; + + default: + WARN_ON(1); + goto bad; + } + +bad: + *_offset = 0; + return 0; +} +EXPORT_SYMBOL(crypto_krb5_how_much_data); + +/** + * crypto_krb5_where_is_the_data - Find the data in a decrypted message + * @krb5: The encoding to use. + * @mode: Mode of operation + * @_offset: Offset of the secure blob in the buffer; updated to data offset. + * @_len: The length of the secure blob; updated to data length. + * + * Find the offset and size of the data in a secure message so that this + * information can be used in the metadata buffer which will get added to the + * digest by crypto_krb5_verify_mic(). + */ +void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_offset, size_t *_len) +{ + switch (mode) { + case KRB5_CHECKSUM_MODE: + *_offset += krb5->cksum_len; + *_len -= krb5->cksum_len; + return; + case KRB5_ENCRYPT_MODE: + *_offset += krb5->conf_len; + *_len -= krb5->conf_len + krb5->cksum_len; + return; + default: + WARN_ON_ONCE(1); + return; + } +} +EXPORT_SYMBOL(crypto_krb5_where_is_the_data); diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index 8fa6715ab35b..b414141b8b42 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -101,5 +101,14 @@ struct krb5_enctype { * krb5_api.c */ const struct krb5_enctype *crypto_krb5_find_enctype(u32 enctype); +size_t crypto_krb5_how_much_buffer(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t data_size, size_t *_offset); +size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_buffer_size, size_t *_offset); +void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_offset, size_t *_len); #endif /* _CRYPTO_KRB5_H */ -- cgit v1.2.3 From a9c27d2d87a388433db100889262841afe771f7a Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 17 Jan 2025 13:29:24 +0000 Subject: crypto/krb5: Add an API to alloc and prepare a crypto object Add an API by which users of the krb5 crypto library can get an allocated and keyed crypto object. For encryption-mode operation, an AEAD object is returned; for checksum-mode operation, a synchronous hash object is returned. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/internal.h | 10 ++++ crypto/krb5/krb5_api.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 7 +++ 3 files changed, 161 insertions(+) (limited to 'crypto') diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 3ede858be4f7..b542d24e5fa5 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -110,3 +110,13 @@ struct krb5_crypto_profile { #define krb5_digest_size(TFM) \ crypto_roundup(crypto_shash_digestsize(TFM)) #define round16(x) (((x) + 15) & ~15) + +/* + * krb5_api.c + */ +struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5, + const struct krb5_buffer *keys, + gfp_t gfp); +struct crypto_shash *krb5_prepare_checksum(const struct krb5_enctype *krb5, + const struct krb5_buffer *Kc, + gfp_t gfp); diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index f6d1bc813daa..f7f2528b3895 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -148,3 +148,147 @@ void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5, } } EXPORT_SYMBOL(crypto_krb5_where_is_the_data); + +/* + * Prepare the encryption with derived key data. + */ +struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5, + const struct krb5_buffer *keys, + gfp_t gfp) +{ + struct crypto_aead *ci = NULL; + int ret = -ENOMEM; + + ci = crypto_alloc_aead(krb5->encrypt_name, 0, 0); + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + if (ret == -ENOENT) + ret = -ENOPKG; + goto err; + } + + ret = crypto_aead_setkey(ci, keys->data, keys->len); + if (ret < 0) { + pr_err("Couldn't set AEAD key %s: %d\n", krb5->encrypt_name, ret); + goto err_ci; + } + + ret = crypto_aead_setauthsize(ci, krb5->cksum_len); + if (ret < 0) { + pr_err("Couldn't set AEAD authsize %s: %d\n", krb5->encrypt_name, ret); + goto err_ci; + } + + return ci; +err_ci: + crypto_free_aead(ci); +err: + return ERR_PTR(ret); +} + +/** + * crypto_krb5_prepare_encryption - Prepare AEAD crypto object for encryption-mode + * @krb5: The encoding to use. + * @TK: The transport key to use. + * @usage: The usage constant for key derivation. + * @gfp: Allocation flags. + * + * Allocate a crypto object that does all the necessary crypto, key it and set + * its parameters and return the crypto handle to it. This can then be used to + * dispatch encrypt and decrypt operations. + */ +struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp) +{ + struct crypto_aead *ci = NULL; + struct krb5_buffer keys = {}; + int ret; + + ret = krb5->profile->derive_encrypt_keys(krb5, TK, usage, &keys, gfp); + if (ret < 0) + goto err; + + ci = krb5_prepare_encryption(krb5, &keys, gfp); + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + goto err; + } + + kfree(keys.data); + return ci; +err: + kfree(keys.data); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(crypto_krb5_prepare_encryption); + +/* + * Prepare the checksum with derived key data. + */ +struct crypto_shash *krb5_prepare_checksum(const struct krb5_enctype *krb5, + const struct krb5_buffer *Kc, + gfp_t gfp) +{ + struct crypto_shash *ci = NULL; + int ret = -ENOMEM; + + ci = crypto_alloc_shash(krb5->cksum_name, 0, 0); + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + if (ret == -ENOENT) + ret = -ENOPKG; + goto err; + } + + ret = crypto_shash_setkey(ci, Kc->data, Kc->len); + if (ret < 0) { + pr_err("Couldn't set shash key %s: %d\n", krb5->cksum_name, ret); + goto err_ci; + } + + return ci; +err_ci: + crypto_free_shash(ci); +err: + return ERR_PTR(ret); +} + +/** + * crypto_krb5_prepare_checksum - Prepare AEAD crypto object for checksum-mode + * @krb5: The encoding to use. + * @TK: The transport key to use. + * @usage: The usage constant for key derivation. + * @gfp: Allocation flags. + * + * Allocate a crypto object that does all the necessary crypto, key it and set + * its parameters and return the crypto handle to it. This can then be used to + * dispatch get_mic and verify_mic operations. + */ +struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp) +{ + struct crypto_shash *ci = NULL; + struct krb5_buffer keys = {}; + int ret; + + ret = krb5->profile->derive_checksum_key(krb5, TK, usage, &keys, gfp); + if (ret < 0) { + pr_err("get_Kc failed %d\n", ret); + goto err; + } + + ci = krb5_prepare_checksum(krb5, &keys, gfp); + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + goto err; + } + + kfree(keys.data); + return ci; +err: + kfree(keys.data); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(crypto_krb5_prepare_checksum); diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index b414141b8b42..94af2c558fa1 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -10,6 +10,7 @@ #include #include +#include struct crypto_shash; struct scatterlist; @@ -110,5 +111,11 @@ size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5, void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5, enum krb5_crypto_mode mode, size_t *_offset, size_t *_len); +struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp); +struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp); #endif /* _CRYPTO_KRB5_H */ -- cgit v1.2.3 From 0392b110ccaf543b31842b04c8142f4f8ce7bdec Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 9 Jan 2025 09:03:35 +0000 Subject: crypto/krb5: Add an API to perform requests Add an API by which users of the krb5 crypto library can perform crypto requests, such as encrypt, decrypt, get_mic and verify_mic. These functions take the previously prepared crypto objects to work on. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/krb5_api.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 21 ++++++++ 2 files changed, 162 insertions(+) (limited to 'crypto') diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index f7f2528b3895..8fc3a1b9d4ad 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -292,3 +292,144 @@ err: return ERR_PTR(ret); } EXPORT_SYMBOL(crypto_krb5_prepare_checksum); + +/** + * crypto_krb5_encrypt - Apply Kerberos encryption and integrity. + * @krb5: The encoding to use. + * @aead: The keyed crypto object to use. + * @sg: Scatterlist defining the crypto buffer. + * @nr_sg: The number of elements in @sg. + * @sg_len: The size of the buffer. + * @data_offset: The offset of the data in the @sg buffer. + * @data_len: The length of the data. + * @preconfounded: True if the confounder is already inserted. + * + * Using the specified Kerberos encoding, insert a confounder and padding as + * needed, encrypt this and the data in place and insert an integrity checksum + * into the buffer. + * + * The buffer must include space for the confounder, the checksum and any + * padding required. The caller can preinsert the confounder into the buffer + * (for testing, for example). + * + * The resulting secured blob may be less than the size of the buffer. + * + * Returns the size of the secure blob if successful, -ENOMEM on an allocation + * failure, -EFAULT if there is insufficient space, -EMSGSIZE if the confounder + * is too short or the data is misaligned. Other errors may also be returned + * from the crypto layer. + */ +ssize_t crypto_krb5_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded) +{ + if (WARN_ON(data_offset > sg_len || + data_len > sg_len || + data_offset > sg_len - data_len)) + return -EMSGSIZE; + return krb5->profile->encrypt(krb5, aead, sg, nr_sg, sg_len, + data_offset, data_len, preconfounded); +} +EXPORT_SYMBOL(crypto_krb5_encrypt); + +/** + * crypto_krb5_decrypt - Validate and remove Kerberos encryption and integrity. + * @krb5: The encoding to use. + * @aead: The keyed crypto object to use. + * @sg: Scatterlist defining the crypto buffer. + * @nr_sg: The number of elements in @sg. + * @_offset: Offset of the secure blob in the buffer; updated to data offset. + * @_len: The length of the secure blob; updated to data length. + * + * Using the specified Kerberos encoding, check and remove the integrity + * checksum and decrypt the secure region, stripping off the confounder. + * + * If successful, @_offset and @_len are updated to outline the region in which + * the data plus the trailing padding are stored. The caller is responsible + * for working out how much padding there is and removing it. + * + * Returns the 0 if successful, -ENOMEM on an allocation failure; sets + * *_error_code and returns -EPROTO if the data cannot be parsed, or -EBADMSG + * if the integrity checksum doesn't match). Other errors may also be returned + * from the crypto layer. + */ +int crypto_krb5_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len) +{ + return krb5->profile->decrypt(krb5, aead, sg, nr_sg, _offset, _len); +} +EXPORT_SYMBOL(crypto_krb5_decrypt); + +/** + * crypto_krb5_get_mic - Apply Kerberos integrity checksum. + * @krb5: The encoding to use. + * @shash: The keyed hash to use. + * @metadata: Metadata to add into the hash before adding the data. + * @sg: Scatterlist defining the crypto buffer. + * @nr_sg: The number of elements in @sg. + * @sg_len: The size of the buffer. + * @data_offset: The offset of the data in the @sg buffer. + * @data_len: The length of the data. + * + * Using the specified Kerberos encoding, calculate and insert an integrity + * checksum into the buffer. + * + * The buffer must include space for the checksum at the front. + * + * Returns the size of the secure blob if successful, -ENOMEM on an allocation + * failure, -EFAULT if there is insufficient space, -EMSGSIZE if the gap for + * the checksum is too short. Other errors may also be returned from the + * crypto layer. + */ +ssize_t crypto_krb5_get_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len) +{ + if (WARN_ON(data_offset > sg_len || + data_len > sg_len || + data_offset > sg_len - data_len)) + return -EMSGSIZE; + return krb5->profile->get_mic(krb5, shash, metadata, sg, nr_sg, sg_len, + data_offset, data_len); +} +EXPORT_SYMBOL(crypto_krb5_get_mic); + +/** + * crypto_krb5_verify_mic - Validate and remove Kerberos integrity checksum. + * @krb5: The encoding to use. + * @shash: The keyed hash to use. + * @metadata: Metadata to add into the hash before adding the data. + * @sg: Scatterlist defining the crypto buffer. + * @nr_sg: The number of elements in @sg. + * @_offset: Offset of the secure blob in the buffer; updated to data offset. + * @_len: The length of the secure blob; updated to data length. + * + * Using the specified Kerberos encoding, check and remove the integrity + * checksum. + * + * If successful, @_offset and @_len are updated to outline the region in which + * the data is stored. + * + * Returns the 0 if successful, -ENOMEM on an allocation failure; sets + * *_error_code and returns -EPROTO if the data cannot be parsed, or -EBADMSG + * if the checksum doesn't match). Other errors may also be returned from the + * crypto layer. + */ +int crypto_krb5_verify_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len) +{ + return krb5->profile->verify_mic(krb5, shash, metadata, sg, nr_sg, + _offset, _len); +} +EXPORT_SYMBOL(crypto_krb5_verify_mic); diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index 94af2c558fa1..81739e9828d3 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -117,5 +117,26 @@ struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *kr struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, u32 usage, gfp_t gfp); +ssize_t crypto_krb5_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded); +int crypto_krb5_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); +ssize_t crypto_krb5_get_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len); +int crypto_krb5_verify_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); #endif /* _CRYPTO_KRB5_H */ -- cgit v1.2.3 From 41cf1d1e8a86c5c675982136f07c519c4b15b157 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 3 Sep 2020 12:05:04 +0100 Subject: crypto/krb5: Provide infrastructure and key derivation Provide key derivation interface functions and a helper to implement the PRF+ function from rfc4402. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Makefile | 1 + crypto/krb5/internal.h | 10 ++++ crypto/krb5/krb5_kdf.c | 145 +++++++++++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 10 ++++ 4 files changed, 166 insertions(+) create mode 100644 crypto/krb5/krb5_kdf.c (limited to 'crypto') diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index c450d0754772..8c2050af8fed 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -4,6 +4,7 @@ # krb5-y += \ + krb5_kdf.o \ krb5_api.o obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index b542d24e5fa5..50abda5169c7 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -120,3 +120,13 @@ struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5, struct crypto_shash *krb5_prepare_checksum(const struct krb5_enctype *krb5, const struct krb5_buffer *Kc, gfp_t gfp); + +/* + * krb5_kdf.c + */ +int krb5_derive_Kc(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp); +int krb5_derive_Ke(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp); +int krb5_derive_Ki(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp); diff --git a/crypto/krb5/krb5_kdf.c b/crypto/krb5/krb5_kdf.c new file mode 100644 index 000000000000..6699e5469d1b --- /dev/null +++ b/crypto/krb5/krb5_kdf.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Kerberos key derivation. + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include "internal.h" + +/** + * crypto_krb5_calc_PRFplus - Calculate PRF+ [RFC4402] + * @krb5: The encryption type to use + * @K: The protocol key for the pseudo-random function + * @L: The length of the output + * @S: The input octet string + * @result: Result buffer, sized to krb5->prf_len + * @gfp: Allocation restrictions + * + * Calculate the kerberos pseudo-random function, PRF+() by the following + * method: + * + * PRF+(K, L, S) = truncate(L, T1 || T2 || .. || Tn) + * Tn = PRF(K, n || S) + * [rfc4402 sec 2] + */ +int crypto_krb5_calc_PRFplus(const struct krb5_enctype *krb5, + const struct krb5_buffer *K, + unsigned int L, + const struct krb5_buffer *S, + struct krb5_buffer *result, + gfp_t gfp) +{ + struct krb5_buffer T_series, Tn, n_S; + void *buffer; + int ret, n = 1; + + Tn.len = krb5->prf_len; + T_series.len = 0; + n_S.len = 4 + S->len; + + buffer = kzalloc(round16(L + Tn.len) + round16(n_S.len), gfp); + if (!buffer) + return -ENOMEM; + + T_series.data = buffer; + n_S.data = buffer + round16(L + Tn.len); + memcpy(n_S.data + 4, S->data, S->len); + + while (T_series.len < L) { + *(__be32 *)(n_S.data) = htonl(n); + Tn.data = T_series.data + Tn.len * (n - 1); + ret = krb5->profile->calc_PRF(krb5, K, &n_S, &Tn, gfp); + if (ret < 0) + goto err; + T_series.len += Tn.len; + n++; + } + + /* Truncate to L */ + memcpy(result->data, T_series.data, L); + ret = 0; + +err: + kfree_sensitive(buffer); + return ret; +} +EXPORT_SYMBOL(crypto_krb5_calc_PRFplus); + +/** + * krb5_derive_Kc - Derive key Kc and install into a hash + * @krb5: The encryption type to use + * @TK: The base key + * @usage: The key usage number + * @key: Prepped buffer to store the key into + * @gfp: Allocation restrictions + * + * Derive the Kerberos Kc checksumming key. The key is stored into the + * prepared buffer. + */ +int krb5_derive_Kc(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp) +{ + u8 buf[5] __aligned(CRYPTO_MINALIGN); + struct krb5_buffer usage_constant = { .len = 5, .data = buf }; + + *(__be32 *)buf = cpu_to_be32(usage); + buf[4] = KEY_USAGE_SEED_CHECKSUM; + + key->len = krb5->Kc_len; + return krb5->profile->calc_Kc(krb5, TK, &usage_constant, key, gfp); +} + +/** + * krb5_derive_Ke - Derive key Ke and install into an skcipher + * @krb5: The encryption type to use + * @TK: The base key + * @usage: The key usage number + * @key: Prepped buffer to store the key into + * @gfp: Allocation restrictions + * + * Derive the Kerberos Ke encryption key. The key is stored into the prepared + * buffer. + */ +int krb5_derive_Ke(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp) +{ + u8 buf[5] __aligned(CRYPTO_MINALIGN); + struct krb5_buffer usage_constant = { .len = 5, .data = buf }; + + *(__be32 *)buf = cpu_to_be32(usage); + buf[4] = KEY_USAGE_SEED_ENCRYPTION; + + key->len = krb5->Ke_len; + return krb5->profile->calc_Ke(krb5, TK, &usage_constant, key, gfp); +} + +/** + * krb5_derive_Ki - Derive key Ki and install into a hash + * @krb5: The encryption type to use + * @TK: The base key + * @usage: The key usage number + * @key: Prepped buffer to store the key into + * @gfp: Allocation restrictions + * + * Derive the Kerberos Ki integrity checksum key. The key is stored into the + * prepared buffer. + */ +int krb5_derive_Ki(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, + u32 usage, struct krb5_buffer *key, gfp_t gfp) +{ + u8 buf[5] __aligned(CRYPTO_MINALIGN); + struct krb5_buffer usage_constant = { .len = 5, .data = buf }; + + *(__be32 *)buf = cpu_to_be32(usage); + buf[4] = KEY_USAGE_SEED_INTEGRITY; + + key->len = krb5->Ki_len; + return krb5->profile->calc_Ki(krb5, TK, &usage_constant, key, gfp); +} diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index 81739e9828d3..b12f012cf354 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -139,4 +139,14 @@ int crypto_krb5_verify_mic(const struct krb5_enctype *krb5, struct scatterlist *sg, unsigned int nr_sg, size_t *_offset, size_t *_len); +/* + * krb5_kdf.c + */ +int crypto_krb5_calc_PRFplus(const struct krb5_enctype *krb5, + const struct krb5_buffer *K, + unsigned int L, + const struct krb5_buffer *S, + struct krb5_buffer *result, + gfp_t gfp); + #endif /* _CRYPTO_KRB5_H */ -- cgit v1.2.3 From c8d8f6af66c3cd7896460da4a0ddd006f391f6d2 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 3 Sep 2020 12:05:04 +0100 Subject: crypto/krb5: Implement the Kerberos5 rfc3961 key derivation Implement the simplified crypto profile for Kerberos 5 rfc3961 with the pseudo-random function, PRF(), from section 5.3 and the key derivation function, DK() from section 5.1. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Makefile | 3 +- crypto/krb5/internal.h | 6 + crypto/krb5/rfc3961_simplified.c | 407 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 415 insertions(+), 1 deletion(-) create mode 100644 crypto/krb5/rfc3961_simplified.c (limited to 'crypto') diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index 8c2050af8fed..8dad8e3bf086 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -5,6 +5,7 @@ krb5-y += \ krb5_kdf.o \ - krb5_api.o + krb5_api.o \ + rfc3961_simplified.o obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 50abda5169c7..7d60977dc0c5 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -5,6 +5,7 @@ * Written by David Howells (dhowells@redhat.com) */ +#include #include /* @@ -130,3 +131,8 @@ int krb5_derive_Ke(const struct krb5_enctype *krb5, const struct krb5_buffer *TK u32 usage, struct krb5_buffer *key, gfp_t gfp); int krb5_derive_Ki(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, u32 usage, struct krb5_buffer *key, gfp_t gfp); + +/* + * rfc3961_simplified.c + */ +extern const struct krb5_crypto_profile rfc3961_simplified_profile; diff --git a/crypto/krb5/rfc3961_simplified.c b/crypto/krb5/rfc3961_simplified.c new file mode 100644 index 000000000000..75eafba059c7 --- /dev/null +++ b/crypto/krb5/rfc3961_simplified.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* rfc3961 Kerberos 5 simplified crypto profile. + * + * Parts borrowed from net/sunrpc/auth_gss/. + */ +/* + * COPYRIGHT (c) 2008 + * The Regents of the University of Michigan + * ALL RIGHTS RESERVED + * + * Permission is granted to use, copy, create derivative works + * and redistribute this software and such derivative works + * for any purpose, so long as the name of The University of + * Michigan is not used in any advertising or publicity + * pertaining to the use of distribution of this software + * without specific, written prior authorization. If the + * above copyright notice or any other identification of the + * University of Michigan is included in any copy of any + * portion of this software, then the disclaimer below must + * also be included. + * + * THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION + * FROM THE UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY + * PURPOSE, AND WITHOUT WARRANTY BY THE UNIVERSITY OF + * MICHIGAN OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING + * WITHOUT LIMITATION THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE + * REGENTS OF THE UNIVERSITY OF MICHIGAN SHALL NOT BE LIABLE + * FOR ANY DAMAGES, INCLUDING SPECIAL, INDIRECT, INCIDENTAL, OR + * CONSEQUENTIAL DAMAGES, WITH RESPECT TO ANY CLAIM ARISING + * OUT OF OR IN CONNECTION WITH THE USE OF THE SOFTWARE, EVEN + * IF IT HAS BEEN OR IS HEREAFTER ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGES. + */ + +/* + * Copyright (C) 1998 by the FundsXpress, INC. + * + * All rights reserved. + * + * Export of this software from the United States of America may require + * a specific license from the United States Government. It is the + * responsibility of any person or organization contemplating export to + * obtain such a license before exporting. + * + * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and + * distribute this software and its documentation for any purpose and + * without fee is hereby granted, provided that the above copyright + * notice appear in all copies and that both that copyright notice and + * this permission notice appear in supporting documentation, and that + * the name of FundsXpress. not be used in advertising or publicity pertaining + * to distribution of the software without specific, written prior + * permission. FundsXpress makes no representations about the suitability of + * this software for any purpose. It is provided "as is" without express + * or implied warranty. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include "internal.h" + +/* Maximum blocksize for the supported crypto algorithms */ +#define KRB5_MAX_BLOCKSIZE (16) + +static int rfc3961_do_encrypt(struct crypto_sync_skcipher *tfm, void *iv, + const struct krb5_buffer *in, struct krb5_buffer *out) +{ + struct scatterlist sg[1]; + u8 local_iv[KRB5_MAX_BLOCKSIZE] __aligned(KRB5_MAX_BLOCKSIZE) = {0}; + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + int ret; + + if (WARN_ON(in->len != out->len)) + return -EINVAL; + if (out->len % crypto_sync_skcipher_blocksize(tfm) != 0) + return -EINVAL; + + if (crypto_sync_skcipher_ivsize(tfm) > KRB5_MAX_BLOCKSIZE) + return -EINVAL; + + if (iv) + memcpy(local_iv, iv, crypto_sync_skcipher_ivsize(tfm)); + + memcpy(out->data, in->data, out->len); + sg_init_one(sg, out->data, out->len); + + skcipher_request_set_sync_tfm(req, tfm); + skcipher_request_set_callback(req, 0, NULL, NULL); + skcipher_request_set_crypt(req, sg, sg, out->len, local_iv); + + ret = crypto_skcipher_encrypt(req); + skcipher_request_zero(req); + return ret; +} + +/* + * Calculate an unkeyed basic hash. + */ +static int rfc3961_calc_H(const struct krb5_enctype *krb5, + const struct krb5_buffer *data, + struct krb5_buffer *digest, + gfp_t gfp) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + size_t desc_size; + int ret = -ENOMEM; + + tfm = crypto_alloc_shash(krb5->hash_name, 0, 0); + if (IS_ERR(tfm)) + return (PTR_ERR(tfm) == -ENOENT) ? -ENOPKG : PTR_ERR(tfm); + + desc_size = crypto_shash_descsize(tfm) + sizeof(*desc); + + desc = kzalloc(desc_size, gfp); + if (!desc) + goto error_tfm; + + digest->len = crypto_shash_digestsize(tfm); + digest->data = kzalloc(digest->len, gfp); + if (!digest->data) + goto error_desc; + + desc->tfm = tfm; + ret = crypto_shash_init(desc); + if (ret < 0) + goto error_digest; + + ret = crypto_shash_finup(desc, data->data, data->len, digest->data); + if (ret < 0) + goto error_digest; + + goto error_desc; + +error_digest: + kfree_sensitive(digest->data); +error_desc: + kfree_sensitive(desc); +error_tfm: + crypto_free_shash(tfm); + return ret; +} + +/* + * This is the n-fold function as described in rfc3961, sec 5.1 + * Taken from MIT Kerberos and modified. + */ +static void rfc3961_nfold(const struct krb5_buffer *source, struct krb5_buffer *result) +{ + const u8 *in = source->data; + u8 *out = result->data; + unsigned long ulcm; + unsigned int inbits, outbits; + int byte, i, msbit; + + /* the code below is more readable if I make these bytes instead of bits */ + inbits = source->len; + outbits = result->len; + + /* first compute lcm(n,k) */ + ulcm = lcm(inbits, outbits); + + /* now do the real work */ + memset(out, 0, outbits); + byte = 0; + + /* this will end up cycling through k lcm(k,n)/k times, which + * is correct. + */ + for (i = ulcm-1; i >= 0; i--) { + /* compute the msbit in k which gets added into this byte */ + msbit = ( + /* first, start with the msbit in the first, + * unrotated byte + */ + ((inbits << 3) - 1) + + /* then, for each byte, shift to the right + * for each repetition + */ + (((inbits << 3) + 13) * (i/inbits)) + + /* last, pick out the correct byte within + * that shifted repetition + */ + ((inbits - (i % inbits)) << 3) + ) % (inbits << 3); + + /* pull out the byte value itself */ + byte += (((in[((inbits - 1) - (msbit >> 3)) % inbits] << 8) | + (in[((inbits) - (msbit >> 3)) % inbits])) + >> ((msbit & 7) + 1)) & 0xff; + + /* do the addition */ + byte += out[i % outbits]; + out[i % outbits] = byte & 0xff; + + /* keep around the carry bit, if any */ + byte >>= 8; + } + + /* if there's a carry bit left over, add it back in */ + if (byte) { + for (i = outbits - 1; i >= 0; i--) { + /* do the addition */ + byte += out[i]; + out[i] = byte & 0xff; + + /* keep around the carry bit, if any */ + byte >>= 8; + } + } +} + +/* + * Calculate a derived key, DK(Base Key, Well-Known Constant) + * + * DK(Key, Constant) = random-to-key(DR(Key, Constant)) + * DR(Key, Constant) = k-truncate(E(Key, Constant, initial-cipher-state)) + * K1 = E(Key, n-fold(Constant), initial-cipher-state) + * K2 = E(Key, K1, initial-cipher-state) + * K3 = E(Key, K2, initial-cipher-state) + * K4 = ... + * DR(Key, Constant) = k-truncate(K1 | K2 | K3 | K4 ...) + * [rfc3961 sec 5.1] + */ +static int rfc3961_calc_DK(const struct krb5_enctype *krb5, + const struct krb5_buffer *inkey, + const struct krb5_buffer *in_constant, + struct krb5_buffer *result, + gfp_t gfp) +{ + unsigned int blocksize, keybytes, keylength, n; + struct krb5_buffer inblock, outblock, rawkey; + struct crypto_sync_skcipher *cipher; + int ret = -EINVAL; + + blocksize = krb5->block_len; + keybytes = krb5->key_bytes; + keylength = krb5->key_len; + + if (inkey->len != keylength || result->len != keylength) + return -EINVAL; + if (!krb5->random_to_key && result->len != keybytes) + return -EINVAL; + + cipher = crypto_alloc_sync_skcipher(krb5->derivation_enc, 0, 0); + if (IS_ERR(cipher)) { + ret = (PTR_ERR(cipher) == -ENOENT) ? -ENOPKG : PTR_ERR(cipher); + goto err_return; + } + ret = crypto_sync_skcipher_setkey(cipher, inkey->data, inkey->len); + if (ret < 0) + goto err_free_cipher; + + ret = -ENOMEM; + inblock.data = kzalloc(blocksize * 2 + keybytes, gfp); + if (!inblock.data) + goto err_free_cipher; + + inblock.len = blocksize; + outblock.data = inblock.data + blocksize; + outblock.len = blocksize; + rawkey.data = outblock.data + blocksize; + rawkey.len = keybytes; + + /* initialize the input block */ + + if (in_constant->len == inblock.len) + memcpy(inblock.data, in_constant->data, inblock.len); + else + rfc3961_nfold(in_constant, &inblock); + + /* loop encrypting the blocks until enough key bytes are generated */ + n = 0; + while (n < rawkey.len) { + rfc3961_do_encrypt(cipher, NULL, &inblock, &outblock); + + if (keybytes - n <= outblock.len) { + memcpy(rawkey.data + n, outblock.data, keybytes - n); + break; + } + + memcpy(rawkey.data + n, outblock.data, outblock.len); + memcpy(inblock.data, outblock.data, outblock.len); + n += outblock.len; + } + + /* postprocess the key */ + if (!krb5->random_to_key) { + /* Identity random-to-key function. */ + memcpy(result->data, rawkey.data, rawkey.len); + ret = 0; + } else { + ret = krb5->random_to_key(krb5, &rawkey, result); + } + + kfree_sensitive(inblock.data); +err_free_cipher: + crypto_free_sync_skcipher(cipher); +err_return: + return ret; +} + +/* + * Calculate single encryption, E() + * + * E(Key, octets) + */ +static int rfc3961_calc_E(const struct krb5_enctype *krb5, + const struct krb5_buffer *key, + const struct krb5_buffer *in_data, + struct krb5_buffer *result, + gfp_t gfp) +{ + struct crypto_sync_skcipher *cipher; + int ret; + + cipher = crypto_alloc_sync_skcipher(krb5->derivation_enc, 0, 0); + if (IS_ERR(cipher)) { + ret = (PTR_ERR(cipher) == -ENOENT) ? -ENOPKG : PTR_ERR(cipher); + goto err; + } + + ret = crypto_sync_skcipher_setkey(cipher, key->data, key->len); + if (ret < 0) + goto err_free; + + ret = rfc3961_do_encrypt(cipher, NULL, in_data, result); + +err_free: + crypto_free_sync_skcipher(cipher); +err: + return ret; +} + +/* + * Calculate the pseudo-random function, PRF(). + * + * tmp1 = H(octet-string) + * tmp2 = truncate tmp1 to multiple of m + * PRF = E(DK(protocol-key, prfconstant), tmp2, initial-cipher-state) + * + * The "prfconstant" used in the PRF operation is the three-octet string + * "prf". + * [rfc3961 sec 5.3] + */ +static int rfc3961_calc_PRF(const struct krb5_enctype *krb5, + const struct krb5_buffer *protocol_key, + const struct krb5_buffer *octet_string, + struct krb5_buffer *result, + gfp_t gfp) +{ + static const struct krb5_buffer prfconstant = { 3, "prf" }; + struct krb5_buffer derived_key; + struct krb5_buffer tmp1, tmp2; + unsigned int m = krb5->block_len; + void *buffer; + int ret; + + if (result->len != krb5->prf_len) + return -EINVAL; + + tmp1.len = krb5->hash_len; + derived_key.len = krb5->key_bytes; + buffer = kzalloc(round16(tmp1.len) + round16(derived_key.len), gfp); + if (!buffer) + return -ENOMEM; + + tmp1.data = buffer; + derived_key.data = buffer + round16(tmp1.len); + + ret = rfc3961_calc_H(krb5, octet_string, &tmp1, gfp); + if (ret < 0) + goto err; + + tmp2.len = tmp1.len & ~(m - 1); + tmp2.data = tmp1.data; + + ret = rfc3961_calc_DK(krb5, protocol_key, &prfconstant, &derived_key, gfp); + if (ret < 0) + goto err; + + ret = rfc3961_calc_E(krb5, &derived_key, &tmp2, result, gfp); + +err: + kfree_sensitive(buffer); + return ret; +} + +const struct krb5_crypto_profile rfc3961_simplified_profile = { + .calc_PRF = rfc3961_calc_PRF, + .calc_Kc = rfc3961_calc_DK, + .calc_Ke = rfc3961_calc_DK, + .calc_Ki = rfc3961_calc_DK, +}; -- cgit v1.2.3 From 8bcdbfa89f4fc1ab7b06e784cb9c3346a943955e Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 17 Jan 2025 13:20:34 +0000 Subject: crypto/krb5: Provide RFC3961 setkey packaging functions Provide functions to derive keys according to RFC3961 (or load the derived keys for the selftester where only derived keys are available) and to package them up appropriately for passing to a krb5enc AEAD setkey or a hash setkey function. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/internal.h | 20 +++++++ crypto/krb5/rfc3961_simplified.c | 122 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 138 insertions(+), 4 deletions(-) (limited to 'crypto') diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 7d60977dc0c5..ae00588619a8 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -136,3 +136,23 @@ int krb5_derive_Ki(const struct krb5_enctype *krb5, const struct krb5_buffer *TK * rfc3961_simplified.c */ extern const struct krb5_crypto_profile rfc3961_simplified_profile; + +int authenc_derive_encrypt_keys(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp); +int authenc_load_encrypt_keys(const struct krb5_enctype *krb5, + const struct krb5_buffer *Ke, + const struct krb5_buffer *Ki, + struct krb5_buffer *setkey, + gfp_t gfp); +int rfc3961_derive_checksum_key(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp); +int rfc3961_load_checksum_key(const struct krb5_enctype *krb5, + const struct krb5_buffer *Kc, + struct krb5_buffer *setkey, + gfp_t gfp); diff --git a/crypto/krb5/rfc3961_simplified.c b/crypto/krb5/rfc3961_simplified.c index 75eafba059c7..335c5bb6904f 100644 --- a/crypto/krb5/rfc3961_simplified.c +++ b/crypto/krb5/rfc3961_simplified.c @@ -68,6 +68,8 @@ #include #include +#include +#include #include #include #include "internal.h" @@ -399,9 +401,121 @@ err: return ret; } +/* + * Derive the Ke and Ki keys and package them into a key parameter that can be + * given to the setkey of a authenc AEAD crypto object. + */ +int authenc_derive_encrypt_keys(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp) +{ + struct crypto_authenc_key_param *param; + struct krb5_buffer Ke, Ki; + struct rtattr *rta; + int ret; + + Ke.len = krb5->Ke_len; + Ki.len = krb5->Ki_len; + setkey->len = RTA_LENGTH(sizeof(*param)) + Ke.len + Ki.len; + setkey->data = kzalloc(setkey->len, GFP_KERNEL); + if (!setkey->data) + return -ENOMEM; + + rta = setkey->data; + rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; + rta->rta_len = RTA_LENGTH(sizeof(*param)); + param = RTA_DATA(rta); + param->enckeylen = htonl(Ke.len); + + Ki.data = (void *)(param + 1); + Ke.data = Ki.data + Ki.len; + + ret = krb5_derive_Ke(krb5, TK, usage, &Ke, gfp); + if (ret < 0) { + pr_err("get_Ke failed %d\n", ret); + return ret; + } + ret = krb5_derive_Ki(krb5, TK, usage, &Ki, gfp); + if (ret < 0) + pr_err("get_Ki failed %d\n", ret); + return ret; +} + +/* + * Package predefined Ke and Ki keys and into a key parameter that can be given + * to the setkey of an authenc AEAD crypto object. + */ +int authenc_load_encrypt_keys(const struct krb5_enctype *krb5, + const struct krb5_buffer *Ke, + const struct krb5_buffer *Ki, + struct krb5_buffer *setkey, + gfp_t gfp) +{ + struct crypto_authenc_key_param *param; + struct rtattr *rta; + + setkey->len = RTA_LENGTH(sizeof(*param)) + Ke->len + Ki->len; + setkey->data = kzalloc(setkey->len, GFP_KERNEL); + if (!setkey->data) + return -ENOMEM; + + rta = setkey->data; + rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM; + rta->rta_len = RTA_LENGTH(sizeof(*param)); + param = RTA_DATA(rta); + param->enckeylen = htonl(Ke->len); + memcpy((void *)(param + 1), Ki->data, Ki->len); + memcpy((void *)(param + 1) + Ki->len, Ke->data, Ke->len); + return 0; +} + +/* + * Derive the Kc key for checksum-only mode and package it into a key parameter + * that can be given to the setkey of a hash crypto object. + */ +int rfc3961_derive_checksum_key(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + unsigned int usage, + struct krb5_buffer *setkey, + gfp_t gfp) +{ + int ret; + + setkey->len = krb5->Kc_len; + setkey->data = kzalloc(setkey->len, GFP_KERNEL); + if (!setkey->data) + return -ENOMEM; + + ret = krb5_derive_Kc(krb5, TK, usage, setkey, gfp); + if (ret < 0) + pr_err("get_Kc failed %d\n", ret); + return ret; +} + +/* + * Package a predefined Kc key for checksum-only mode into a key parameter that + * can be given to the setkey of a hash crypto object. + */ +int rfc3961_load_checksum_key(const struct krb5_enctype *krb5, + const struct krb5_buffer *Kc, + struct krb5_buffer *setkey, + gfp_t gfp) +{ + setkey->len = krb5->Kc_len; + setkey->data = kmemdup(Kc->data, Kc->len, GFP_KERNEL); + if (!setkey->data) + return -ENOMEM; + return 0; +} const struct krb5_crypto_profile rfc3961_simplified_profile = { - .calc_PRF = rfc3961_calc_PRF, - .calc_Kc = rfc3961_calc_DK, - .calc_Ke = rfc3961_calc_DK, - .calc_Ki = rfc3961_calc_DK, + .calc_PRF = rfc3961_calc_PRF, + .calc_Kc = rfc3961_calc_DK, + .calc_Ke = rfc3961_calc_DK, + .calc_Ki = rfc3961_calc_DK, + .derive_encrypt_keys = authenc_derive_encrypt_keys, + .load_encrypt_keys = authenc_load_encrypt_keys, + .derive_checksum_key = rfc3961_derive_checksum_key, + .load_checksum_key = rfc3961_load_checksum_key, }; -- cgit v1.2.3 From 00244da40f7821b242c4612428d4192230dba27f Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 24 Sep 2020 08:31:06 +0100 Subject: crypto/krb5: Implement the Kerberos5 rfc3961 encrypt and decrypt functions Add functions that encrypt and decrypt a message according to rfc3961 sec 5.3, using Ki to checksum the data to be secured and Ke to encrypt it during the encryption phase, then decrypting with Ke and verifying the checksum with Ki in the decryption phase. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/internal.h | 13 ++++ crypto/krb5/rfc3961_simplified.c | 146 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) (limited to 'crypto') diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index ae00588619a8..c8deb112b604 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -7,6 +7,8 @@ #include #include +#include +#include /* * Profile used for key derivation and encryption. @@ -137,6 +139,8 @@ int krb5_derive_Ki(const struct krb5_enctype *krb5, const struct krb5_buffer *TK */ extern const struct krb5_crypto_profile rfc3961_simplified_profile; +int crypto_shash_update_sg(struct shash_desc *desc, struct scatterlist *sg, + size_t offset, size_t len); int authenc_derive_encrypt_keys(const struct krb5_enctype *krb5, const struct krb5_buffer *TK, unsigned int usage, @@ -156,3 +160,12 @@ int rfc3961_load_checksum_key(const struct krb5_enctype *krb5, const struct krb5_buffer *Kc, struct krb5_buffer *setkey, gfp_t gfp); +ssize_t krb5_aead_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded); +int krb5_aead_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); diff --git a/crypto/krb5/rfc3961_simplified.c b/crypto/krb5/rfc3961_simplified.c index 335c5bb6904f..58323ce5838a 100644 --- a/crypto/krb5/rfc3961_simplified.c +++ b/crypto/krb5/rfc3961_simplified.c @@ -66,7 +66,10 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include #include +#include #include #include #include @@ -77,6 +80,31 @@ /* Maximum blocksize for the supported crypto algorithms */ #define KRB5_MAX_BLOCKSIZE (16) +int crypto_shash_update_sg(struct shash_desc *desc, struct scatterlist *sg, + size_t offset, size_t len) +{ + do { + int ret; + + if (offset < sg->length) { + struct page *page = sg_page(sg); + void *p = kmap_local_page(page); + void *q = p + sg->offset + offset; + size_t seg = min_t(size_t, len, sg->length - offset); + + ret = crypto_shash_update(desc, q, seg); + kunmap_local(p); + if (ret < 0) + return ret; + len -= seg; + offset = 0; + } else { + offset -= sg->length; + } + } while (len > 0 && (sg = sg_next(sg))); + return 0; +} + static int rfc3961_do_encrypt(struct crypto_sync_skcipher *tfm, void *iv, const struct krb5_buffer *in, struct krb5_buffer *out) { @@ -509,6 +537,122 @@ int rfc3961_load_checksum_key(const struct krb5_enctype *krb5, return -ENOMEM; return 0; } + +/* + * Apply encryption and checksumming functions to part of a scatterlist. + */ +ssize_t krb5_aead_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded) +{ + struct aead_request *req; + ssize_t ret, done; + size_t bsize, base_len, secure_offset, secure_len, pad_len, cksum_offset; + void *buffer; + u8 *iv; + + if (WARN_ON(data_offset != krb5->conf_len)) + return -EINVAL; /* Data is in wrong place */ + + secure_offset = 0; + base_len = krb5->conf_len + data_len; + pad_len = 0; + secure_len = base_len + pad_len; + cksum_offset = secure_len; + if (WARN_ON(cksum_offset + krb5->cksum_len > sg_len)) + return -EFAULT; + + bsize = krb5_aead_size(aead) + + krb5_aead_ivsize(aead); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + /* Insert the confounder into the buffer */ + ret = -EFAULT; + if (!preconfounded) { + get_random_bytes(buffer, krb5->conf_len); + done = sg_pcopy_from_buffer(sg, nr_sg, buffer, krb5->conf_len, + secure_offset); + if (done != krb5->conf_len) + goto error; + } + + /* We may need to pad out to the crypto blocksize. */ + if (pad_len) { + done = sg_zero_buffer(sg, nr_sg, pad_len, data_offset + data_len); + if (done != pad_len) + goto error; + } + + /* Hash and encrypt the message. */ + req = buffer; + iv = buffer + krb5_aead_size(aead); + + aead_request_set_tfm(req, aead); + aead_request_set_callback(req, 0, NULL, NULL); + aead_request_set_crypt(req, sg, sg, secure_len, iv); + ret = crypto_aead_encrypt(req); + if (ret < 0) + goto error; + + ret = secure_len + krb5->cksum_len; + +error: + kfree_sensitive(buffer); + return ret; +} + +/* + * Apply decryption and checksumming functions to a message. The offset and + * length are updated to reflect the actual content of the encrypted region. + */ +int krb5_aead_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len) +{ + struct aead_request *req; + size_t bsize; + void *buffer; + int ret; + u8 *iv; + + if (WARN_ON(*_offset != 0)) + return -EINVAL; /* Can't set offset on aead */ + + if (*_len < krb5->conf_len + krb5->cksum_len) + return -EPROTO; + + bsize = krb5_aead_size(aead) + + krb5_aead_ivsize(aead); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + /* Decrypt the message and verify its checksum. */ + req = buffer; + iv = buffer + krb5_aead_size(aead); + + aead_request_set_tfm(req, aead); + aead_request_set_callback(req, 0, NULL, NULL); + aead_request_set_crypt(req, sg, sg, *_len, iv); + ret = crypto_aead_decrypt(req); + if (ret < 0) + goto error; + + /* Adjust the boundaries of the data. */ + *_offset += krb5->conf_len; + *_len -= krb5->conf_len + krb5->cksum_len; + ret = 0; + +error: + kfree_sensitive(buffer); + return ret; +} + const struct krb5_crypto_profile rfc3961_simplified_profile = { .calc_PRF = rfc3961_calc_PRF, .calc_Kc = rfc3961_calc_DK, @@ -518,4 +662,6 @@ const struct krb5_crypto_profile rfc3961_simplified_profile = { .load_encrypt_keys = authenc_load_encrypt_keys, .derive_checksum_key = rfc3961_derive_checksum_key, .load_checksum_key = rfc3961_load_checksum_key, + .encrypt = krb5_aead_encrypt, + .decrypt = krb5_aead_decrypt, }; -- cgit v1.2.3 From 348f5669d1f6c1c210b9017ebb8b82282eca6f25 Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 24 Sep 2020 10:23:48 +0100 Subject: crypto/krb5: Implement the Kerberos5 rfc3961 get_mic and verify_mic Add functions that sign and verify a message according to rfc3961 sec 5.4, using Kc to generate a checksum and insert it into the MIC field in the skbuff in the sign phase then checksum the data and compare it to the MIC in the verify phase. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/internal.h | 10 +++ crypto/krb5/rfc3961_simplified.c | 130 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) (limited to 'crypto') diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index c8deb112b604..07a47ddf3ea9 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -169,3 +169,13 @@ int krb5_aead_decrypt(const struct krb5_enctype *krb5, struct crypto_aead *aead, struct scatterlist *sg, unsigned int nr_sg, size_t *_offset, size_t *_len); +ssize_t rfc3961_get_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, size_t sg_len, + size_t data_offset, size_t data_len); +int rfc3961_verify_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); diff --git a/crypto/krb5/rfc3961_simplified.c b/crypto/krb5/rfc3961_simplified.c index 58323ce5838a..c1dcb0dd3a00 100644 --- a/crypto/krb5/rfc3961_simplified.c +++ b/crypto/krb5/rfc3961_simplified.c @@ -653,6 +653,134 @@ error: return ret; } +/* + * Generate a checksum over some metadata and part of an skbuff and insert the + * MIC into the skbuff immediately prior to the data. + */ +ssize_t rfc3961_get_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, size_t sg_len, + size_t data_offset, size_t data_len) +{ + struct shash_desc *desc; + ssize_t ret, done; + size_t bsize; + void *buffer, *digest; + + if (WARN_ON(data_offset != krb5->cksum_len)) + return -EMSGSIZE; + + bsize = krb5_shash_size(shash) + + krb5_digest_size(shash); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + /* Calculate the MIC with key Kc and store it into the skb */ + desc = buffer; + desc->tfm = shash; + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + + if (metadata) { + ret = crypto_shash_update(desc, metadata->data, metadata->len); + if (ret < 0) + goto error; + } + + ret = crypto_shash_update_sg(desc, sg, data_offset, data_len); + if (ret < 0) + goto error; + + digest = buffer + krb5_shash_size(shash); + ret = crypto_shash_final(desc, digest); + if (ret < 0) + goto error; + + ret = -EFAULT; + done = sg_pcopy_from_buffer(sg, nr_sg, digest, krb5->cksum_len, + data_offset - krb5->cksum_len); + if (done != krb5->cksum_len) + goto error; + + ret = krb5->cksum_len + data_len; + +error: + kfree_sensitive(buffer); + return ret; +} + +/* + * Check the MIC on a region of an skbuff. The offset and length are updated + * to reflect the actual content of the secure region. + */ +int rfc3961_verify_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len) +{ + struct shash_desc *desc; + ssize_t done; + size_t bsize, data_offset, data_len, offset = *_offset, len = *_len; + void *buffer = NULL; + int ret; + u8 *cksum, *cksum2; + + if (len < krb5->cksum_len) + return -EPROTO; + data_offset = offset + krb5->cksum_len; + data_len = len - krb5->cksum_len; + + bsize = krb5_shash_size(shash) + + krb5_digest_size(shash) * 2; + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + cksum = buffer + + krb5_shash_size(shash); + cksum2 = buffer + + krb5_shash_size(shash) + + krb5_digest_size(shash); + + /* Calculate the MIC */ + desc = buffer; + desc->tfm = shash; + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + + if (metadata) { + ret = crypto_shash_update(desc, metadata->data, metadata->len); + if (ret < 0) + goto error; + } + + crypto_shash_update_sg(desc, sg, data_offset, data_len); + crypto_shash_final(desc, cksum); + + ret = -EFAULT; + done = sg_pcopy_to_buffer(sg, nr_sg, cksum2, krb5->cksum_len, offset); + if (done != krb5->cksum_len) + goto error; + + if (memcmp(cksum, cksum2, krb5->cksum_len) != 0) { + ret = -EBADMSG; + goto error; + } + + *_offset += krb5->cksum_len; + *_len -= krb5->cksum_len; + ret = 0; + +error: + kfree_sensitive(buffer); + return ret; +} + const struct krb5_crypto_profile rfc3961_simplified_profile = { .calc_PRF = rfc3961_calc_PRF, .calc_Kc = rfc3961_calc_DK, @@ -664,4 +792,6 @@ const struct krb5_crypto_profile rfc3961_simplified_profile = { .load_checksum_key = rfc3961_load_checksum_key, .encrypt = krb5_aead_encrypt, .decrypt = krb5_aead_decrypt, + .get_mic = rfc3961_get_mic, + .verify_mic = rfc3961_verify_mic, }; -- cgit v1.2.3 From 7c164b66b276c49b3888d3280e1b70a85732a38c Mon Sep 17 00:00:00 2001 From: David Howells Date: Thu, 3 Sep 2020 12:05:04 +0100 Subject: crypto/krb5: Implement the AES enctypes from rfc3962 Implement the aes128-cts-hmac-sha1-96 and aes256-cts-hmac-sha1-96 enctypes from rfc3962, using the rfc3961 kerberos 5 simplified crypto scheme. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Kconfig | 1 + crypto/krb5/Makefile | 3 +- crypto/krb5/internal.h | 6 +++ crypto/krb5/krb5_api.c | 2 + crypto/krb5/rfc3962_aes.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 crypto/krb5/rfc3962_aes.c (limited to 'crypto') diff --git a/crypto/krb5/Kconfig b/crypto/krb5/Kconfig index 079873618abf..2ad874990dc8 100644 --- a/crypto/krb5/Kconfig +++ b/crypto/krb5/Kconfig @@ -5,6 +5,7 @@ config CRYPTO_KRB5 select CRYPTO_AUTHENC select CRYPTO_SKCIPHER select CRYPTO_HASH_INFO + select CRYPTO_HMAC select CRYPTO_SHA1 select CRYPTO_CBC select CRYPTO_CTS diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index 8dad8e3bf086..35f21411abf8 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -6,6 +6,7 @@ krb5-y += \ krb5_kdf.o \ krb5_api.o \ - rfc3961_simplified.o + rfc3961_simplified.o \ + rfc3962_aes.o obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 07a47ddf3ea9..43f904a69e32 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -179,3 +179,9 @@ int rfc3961_verify_mic(const struct krb5_enctype *krb5, const struct krb5_buffer *metadata, struct scatterlist *sg, unsigned int nr_sg, size_t *_offset, size_t *_len); + +/* + * rfc3962_aes.c + */ +extern const struct krb5_enctype krb5_aes128_cts_hmac_sha1_96; +extern const struct krb5_enctype krb5_aes256_cts_hmac_sha1_96; diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 8fc3a1b9d4ad..ecc6655953d5 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -17,6 +17,8 @@ MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); static const struct krb5_enctype *const krb5_supported_enctypes[] = { + &krb5_aes128_cts_hmac_sha1_96, + &krb5_aes256_cts_hmac_sha1_96, }; /** diff --git a/crypto/krb5/rfc3962_aes.c b/crypto/krb5/rfc3962_aes.c new file mode 100644 index 000000000000..5cbf8f4638b9 --- /dev/null +++ b/crypto/krb5/rfc3962_aes.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* rfc3962 Advanced Encryption Standard (AES) Encryption for Kerberos 5 + * + * Parts borrowed from net/sunrpc/auth_gss/. + */ +/* + * COPYRIGHT (c) 2008 + * The Regents of the University of Michigan + * ALL RIGHTS RESERVED + * + * Permission is granted to use, copy, create derivative works + * and redistribute this software and such derivative works + * for any purpose, so long as the name of The University of + * Michigan is not used in any advertising or publicity + * pertaining to the use of distribution of this software + * without specific, written prior authorization. If the + * above copyright notice or any other identification of the + * University of Michigan is included in any copy of any + * portion of this software, then the disclaimer below must + * also be included. + * + * THIS SOFTWARE IS PROVIDED AS IS, WITHOUT REPRESENTATION + * FROM THE UNIVERSITY OF MICHIGAN AS TO ITS FITNESS FOR ANY + * PURPOSE, AND WITHOUT WARRANTY BY THE UNIVERSITY OF + * MICHIGAN OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING + * WITHOUT LIMITATION THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE + * REGENTS OF THE UNIVERSITY OF MICHIGAN SHALL NOT BE LIABLE + * FOR ANY DAMAGES, INCLUDING SPECIAL, INDIRECT, INCIDENTAL, OR + * CONSEQUENTIAL DAMAGES, WITH RESPECT TO ANY CLAIM ARISING + * OUT OF OR IN CONNECTION WITH THE USE OF THE SOFTWARE, EVEN + * IF IT HAS BEEN OR IS HEREAFTER ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGES. + */ + +/* + * Copyright (C) 1998 by the FundsXpress, INC. + * + * All rights reserved. + * + * Export of this software from the United States of America may require + * a specific license from the United States Government. It is the + * responsibility of any person or organization contemplating export to + * obtain such a license before exporting. + * + * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and + * distribute this software and its documentation for any purpose and + * without fee is hereby granted, provided that the above copyright + * notice appear in all copies and that both that copyright notice and + * this permission notice appear in supporting documentation, and that + * the name of FundsXpress. not be used in advertising or publicity pertaining + * to distribution of the software without specific, written prior + * permission. FundsXpress makes no representations about the suitability of + * this software for any purpose. It is provided "as is" without express + * or implied warranty. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE. + */ + +/* + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "internal.h" + +const struct krb5_enctype krb5_aes128_cts_hmac_sha1_96 = { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA1_96, + .ctype = KRB5_CKSUMTYPE_HMAC_SHA1_96_AES128, + .name = "aes128-cts-hmac-sha1-96", + .encrypt_name = "krb5enc(hmac(sha1),cts(cbc(aes)))", + .cksum_name = "hmac(sha1)", + .hash_name = "sha1", + .derivation_enc = "cts(cbc(aes))", + .key_bytes = 16, + .key_len = 16, + .Kc_len = 16, + .Ke_len = 16, + .Ki_len = 16, + .block_len = 16, + .conf_len = 16, + .cksum_len = 12, + .hash_len = 20, + .prf_len = 16, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc3961_simplified_profile, +}; + +const struct krb5_enctype krb5_aes256_cts_hmac_sha1_96 = { + .etype = KRB5_ENCTYPE_AES256_CTS_HMAC_SHA1_96, + .ctype = KRB5_CKSUMTYPE_HMAC_SHA1_96_AES256, + .name = "aes256-cts-hmac-sha1-96", + .encrypt_name = "krb5enc(hmac(sha1),cts(cbc(aes)))", + .cksum_name = "hmac(sha1)", + .hash_name = "sha1", + .derivation_enc = "cts(cbc(aes))", + .key_bytes = 32, + .key_len = 32, + .Kc_len = 32, + .Ke_len = 32, + .Ki_len = 32, + .block_len = 16, + .conf_len = 16, + .cksum_len = 12, + .hash_len = 20, + .prf_len = 16, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc3961_simplified_profile, +}; -- cgit v1.2.3 From 6c3c0e86c2acf53bf67c095c67335a0bec2a16af Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 3 Feb 2025 13:42:41 +0000 Subject: crypto/krb5: Implement the AES enctypes from rfc8009 Implement the aes128-cts-hmac-sha256-128 and aes256-cts-hmac-sha384-192 enctypes from rfc8009, overriding the rfc3961 kerberos 5 simplified crypto scheme. Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Kconfig | 2 + crypto/krb5/Makefile | 3 +- crypto/krb5/internal.h | 6 + crypto/krb5/krb5_api.c | 2 + crypto/krb5/rfc8009_aes2.c | 362 +++++++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 4 + 6 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 crypto/krb5/rfc8009_aes2.c (limited to 'crypto') diff --git a/crypto/krb5/Kconfig b/crypto/krb5/Kconfig index 2ad874990dc8..52f0ed2d7820 100644 --- a/crypto/krb5/Kconfig +++ b/crypto/krb5/Kconfig @@ -7,6 +7,8 @@ config CRYPTO_KRB5 select CRYPTO_HASH_INFO select CRYPTO_HMAC select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_SHA512 select CRYPTO_CBC select CRYPTO_CTS select CRYPTO_AES diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index 35f21411abf8..7fd215ec3a85 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -7,6 +7,7 @@ krb5-y += \ krb5_kdf.o \ krb5_api.o \ rfc3961_simplified.o \ - rfc3962_aes.o + rfc3962_aes.o \ + rfc8009_aes2.o obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 43f904a69e32..f537f6eb86eb 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -185,3 +185,9 @@ int rfc3961_verify_mic(const struct krb5_enctype *krb5, */ extern const struct krb5_enctype krb5_aes128_cts_hmac_sha1_96; extern const struct krb5_enctype krb5_aes256_cts_hmac_sha1_96; + +/* + * rfc8009_aes2.c + */ +extern const struct krb5_enctype krb5_aes128_cts_hmac_sha256_128; +extern const struct krb5_enctype krb5_aes256_cts_hmac_sha384_192; diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index ecc6655953d5..5b94cc5db461 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -19,6 +19,8 @@ MODULE_LICENSE("GPL"); static const struct krb5_enctype *const krb5_supported_enctypes[] = { &krb5_aes128_cts_hmac_sha1_96, &krb5_aes256_cts_hmac_sha1_96, + &krb5_aes128_cts_hmac_sha256_128, + &krb5_aes256_cts_hmac_sha384_192, }; /** diff --git a/crypto/krb5/rfc8009_aes2.c b/crypto/krb5/rfc8009_aes2.c new file mode 100644 index 000000000000..d39851fc3a4e --- /dev/null +++ b/crypto/krb5/rfc8009_aes2.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* rfc8009 AES Encryption with HMAC-SHA2 for Kerberos 5 + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include "internal.h" + +static const struct krb5_buffer rfc8009_no_context = { .len = 0, .data = "" }; + +/* + * Calculate the key derivation function KDF-HMAC-SHA2(key, label, [context,] k) + * + * KDF-HMAC-SHA2(key, label, [context,] k) = k-truncate(K1) + * + * Using the appropriate one of: + * K1 = HMAC-SHA-256(key, 0x00000001 | label | 0x00 | k) + * K1 = HMAC-SHA-384(key, 0x00000001 | label | 0x00 | k) + * K1 = HMAC-SHA-256(key, 0x00000001 | label | 0x00 | context | k) + * K1 = HMAC-SHA-384(key, 0x00000001 | label | 0x00 | context | k) + * [rfc8009 sec 3] + */ +static int rfc8009_calc_KDF_HMAC_SHA2(const struct krb5_enctype *krb5, + const struct krb5_buffer *key, + const struct krb5_buffer *label, + const struct krb5_buffer *context, + unsigned int k, + struct krb5_buffer *result, + gfp_t gfp) +{ + struct crypto_shash *shash; + struct krb5_buffer K1, data; + struct shash_desc *desc; + __be32 tmp; + size_t bsize; + void *buffer; + u8 *p; + int ret = -ENOMEM; + + if (WARN_ON(result->len != k / 8)) + return -EINVAL; + + shash = crypto_alloc_shash(krb5->cksum_name, 0, 0); + if (IS_ERR(shash)) + return (PTR_ERR(shash) == -ENOENT) ? -ENOPKG : PTR_ERR(shash); + ret = crypto_shash_setkey(shash, key->data, key->len); + if (ret < 0) + goto error_shash; + + ret = -EINVAL; + if (WARN_ON(crypto_shash_digestsize(shash) * 8 < k)) + goto error_shash; + + ret = -ENOMEM; + data.len = 4 + label->len + 1 + context->len + 4; + bsize = krb5_shash_size(shash) + + krb5_digest_size(shash) + + crypto_roundup(data.len); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + goto error_shash; + + desc = buffer; + desc->tfm = shash; + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + + p = data.data = buffer + + krb5_shash_size(shash) + + krb5_digest_size(shash); + *(__be32 *)p = htonl(0x00000001); + p += 4; + memcpy(p, label->data, label->len); + p += label->len; + *p++ = 0; + memcpy(p, context->data, context->len); + p += context->len; + tmp = htonl(k); + memcpy(p, &tmp, 4); + p += 4; + + ret = -EINVAL; + if (WARN_ON(p - (u8 *)data.data != data.len)) + goto error; + + K1.len = crypto_shash_digestsize(shash); + K1.data = buffer + + krb5_shash_size(shash); + + ret = crypto_shash_finup(desc, data.data, data.len, K1.data); + if (ret < 0) + goto error; + + memcpy(result->data, K1.data, result->len); + +error: + kfree_sensitive(buffer); +error_shash: + crypto_free_shash(shash); + return ret; +} + +/* + * Calculate the pseudo-random function, PRF(). + * + * PRF = KDF-HMAC-SHA2(input-key, "prf", octet-string, 256) + * PRF = KDF-HMAC-SHA2(input-key, "prf", octet-string, 384) + * + * The "prfconstant" used in the PRF operation is the three-octet string + * "prf". + * [rfc8009 sec 5] + */ +static int rfc8009_calc_PRF(const struct krb5_enctype *krb5, + const struct krb5_buffer *input_key, + const struct krb5_buffer *octet_string, + struct krb5_buffer *result, + gfp_t gfp) +{ + static const struct krb5_buffer prfconstant = { 3, "prf" }; + + return rfc8009_calc_KDF_HMAC_SHA2(krb5, input_key, &prfconstant, + octet_string, krb5->prf_len * 8, + result, gfp); +} + +/* + * Derive Ke. + * Ke = KDF-HMAC-SHA2(base-key, usage | 0xAA, 128) + * Ke = KDF-HMAC-SHA2(base-key, usage | 0xAA, 256) + * [rfc8009 sec 5] + */ +static int rfc8009_calc_Ke(const struct krb5_enctype *krb5, + const struct krb5_buffer *base_key, + const struct krb5_buffer *usage_constant, + struct krb5_buffer *result, + gfp_t gfp) +{ + return rfc8009_calc_KDF_HMAC_SHA2(krb5, base_key, usage_constant, + &rfc8009_no_context, krb5->key_bytes * 8, + result, gfp); +} + +/* + * Derive Kc/Ki + * Kc = KDF-HMAC-SHA2(base-key, usage | 0x99, 128) + * Ki = KDF-HMAC-SHA2(base-key, usage | 0x55, 128) + * Kc = KDF-HMAC-SHA2(base-key, usage | 0x99, 192) + * Ki = KDF-HMAC-SHA2(base-key, usage | 0x55, 192) + * [rfc8009 sec 5] + */ +static int rfc8009_calc_Ki(const struct krb5_enctype *krb5, + const struct krb5_buffer *base_key, + const struct krb5_buffer *usage_constant, + struct krb5_buffer *result, + gfp_t gfp) +{ + return rfc8009_calc_KDF_HMAC_SHA2(krb5, base_key, usage_constant, + &rfc8009_no_context, krb5->cksum_len * 8, + result, gfp); +} + +/* + * Apply encryption and checksumming functions to a message. Unlike for + * RFC3961, for RFC8009, we have to chuck the starting IV into the hash first. + */ +static ssize_t rfc8009_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded) +{ + struct aead_request *req; + struct scatterlist bsg[2]; + ssize_t ret, done; + size_t bsize, base_len, secure_offset, secure_len, pad_len, cksum_offset; + void *buffer; + u8 *iv, *ad; + + if (WARN_ON(data_offset != krb5->conf_len)) + return -EINVAL; /* Data is in wrong place */ + + secure_offset = 0; + base_len = krb5->conf_len + data_len; + pad_len = 0; + secure_len = base_len + pad_len; + cksum_offset = secure_len; + if (WARN_ON(cksum_offset + krb5->cksum_len > sg_len)) + return -EFAULT; + + bsize = krb5_aead_size(aead) + + krb5_aead_ivsize(aead) * 2; + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + req = buffer; + iv = buffer + krb5_aead_size(aead); + ad = buffer + krb5_aead_size(aead) + krb5_aead_ivsize(aead); + + /* Insert the confounder into the buffer */ + ret = -EFAULT; + if (!preconfounded) { + get_random_bytes(buffer, krb5->conf_len); + done = sg_pcopy_from_buffer(sg, nr_sg, buffer, krb5->conf_len, + secure_offset); + if (done != krb5->conf_len) + goto error; + } + + /* We may need to pad out to the crypto blocksize. */ + if (pad_len) { + done = sg_zero_buffer(sg, nr_sg, pad_len, data_offset + data_len); + if (done != pad_len) + goto error; + } + + /* We need to include the starting IV in the hash. */ + sg_init_table(bsg, 2); + sg_set_buf(&bsg[0], ad, krb5_aead_ivsize(aead)); + sg_chain(bsg, 2, sg); + + /* Hash and encrypt the message. */ + aead_request_set_tfm(req, aead); + aead_request_set_callback(req, 0, NULL, NULL); + aead_request_set_ad(req, krb5_aead_ivsize(aead)); + aead_request_set_crypt(req, bsg, bsg, secure_len, iv); + ret = crypto_aead_encrypt(req); + if (ret < 0) + goto error; + + ret = secure_len + krb5->cksum_len; + +error: + kfree_sensitive(buffer); + return ret; +} + +/* + * Apply decryption and checksumming functions to a message. Unlike for + * RFC3961, for RFC8009, we have to chuck the starting IV into the hash first. + * + * The offset and length are updated to reflect the actual content of the + * encrypted region. + */ +static int rfc8009_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len) +{ + struct aead_request *req; + struct scatterlist bsg[2]; + size_t bsize; + void *buffer; + int ret; + u8 *iv, *ad; + + if (WARN_ON(*_offset != 0)) + return -EINVAL; /* Can't set offset on aead */ + + if (*_len < krb5->conf_len + krb5->cksum_len) + return -EPROTO; + + bsize = krb5_aead_size(aead) + + krb5_aead_ivsize(aead) * 2; + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + return -ENOMEM; + + req = buffer; + iv = buffer + krb5_aead_size(aead); + ad = buffer + krb5_aead_size(aead) + krb5_aead_ivsize(aead); + + /* We need to include the starting IV in the hash. */ + sg_init_table(bsg, 2); + sg_set_buf(&bsg[0], ad, krb5_aead_ivsize(aead)); + sg_chain(bsg, 2, sg); + + /* Decrypt the message and verify its checksum. */ + aead_request_set_tfm(req, aead); + aead_request_set_callback(req, 0, NULL, NULL); + aead_request_set_ad(req, krb5_aead_ivsize(aead)); + aead_request_set_crypt(req, bsg, bsg, *_len, iv); + ret = crypto_aead_decrypt(req); + if (ret < 0) + goto error; + + /* Adjust the boundaries of the data. */ + *_offset += krb5->conf_len; + *_len -= krb5->conf_len + krb5->cksum_len; + ret = 0; + +error: + kfree_sensitive(buffer); + return ret; +} + +static const struct krb5_crypto_profile rfc8009_crypto_profile = { + .calc_PRF = rfc8009_calc_PRF, + .calc_Kc = rfc8009_calc_Ki, + .calc_Ke = rfc8009_calc_Ke, + .calc_Ki = rfc8009_calc_Ki, + .derive_encrypt_keys = authenc_derive_encrypt_keys, + .load_encrypt_keys = authenc_load_encrypt_keys, + .derive_checksum_key = rfc3961_derive_checksum_key, + .load_checksum_key = rfc3961_load_checksum_key, + .encrypt = rfc8009_encrypt, + .decrypt = rfc8009_decrypt, + .get_mic = rfc3961_get_mic, + .verify_mic = rfc3961_verify_mic, +}; + +const struct krb5_enctype krb5_aes128_cts_hmac_sha256_128 = { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128, + .ctype = KRB5_CKSUMTYPE_HMAC_SHA256_128_AES128, + .name = "aes128-cts-hmac-sha256-128", + .encrypt_name = "authenc(hmac(sha256),cts(cbc(aes)))", + .cksum_name = "hmac(sha256)", + .hash_name = "sha256", + .derivation_enc = "cts(cbc(aes))", + .key_bytes = 16, + .key_len = 16, + .Kc_len = 16, + .Ke_len = 16, + .Ki_len = 16, + .block_len = 16, + .conf_len = 16, + .cksum_len = 16, + .hash_len = 20, + .prf_len = 32, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc8009_crypto_profile, +}; + +const struct krb5_enctype krb5_aes256_cts_hmac_sha384_192 = { + .etype = KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192, + .ctype = KRB5_CKSUMTYPE_HMAC_SHA384_192_AES256, + .name = "aes256-cts-hmac-sha384-192", + .encrypt_name = "authenc(hmac(sha384),cts(cbc(aes)))", + .cksum_name = "hmac(sha384)", + .hash_name = "sha384", + .derivation_enc = "cts(cbc(aes))", + .key_bytes = 32, + .key_len = 32, + .Kc_len = 24, + .Ke_len = 32, + .Ki_len = 24, + .block_len = 16, + .conf_len = 16, + .cksum_len = 24, + .hash_len = 20, + .prf_len = 48, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc8009_crypto_profile, +}; diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index b12f012cf354..b8fda81379ab 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -31,6 +31,8 @@ struct scatterlist; #define KRB5_ENCTYPE_DES3_CBC_SHA1 0x0010 #define KRB5_ENCTYPE_AES128_CTS_HMAC_SHA1_96 0x0011 #define KRB5_ENCTYPE_AES256_CTS_HMAC_SHA1_96 0x0012 +#define KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128 0x0013 +#define KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192 0x0014 #define KRB5_ENCTYPE_ARCFOUR_HMAC 0x0017 #define KRB5_ENCTYPE_ARCFOUR_HMAC_EXP 0x0018 #define KRB5_ENCTYPE_UNKNOWN 0x01ff @@ -45,6 +47,8 @@ struct scatterlist; #define KRB5_CKSUMTYPE_HMAC_SHA1_DES3 0x000c #define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES128 0x000f #define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES256 0x0010 +#define KRB5_CKSUMTYPE_HMAC_SHA256_128_AES128 0x0013 +#define KRB5_CKSUMTYPE_HMAC_SHA384_192_AES256 0x0014 #define KRB5_CKSUMTYPE_HMAC_MD5_ARCFOUR -138 /* Microsoft md5 hmac cksumtype */ /* -- cgit v1.2.3 From 742e38d4d4033e7ff53178acf7edd2b1fe0142ef Mon Sep 17 00:00:00 2001 From: David Howells Date: Fri, 25 Sep 2020 12:24:50 +0100 Subject: crypto/krb5: Implement the Camellia enctypes from rfc6803 Implement the camellia128-cts-cmac and camellia256-cts-cmac enctypes from rfc6803. Note that the test vectors in rfc6803 for encryption are incomplete, lacking the key usage number needed to derive Ke and Ki, and there are errata for this: https://www.rfc-editor.org/errata_search.php?rfc=6803 Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Kconfig | 2 + crypto/krb5/Makefile | 1 + crypto/krb5/internal.h | 6 ++ crypto/krb5/krb5_api.c | 2 + crypto/krb5/rfc6803_camellia.c | 237 +++++++++++++++++++++++++++++++++++++++++ include/crypto/krb5.h | 4 + 6 files changed, 252 insertions(+) create mode 100644 crypto/krb5/rfc6803_camellia.c (limited to 'crypto') diff --git a/crypto/krb5/Kconfig b/crypto/krb5/Kconfig index 52f0ed2d7820..5b339690905c 100644 --- a/crypto/krb5/Kconfig +++ b/crypto/krb5/Kconfig @@ -6,12 +6,14 @@ config CRYPTO_KRB5 select CRYPTO_SKCIPHER select CRYPTO_HASH_INFO select CRYPTO_HMAC + select CRYPTO_CMAC select CRYPTO_SHA1 select CRYPTO_SHA256 select CRYPTO_SHA512 select CRYPTO_CBC select CRYPTO_CTS select CRYPTO_AES + select CRYPTO_CAMELLIA help Provide a library for provision of Kerberos-5-based crypto. This is intended for network filesystems to use. diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index 7fd215ec3a85..7cbe5e5ded19 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -8,6 +8,7 @@ krb5-y += \ krb5_api.o \ rfc3961_simplified.o \ rfc3962_aes.o \ + rfc6803_camellia.o \ rfc8009_aes2.o obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index f537f6eb86eb..8679140ef90d 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -186,6 +186,12 @@ int rfc3961_verify_mic(const struct krb5_enctype *krb5, extern const struct krb5_enctype krb5_aes128_cts_hmac_sha1_96; extern const struct krb5_enctype krb5_aes256_cts_hmac_sha1_96; +/* + * rfc6803_camellia.c + */ +extern const struct krb5_enctype krb5_camellia128_cts_cmac; +extern const struct krb5_enctype krb5_camellia256_cts_cmac; + /* * rfc8009_aes2.c */ diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 5b94cc5db461..02e21c8f4d14 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -21,6 +21,8 @@ static const struct krb5_enctype *const krb5_supported_enctypes[] = { &krb5_aes256_cts_hmac_sha1_96, &krb5_aes128_cts_hmac_sha256_128, &krb5_aes256_cts_hmac_sha384_192, + &krb5_camellia128_cts_cmac, + &krb5_camellia256_cts_cmac, }; /** diff --git a/crypto/krb5/rfc6803_camellia.c b/crypto/krb5/rfc6803_camellia.c new file mode 100644 index 000000000000..77cd4ce023f1 --- /dev/null +++ b/crypto/krb5/rfc6803_camellia.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* rfc6803 Camellia Encryption for Kerberos 5 + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include "internal.h" + +/* + * Calculate the key derivation function KDF-FEEDBACK_CMAC(key, constant) + * + * n = ceiling(k / 128) + * K(0) = zeros + * K(i) = CMAC(key, K(i-1) | i | constant | 0x00 | k) + * DR(key, constant) = k-truncate(K(1) | K(2) | ... | K(n)) + * KDF-FEEDBACK-CMAC(key, constant) = random-to-key(DR(key, constant)) + * + * [rfc6803 sec 3] + */ +static int rfc6803_calc_KDF_FEEDBACK_CMAC(const struct krb5_enctype *krb5, + const struct krb5_buffer *key, + const struct krb5_buffer *constant, + struct krb5_buffer *result, + gfp_t gfp) +{ + struct crypto_shash *shash; + struct krb5_buffer K, data; + struct shash_desc *desc; + __be32 tmp; + size_t bsize, offset, seg; + void *buffer; + u32 i = 0, k = result->len * 8; + u8 *p; + int ret = -ENOMEM; + + shash = crypto_alloc_shash(krb5->cksum_name, 0, 0); + if (IS_ERR(shash)) + return (PTR_ERR(shash) == -ENOENT) ? -ENOPKG : PTR_ERR(shash); + ret = crypto_shash_setkey(shash, key->data, key->len); + if (ret < 0) + goto error_shash; + + ret = -ENOMEM; + K.len = crypto_shash_digestsize(shash); + data.len = K.len + 4 + constant->len + 1 + 4; + bsize = krb5_shash_size(shash) + + krb5_digest_size(shash) + + crypto_roundup(K.len) + + crypto_roundup(data.len); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + goto error_shash; + + desc = buffer; + desc->tfm = shash; + + K.data = buffer + + krb5_shash_size(shash) + + krb5_digest_size(shash); + data.data = buffer + + krb5_shash_size(shash) + + krb5_digest_size(shash) + + crypto_roundup(K.len); + + p = data.data + K.len + 4; + memcpy(p, constant->data, constant->len); + p += constant->len; + *p++ = 0x00; + tmp = htonl(k); + memcpy(p, &tmp, 4); + p += 4; + + ret = -EINVAL; + if (WARN_ON(p - (u8 *)data.data != data.len)) + goto error; + + offset = 0; + do { + i++; + p = data.data; + memcpy(p, K.data, K.len); + p += K.len; + *(__be32 *)p = htonl(i); + + ret = crypto_shash_init(desc); + if (ret < 0) + goto error; + ret = crypto_shash_finup(desc, data.data, data.len, K.data); + if (ret < 0) + goto error; + + seg = min_t(size_t, result->len - offset, K.len); + memcpy(result->data + offset, K.data, seg); + offset += seg; + } while (offset < result->len); + +error: + kfree_sensitive(buffer); +error_shash: + crypto_free_shash(shash); + return ret; +} + +/* + * Calculate the pseudo-random function, PRF(). + * + * Kp = KDF-FEEDBACK-CMAC(protocol-key, "prf") + * PRF = CMAC(Kp, octet-string) + * [rfc6803 sec 6] + */ +static int rfc6803_calc_PRF(const struct krb5_enctype *krb5, + const struct krb5_buffer *protocol_key, + const struct krb5_buffer *octet_string, + struct krb5_buffer *result, + gfp_t gfp) +{ + static const struct krb5_buffer prfconstant = { 3, "prf" }; + struct crypto_shash *shash; + struct krb5_buffer Kp; + struct shash_desc *desc; + size_t bsize; + void *buffer; + int ret; + + Kp.len = krb5->prf_len; + + shash = crypto_alloc_shash(krb5->cksum_name, 0, 0); + if (IS_ERR(shash)) + return (PTR_ERR(shash) == -ENOENT) ? -ENOPKG : PTR_ERR(shash); + + ret = -EINVAL; + if (result->len != crypto_shash_digestsize(shash)) + goto out_shash; + + ret = -ENOMEM; + bsize = krb5_shash_size(shash) + + krb5_digest_size(shash) + + crypto_roundup(Kp.len); + buffer = kzalloc(bsize, GFP_NOFS); + if (!buffer) + goto out_shash; + + Kp.data = buffer + + krb5_shash_size(shash) + + krb5_digest_size(shash); + + ret = rfc6803_calc_KDF_FEEDBACK_CMAC(krb5, protocol_key, &prfconstant, + &Kp, gfp); + if (ret < 0) + goto out; + + ret = crypto_shash_setkey(shash, Kp.data, Kp.len); + if (ret < 0) + goto out; + + desc = buffer; + desc->tfm = shash; + ret = crypto_shash_init(desc); + if (ret < 0) + goto out; + + ret = crypto_shash_finup(desc, octet_string->data, octet_string->len, result->data); + if (ret < 0) + goto out; + +out: + kfree_sensitive(buffer); +out_shash: + crypto_free_shash(shash); + return ret; +} + + +static const struct krb5_crypto_profile rfc6803_crypto_profile = { + .calc_PRF = rfc6803_calc_PRF, + .calc_Kc = rfc6803_calc_KDF_FEEDBACK_CMAC, + .calc_Ke = rfc6803_calc_KDF_FEEDBACK_CMAC, + .calc_Ki = rfc6803_calc_KDF_FEEDBACK_CMAC, + .derive_encrypt_keys = authenc_derive_encrypt_keys, + .load_encrypt_keys = authenc_load_encrypt_keys, + .derive_checksum_key = rfc3961_derive_checksum_key, + .load_checksum_key = rfc3961_load_checksum_key, + .encrypt = krb5_aead_encrypt, + .decrypt = krb5_aead_decrypt, + .get_mic = rfc3961_get_mic, + .verify_mic = rfc3961_verify_mic, +}; + +const struct krb5_enctype krb5_camellia128_cts_cmac = { + .etype = KRB5_ENCTYPE_CAMELLIA128_CTS_CMAC, + .ctype = KRB5_CKSUMTYPE_CMAC_CAMELLIA128, + .name = "camellia128-cts-cmac", + .encrypt_name = "krb5enc(cmac(camellia),cts(cbc(camellia)))", + .cksum_name = "cmac(camellia)", + .hash_name = NULL, + .derivation_enc = "cts(cbc(camellia))", + .key_bytes = 16, + .key_len = 16, + .Kc_len = 16, + .Ke_len = 16, + .Ki_len = 16, + .block_len = 16, + .conf_len = 16, + .cksum_len = 16, + .hash_len = 16, + .prf_len = 16, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc6803_crypto_profile, +}; + +const struct krb5_enctype krb5_camellia256_cts_cmac = { + .etype = KRB5_ENCTYPE_CAMELLIA256_CTS_CMAC, + .ctype = KRB5_CKSUMTYPE_CMAC_CAMELLIA256, + .name = "camellia256-cts-cmac", + .encrypt_name = "krb5enc(cmac(camellia),cts(cbc(camellia)))", + .cksum_name = "cmac(camellia)", + .hash_name = NULL, + .derivation_enc = "cts(cbc(camellia))", + .key_bytes = 32, + .key_len = 32, + .Kc_len = 32, + .Ke_len = 32, + .Ki_len = 32, + .block_len = 16, + .conf_len = 16, + .cksum_len = 16, + .hash_len = 16, + .prf_len = 16, + .keyed_cksum = true, + .random_to_key = NULL, /* Identity */ + .profile = &rfc6803_crypto_profile, +}; diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h index b8fda81379ab..62d998e62f47 100644 --- a/include/crypto/krb5.h +++ b/include/crypto/krb5.h @@ -35,6 +35,8 @@ struct scatterlist; #define KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192 0x0014 #define KRB5_ENCTYPE_ARCFOUR_HMAC 0x0017 #define KRB5_ENCTYPE_ARCFOUR_HMAC_EXP 0x0018 +#define KRB5_ENCTYPE_CAMELLIA128_CTS_CMAC 0x0019 +#define KRB5_ENCTYPE_CAMELLIA256_CTS_CMAC 0x001a #define KRB5_ENCTYPE_UNKNOWN 0x01ff #define KRB5_CKSUMTYPE_CRC32 0x0001 @@ -47,6 +49,8 @@ struct scatterlist; #define KRB5_CKSUMTYPE_HMAC_SHA1_DES3 0x000c #define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES128 0x000f #define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES256 0x0010 +#define KRB5_CKSUMTYPE_CMAC_CAMELLIA128 0x0011 +#define KRB5_CKSUMTYPE_CMAC_CAMELLIA256 0x0012 #define KRB5_CKSUMTYPE_HMAC_SHA256_128_AES128 0x0013 #define KRB5_CKSUMTYPE_HMAC_SHA384_192_AES256 0x0014 #define KRB5_CKSUMTYPE_HMAC_MD5_ARCFOUR -138 /* Microsoft md5 hmac cksumtype */ -- cgit v1.2.3 From fc0cf10c04f49ddba1925b630467f49ea993569e Mon Sep 17 00:00:00 2001 From: David Howells Date: Mon, 3 Feb 2025 13:44:37 +0000 Subject: crypto/krb5: Implement crypto self-testing Implement self-testing infrastructure to test the pseudo-random function, key derivation, encryption and checksumming. Add the testing data from rfc8009 to test AES + HMAC-SHA2. Add the testing data from rfc6803 to test Camellia. Note some encryption test vectors here are incomplete, lacking the key usage number needed to derive Ke and Ki, and there are errata for this: https://www.rfc-editor.org/errata_search.php?rfc=6803 Signed-off-by: David Howells cc: Herbert Xu cc: "David S. Miller" cc: Chuck Lever cc: Marc Dionne cc: Eric Dumazet cc: Jakub Kicinski cc: Paolo Abeni cc: Simon Horman cc: linux-afs@lists.infradead.org cc: linux-nfs@vger.kernel.org cc: linux-crypto@vger.kernel.org cc: netdev@vger.kernel.org --- crypto/krb5/Kconfig | 7 + crypto/krb5/Makefile | 4 + crypto/krb5/internal.h | 48 ++++ crypto/krb5/krb5_api.c | 11 + crypto/krb5/selftest.c | 544 ++++++++++++++++++++++++++++++++++++++++++++ crypto/krb5/selftest_data.c | 291 ++++++++++++++++++++++++ 6 files changed, 905 insertions(+) create mode 100644 crypto/krb5/selftest.c create mode 100644 crypto/krb5/selftest_data.c (limited to 'crypto') diff --git a/crypto/krb5/Kconfig b/crypto/krb5/Kconfig index 5b339690905c..4d0476e13f3c 100644 --- a/crypto/krb5/Kconfig +++ b/crypto/krb5/Kconfig @@ -17,3 +17,10 @@ config CRYPTO_KRB5 help Provide a library for provision of Kerberos-5-based crypto. This is intended for network filesystems to use. + +config CRYPTO_KRB5_SELFTESTS + bool "Kerberos 5 crypto selftests" + depends on CRYPTO_KRB5 + help + Turn on some self-testing for the kerberos 5 crypto functions. These + will be performed on module load or boot, if compiled in. diff --git a/crypto/krb5/Makefile b/crypto/krb5/Makefile index 7cbe5e5ded19..d38890c0b247 100644 --- a/crypto/krb5/Makefile +++ b/crypto/krb5/Makefile @@ -11,4 +11,8 @@ krb5-y += \ rfc6803_camellia.o \ rfc8009_aes2.o +krb5-$(CONFIG_CRYPTO_KRB5_SELFTESTS) += \ + selftest.o \ + selftest_data.o + obj-$(CONFIG_CRYPTO_KRB5) += krb5.o diff --git a/crypto/krb5/internal.h b/crypto/krb5/internal.h index 8679140ef90d..a59084ffafe8 100644 --- a/crypto/krb5/internal.h +++ b/crypto/krb5/internal.h @@ -114,6 +114,37 @@ struct krb5_crypto_profile { crypto_roundup(crypto_shash_digestsize(TFM)) #define round16(x) (((x) + 15) & ~15) +/* + * Self-testing data. + */ +struct krb5_prf_test { + u32 etype; + const char *name, *key, *octet, *prf; +}; + +struct krb5_key_test_one { + u32 use; + const char *key; +}; + +struct krb5_key_test { + u32 etype; + const char *name, *key; + struct krb5_key_test_one Kc, Ke, Ki; +}; + +struct krb5_enc_test { + u32 etype; + u32 usage; + const char *name, *plain, *conf, *K0, *Ke, *Ki, *ct; +}; + +struct krb5_mic_test { + u32 etype; + u32 usage; + const char *name, *plain, *K0, *Kc, *mic; +}; + /* * krb5_api.c */ @@ -197,3 +228,20 @@ extern const struct krb5_enctype krb5_camellia256_cts_cmac; */ extern const struct krb5_enctype krb5_aes128_cts_hmac_sha256_128; extern const struct krb5_enctype krb5_aes256_cts_hmac_sha384_192; + +/* + * selftest.c + */ +#ifdef CONFIG_CRYPTO_KRB5_SELFTESTS +int krb5_selftest(void); +#else +static inline int krb5_selftest(void) { return 0; } +#endif + +/* + * selftest_data.c + */ +extern const struct krb5_prf_test krb5_prf_tests[]; +extern const struct krb5_key_test krb5_key_tests[]; +extern const struct krb5_enc_test krb5_enc_tests[]; +extern const struct krb5_mic_test krb5_mic_tests[]; diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 02e21c8f4d14..23026d4206c8 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -439,3 +439,14 @@ int crypto_krb5_verify_mic(const struct krb5_enctype *krb5, _offset, _len); } EXPORT_SYMBOL(crypto_krb5_verify_mic); + +static int __init crypto_krb5_init(void) +{ + return krb5_selftest(); +} +module_init(crypto_krb5_init); + +static void __exit crypto_krb5_exit(void) +{ +} +module_exit(crypto_krb5_exit); diff --git a/crypto/krb5/selftest.c b/crypto/krb5/selftest.c new file mode 100644 index 000000000000..2a81a6315a0d --- /dev/null +++ b/crypto/krb5/selftest.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Kerberos library self-testing + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include "internal.h" + +#define VALID(X) \ + ({ \ + bool __x = (X); \ + if (__x) { \ + pr_warn("!!! TESTINVAL %s:%u\n", __FILE__, __LINE__); \ + ret = -EBADMSG; \ + } \ + __x; \ + }) + +#define CHECK(X) \ + ({ \ + bool __x = (X); \ + if (__x) { \ + pr_warn("!!! TESTFAIL %s:%u\n", __FILE__, __LINE__); \ + ret = -EBADMSG; \ + } \ + __x; \ + }) + +enum which_key { + TEST_KC, TEST_KE, TEST_KI, +}; + +#if 0 +static void dump_sg(struct scatterlist *sg, unsigned int limit) +{ + unsigned int index = 0, n = 0; + + for (; sg && limit > 0; sg = sg_next(sg)) { + unsigned int off = sg->offset, len = umin(sg->length, limit); + const void *p = kmap_local_page(sg_page(sg)); + + limit -= len; + while (len > 0) { + unsigned int part = umin(len, 32); + + pr_notice("[%x] %04x: %*phN\n", n, index, part, p + off); + index += part; + off += part; + len -= part; + } + + kunmap_local(p); + n++; + } +} +#endif + +static int prep_buf(struct krb5_buffer *buf) +{ + buf->data = kmalloc(buf->len, GFP_KERNEL); + if (!buf->data) + return -ENOMEM; + return 0; +} + +#define PREP_BUF(BUF, LEN) \ + do { \ + (BUF)->len = (LEN); \ + ret = prep_buf((BUF)); \ + if (ret < 0) \ + goto out; \ + } while (0) + +static int load_buf(struct krb5_buffer *buf, const char *from) +{ + size_t len = strlen(from); + int ret; + + if (len > 1 && from[0] == '\'') { + PREP_BUF(buf, len - 1); + memcpy(buf->data, from + 1, len - 1); + ret = 0; + goto out; + } + + if (VALID(len & 1)) + return -EINVAL; + + PREP_BUF(buf, len / 2); + ret = hex2bin(buf->data, from, buf->len); + if (ret < 0) { + VALID(1); + goto out; + } +out: + return ret; +} + +#define LOAD_BUF(BUF, FROM) do { ret = load_buf(BUF, FROM); if (ret < 0) goto out; } while (0) + +static void clear_buf(struct krb5_buffer *buf) +{ + kfree(buf->data); + buf->len = 0; + buf->data = NULL; +} + +/* + * Perform a pseudo-random function check. + */ +static int krb5_test_one_prf(const struct krb5_prf_test *test) +{ + const struct krb5_enctype *krb5 = crypto_krb5_find_enctype(test->etype); + struct krb5_buffer key = {}, octet = {}, result = {}, prf = {}; + int ret; + + if (!krb5) + return -EOPNOTSUPP; + + pr_notice("Running %s %s\n", krb5->name, test->name); + + LOAD_BUF(&key, test->key); + LOAD_BUF(&octet, test->octet); + LOAD_BUF(&prf, test->prf); + PREP_BUF(&result, krb5->prf_len); + + if (VALID(result.len != prf.len)) { + ret = -EINVAL; + goto out; + } + + ret = krb5->profile->calc_PRF(krb5, &key, &octet, &result, GFP_KERNEL); + if (ret < 0) { + CHECK(1); + pr_warn("PRF calculation failed %d\n", ret); + goto out; + } + + if (memcmp(result.data, prf.data, result.len) != 0) { + CHECK(1); + ret = -EKEYREJECTED; + goto out; + } + + ret = 0; + +out: + clear_buf(&result); + clear_buf(&octet); + clear_buf(&key); + return ret; +} + +/* + * Perform a key derivation check. + */ +static int krb5_test_key(const struct krb5_enctype *krb5, + const struct krb5_buffer *base_key, + const struct krb5_key_test_one *test, + enum which_key which) +{ + struct krb5_buffer key = {}, result = {}; + int ret; + + LOAD_BUF(&key, test->key); + PREP_BUF(&result, key.len); + + switch (which) { + case TEST_KC: + ret = krb5_derive_Kc(krb5, base_key, test->use, &result, GFP_KERNEL); + break; + case TEST_KE: + ret = krb5_derive_Ke(krb5, base_key, test->use, &result, GFP_KERNEL); + break; + case TEST_KI: + ret = krb5_derive_Ki(krb5, base_key, test->use, &result, GFP_KERNEL); + break; + default: + VALID(1); + ret = -EINVAL; + goto out; + } + + if (ret < 0) { + CHECK(1); + pr_warn("Key derivation failed %d\n", ret); + goto out; + } + + if (memcmp(result.data, key.data, result.len) != 0) { + CHECK(1); + ret = -EKEYREJECTED; + goto out; + } + +out: + clear_buf(&key); + clear_buf(&result); + return ret; +} + +static int krb5_test_one_key(const struct krb5_key_test *test) +{ + const struct krb5_enctype *krb5 = crypto_krb5_find_enctype(test->etype); + struct krb5_buffer base_key = {}; + int ret; + + if (!krb5) + return -EOPNOTSUPP; + + pr_notice("Running %s %s\n", krb5->name, test->name); + + LOAD_BUF(&base_key, test->key); + + ret = krb5_test_key(krb5, &base_key, &test->Kc, TEST_KC); + if (ret < 0) + goto out; + ret = krb5_test_key(krb5, &base_key, &test->Ke, TEST_KE); + if (ret < 0) + goto out; + ret = krb5_test_key(krb5, &base_key, &test->Ki, TEST_KI); + if (ret < 0) + goto out; + +out: + clear_buf(&base_key); + return ret; +} + +/* + * Perform an encryption test. + */ +static int krb5_test_one_enc(const struct krb5_enc_test *test, void *buf) +{ + const struct krb5_enctype *krb5 = crypto_krb5_find_enctype(test->etype); + struct crypto_aead *ci = NULL; + struct krb5_buffer K0 = {}, Ke = {}, Ki = {}, keys = {}; + struct krb5_buffer conf = {}, plain = {}, ct = {}; + struct scatterlist sg[1]; + size_t data_len, data_offset, message_len; + int ret; + + if (!krb5) + return -EOPNOTSUPP; + + pr_notice("Running %s %s\n", krb5->name, test->name); + + /* Load the test data into binary buffers. */ + LOAD_BUF(&conf, test->conf); + LOAD_BUF(&plain, test->plain); + LOAD_BUF(&ct, test->ct); + + if (test->K0) { + LOAD_BUF(&K0, test->K0); + } else { + LOAD_BUF(&Ke, test->Ke); + LOAD_BUF(&Ki, test->Ki); + + ret = krb5->profile->load_encrypt_keys(krb5, &Ke, &Ki, &keys, GFP_KERNEL); + if (ret < 0) + goto out; + } + + if (VALID(conf.len != krb5->conf_len) || + VALID(ct.len != krb5->conf_len + plain.len + krb5->cksum_len)) + goto out; + + data_len = plain.len; + message_len = crypto_krb5_how_much_buffer(krb5, KRB5_ENCRYPT_MODE, + data_len, &data_offset); + + if (CHECK(message_len != ct.len)) { + pr_warn("Encrypted length mismatch %zu != %u\n", message_len, ct.len); + goto out; + } + if (CHECK(data_offset != conf.len)) { + pr_warn("Data offset mismatch %zu != %u\n", data_offset, conf.len); + goto out; + } + + memcpy(buf, conf.data, conf.len); + memcpy(buf + data_offset, plain.data, plain.len); + + /* Allocate a crypto object and set its key. */ + if (test->K0) + ci = crypto_krb5_prepare_encryption(krb5, &K0, test->usage, GFP_KERNEL); + else + ci = krb5_prepare_encryption(krb5, &keys, GFP_KERNEL); + + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + ci = NULL; + pr_err("Couldn't alloc AEAD %s: %d\n", krb5->encrypt_name, ret); + goto out; + } + + /* Encrypt the message. */ + sg_init_one(sg, buf, message_len); + ret = crypto_krb5_encrypt(krb5, ci, sg, 1, message_len, + data_offset, data_len, true); + if (ret < 0) { + CHECK(1); + pr_warn("Encryption failed %d\n", ret); + goto out; + } + if (ret != message_len) { + CHECK(1); + pr_warn("Encrypted message wrong size %x != %zx\n", ret, message_len); + goto out; + } + + if (memcmp(buf, ct.data, ct.len) != 0) { + CHECK(1); + pr_warn("Ciphertext mismatch\n"); + pr_warn("BUF %*phN\n", ct.len, buf); + pr_warn("CT %*phN\n", ct.len, ct.data); + pr_warn("PT %*phN%*phN\n", conf.len, conf.data, plain.len, plain.data); + ret = -EKEYREJECTED; + goto out; + } + + /* Decrypt the encrypted message. */ + data_offset = 0; + data_len = message_len; + ret = crypto_krb5_decrypt(krb5, ci, sg, 1, &data_offset, &data_len); + if (ret < 0) { + CHECK(1); + pr_warn("Decryption failed %d\n", ret); + goto out; + } + + if (CHECK(data_offset != conf.len) || + CHECK(data_len != plain.len)) + goto out; + + if (memcmp(buf, conf.data, conf.len) != 0) { + CHECK(1); + pr_warn("Confounder mismatch\n"); + pr_warn("ENC %*phN\n", conf.len, buf); + pr_warn("DEC %*phN\n", conf.len, conf.data); + ret = -EKEYREJECTED; + goto out; + } + + if (memcmp(buf + conf.len, plain.data, plain.len) != 0) { + CHECK(1); + pr_warn("Plaintext mismatch\n"); + pr_warn("BUF %*phN\n", plain.len, buf + conf.len); + pr_warn("PT %*phN\n", plain.len, plain.data); + ret = -EKEYREJECTED; + goto out; + } + + ret = 0; + +out: + clear_buf(&ct); + clear_buf(&plain); + clear_buf(&conf); + clear_buf(&keys); + clear_buf(&Ki); + clear_buf(&Ke); + clear_buf(&K0); + if (ci) + crypto_free_aead(ci); + return ret; +} + +/* + * Perform a checksum test. + */ +static int krb5_test_one_mic(const struct krb5_mic_test *test, void *buf) +{ + const struct krb5_enctype *krb5 = crypto_krb5_find_enctype(test->etype); + struct crypto_shash *ci = NULL; + struct scatterlist sg[1]; + struct krb5_buffer K0 = {}, Kc = {}, keys = {}, plain = {}, mic = {}; + size_t offset, len, message_len; + int ret; + + if (!krb5) + return -EOPNOTSUPP; + + pr_notice("Running %s %s\n", krb5->name, test->name); + + /* Allocate a crypto object and set its key. */ + if (test->K0) { + LOAD_BUF(&K0, test->K0); + ci = crypto_krb5_prepare_checksum(krb5, &K0, test->usage, GFP_KERNEL); + } else { + LOAD_BUF(&Kc, test->Kc); + + ret = krb5->profile->load_checksum_key(krb5, &Kc, &keys, GFP_KERNEL); + if (ret < 0) + goto out; + + ci = krb5_prepare_checksum(krb5, &Kc, GFP_KERNEL); + } + if (IS_ERR(ci)) { + ret = PTR_ERR(ci); + ci = NULL; + pr_err("Couldn't alloc shash %s: %d\n", krb5->cksum_name, ret); + goto out; + } + + /* Load the test data into binary buffers. */ + LOAD_BUF(&plain, test->plain); + LOAD_BUF(&mic, test->mic); + + len = plain.len; + message_len = crypto_krb5_how_much_buffer(krb5, KRB5_CHECKSUM_MODE, + len, &offset); + + if (CHECK(message_len != mic.len + plain.len)) { + pr_warn("MIC length mismatch %zu != %u\n", + message_len, mic.len + plain.len); + goto out; + } + + memcpy(buf + offset, plain.data, plain.len); + + /* Generate a MIC generation request. */ + sg_init_one(sg, buf, 1024); + + ret = crypto_krb5_get_mic(krb5, ci, NULL, sg, 1, 1024, + krb5->cksum_len, plain.len); + if (ret < 0) { + CHECK(1); + pr_warn("Get MIC failed %d\n", ret); + goto out; + } + len = ret; + + if (CHECK(len != plain.len + mic.len)) { + pr_warn("MIC length mismatch %zu != %u\n", len, plain.len + mic.len); + goto out; + } + + if (memcmp(buf, mic.data, mic.len) != 0) { + CHECK(1); + pr_warn("MIC mismatch\n"); + pr_warn("BUF %*phN\n", mic.len, buf); + pr_warn("MIC %*phN\n", mic.len, mic.data); + ret = -EKEYREJECTED; + goto out; + } + + /* Generate a verification request. */ + offset = 0; + ret = crypto_krb5_verify_mic(krb5, ci, NULL, sg, 1, &offset, &len); + if (ret < 0) { + CHECK(1); + pr_warn("Verify MIC failed %d\n", ret); + goto out; + } + + if (CHECK(offset != mic.len) || + CHECK(len != plain.len)) + goto out; + + if (memcmp(buf + offset, plain.data, plain.len) != 0) { + CHECK(1); + pr_warn("Plaintext mismatch\n"); + pr_warn("BUF %*phN\n", plain.len, buf + offset); + pr_warn("PT %*phN\n", plain.len, plain.data); + ret = -EKEYREJECTED; + goto out; + } + + ret = 0; + +out: + clear_buf(&mic); + clear_buf(&plain); + clear_buf(&keys); + clear_buf(&K0); + clear_buf(&Kc); + if (ci) + crypto_free_shash(ci); + return ret; +} + +int krb5_selftest(void) +{ + void *buf; + int ret = 0, i; + + buf = kmalloc(4096, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + pr_notice("\n"); + pr_notice("Running selftests\n"); + + for (i = 0; krb5_prf_tests[i].name; i++) { + ret = krb5_test_one_prf(&krb5_prf_tests[i]); + if (ret < 0) { + if (ret != -EOPNOTSUPP) + goto out; + pr_notice("Skipping %s\n", krb5_prf_tests[i].name); + } + } + + for (i = 0; krb5_key_tests[i].name; i++) { + ret = krb5_test_one_key(&krb5_key_tests[i]); + if (ret < 0) { + if (ret != -EOPNOTSUPP) + goto out; + pr_notice("Skipping %s\n", krb5_key_tests[i].name); + } + } + + for (i = 0; krb5_enc_tests[i].name; i++) { + memset(buf, 0x5a, 4096); + ret = krb5_test_one_enc(&krb5_enc_tests[i], buf); + if (ret < 0) { + if (ret != -EOPNOTSUPP) + goto out; + pr_notice("Skipping %s\n", krb5_enc_tests[i].name); + } + } + + for (i = 0; krb5_mic_tests[i].name; i++) { + memset(buf, 0x5a, 4096); + ret = krb5_test_one_mic(&krb5_mic_tests[i], buf); + if (ret < 0) { + if (ret != -EOPNOTSUPP) + goto out; + pr_notice("Skipping %s\n", krb5_mic_tests[i].name); + } + } + + ret = 0; +out: + pr_notice("Selftests %s\n", ret == 0 ? "succeeded" : "failed"); + kfree(buf); + return ret; +} diff --git a/crypto/krb5/selftest_data.c b/crypto/krb5/selftest_data.c new file mode 100644 index 000000000000..24447ee8bf07 --- /dev/null +++ b/crypto/krb5/selftest_data.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Data for Kerberos library self-testing + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "internal.h" + +/* + * Pseudo-random function tests. + */ +const struct krb5_prf_test krb5_prf_tests[] = { + /* rfc8009 Appendix A */ + { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128, + .name = "prf", + .key = "3705D96080C17728A0E800EAB6E0D23C", + .octet = "74657374", + .prf = "9D188616F63852FE86915BB840B4A886FF3E6BB0F819B49B893393D393854295", + }, { + .etype = KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192, + .name = "prf", + .key = "6D404D37FAF79F9DF0D33568D320669800EB4836472EA8A026D16B7182460C52", + .octet = "74657374", + .prf = + "9801F69A368C2BF675E59521E177D9A07F67EFE1CFDE8D3C8D6F6A0256E3B17D" + "B3C1B62AD1B8553360D17367EB1514D2", + }, + {/* END */} +}; + +/* + * Key derivation tests. + */ +const struct krb5_key_test krb5_key_tests[] = { + /* rfc8009 Appendix A */ + { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128, + .name = "key", + .key = "3705D96080C17728A0E800EAB6E0D23C", + .Kc.use = 0x00000002, + .Kc.key = "B31A018A48F54776F403E9A396325DC3", + .Ke.use = 0x00000002, + .Ke.key = "9B197DD1E8C5609D6E67C3E37C62C72E", + .Ki.use = 0x00000002, + .Ki.key = "9FDA0E56AB2D85E1569A688696C26A6C", + }, { + .etype = KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192, + .name = "key", + .key = "6D404D37FAF79F9DF0D33568D320669800EB4836472EA8A026D16B7182460C52", + .Kc.use = 0x00000002, + .Kc.key = "EF5718BE86CC84963D8BBB5031E9F5C4BA41F28FAF69E73D", + .Ke.use = 0x00000002, + .Ke.key = "56AB22BEE63D82D7BC5227F6773F8EA7A5EB1C825160C38312980C442E5C7E49", + .Ki.use = 0x00000002, + .Ki.key = "69B16514E3CD8E56B82010D5C73012B622C4D00FFC23ED1F", + }, + /* rfc6803 sec 10 */ + { + .etype = KRB5_ENCTYPE_CAMELLIA128_CTS_CMAC, + .name = "key", + .key = "57D0297298FFD9D35DE5A47FB4BDE24B", + .Kc.use = 0x00000002, + .Kc.key = "D155775A209D05F02B38D42A389E5A56", + .Ke.use = 0x00000002, + .Ke.key = "64DF83F85A532F17577D8C37035796AB", + .Ki.use = 0x00000002, + .Ki.key = "3E4FBDF30FB8259C425CB6C96F1F4635", + }, + { + .etype = KRB5_ENCTYPE_CAMELLIA256_CTS_CMAC, + .name = "key", + .key = "B9D6828B2056B7BE656D88A123B1FAC68214AC2B727ECF5F69AFE0C4DF2A6D2C", + .Kc.use = 0x00000002, + .Kc.key = "E467F9A9552BC7D3155A6220AF9C19220EEED4FF78B0D1E6A1544991461A9E50", + .Ke.use = 0x00000002, + .Ke.key = "412AEFC362A7285FC3966C6A5181E7605AE675235B6D549FBFC9AB6630A4C604", + .Ki.use = 0x00000002, + .Ki.key = "FA624FA0E523993FA388AEFDC67E67EBCD8C08E8A0246B1D73B0D1DD9FC582B0", + }, + {/* END */} +}; + +/* + * Encryption tests. + */ +const struct krb5_enc_test krb5_enc_tests[] = { + /* rfc8009 Appendix A */ + { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128, + .name = "enc no plain", + .plain = "", + .conf = "7E5895EAF2672435BAD817F545A37148", + .Ke = "9B197DD1E8C5609D6E67C3E37C62C72E", + .Ki = "9FDA0E56AB2D85E1569A688696C26A6C", + .ct = "EF85FB890BB8472F4DAB20394DCA781DAD877EDA39D50C870C0D5A0A8E48C718", + }, { + .etype = KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128, + .name = "enc plain Date: Tue, 25 Feb 2025 13:03:26 +0800 Subject: crypto: api - Move struct crypto_type into internal.h Move the definition of struct crypto_type into internal.h as it is only used by API implementors and not algorithm implementors. Signed-off-by: Herbert Xu --- crypto/internal.h | 14 ++++++++++++++ include/crypto/algapi.h | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) (limited to 'crypto') diff --git a/crypto/internal.h b/crypto/internal.h index 46b661be0f90..08d43b40e7db 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -33,6 +33,20 @@ struct crypto_larval { bool test_started; }; +struct crypto_type { + unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); + unsigned int (*extsize)(struct crypto_alg *alg); + int (*init_tfm)(struct crypto_tfm *tfm); + void (*show)(struct seq_file *m, struct crypto_alg *alg); + int (*report)(struct sk_buff *skb, struct crypto_alg *alg); + void (*free)(struct crypto_instance *inst); + + unsigned int type; + unsigned int maskclear; + unsigned int maskset; + unsigned int tfmsize; +}; + enum { CRYPTOA_UNSPEC, CRYPTOA_ALG, diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 11065978d360..94989b2e1350 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -55,20 +55,6 @@ struct scatterlist; struct seq_file; struct sk_buff; -struct crypto_type { - unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); - unsigned int (*extsize)(struct crypto_alg *alg); - int (*init_tfm)(struct crypto_tfm *tfm); - void (*show)(struct seq_file *m, struct crypto_alg *alg); - int (*report)(struct sk_buff *skb, struct crypto_alg *alg); - void (*free)(struct crypto_instance *inst); - - unsigned int type; - unsigned int maskclear; - unsigned int maskset; - unsigned int tfmsize; -}; - struct crypto_instance { struct crypto_alg alg; -- cgit v1.2.3 From cc47f07234f72cbd8e2c973cdbf2a6730660a463 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 27 Feb 2025 17:04:46 +0800 Subject: crypto: lzo - Fix compression buffer overrun Unlike the decompression code, the compression code in LZO never checked for output overruns. It instead assumes that the caller always provides enough buffer space, disregarding the buffer length provided by the caller. Add a safe compression interface that checks for the end of buffer before each write. Use the safe interface in crypto/lzo. Signed-off-by: Herbert Xu Reviewed-by: David Sterba Signed-off-by: Herbert Xu --- crypto/lzo-rle.c | 2 +- crypto/lzo.c | 2 +- include/linux/lzo.h | 8 ++++ lib/lzo/Makefile | 2 +- lib/lzo/lzo1x_compress.c | 102 +++++++++++++++++++++++++++++++----------- lib/lzo/lzo1x_compress_safe.c | 18 ++++++++ 6 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 lib/lzo/lzo1x_compress_safe.c (limited to 'crypto') diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 0631d975bfac..0abc2d87f042 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -55,7 +55,7 @@ static int __lzorle_compress(const u8 *src, unsigned int slen, size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */ int err; - err = lzorle1x_1_compress(src, slen, dst, &tmp_len, ctx); + err = lzorle1x_1_compress_safe(src, slen, dst, &tmp_len, ctx); if (err != LZO_E_OK) return -EINVAL; diff --git a/crypto/lzo.c b/crypto/lzo.c index ebda132dd22b..8338851c7406 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -55,7 +55,7 @@ static int __lzo_compress(const u8 *src, unsigned int slen, size_t tmp_len = *dlen; /* size_t(ulong) <-> uint on 64 bit */ int err; - err = lzo1x_1_compress(src, slen, dst, &tmp_len, ctx); + err = lzo1x_1_compress_safe(src, slen, dst, &tmp_len, ctx); if (err != LZO_E_OK) return -EINVAL; diff --git a/include/linux/lzo.h b/include/linux/lzo.h index e95c7d1092b2..4d30e3624acd 100644 --- a/include/linux/lzo.h +++ b/include/linux/lzo.h @@ -24,10 +24,18 @@ int lzo1x_1_compress(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len, void *wrkmem); +/* Same as above but does not write more than dst_len to dst. */ +int lzo1x_1_compress_safe(const unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len, void *wrkmem); + /* This requires 'wrkmem' of size LZO1X_1_MEM_COMPRESS */ int lzorle1x_1_compress(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len, void *wrkmem); +/* Same as above but does not write more than dst_len to dst. */ +int lzorle1x_1_compress_safe(const unsigned char *src, size_t src_len, + unsigned char *dst, size_t *dst_len, void *wrkmem); + /* safe decompression with overrun testing */ int lzo1x_decompress_safe(const unsigned char *src, size_t src_len, unsigned char *dst, size_t *dst_len); diff --git a/lib/lzo/Makefile b/lib/lzo/Makefile index 2f58fafbbddd..fc7b2b7ef4b2 100644 --- a/lib/lzo/Makefile +++ b/lib/lzo/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -lzo_compress-objs := lzo1x_compress.o +lzo_compress-objs := lzo1x_compress.o lzo1x_compress_safe.o lzo_decompress-objs := lzo1x_decompress_safe.o obj-$(CONFIG_LZO_COMPRESS) += lzo_compress.o diff --git a/lib/lzo/lzo1x_compress.c b/lib/lzo/lzo1x_compress.c index 47d6d43ea957..7b10ca86a893 100644 --- a/lib/lzo/lzo1x_compress.c +++ b/lib/lzo/lzo1x_compress.c @@ -18,11 +18,22 @@ #include #include "lzodefs.h" -static noinline size_t -lzo1x_1_do_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - size_t ti, void *wrkmem, signed char *state_offset, - const unsigned char bitstream_version) +#undef LZO_UNSAFE + +#ifndef LZO_SAFE +#define LZO_UNSAFE 1 +#define LZO_SAFE(name) name +#define HAVE_OP(x) 1 +#endif + +#define NEED_OP(x) if (!HAVE_OP(x)) goto output_overrun + +static noinline int +LZO_SAFE(lzo1x_1_do_compress)(const unsigned char *in, size_t in_len, + unsigned char **out, unsigned char *op_end, + size_t *tp, void *wrkmem, + signed char *state_offset, + const unsigned char bitstream_version) { const unsigned char *ip; unsigned char *op; @@ -30,8 +41,9 @@ lzo1x_1_do_compress(const unsigned char *in, size_t in_len, const unsigned char * const ip_end = in + in_len - 20; const unsigned char *ii; lzo_dict_t * const dict = (lzo_dict_t *) wrkmem; + size_t ti = *tp; - op = out; + op = *out; ip = in; ii = ip; ip += ti < 4 ? 4 - ti : 0; @@ -116,25 +128,32 @@ next: if (t != 0) { if (t <= 3) { op[*state_offset] |= t; + NEED_OP(4); COPY4(op, ii); op += t; } else if (t <= 16) { + NEED_OP(17); *op++ = (t - 3); COPY8(op, ii); COPY8(op + 8, ii + 8); op += t; } else { if (t <= 18) { + NEED_OP(1); *op++ = (t - 3); } else { size_t tt = t - 18; + NEED_OP(1); *op++ = 0; while (unlikely(tt > 255)) { tt -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = tt; } + NEED_OP(t); do { COPY8(op, ii); COPY8(op + 8, ii + 8); @@ -151,6 +170,7 @@ next: if (unlikely(run_length)) { ip += run_length; run_length -= MIN_ZERO_RUN_LENGTH; + NEED_OP(4); put_unaligned_le32((run_length << 21) | 0xfffc18 | (run_length & 0x7), op); op += 4; @@ -243,10 +263,12 @@ m_len_done: ip += m_len; if (m_len <= M2_MAX_LEN && m_off <= M2_MAX_OFFSET) { m_off -= 1; + NEED_OP(2); *op++ = (((m_len - 1) << 5) | ((m_off & 7) << 2)); *op++ = (m_off >> 3); } else if (m_off <= M3_MAX_OFFSET) { m_off -= 1; + NEED_OP(1); if (m_len <= M3_MAX_LEN) *op++ = (M3_MARKER | (m_len - 2)); else { @@ -254,14 +276,18 @@ m_len_done: *op++ = M3_MARKER | 0; while (unlikely(m_len > 255)) { m_len -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = (m_len); } + NEED_OP(2); *op++ = (m_off << 2); *op++ = (m_off >> 6); } else { m_off -= 0x4000; + NEED_OP(1); if (m_len <= M4_MAX_LEN) *op++ = (M4_MARKER | ((m_off >> 11) & 8) | (m_len - 2)); @@ -282,11 +308,14 @@ m_len_done: m_len -= M4_MAX_LEN; *op++ = (M4_MARKER | ((m_off >> 11) & 8)); while (unlikely(m_len > 255)) { + NEED_OP(1); m_len -= 255; *op++ = 0; } + NEED_OP(1); *op++ = (m_len); } + NEED_OP(2); *op++ = (m_off << 2); *op++ = (m_off >> 6); } @@ -295,14 +324,20 @@ finished_writing_instruction: ii = ip; goto next; } - *out_len = op - out; - return in_end - (ii - ti); + *out = op; + *tp = in_end - (ii - ti); + return LZO_E_OK; + +output_overrun: + return LZO_E_OUTPUT_OVERRUN; } -static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem, const unsigned char bitstream_version) +static int LZO_SAFE(lzogeneric1x_1_compress)( + const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem, const unsigned char bitstream_version) { + unsigned char * const op_end = out + *out_len; const unsigned char *ip = in; unsigned char *op = out; unsigned char *data_start; @@ -326,14 +361,18 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, while (l > 20) { size_t ll = min_t(size_t, l, m4_max_offset + 1); uintptr_t ll_end = (uintptr_t) ip + ll; + int err; + if ((ll_end + ((t + ll) >> 5)) <= ll_end) break; BUILD_BUG_ON(D_SIZE * sizeof(lzo_dict_t) > LZO1X_1_MEM_COMPRESS); memset(wrkmem, 0, D_SIZE * sizeof(lzo_dict_t)); - t = lzo1x_1_do_compress(ip, ll, op, out_len, t, wrkmem, - &state_offset, bitstream_version); + err = LZO_SAFE(lzo1x_1_do_compress)( + ip, ll, &op, op_end, &t, wrkmem, + &state_offset, bitstream_version); + if (err != LZO_E_OK) + return err; ip += ll; - op += *out_len; l -= ll; } t += l; @@ -342,20 +381,26 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, const unsigned char *ii = in + in_len - t; if (op == data_start && t <= 238) { + NEED_OP(1); *op++ = (17 + t); } else if (t <= 3) { op[state_offset] |= t; } else if (t <= 18) { + NEED_OP(1); *op++ = (t - 3); } else { size_t tt = t - 18; + NEED_OP(1); *op++ = 0; while (tt > 255) { tt -= 255; + NEED_OP(1); *op++ = 0; } + NEED_OP(1); *op++ = tt; } + NEED_OP(t); if (t >= 16) do { COPY8(op, ii); COPY8(op + 8, ii + 8); @@ -368,31 +413,38 @@ static int lzogeneric1x_1_compress(const unsigned char *in, size_t in_len, } while (--t > 0); } + NEED_OP(3); *op++ = M4_MARKER | 1; *op++ = 0; *op++ = 0; *out_len = op - out; return LZO_E_OK; + +output_overrun: + return LZO_E_OUTPUT_OVERRUN; } -int lzo1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem) +int LZO_SAFE(lzo1x_1_compress)(const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem) { - return lzogeneric1x_1_compress(in, in_len, out, out_len, wrkmem, 0); + return LZO_SAFE(lzogeneric1x_1_compress)( + in, in_len, out, out_len, wrkmem, 0); } -int lzorle1x_1_compress(const unsigned char *in, size_t in_len, - unsigned char *out, size_t *out_len, - void *wrkmem) +int LZO_SAFE(lzorle1x_1_compress)(const unsigned char *in, size_t in_len, + unsigned char *out, size_t *out_len, + void *wrkmem) { - return lzogeneric1x_1_compress(in, in_len, out, out_len, - wrkmem, LZO_VERSION); + return LZO_SAFE(lzogeneric1x_1_compress)( + in, in_len, out, out_len, wrkmem, LZO_VERSION); } -EXPORT_SYMBOL_GPL(lzo1x_1_compress); -EXPORT_SYMBOL_GPL(lzorle1x_1_compress); +EXPORT_SYMBOL_GPL(LZO_SAFE(lzo1x_1_compress)); +EXPORT_SYMBOL_GPL(LZO_SAFE(lzorle1x_1_compress)); +#ifndef LZO_UNSAFE MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("LZO1X-1 Compressor"); +#endif diff --git a/lib/lzo/lzo1x_compress_safe.c b/lib/lzo/lzo1x_compress_safe.c new file mode 100644 index 000000000000..371c9f849492 --- /dev/null +++ b/lib/lzo/lzo1x_compress_safe.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * LZO1X Compressor from LZO + * + * Copyright (C) 1996-2012 Markus F.X.J. Oberhumer + * + * The full LZO package can be found at: + * http://www.oberhumer.com/opensource/lzo/ + * + * Changed for Linux kernel use by: + * Nitin Gupta + * Richard Purdie + */ + +#define LZO_SAFE(name) name##_safe +#define HAVE_OP(x) ((size_t)(op_end - op) >= (size_t)(x)) + +#include "lzo1x_compress.c" -- cgit v1.2.3 From 8f3332eecdd420c4cfc8861c7b63508cac07e227 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 27 Feb 2025 18:14:57 +0800 Subject: crypto: acomp - Remove acomp request flags The acomp request flags field duplicates the base request flags and is confusing. Remove it. Signed-off-by: Herbert Xu --- crypto/acompress.c | 2 +- include/crypto/acompress.h | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 6fdf0ff9f3c0..30176316140a 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -144,7 +144,7 @@ void acomp_request_free(struct acomp_req *req) if (tfm->__crt_alg->cra_type != &crypto_acomp_type) crypto_acomp_scomp_free_ctx(req); - if (req->flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { + if (req->base.flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { acomp->dst_free(req->dst); req->dst = NULL; } diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 54937b615239..b6d5136e689d 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -24,7 +24,6 @@ * @dst: Destination data * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced - * @flags: Internal flags * @__ctx: Start of private context data */ struct acomp_req { @@ -33,7 +32,6 @@ struct acomp_req { struct scatterlist *dst; unsigned int slen; unsigned int dlen; - u32 flags; void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -232,9 +230,9 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->slen = slen; req->dlen = dlen; - req->flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; if (!req->dst) - req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; } /** -- cgit v1.2.3 From 98330b9a61506de7df0d1725122111909c157864 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 3 Mar 2025 11:09:06 +0800 Subject: crypto: Kconfig - Select LIB generic option Select the generic LIB options if the Crypto API algorithm is enabled. Otherwise this may lead to a build failure as the Crypto API algorithm always uses the generic implementation. Fixes: 17ec3e71ba79 ("crypto: lib/Kconfig - Hide arch options from user") Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202503022113.79uEtUuy-lkp@intel.com/ Closes: https://lore.kernel.org/oe-kbuild-all/202503022115.9OOyDR5A-lkp@intel.com/ Signed-off-by: Herbert Xu --- crypto/Kconfig | 3 +++ 1 file changed, 3 insertions(+) (limited to 'crypto') diff --git a/crypto/Kconfig b/crypto/Kconfig index f0119d92decc..db99fec7d161 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -329,6 +329,7 @@ config CRYPTO_ECRDSA config CRYPTO_CURVE25519 tristate "Curve25519" select CRYPTO_KPP + select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 elliptic curve (RFC7748) @@ -627,6 +628,7 @@ config CRYPTO_ARC4 config CRYPTO_CHACHA20 tristate "ChaCha" + select CRYPTO_LIB_CHACHA_GENERIC select CRYPTO_LIB_CHACHA_INTERNAL select CRYPTO_SKCIPHER help @@ -948,6 +950,7 @@ config CRYPTO_POLYVAL config CRYPTO_POLY1305 tristate "Poly1305" select CRYPTO_HASH + select CRYPTO_LIB_POLY1305_GENERIC select CRYPTO_LIB_POLY1305_INTERNAL help Poly1305 authenticator algorithm (RFC7539) -- cgit v1.2.3 From eca6828403b80343647de39d4782ee56cc9e36dd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Wed, 5 Mar 2025 19:33:05 -0800 Subject: crypto: skcipher - fix mismatch between mapping and unmapping order Local kunmaps have to be unmapped in the opposite order from which they were mapped. My recent change flipped the unmap order in the SKCIPHER_WALK_DIFF case. Adjust the mapping side to match. This fixes a WARN_ON_ONCE that was triggered when running the crypto-self tests on a 32-bit kernel with CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP=y. Fixes: 95dbd711b1d8 ("crypto: skcipher - use the new scatterwalk functions") Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/skcipher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 53123d3685d5..66d19c360dd8 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -209,12 +209,12 @@ static int skcipher_next_fast(struct skcipher_walk *walk) diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); - skcipher_map_src(walk); - walk->dst.virt.addr = walk->src.virt.addr; + skcipher_map_dst(walk); + walk->src.virt.addr = walk->dst.virt.addr; if (diff) { walk->flags |= SKCIPHER_WALK_DIFF; - skcipher_map_dst(walk); + skcipher_map_src(walk); } return 0; -- cgit v1.2.3 From 20238d49448cdb406da2b9bd3e50f892b26da318 Mon Sep 17 00:00:00 2001 From: "Dr. David Alan Gilbert" Date: Sun, 29 Sep 2024 14:21:48 +0100 Subject: async_xor: Remove unused 'async_xor_val' async_xor_val has been unused since commit a7c224a820c3 ("md/raid5: convert to new xor compution interface") Remove it. Signed-off-by: Dr. David Alan Gilbert Signed-off-by: Herbert Xu --- crypto/async_tx/async_xor.c | 26 -------------------------- include/linux/async_tx.h | 5 ----- 2 files changed, 31 deletions(-) (limited to 'crypto') diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 1a3855284091..2c499654a36c 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -389,32 +389,6 @@ async_xor_val_offs(struct page *dest, unsigned int offset, } EXPORT_SYMBOL_GPL(async_xor_val_offs); -/** - * async_xor_val - attempt a xor parity check with a dma engine. - * @dest: destination page used if the xor is performed synchronously - * @src_list: array of source pages - * @offset: offset in pages to start transaction - * @src_cnt: number of source pages - * @len: length in bytes - * @result: 0 if sum == 0 else non-zero - * @submit: submission / completion modifiers - * - * honored flags: ASYNC_TX_ACK - * - * src_list note: if the dest is also a source it must be at index zero. - * The contents of this array will be overwritten if a scribble region - * is not specified. - */ -struct dma_async_tx_descriptor * -async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum sum_check_flags *result, - struct async_submit_ctl *submit) -{ - return async_xor_val_offs(dest, offset, src_list, NULL, src_cnt, - len, result, submit); -} -EXPORT_SYMBOL_GPL(async_xor_val); - MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); MODULE_LICENSE("GPL"); diff --git a/include/linux/async_tx.h b/include/linux/async_tx.h index 5cc73d7e5b52..1ca9f9e05f4f 100644 --- a/include/linux/async_tx.h +++ b/include/linux/async_tx.h @@ -167,11 +167,6 @@ async_xor_offs(struct page *dest, unsigned int offset, struct page **src_list, unsigned int *src_offset, int src_cnt, size_t len, struct async_submit_ctl *submit); -struct dma_async_tx_descriptor * -async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum sum_check_flags *result, - struct async_submit_ctl *submit); - struct dma_async_tx_descriptor * async_xor_val_offs(struct page *dest, unsigned int offset, struct page **src_list, unsigned int *src_offset, -- cgit v1.2.3 From 65775cf313987926e9746b0ca7f5519d297af2da Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:21 +0800 Subject: crypto: scatterwalk - Change scatterwalk_next calling convention Rather than returning the address and storing the length into an argument pointer, add an address field to the walk struct and use that to store the address. The length is returned directly. Change the done functions to use this stored address instead of getting them from the caller. Split the address into two using a union. The user should only access the const version so that it is never changed. Signed-off-by: Herbert Xu --- arch/arm/crypto/ghash-ce-glue.c | 7 +++---- arch/arm64/crypto/aes-ce-ccm-glue.c | 9 ++++----- arch/arm64/crypto/ghash-ce-glue.c | 7 +++---- arch/arm64/crypto/sm4-ce-ccm-glue.c | 8 ++++---- arch/arm64/crypto/sm4-ce-gcm-glue.c | 8 ++++---- arch/s390/crypto/aes_s390.c | 21 +++++++++------------ arch/x86/crypto/aegis128-aesni-glue.c | 7 +++---- arch/x86/crypto/aesni-intel_glue.c | 9 ++++----- crypto/aegis128-core.c | 7 +++---- crypto/scatterwalk.c | 14 ++++++-------- crypto/skcipher.c | 10 +++++++--- drivers/crypto/nx/nx.c | 7 +++---- include/crypto/algapi.h | 7 +++++++ include/crypto/scatterwalk.h | 35 ++++++++++++++++++----------------- 14 files changed, 78 insertions(+), 78 deletions(-) (limited to 'crypto') diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 9613ffed84f9..dab66b520b6e 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -460,11 +460,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - gcm_update_mac(dg, p, n, buf, &buf_count, ctx); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); + scatterwalk_done_src(&walk, n); if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { kernel_neon_end(); diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 1c29546983bf..2d791d51891b 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -157,12 +157,11 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc, - num_rounds(ctx)); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + macp = ce_aes_ccm_auth_data(mac, walk.addr, n, macp, + ctx->key_enc, num_rounds(ctx)); + scatterwalk_done_src(&walk, n); len -= n; } while (len); } diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 69d4fb78c30d..071e122f9c37 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -309,11 +309,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - gcm_update_mac(dg, p, n, buf, &buf_count, ctx); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); + scatterwalk_done_src(&walk, n); len -= n; } while (len); diff --git a/arch/arm64/crypto/sm4-ce-ccm-glue.c b/arch/arm64/crypto/sm4-ce-ccm-glue.c index 119f86eb7cc9..e9cc1c1364ec 100644 --- a/arch/arm64/crypto/sm4-ce-ccm-glue.c +++ b/arch/arm64/crypto/sm4-ce-ccm-glue.c @@ -113,10 +113,10 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) do { unsigned int n, orig_n; - const u8 *p, *orig_p; + const u8 *p; - orig_p = scatterwalk_next(&walk, assoclen, &orig_n); - p = orig_p; + orig_n = scatterwalk_next(&walk, assoclen); + p = walk.addr; n = orig_n; while (n > 0) { @@ -149,7 +149,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) } } - scatterwalk_done_src(&walk, orig_p, orig_n); + scatterwalk_done_src(&walk, orig_n); assoclen -= orig_n; } while (assoclen); } diff --git a/arch/arm64/crypto/sm4-ce-gcm-glue.c b/arch/arm64/crypto/sm4-ce-gcm-glue.c index 2e27d7752d4f..c2ea3d5f690b 100644 --- a/arch/arm64/crypto/sm4-ce-gcm-glue.c +++ b/arch/arm64/crypto/sm4-ce-gcm-glue.c @@ -83,10 +83,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[]) do { unsigned int n, orig_n; - const u8 *p, *orig_p; + const u8 *p; - orig_p = scatterwalk_next(&walk, assoclen, &orig_n); - p = orig_p; + orig_n = scatterwalk_next(&walk, assoclen); + p = walk.addr; n = orig_n; if (n + buflen < GHASH_BLOCK_SIZE) { @@ -118,7 +118,7 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[]) memcpy(&buffer[0], p, buflen); } - scatterwalk_done_src(&walk, orig_p, orig_n); + scatterwalk_done_src(&walk, orig_n); assoclen -= orig_n; } while (assoclen); diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 7fd303df05ab..ed85bd6e298f 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -66,7 +66,6 @@ struct s390_xts_ctx { struct gcm_sg_walk { struct scatter_walk walk; unsigned int walk_bytes; - u8 *walk_ptr; unsigned int walk_bytes_remain; u8 buf[AES_BLOCK_SIZE]; unsigned int buf_bytes; @@ -789,8 +788,7 @@ static inline unsigned int _gcm_sg_clamp_and_map(struct gcm_sg_walk *gw) { if (gw->walk_bytes_remain == 0) return 0; - gw->walk_ptr = scatterwalk_next(&gw->walk, gw->walk_bytes_remain, - &gw->walk_bytes); + gw->walk_bytes = scatterwalk_next(&gw->walk, gw->walk_bytes_remain); return gw->walk_bytes; } @@ -799,10 +797,9 @@ static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw, { gw->walk_bytes_remain -= nbytes; if (out) - scatterwalk_done_dst(&gw->walk, gw->walk_ptr, nbytes); + scatterwalk_done_dst(&gw->walk, nbytes); else - scatterwalk_done_src(&gw->walk, gw->walk_ptr, nbytes); - gw->walk_ptr = NULL; + scatterwalk_done_src(&gw->walk, nbytes); } static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) @@ -828,14 +825,14 @@ static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) } if (!gw->buf_bytes && gw->walk_bytes >= minbytesneeded) { - gw->ptr = gw->walk_ptr; + gw->ptr = gw->walk.addr; gw->nbytes = gw->walk_bytes; goto out; } while (1) { n = min(gw->walk_bytes, AES_BLOCK_SIZE - gw->buf_bytes); - memcpy(gw->buf + gw->buf_bytes, gw->walk_ptr, n); + memcpy(gw->buf + gw->buf_bytes, gw->walk.addr, n); gw->buf_bytes += n; _gcm_sg_unmap_and_advance(gw, n, false); if (gw->buf_bytes >= minbytesneeded) { @@ -869,13 +866,13 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) } if (gw->walk_bytes >= minbytesneeded) { - gw->ptr = gw->walk_ptr; + gw->ptr = gw->walk.addr; gw->nbytes = gw->walk_bytes; goto out; } - scatterwalk_unmap(gw->walk_ptr); - gw->walk_ptr = NULL; + /* XXX */ + scatterwalk_unmap(gw->walk.addr); gw->ptr = gw->buf; gw->nbytes = sizeof(gw->buf); @@ -914,7 +911,7 @@ static int gcm_out_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone) if (!_gcm_sg_clamp_and_map(gw)) return i; n = min(gw->walk_bytes, bytesdone - i); - memcpy(gw->walk_ptr, gw->buf + i, n); + memcpy(gw->walk.addr, gw->buf + i, n); _gcm_sg_unmap_and_advance(gw, n, true); } } else diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 1bd093d073ed..26786e15abac 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -71,10 +71,9 @@ static void crypto_aegis128_aesni_process_ad( scatterwalk_start(&walk, sg_src); while (assoclen != 0) { - unsigned int size; - const u8 *mapped = scatterwalk_next(&walk, assoclen, &size); + unsigned int size = scatterwalk_next(&walk, assoclen); + const u8 *src = walk.addr; unsigned int left = size; - const u8 *src = mapped; if (pos + size >= AEGIS128_BLOCK_SIZE) { if (pos > 0) { @@ -97,7 +96,7 @@ static void crypto_aegis128_aesni_process_ad( pos += left; assoclen -= size; - scatterwalk_done_src(&walk, mapped, size); + scatterwalk_done_src(&walk, size); } if (pos > 0) { diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index c4bd05688b55..e141b7995304 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1306,12 +1306,11 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], scatterwalk_start(&walk, sg_src); while (assoclen) { - unsigned int orig_len_this_step; - const u8 *orig_src = scatterwalk_next(&walk, assoclen, - &orig_len_this_step); + unsigned int orig_len_this_step = scatterwalk_next( + &walk, assoclen); unsigned int len_this_step = orig_len_this_step; unsigned int len; - const u8 *src = orig_src; + const u8 *src = walk.addr; if (unlikely(pos)) { len = min(len_this_step, 16 - pos); @@ -1335,7 +1334,7 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], pos = len_this_step; } next: - scatterwalk_done_src(&walk, orig_src, orig_len_this_step); + scatterwalk_done_src(&walk, orig_len_this_step); if (need_resched()) { kernel_fpu_end(); kernel_fpu_begin(); diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 15d64d836356..72f6ee1345ef 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -284,10 +284,9 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, scatterwalk_start(&walk, sg_src); while (assoclen != 0) { - unsigned int size; - const u8 *mapped = scatterwalk_next(&walk, assoclen, &size); + unsigned int size = scatterwalk_next(&walk, assoclen); + const u8 *src = walk.addr; unsigned int left = size; - const u8 *src = mapped; if (pos + size >= AEGIS_BLOCK_SIZE) { if (pos > 0) { @@ -308,7 +307,7 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, pos += left; assoclen -= size; - scatterwalk_done_src(&walk, mapped, size); + scatterwalk_done_src(&walk, size); } if (pos > 0) { diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 87c080f565d4..20a28c6d94da 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -34,12 +34,11 @@ inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, unsigned int nbytes) { do { - const void *src_addr; unsigned int to_copy; - src_addr = scatterwalk_next(walk, nbytes, &to_copy); - memcpy(buf, src_addr, to_copy); - scatterwalk_done_src(walk, src_addr, to_copy); + to_copy = scatterwalk_next(walk, nbytes); + memcpy(buf, walk->addr, to_copy); + scatterwalk_done_src(walk, to_copy); buf += to_copy; nbytes -= to_copy; } while (nbytes); @@ -50,12 +49,11 @@ inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, unsigned int nbytes) { do { - void *dst_addr; unsigned int to_copy; - dst_addr = scatterwalk_next(walk, nbytes, &to_copy); - memcpy(dst_addr, buf, to_copy); - scatterwalk_done_dst(walk, dst_addr, to_copy); + to_copy = scatterwalk_next(walk, nbytes); + memcpy(walk->addr, buf, to_copy); + scatterwalk_done_dst(walk, to_copy); buf += to_copy; nbytes -= to_copy; } while (nbytes); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 66d19c360dd8..0c6911154241 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -41,12 +41,16 @@ static int skcipher_walk_next(struct skcipher_walk *walk); static inline void skcipher_map_src(struct skcipher_walk *walk) { - walk->src.virt.addr = scatterwalk_map(&walk->in); + /* XXX */ + walk->in.__addr = scatterwalk_map(&walk->in); + walk->src.virt.addr = walk->in.addr; } static inline void skcipher_map_dst(struct skcipher_walk *walk) { - walk->dst.virt.addr = scatterwalk_map(&walk->out); + /* XXX */ + walk->out.__addr = scatterwalk_map(&walk->out); + walk->dst.virt.addr = walk->out.addr; } static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) @@ -120,7 +124,7 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) goto dst_done; } - scatterwalk_done_dst(&walk->out, walk->dst.virt.addr, n); + scatterwalk_done_dst(&walk->out, n); dst_done: if (res > 0) diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index dd95e5361d88..a3b979193d9b 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -154,17 +154,16 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, struct scatter_walk walk; struct nx_sg *nx_sg = nx_dst; unsigned int n, len = *src_len; - char *dst; /* we need to fast forward through @start bytes first */ scatterwalk_start_at_pos(&walk, sg_src, start); while (len && (nx_sg - nx_dst) < sglen) { - dst = scatterwalk_next(&walk, len, &n); + n = scatterwalk_next(&walk, len); - nx_sg = nx_build_sg_list(nx_sg, dst, &n, sglen - (nx_sg - nx_dst)); + nx_sg = nx_build_sg_list(nx_sg, walk.addr, &n, sglen - (nx_sg - nx_dst)); - scatterwalk_done_src(&walk, dst, n); + scatterwalk_done_src(&walk, n); len -= n; } /* update to_process */ diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 94989b2e1350..f92e22686a68 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -54,6 +54,7 @@ struct rtattr; struct scatterlist; struct seq_file; struct sk_buff; +union crypto_no_such_thing; struct crypto_instance { struct crypto_alg alg; @@ -108,6 +109,12 @@ struct crypto_queue { struct scatter_walk { struct scatterlist *sg; unsigned int offset; + union { + void *const addr; + + /* Private API field, do not touch. */ + union crypto_no_such_thing *__addr; + }; }; struct crypto_attr_alg { diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 3024adbdd443..8523c7591d95 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -120,18 +120,20 @@ static inline void *scatterwalk_map(struct scatter_walk *walk) * scatterwalk_next() - Get the next data buffer in a scatterlist walk * @walk: the scatter_walk * @total: the total number of bytes remaining, > 0 - * @nbytes_ret: (out) the next number of bytes available, <= @total * - * Return: A virtual address for the next segment of data from the scatterlist. - * The caller must call scatterwalk_done_src() or scatterwalk_done_dst() - * when it is done using this virtual address. + * A virtual address for the next segment of data from the scatterlist will + * be placed into @walk->addr. The caller must call scatterwalk_done_src() + * or scatterwalk_done_dst() when it is done using this virtual address. + * + * Returns: the next number of bytes available, <= @total */ -static inline void *scatterwalk_next(struct scatter_walk *walk, - unsigned int total, - unsigned int *nbytes_ret) +static inline unsigned int scatterwalk_next(struct scatter_walk *walk, + unsigned int total) { - *nbytes_ret = scatterwalk_clamp(walk, total); - return scatterwalk_map(walk); + unsigned int nbytes = scatterwalk_clamp(walk, total); + + walk->__addr = scatterwalk_map(walk); + return nbytes; } static inline void scatterwalk_unmap(const void *vaddr) @@ -149,32 +151,31 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, /** * scatterwalk_done_src() - Finish one step of a walk of source scatterlist * @walk: the scatter_walk - * @vaddr: the address returned by scatterwalk_next() * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * - * Use this if the @vaddr was not written to, i.e. it is source data. + * Use this if the mapped address was not written to, i.e. it is source data. */ static inline void scatterwalk_done_src(struct scatter_walk *walk, - const void *vaddr, unsigned int nbytes) + unsigned int nbytes) { - scatterwalk_unmap(vaddr); + scatterwalk_unmap(walk->addr); scatterwalk_advance(walk, nbytes); } /** * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist * @walk: the scatter_walk - * @vaddr: the address returned by scatterwalk_next() * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * - * Use this if the @vaddr may have been written to, i.e. it is destination data. + * Use this if the mapped address may have been written to, i.e. it is + * destination data. */ static inline void scatterwalk_done_dst(struct scatter_walk *walk, - void *vaddr, unsigned int nbytes) + unsigned int nbytes) { - scatterwalk_unmap(vaddr); + scatterwalk_unmap(walk->addr); /* * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just * relying on flush_dcache_page() being a no-op when not implemented, -- cgit v1.2.3 From 131bdceca1f0a2d9381270dc40f898458e5e184b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:23 +0800 Subject: crypto: scatterwalk - Add memcpy_sglist Add memcpy_sglist which copies one SG list to another. Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 27 +++++++++++++++++++++++++++ include/crypto/scatterwalk.h | 3 +++ 2 files changed, 30 insertions(+) (limited to 'crypto') diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 20a28c6d94da..8225801488d5 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -86,6 +86,33 @@ void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, } EXPORT_SYMBOL_GPL(memcpy_to_sglist); +void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct scatter_walk swalk; + struct scatter_walk dwalk; + + if (unlikely(nbytes == 0)) /* in case sg == NULL */ + return; + + scatterwalk_start(&swalk, src); + scatterwalk_start(&dwalk, dst); + + do { + unsigned int slen, dlen; + unsigned int len; + + slen = scatterwalk_next(&swalk, nbytes); + dlen = scatterwalk_next(&dwalk, nbytes); + len = min(slen, dlen); + memcpy(dwalk.addr, swalk.addr, len); + scatterwalk_done_dst(&dwalk, len); + scatterwalk_done_src(&swalk, len); + nbytes -= len; + } while (nbytes); +} +EXPORT_SYMBOL_GPL(memcpy_sglist); + struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, unsigned int len) diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 8523c7591d95..c62f47d04eb1 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -210,6 +210,9 @@ void memcpy_from_sglist(void *buf, struct scatterlist *sg, void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, const void *buf, unsigned int nbytes); +void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes); + /* In new code, please use memcpy_{from,to}_sglist() directly instead. */ static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, unsigned int start, -- cgit v1.2.3 From db873be6f0549597f92c72986b1939643a7f9a75 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:25 +0800 Subject: crypto: skcipher - Eliminate duplicate virt.addr field Reuse the addr field from struct scatter_walk for skcipher_walk. Keep the existing virt.addr fields but make them const for the user to access the mapped address. Signed-off-by: Herbert Xu --- crypto/skcipher.c | 29 ++++++++++++----------------- include/crypto/algapi.h | 5 +++-- include/crypto/internal/skcipher.h | 26 +++++++++++++++++++++----- 3 files changed, 36 insertions(+), 24 deletions(-) (limited to 'crypto') diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 0c6911154241..ab5d852febcd 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -43,14 +43,12 @@ static inline void skcipher_map_src(struct skcipher_walk *walk) { /* XXX */ walk->in.__addr = scatterwalk_map(&walk->in); - walk->src.virt.addr = walk->in.addr; } static inline void skcipher_map_dst(struct skcipher_walk *walk) { /* XXX */ walk->out.__addr = scatterwalk_map(&walk->out); - walk->dst.virt.addr = walk->out.addr; } static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) @@ -100,8 +98,7 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) SKCIPHER_WALK_DIFF)))) { scatterwalk_advance(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_DIFF) { - scatterwalk_unmap(walk->src.virt.addr); - scatterwalk_advance(&walk->in, n); + scatterwalk_done_src(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_COPY) { scatterwalk_advance(&walk->in, n); skcipher_map_dst(walk); @@ -116,11 +113,8 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) */ res = -EINVAL; total = 0; - } else { - u8 *buf = PTR_ALIGN(walk->buffer, walk->alignmask + 1); - - memcpy_to_scatterwalk(&walk->out, buf, n); - } + } else + memcpy_to_scatterwalk(&walk->out, walk->out.addr, n); goto dst_done; } @@ -162,7 +156,7 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) { unsigned alignmask = walk->alignmask; unsigned n; - u8 *buffer; + void *buffer; if (!walk->buffer) walk->buffer = walk->page; @@ -176,10 +170,11 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) return skcipher_walk_done(walk, -ENOMEM); walk->buffer = buffer; } - walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1); - walk->src.virt.addr = walk->dst.virt.addr; - memcpy_from_scatterwalk(walk->src.virt.addr, &walk->in, bsize); + buffer = PTR_ALIGN(buffer, alignmask + 1); + memcpy_from_scatterwalk(buffer, &walk->in, bsize); + walk->out.__addr = buffer; + walk->in.__addr = walk->out.addr; walk->nbytes = bsize; walk->flags |= SKCIPHER_WALK_SLOW; @@ -189,7 +184,7 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) static int skcipher_next_copy(struct skcipher_walk *walk) { - u8 *tmp = walk->page; + void *tmp = walk->page; skcipher_map_src(walk); memcpy(tmp, walk->src.virt.addr, walk->nbytes); @@ -199,8 +194,8 @@ static int skcipher_next_copy(struct skcipher_walk *walk) * processed (which might be less than walk->nbytes) is known. */ - walk->src.virt.addr = tmp; - walk->dst.virt.addr = tmp; + walk->in.__addr = tmp; + walk->out.__addr = tmp; return 0; } @@ -214,7 +209,7 @@ static int skcipher_next_fast(struct skcipher_walk *walk) (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); skcipher_map_dst(walk); - walk->src.virt.addr = walk->dst.virt.addr; + walk->in.__addr = walk->dst.virt.addr; if (diff) { walk->flags |= SKCIPHER_WALK_DIFF; diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index f92e22686a68..6e07bbc04089 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -107,14 +107,15 @@ struct crypto_queue { }; struct scatter_walk { - struct scatterlist *sg; - unsigned int offset; + /* Must be the first member, see struct skcipher_walk. */ union { void *const addr; /* Private API field, do not touch. */ union crypto_no_such_thing *__addr; }; + struct scatterlist *sg; + unsigned int offset; }; struct crypto_attr_alg { diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index d6ae7a86fed2..c705124432c5 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -56,15 +56,31 @@ struct crypto_lskcipher_spawn { struct skcipher_walk { union { + /* Virtual address of the source. */ struct { - void *addr; - } virt; - } src, dst; + struct { + void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; - struct scatter_walk in; unsigned int nbytes; - struct scatter_walk out; + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + unsigned int total; u8 *page; -- cgit v1.2.3 From 37d451809f572ec197d3c18d9638c8715274255f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:53:13 +0800 Subject: crypto: skcipher - Make skcipher_walk src.virt.addr const Mark the src.virt.addr field in struct skcipher_walk as a pointer to const data. This guarantees that the user won't modify the data which should be done through dst.virt.addr to ensure that flushing is done when necessary. Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-glue.c | 2 +- arch/arm64/crypto/aes-neonbs-glue.c | 3 ++- arch/powerpc/crypto/aes-gcm-p10-glue.c | 6 +++--- arch/powerpc/crypto/aes_ctr.c | 2 +- arch/sparc/crypto/aes_glue.c | 2 +- arch/x86/crypto/des3_ede_glue.c | 2 +- crypto/ctr.c | 10 +++++----- crypto/lrw.c | 2 +- crypto/pcbc.c | 28 ++++++++++++++-------------- crypto/xctr.c | 2 +- crypto/xts.c | 2 +- include/crypto/ctr.h | 2 +- include/crypto/internal/skcipher.h | 2 +- 13 files changed, 33 insertions(+), 32 deletions(-) (limited to 'crypto') diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index 21df5e7f51f9..1cf61f51e766 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -399,9 +399,9 @@ static int ctr_encrypt(struct skcipher_request *req) } if (walk.nbytes) { u8 __aligned(8) tail[AES_BLOCK_SIZE]; + const u8 *tsrc = walk.src.virt.addr; unsigned int nbytes = walk.nbytes; u8 *tdst = walk.dst.virt.addr; - u8 *tsrc = walk.src.virt.addr; /* * Tell aes_ctr_encrypt() to process a tail block. diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index 46425e7b9755..c4a623e86593 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -287,7 +287,8 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, struct skcipher_walk walk; int nbytes, err; int first = 1; - u8 *out, *in; + const u8 *in; + u8 *out; if (req->cryptlen < AES_BLOCK_SIZE) return -EINVAL; diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c index 679f52794baf..85f4fd4b1bdc 100644 --- a/arch/powerpc/crypto/aes-gcm-p10-glue.c +++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c @@ -35,9 +35,9 @@ MODULE_ALIAS_CRYPTO("aes"); asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, void *key); asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); -asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_encrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); -asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_decrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, @@ -261,7 +261,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, return ret; while ((nbytes = walk.nbytes) > 0 && ret == 0) { - u8 *src = walk.src.virt.addr; + const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c index 9a3da8cd62f3..3da75f42529a 100644 --- a/arch/powerpc/crypto/aes_ctr.c +++ b/arch/powerpc/crypto/aes_ctr.c @@ -69,9 +69,9 @@ static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, struct skcipher_walk *walk) { + const u8 *src = walk->src.virt.addr; u8 *ctrblk = walk->iv; u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index e3d2138ff9e2..683150830356 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -321,7 +321,7 @@ static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx, { u8 *ctrblk = walk->iv; u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index e88439d3828e..34600f90d8a6 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -73,7 +73,7 @@ static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) err = skcipher_walk_virt(&walk, req, false); while ((nbytes = walk.nbytes)) { - u8 *wsrc = walk.src.virt.addr; + const u8 *wsrc = walk.src.virt.addr; u8 *wdst = walk.dst.virt.addr; /* Process four block batch */ diff --git a/crypto/ctr.c b/crypto/ctr.c index 73c0d6e53b2f..97a947b0a876 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -33,7 +33,7 @@ static void crypto_ctr_crypt_final(struct skcipher_walk *walk, u8 *ctrblk = walk->iv; u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; @@ -50,7 +50,7 @@ static int crypto_ctr_crypt_segment(struct skcipher_walk *walk, crypto_cipher_alg(tfm)->cia_encrypt; unsigned int bsize = crypto_cipher_blocksize(tfm); u8 *ctrblk = walk->iv; - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; @@ -77,20 +77,20 @@ static int crypto_ctr_crypt_inplace(struct skcipher_walk *walk, unsigned int bsize = crypto_cipher_blocksize(tfm); unsigned long alignmask = crypto_cipher_alignmask(tfm); unsigned int nbytes = walk->nbytes; + u8 *dst = walk->dst.virt.addr; u8 *ctrblk = walk->iv; - u8 *src = walk->src.virt.addr; u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); do { /* create keystream */ fn(crypto_cipher_tfm(tfm), keystream, ctrblk); - crypto_xor(src, keystream, bsize); + crypto_xor(dst, keystream, bsize); /* increment counter in counterblock */ crypto_inc(ctrblk, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; diff --git a/crypto/lrw.c b/crypto/lrw.c index e216fbf2b786..391ae0f7641f 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -167,7 +167,7 @@ static int lrw_xor_tweak(struct skcipher_request *req, bool second_pass) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wsrc; + const be128 *wsrc; be128 *wdst; wsrc = w.src.virt.addr; diff --git a/crypto/pcbc.c b/crypto/pcbc.c index cbfb3ac14b3a..9d2e56d6744a 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -22,8 +22,8 @@ static int crypto_pcbc_encrypt_segment(struct skcipher_request *req, struct crypto_cipher *tfm) { int bsize = crypto_cipher_blocksize(tfm); + const u8 *src = walk->src.virt.addr; unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; @@ -45,17 +45,17 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req, { int bsize = crypto_cipher_blocksize(tfm); unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; u8 tmpbuf[MAX_CIPHER_BLOCKSIZE]; do { - memcpy(tmpbuf, src, bsize); - crypto_xor(iv, src, bsize); - crypto_cipher_encrypt_one(tfm, src, iv); - crypto_xor_cpy(iv, tmpbuf, src, bsize); + memcpy(tmpbuf, dst, bsize); + crypto_xor(iv, dst, bsize); + crypto_cipher_encrypt_one(tfm, dst, iv); + crypto_xor_cpy(iv, tmpbuf, dst, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; @@ -89,8 +89,8 @@ static int crypto_pcbc_decrypt_segment(struct skcipher_request *req, struct crypto_cipher *tfm) { int bsize = crypto_cipher_blocksize(tfm); + const u8 *src = walk->src.virt.addr; unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; @@ -112,17 +112,17 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req, { int bsize = crypto_cipher_blocksize(tfm); unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; u8 tmpbuf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(u32)); do { - memcpy(tmpbuf, src, bsize); - crypto_cipher_decrypt_one(tfm, src, src); - crypto_xor(src, iv, bsize); - crypto_xor_cpy(iv, src, tmpbuf, bsize); + memcpy(tmpbuf, dst, bsize); + crypto_cipher_decrypt_one(tfm, dst, dst); + crypto_xor(dst, iv, bsize); + crypto_xor_cpy(iv, dst, tmpbuf, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; diff --git a/crypto/xctr.c b/crypto/xctr.c index 6ed9c85ededa..9c536ab6d2e5 100644 --- a/crypto/xctr.c +++ b/crypto/xctr.c @@ -78,7 +78,7 @@ static int crypto_xctr_crypt_inplace(struct skcipher_walk *walk, crypto_cipher_alg(tfm)->cia_encrypt; unsigned long alignmask = crypto_cipher_alignmask(tfm); unsigned int nbytes = walk->nbytes; - u8 *data = walk->src.virt.addr; + u8 *data = walk->dst.virt.addr; u8 tmp[XCTR_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); __le32 ctr32 = cpu_to_le32(byte_ctr / XCTR_BLOCKSIZE + 1); diff --git a/crypto/xts.c b/crypto/xts.c index 821060ede2cf..31529c9ef08f 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -99,7 +99,7 @@ static int xts_xor_tweak(struct skcipher_request *req, bool second_pass, while (w.nbytes) { unsigned int avail = w.nbytes; - le128 *wsrc; + const le128 *wsrc; le128 *wdst; wsrc = w.src.virt.addr; diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index a1c66d1001af..da1ee73e9ce9 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -34,8 +34,8 @@ static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes > 0) { + const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; - u8 *src = walk.src.virt.addr; int nbytes = walk.nbytes; int tail = 0; diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index c705124432c5..a958ab0636ad 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -59,7 +59,7 @@ struct skcipher_walk { /* Virtual address of the source. */ struct { struct { - void *const addr; + const void *const addr; } virt; } src; -- cgit v1.2.3 From 3d6979bf3bd51f47e889327e10e4653d55168c21 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:12 +0800 Subject: crypto: api - Add cra_type->destroy hook Add a cra_type->destroy hook so that resources can be freed after the last user of a registered algorithm is gone. Signed-off-by: Herbert Xu --- crypto/api.c | 10 ++++++++++ crypto/internal.h | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/api.c b/crypto/api.c index c2c4eb14ef95..91957bb52f3f 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -707,5 +707,15 @@ void crypto_req_done(void *data, int err) } EXPORT_SYMBOL_GPL(crypto_req_done); +void crypto_destroy_alg(struct crypto_alg *alg) +{ + if (alg->cra_type && alg->cra_type->destroy) + alg->cra_type->destroy(alg); + + if (alg->cra_destroy) + alg->cra_destroy(alg); +} +EXPORT_SYMBOL_GPL(crypto_destroy_alg); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/crypto/internal.h b/crypto/internal.h index 08d43b40e7db..11567ea24fc3 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -40,6 +40,7 @@ struct crypto_type { void (*show)(struct seq_file *m, struct crypto_alg *alg); int (*report)(struct sk_buff *skb, struct crypto_alg *alg); void (*free)(struct crypto_instance *inst); + void (*destroy)(struct crypto_alg *alg); unsigned int type; unsigned int maskclear; @@ -127,6 +128,7 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, const struct crypto_type *frontend, int node); void *crypto_clone_tfm(const struct crypto_type *frontend, struct crypto_tfm *otfm); +void crypto_destroy_alg(struct crypto_alg *alg); static inline void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend) @@ -163,8 +165,8 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) static inline void crypto_alg_put(struct crypto_alg *alg) { - if (refcount_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) - alg->cra_destroy(alg); + if (refcount_dec_and_test(&alg->cra_refcnt)) + crypto_destroy_alg(alg); } static inline int crypto_tmpl_get(struct crypto_template *tmpl) -- cgit v1.2.3 From 0af7304c0696ec5b3589af6973b7f27e014c2903 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:14 +0800 Subject: crypto: scomp - Remove tfm argument from alloc/free_ctx The tfm argument is completely unused and meaningless as the same stream object is identical over all transforms of a given algorithm. Remove it. Signed-off-by: Herbert Xu --- crypto/842.c | 8 ++++---- crypto/deflate.c | 4 ++-- crypto/lz4.c | 8 ++++---- crypto/lz4hc.c | 8 ++++---- crypto/lzo-rle.c | 8 ++++---- crypto/lzo.c | 8 ++++---- crypto/zstd.c | 4 ++-- drivers/crypto/cavium/zip/zip_crypto.c | 6 +++--- drivers/crypto/cavium/zip/zip_crypto.h | 6 +++--- include/crypto/internal/scompress.h | 8 ++++---- 10 files changed, 34 insertions(+), 34 deletions(-) (limited to 'crypto') diff --git a/crypto/842.c b/crypto/842.c index e59e54d76960..2238478c3493 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -28,7 +28,7 @@ struct crypto842_ctx { void *wmem; /* working memory for compress */ }; -static void *crypto842_alloc_ctx(struct crypto_scomp *tfm) +static void *crypto842_alloc_ctx(void) { void *ctx; @@ -43,14 +43,14 @@ static int crypto842_init(struct crypto_tfm *tfm) { struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->wmem = crypto842_alloc_ctx(NULL); + ctx->wmem = crypto842_alloc_ctx(); if (IS_ERR(ctx->wmem)) return -ENOMEM; return 0; } -static void crypto842_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void crypto842_free_ctx(void *ctx) { kfree(ctx); } @@ -59,7 +59,7 @@ static void crypto842_exit(struct crypto_tfm *tfm) { struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - crypto842_free_ctx(NULL, ctx->wmem); + crypto842_free_ctx(ctx->wmem); } static int crypto842_compress(struct crypto_tfm *tfm, diff --git a/crypto/deflate.c b/crypto/deflate.c index 98e8bcb81a6a..1bf7184ad670 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -112,7 +112,7 @@ out: return ret; } -static void *deflate_alloc_ctx(struct crypto_scomp *tfm) +static void *deflate_alloc_ctx(void) { struct deflate_ctx *ctx; int ret; @@ -143,7 +143,7 @@ static void __deflate_exit(void *ctx) deflate_decomp_exit(ctx); } -static void deflate_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void deflate_free_ctx(void *ctx) { __deflate_exit(ctx); kfree_sensitive(ctx); diff --git a/crypto/lz4.c b/crypto/lz4.c index 0606f8862e78..e66c6d1ba34f 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -16,7 +16,7 @@ struct lz4_ctx { void *lz4_comp_mem; }; -static void *lz4_alloc_ctx(struct crypto_scomp *tfm) +static void *lz4_alloc_ctx(void) { void *ctx; @@ -31,14 +31,14 @@ static int lz4_init(struct crypto_tfm *tfm) { struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lz4_comp_mem = lz4_alloc_ctx(NULL); + ctx->lz4_comp_mem = lz4_alloc_ctx(); if (IS_ERR(ctx->lz4_comp_mem)) return -ENOMEM; return 0; } -static void lz4_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lz4_free_ctx(void *ctx) { vfree(ctx); } @@ -47,7 +47,7 @@ static void lz4_exit(struct crypto_tfm *tfm) { struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - lz4_free_ctx(NULL, ctx->lz4_comp_mem); + lz4_free_ctx(ctx->lz4_comp_mem); } static int __lz4_compress_crypto(const u8 *src, unsigned int slen, diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index d7cc94aa2fcf..25a95b65aca5 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -15,7 +15,7 @@ struct lz4hc_ctx { void *lz4hc_comp_mem; }; -static void *lz4hc_alloc_ctx(struct crypto_scomp *tfm) +static void *lz4hc_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lz4hc_init(struct crypto_tfm *tfm) { struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(NULL); + ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(); if (IS_ERR(ctx->lz4hc_comp_mem)) return -ENOMEM; return 0; } -static void lz4hc_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lz4hc_free_ctx(void *ctx) { vfree(ctx); } @@ -46,7 +46,7 @@ static void lz4hc_exit(struct crypto_tfm *tfm) { struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - lz4hc_free_ctx(NULL, ctx->lz4hc_comp_mem); + lz4hc_free_ctx(ctx->lz4hc_comp_mem); } static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen, diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 0abc2d87f042..6c845e7d32f5 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -15,7 +15,7 @@ struct lzorle_ctx { void *lzorle_comp_mem; }; -static void *lzorle_alloc_ctx(struct crypto_scomp *tfm) +static void *lzorle_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lzorle_init(struct crypto_tfm *tfm) { struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lzorle_comp_mem = lzorle_alloc_ctx(NULL); + ctx->lzorle_comp_mem = lzorle_alloc_ctx(); if (IS_ERR(ctx->lzorle_comp_mem)) return -ENOMEM; return 0; } -static void lzorle_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lzorle_free_ctx(void *ctx) { kvfree(ctx); } @@ -46,7 +46,7 @@ static void lzorle_exit(struct crypto_tfm *tfm) { struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - lzorle_free_ctx(NULL, ctx->lzorle_comp_mem); + lzorle_free_ctx(ctx->lzorle_comp_mem); } static int __lzorle_compress(const u8 *src, unsigned int slen, diff --git a/crypto/lzo.c b/crypto/lzo.c index 8338851c7406..035d62e2afe0 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -15,7 +15,7 @@ struct lzo_ctx { void *lzo_comp_mem; }; -static void *lzo_alloc_ctx(struct crypto_scomp *tfm) +static void *lzo_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lzo_init(struct crypto_tfm *tfm) { struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lzo_comp_mem = lzo_alloc_ctx(NULL); + ctx->lzo_comp_mem = lzo_alloc_ctx(); if (IS_ERR(ctx->lzo_comp_mem)) return -ENOMEM; return 0; } -static void lzo_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lzo_free_ctx(void *ctx) { kvfree(ctx); } @@ -46,7 +46,7 @@ static void lzo_exit(struct crypto_tfm *tfm) { struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - lzo_free_ctx(NULL, ctx->lzo_comp_mem); + lzo_free_ctx(ctx->lzo_comp_mem); } static int __lzo_compress(const u8 *src, unsigned int slen, diff --git a/crypto/zstd.c b/crypto/zstd.c index 154a969c83a8..68a093427944 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -103,7 +103,7 @@ static int __zstd_init(void *ctx) return ret; } -static void *zstd_alloc_ctx(struct crypto_scomp *tfm) +static void *zstd_alloc_ctx(void) { int ret; struct zstd_ctx *ctx; @@ -134,7 +134,7 @@ static void __zstd_exit(void *ctx) zstd_decomp_exit(ctx); } -static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void zstd_free_ctx(void *ctx) { __zstd_exit(ctx); kfree_sensitive(ctx); diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c index 1046a746d36f..a9c3efce8f2d 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -236,7 +236,7 @@ int zip_comp_decompress(struct crypto_tfm *tfm, } /* Legacy compress framework end */ /* SCOMP framework start */ -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) +void *zip_alloc_scomp_ctx_deflate(void) { int ret; struct zip_kernel_ctx *zip_ctx; @@ -255,7 +255,7 @@ void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) return zip_ctx; } -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) +void *zip_alloc_scomp_ctx_lzs(void) { int ret; struct zip_kernel_ctx *zip_ctx; @@ -274,7 +274,7 @@ void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) return zip_ctx; } -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *ctx) +void zip_free_scomp_ctx(void *ctx) { struct zip_kernel_ctx *zip_ctx = ctx; diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h index b59ddfcacd34..dbe20bfeb3e9 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -67,9 +67,9 @@ int zip_comp_decompress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); +void *zip_alloc_scomp_ctx_deflate(void); +void *zip_alloc_scomp_ctx_lzs(void); +void zip_free_scomp_ctx(void *zip_ctx); int zip_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 07a10fd2d321..6ba9974df7d3 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -31,8 +31,8 @@ struct crypto_scomp { * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { - void *(*alloc_ctx)(struct crypto_scomp *tfm); - void (*free_ctx)(struct crypto_scomp *tfm, void *ctx); + void *(*alloc_ctx)(void); + void (*free_ctx)(void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -73,13 +73,13 @@ static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) { - return crypto_scomp_alg(tfm)->alloc_ctx(tfm); + return crypto_scomp_alg(tfm)->alloc_ctx(); } static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, void *ctx) { - return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx); + return crypto_scomp_alg(tfm)->free_ctx(ctx); } static inline int crypto_scomp_compress(struct crypto_scomp *tfm, -- cgit v1.2.3 From 3d72ad46a23ae42450d1f475bb472151dede5b93 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:17 +0800 Subject: crypto: acomp - Move stream management into scomp layer Rather than allocating the stream memory in the request object, move it into a per-cpu buffer managed by scomp. This takes the stress off the user from having to manage large request objects and setting up their own per-cpu buffers in order to do so. Signed-off-by: Herbert Xu --- crypto/acompress.c | 30 ------------- crypto/compress.h | 2 - crypto/scompress.c | 90 ++++++++++++++++++++++++------------- include/crypto/acompress.h | 26 ++++++++++- include/crypto/internal/acompress.h | 17 +------ include/crypto/internal/scompress.h | 12 +---- 6 files changed, 84 insertions(+), 93 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 30176316140a..ef36ec31d73d 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -123,36 +123,6 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); -struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp) -{ - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct acomp_req *req; - - req = __acomp_request_alloc(acomp); - if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type)) - return crypto_acomp_scomp_alloc_ctx(req); - - return req; -} -EXPORT_SYMBOL_GPL(acomp_request_alloc); - -void acomp_request_free(struct acomp_req *req) -{ - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - - if (tfm->__crt_alg->cra_type != &crypto_acomp_type) - crypto_acomp_scomp_free_ctx(req); - - if (req->base.flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { - acomp->dst_free(req->dst); - req->dst = NULL; - } - - __acomp_request_free(req); -} -EXPORT_SYMBOL_GPL(acomp_request_free); - void comp_prepare_alg(struct comp_alg_common *alg) { struct crypto_alg *base = &alg->base; diff --git a/crypto/compress.h b/crypto/compress.h index c3cedfb5e606..f7737a1fcbbd 100644 --- a/crypto/compress.h +++ b/crypto/compress.h @@ -15,8 +15,6 @@ struct acomp_req; struct comp_alg_common; int crypto_init_scomp_ops_async(struct crypto_tfm *tfm); -struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req); -void crypto_acomp_scomp_free_ctx(struct acomp_req *req); void comp_prepare_alg(struct comp_alg_common *alg); diff --git a/crypto/scompress.c b/crypto/scompress.c index 1cef6bb06a81..9b6d9bbbc73a 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -98,13 +98,62 @@ error: return -ENOMEM; } +static void scomp_free_streams(struct scomp_alg *alg) +{ + struct crypto_acomp_stream __percpu *stream = alg->stream; + int i; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); + + if (!ps->ctx) + break; + + alg->free_ctx(ps); + } + + free_percpu(stream); +} + +static int scomp_alloc_streams(struct scomp_alg *alg) +{ + struct crypto_acomp_stream __percpu *stream; + int i; + + stream = alloc_percpu(struct crypto_acomp_stream); + if (!stream) + return -ENOMEM; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); + + ps->ctx = alg->alloc_ctx(); + if (IS_ERR(ps->ctx)) { + scomp_free_streams(alg); + return PTR_ERR(ps->ctx); + } + + spin_lock_init(&ps->lock); + } + + alg->stream = stream; + return 0; +} + static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { + struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); int ret = 0; mutex_lock(&scomp_lock); + if (!alg->stream) { + ret = scomp_alloc_streams(alg); + if (ret) + goto unlock; + } if (!scomp_scratch_users++) ret = crypto_scomp_alloc_scratches(); +unlock: mutex_unlock(&scomp_lock); return ret; @@ -115,7 +164,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); void **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; - void **ctx = acomp_request_ctx(req); + struct crypto_acomp_stream *stream; struct scomp_scratch *scratch; void *src, *dst; unsigned int dlen; @@ -148,12 +197,15 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) else dst = scratch->dst; + stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); + spin_lock(&stream->lock); if (dir) ret = crypto_scomp_compress(scomp, src, req->slen, - dst, &req->dlen, *ctx); + dst, &req->dlen, stream->ctx); else ret = crypto_scomp_decompress(scomp, src, req->slen, - dst, &req->dlen, *ctx); + dst, &req->dlen, stream->ctx); + spin_unlock(&stream->lock); if (!ret) { if (!req->dst) { req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL); @@ -226,45 +278,19 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) crt->compress = scomp_acomp_compress; crt->decompress = scomp_acomp_decompress; crt->dst_free = sgl_free; - crt->reqsize = sizeof(void *); return 0; } -struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req) +static void crypto_scomp_destroy(struct crypto_alg *alg) { - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); - struct crypto_scomp *scomp = *tfm_ctx; - void *ctx; - - ctx = crypto_scomp_alloc_ctx(scomp); - if (IS_ERR(ctx)) { - kfree(req); - return NULL; - } - - *req->__ctx = ctx; - - return req; -} - -void crypto_acomp_scomp_free_ctx(struct acomp_req *req) -{ - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); - struct crypto_scomp *scomp = *tfm_ctx; - void *ctx = *req->__ctx; - - if (ctx) - crypto_scomp_free_ctx(scomp, ctx); + scomp_free_streams(__crypto_scomp_alg(alg)); } static const struct crypto_type crypto_scomp_type = { .extsize = crypto_alg_extsize, .init_tfm = crypto_scomp_init_tfm, + .destroy = crypto_scomp_destroy, #ifdef CONFIG_PROC_FS .show = crypto_scomp_show, #endif diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index b6d5136e689d..c4937709ad0e 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -10,8 +10,12 @@ #define _CRYPTO_ACOMP_H #include +#include #include #include +#include +#include +#include #define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 #define CRYPTO_ACOMP_DST_MAX 131072 @@ -54,8 +58,14 @@ struct crypto_acomp { struct crypto_tfm base; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + #define COMP_ALG_COMMON { \ struct crypto_alg base; \ + struct crypto_acomp_stream __percpu *stream; \ } struct comp_alg_common COMP_ALG_COMMON; @@ -173,7 +183,16 @@ static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) * * Return: allocated handle in case of success or NULL in case of an error */ -struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); +static inline struct acomp_req *acomp_request_alloc_noprof(struct crypto_acomp *tfm) +{ + struct acomp_req *req; + + req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); + if (likely(req)) + acomp_request_set_tfm(req, tfm); + return req; +} +#define acomp_request_alloc(...) alloc_hooks(acomp_request_alloc_noprof(__VA_ARGS__)) /** * acomp_request_free() -- zeroize and free asynchronous (de)compression @@ -182,7 +201,10 @@ struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); * * @req: request to free */ -void acomp_request_free(struct acomp_req *req); +static inline void acomp_request_free(struct acomp_req *req) +{ + kfree_sensitive(req); +} /** * acomp_request_set_callback() -- Sets an asynchronous callback diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8831edaafc05..4a8f7e3beaa1 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -32,6 +32,7 @@ * * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure + * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with scomp */ struct acomp_alg { @@ -68,22 +69,6 @@ static inline void acomp_request_complete(struct acomp_req *req, crypto_request_complete(&req->base, err); } -static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm) -{ - struct acomp_req *req; - - req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); - if (likely(req)) - acomp_request_set_tfm(req, tfm); - return req; -} -#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__)) - -static inline void __acomp_request_free(struct acomp_req *req) -{ - kfree_sensitive(req); -} - /** * crypto_register_acomp() -- Register asynchronous compression algorithm * diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 6ba9974df7d3..88986ab8ce15 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -28,6 +28,7 @@ struct crypto_scomp { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure + * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { @@ -71,17 +72,6 @@ static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg); } -static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) -{ - return crypto_scomp_alg(tfm)->alloc_ctx(); -} - -static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, - void *ctx) -{ - return crypto_scomp_alg(tfm)->free_ctx(ctx); -} - static inline int crypto_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) -- cgit v1.2.3 From cff12830e2cb2044067167f95a537074240347b7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:19 +0800 Subject: crypto: scomp - Disable BH when taking per-cpu spin lock Disable BH when taking per-cpu spin locks. This isn't an issue right now because the only user zswap calls scomp from process context. However, if scomp is called from softirq context the spin lock may dead-lock. Signed-off-by: Herbert Xu --- crypto/scompress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/scompress.c b/crypto/scompress.c index 9b6d9bbbc73a..a2ce481a10bb 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -182,7 +182,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dlen = req->dlen; scratch = raw_cpu_ptr(&scomp_scratch); - spin_lock(&scratch->lock); + spin_lock_bh(&scratch->lock); if (sg_nents(req->src) == 1 && !PageHighMem(sg_page(req->src))) { src = page_to_virt(sg_page(req->src)) + req->src->offset; @@ -230,7 +230,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) } } out: - spin_unlock(&scratch->lock); + spin_unlock_bh(&scratch->lock); return ret; } -- cgit v1.2.3 From b67a026003725a5d2496eba691c293694ab4847a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:21 +0800 Subject: crypto: acomp - Add request chaining and virtual addresses This adds request chaining and virtual address support to the acomp interface. It is identical to the ahash interface, except that a new flag CRYPTO_ACOMP_REQ_NONDMA has been added to indicate that the virtual addresses are not suitable for DMA. This is because all existing and potential acomp users can provide memory that is suitable for DMA so there is no need for a fall-back copy path. Signed-off-by: Herbert Xu --- crypto/acompress.c | 197 +++++++++++++++++++++++++++++++++++ include/crypto/acompress.h | 198 +++++++++++++++++++++++++++++++++--- include/crypto/internal/acompress.h | 42 ++++++++ 3 files changed, 424 insertions(+), 13 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index ef36ec31d73d..45444e99a9db 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -23,6 +23,8 @@ struct crypto_scomp; static const struct crypto_type crypto_acomp_type; +static void acomp_reqchain_done(void *data, int err); + static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg) { return container_of(alg, struct acomp_alg, calg.base); @@ -123,6 +125,201 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); +static bool acomp_request_has_nondma(struct acomp_req *req) +{ + struct acomp_req *r2; + + if (acomp_request_isnondma(req)) + return true; + + list_for_each_entry(r2, &req->base.list, base.list) + if (acomp_request_isnondma(r2)) + return true; + + return false; +} + +static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req_chain *state = &req->chain; + + if (!acomp_is_async(tfm)) + return; + + state->compl = req->base.complete; + state->data = req->base.data; + req->base.complete = cplt; + req->base.data = state; + state->req0 = req; +} + +static void acomp_restore_req(struct acomp_req_chain *state) +{ + struct acomp_req *req = state->req0; + struct crypto_acomp *tfm; + + tfm = crypto_acomp_reqtfm(req); + if (!acomp_is_async(tfm)) + return; + + req->base.complete = state->compl; + req->base.data = state->data; +} + +static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +{ + struct acomp_req *req = state->cur; + unsigned int slen = req->slen; + unsigned int dlen = req->dlen; + + req->base.err = err; + state = &req->chain; + + if (state->src) + acomp_request_set_src_dma(req, state->src, slen); + if (state->dst) + acomp_request_set_dst_dma(req, state->dst, dlen); + state->src = NULL; + state->dst = NULL; +} + +static void acomp_virt_to_sg(struct acomp_req *req) +{ + struct acomp_req_chain *state = &req->chain; + + if (acomp_request_src_isvirt(req)) { + unsigned int slen = req->slen; + const u8 *svirt = req->svirt; + + state->src = svirt; + sg_init_one(&state->ssg, svirt, slen); + acomp_request_set_src_sg(req, &state->ssg, slen); + } + + if (acomp_request_dst_isvirt(req)) { + unsigned int dlen = req->dlen; + u8 *dvirt = req->dvirt; + + state->dst = dvirt; + sg_init_one(&state->dsg, dvirt, dlen); + acomp_request_set_dst_sg(req, &state->dsg, dlen); + } +} + +static int acomp_reqchain_finish(struct acomp_req_chain *state, + int err, u32 mask) +{ + struct acomp_req *req0 = state->req0; + struct acomp_req *req = state->cur; + struct acomp_req *n; + + acomp_reqchain_virt(state, err); + + if (req != req0) + list_add_tail(&req->base.list, &req0->base.list); + + list_for_each_entry_safe(req, n, &state->head, base.list) { + list_del_init(&req->base.list); + + req->base.flags &= mask; + req->base.complete = acomp_reqchain_done; + req->base.data = state; + state->cur = req; + + acomp_virt_to_sg(req); + err = state->op(req); + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + err = -EBUSY; + goto out; + } + + if (err == -EBUSY) + goto out; + + acomp_reqchain_virt(state, err); + list_add_tail(&req->base.list, &req0->base.list); + } + + acomp_restore_req(state); + +out: + return err; +} + +static void acomp_reqchain_done(void *data, int err) +{ + struct acomp_req_chain *state = data; + crypto_completion_t compl = state->compl; + + data = state->data; + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + return; + goto notify; + } + + err = acomp_reqchain_finish(state, err, CRYPTO_TFM_REQ_MAY_BACKLOG); + if (err == -EBUSY) + return; + +notify: + compl(data, err); +} + +static int acomp_do_req_chain(struct acomp_req *req, + int (*op)(struct acomp_req *req)) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req_chain *state = &req->chain; + int err; + + if (crypto_acomp_req_chain(tfm) || + (!acomp_request_chained(req) && !acomp_request_isvirt(req))) + return op(req); + + /* + * There are no in-kernel users that do this. If and ever + * such users come into being then we could add a fall-back + * path. + */ + if (acomp_request_has_nondma(req)) + return -EINVAL; + + if (acomp_is_async(tfm)) { + acomp_save_req(req, acomp_reqchain_done); + state = req->base.data; + } + + state->op = op; + state->cur = req; + state->src = NULL; + INIT_LIST_HEAD(&state->head); + list_splice_init(&req->base.list, &state->head); + + acomp_virt_to_sg(req); + err = op(req); + if (err == -EBUSY || err == -EINPROGRESS) + return -EBUSY; + + return acomp_reqchain_finish(state, err, ~0); +} + +int crypto_acomp_compress(struct acomp_req *req) +{ + return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); +} +EXPORT_SYMBOL_GPL(crypto_acomp_compress); + +int crypto_acomp_decompress(struct acomp_req *req) +{ + return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); +} +EXPORT_SYMBOL_GPL(crypto_acomp_decompress); + void comp_prepare_alg(struct comp_alg_common *alg) { struct crypto_alg *base = &alg->base; diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index c4937709ad0e..c4d8a29274c6 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -13,13 +13,42 @@ #include #include #include +#include #include #include #include #define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 + +/* Set this bit if source is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_SRC_VIRT 0x00000002 + +/* Set this bit for if virtual address source cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_SRC_NONDMA 0x00000004 + +/* Set this bit if destination is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_DST_VIRT 0x00000008 + +/* Set this bit for if virtual address destination cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 + #define CRYPTO_ACOMP_DST_MAX 131072 +struct acomp_req; + +struct acomp_req_chain { + struct list_head head; + struct acomp_req *req0; + struct acomp_req *cur; + int (*op)(struct acomp_req *req); + crypto_completion_t compl; + void *data; + struct scatterlist ssg; + struct scatterlist dsg; + const u8 *src; + u8 *dst; +}; + /** * struct acomp_req - asynchronous (de)compression request * @@ -28,14 +57,24 @@ * @dst: Destination data * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced + * @chain: Private API code data, do not use * @__ctx: Start of private context data */ struct acomp_req { struct crypto_async_request base; - struct scatterlist *src; - struct scatterlist *dst; + union { + struct scatterlist *src; + const u8 *svirt; + }; + union { + struct scatterlist *dst; + u8 *dvirt; + }; unsigned int slen; unsigned int dlen; + + struct acomp_req_chain chain; + void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -222,10 +261,16 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { + u32 keep = CRYPTO_ACOMP_ALLOC_OUTPUT | CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.complete = cmpl; req->base.data = data; - req->base.flags &= CRYPTO_ACOMP_ALLOC_OUTPUT; - req->base.flags |= flgs & ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= keep; + req->base.flags |= flgs & ~keep; + + crypto_reqchain_init(&req->base); } /** @@ -252,11 +297,144 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->slen = slen; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~(CRYPTO_ACOMP_ALLOC_OUTPUT | + CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA); if (!req->dst) req->base.flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; } +/** + * acomp_request_set_src_sg() -- Sets source scatterlist + * + * Sets source scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @src: pointer to input buffer scatterlist + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_sg(struct acomp_req *req, + struct scatterlist *src, + unsigned int slen) +{ + req->src = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_dma() -- Sets DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_dma(struct acomp_req *req, + const u8 *src, unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_nondma() -- Sets non-DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_nondma(struct acomp_req *req, + const u8 *src, + unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_dst_sg() -- Sets destination scatterlist + * + * Sets destination scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @dst: pointer to output buffer scatterlist + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_sg(struct acomp_req *req, + struct scatterlist *dst, + unsigned int dlen) +{ + req->dst = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_dma() -- Sets DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_dma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_nondma() -- Sets non-DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_nondma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +static inline void acomp_request_chain(struct acomp_req *req, + struct acomp_req *head) +{ + crypto_request_chain(&req->base, &head->base); +} + /** * crypto_acomp_compress() -- Invoke asynchronous compress operation * @@ -266,10 +444,7 @@ static inline void acomp_request_set_params(struct acomp_req *req, * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_compress(struct acomp_req *req) -{ - return crypto_acomp_reqtfm(req)->compress(req); -} +int crypto_acomp_compress(struct acomp_req *req); /** * crypto_acomp_decompress() -- Invoke asynchronous decompress operation @@ -280,9 +455,6 @@ static inline int crypto_acomp_compress(struct acomp_req *req) * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_decompress(struct acomp_req *req) -{ - return crypto_acomp_reqtfm(req)->decompress(req); -} +int crypto_acomp_decompress(struct acomp_req *req); #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 4a8f7e3beaa1..957a5ed7c7f1 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -94,4 +94,46 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); +static inline bool acomp_request_chained(struct acomp_req *req) +{ + return crypto_request_chained(&req->base); +} + +static inline bool acomp_request_src_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +static inline bool acomp_request_dst_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT; +} + +static inline bool acomp_request_isvirt(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT); +} + +static inline bool acomp_request_src_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_NONDMA; +} + +static inline bool acomp_request_dst_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_NONDMA; +} + +static inline bool acomp_request_isnondma(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_NONDMA); +} + +static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) +{ + return crypto_tfm_req_chain(&tfm->base); +} + #endif -- cgit v1.2.3 From d2d072a313c1817a0d72d7b8301eaf29ce7f83fc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:24 +0800 Subject: crypto: testmgr - Remove NULL dst acomp tests In preparation for the partial removal of NULL dst acomp support, remove the tests for them. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 29 ----------------------------- 1 file changed, 29 deletions(-) (limited to 'crypto') diff --git a/crypto/testmgr.c b/crypto/testmgr.c index cbce769b16ef..140872765dcd 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3522,21 +3522,6 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - crypto_init_wait(&wait); - sg_init_one(&src, input_vec, ilen); - acomp_request_set_params(req, &src, NULL, ilen, 0); - - ret = crypto_wait_req(crypto_acomp_compress(req), &wait); - if (ret) { - pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n", - i + 1, algo, -ret); - kfree(input_vec); - acomp_request_free(req); - goto out; - } -#endif - kfree(input_vec); acomp_request_free(req); } @@ -3598,20 +3583,6 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - crypto_init_wait(&wait); - acomp_request_set_params(req, &src, NULL, ilen, 0); - - ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); - if (ret) { - pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n", - i + 1, algo, -ret); - kfree(input_vec); - acomp_request_free(req); - goto out; - } -#endif - kfree(input_vec); acomp_request_free(req); } -- cgit v1.2.3 From 7450ebd29cd9b9745f005f2609badacea15fbe30 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 10 Mar 2025 10:20:16 -0700 Subject: crypto: scatterwalk - simplify map and unmap calling convention Now that the address returned by scatterwalk_map() is always being stored into the same struct scatter_walk that is passed in, make scatterwalk_map() do so itself and return void. Similarly, now that scatterwalk_unmap() is always being passed the address field within a struct scatter_walk, make scatterwalk_unmap() take a pointer to struct scatter_walk instead of the address directly. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/aes_s390.c | 3 +-- crypto/skcipher.c | 28 ++++++++-------------------- include/crypto/scatterwalk.h | 43 ++++++++++++++++++++++++------------------- 3 files changed, 33 insertions(+), 41 deletions(-) (limited to 'crypto') diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index ed85bd6e298f..5d36f4020dfa 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -871,8 +871,7 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) goto out; } - /* XXX */ - scatterwalk_unmap(gw->walk.addr); + scatterwalk_unmap(&gw->walk); gw->ptr = gw->buf; gw->nbytes = sizeof(gw->buf); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index ab5d852febcd..132075a905d9 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -39,18 +39,6 @@ static const struct crypto_type crypto_skcipher_type; static int skcipher_walk_next(struct skcipher_walk *walk); -static inline void skcipher_map_src(struct skcipher_walk *walk) -{ - /* XXX */ - walk->in.__addr = scatterwalk_map(&walk->in); -} - -static inline void skcipher_map_dst(struct skcipher_walk *walk) -{ - /* XXX */ - walk->out.__addr = scatterwalk_map(&walk->out); -} - static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) { return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -101,8 +89,8 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) scatterwalk_done_src(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_COPY) { scatterwalk_advance(&walk->in, n); - skcipher_map_dst(walk); - memcpy(walk->dst.virt.addr, walk->page, n); + scatterwalk_map(&walk->out); + memcpy(walk->out.addr, walk->page, n); } else { /* SKCIPHER_WALK_SLOW */ if (res > 0) { /* @@ -186,9 +174,9 @@ static int skcipher_next_copy(struct skcipher_walk *walk) { void *tmp = walk->page; - skcipher_map_src(walk); - memcpy(tmp, walk->src.virt.addr, walk->nbytes); - scatterwalk_unmap(walk->src.virt.addr); + scatterwalk_map(&walk->in); + memcpy(tmp, walk->in.addr, walk->nbytes); + scatterwalk_unmap(&walk->in); /* * walk->in is advanced later when the number of bytes actually * processed (which might be less than walk->nbytes) is known. @@ -208,12 +196,12 @@ static int skcipher_next_fast(struct skcipher_walk *walk) diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); - skcipher_map_dst(walk); - walk->in.__addr = walk->dst.virt.addr; + scatterwalk_map(&walk->out); + walk->in.__addr = walk->out.__addr; if (diff) { walk->flags |= SKCIPHER_WALK_DIFF; - skcipher_map_src(walk); + scatterwalk_map(&walk->in); } return 0; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index c62f47d04eb1..b7e617ae4442 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -97,23 +97,28 @@ static inline void scatterwalk_get_sglist(struct scatter_walk *walk, scatterwalk_crypto_chain(sg_out, sg_next(walk->sg), 2); } -static inline void *scatterwalk_map(struct scatter_walk *walk) +static inline void scatterwalk_map(struct scatter_walk *walk) { struct page *base_page = sg_page(walk->sg); - if (IS_ENABLED(CONFIG_HIGHMEM)) - return kmap_local_page(base_page + (walk->offset >> PAGE_SHIFT)) + - offset_in_page(walk->offset); - /* - * When !HIGHMEM we allow the walker to return segments that span a page - * boundary; see scatterwalk_clamp(). To make it clear that in this - * case we're working in the linear buffer of the whole sg entry in the - * kernel's direct map rather than within the mapped buffer of a single - * page, compute the address as an offset from the page_address() of the - * first page of the sg entry. Either way the result is the address in - * the direct map, but this makes it clearer what is really going on. - */ - return page_address(base_page) + walk->offset; + if (IS_ENABLED(CONFIG_HIGHMEM)) { + walk->__addr = kmap_local_page(base_page + + (walk->offset >> PAGE_SHIFT)) + + offset_in_page(walk->offset); + } else { + /* + * When !HIGHMEM we allow the walker to return segments that + * span a page boundary; see scatterwalk_clamp(). To make it + * clear that in this case we're working in the linear buffer of + * the whole sg entry in the kernel's direct map rather than + * within the mapped buffer of a single page, compute the + * address as an offset from the page_address() of the first + * page of the sg entry. Either way the result is the address + * in the direct map, but this makes it clearer what is really + * going on. + */ + walk->__addr = page_address(base_page) + walk->offset; + } } /** @@ -132,14 +137,14 @@ static inline unsigned int scatterwalk_next(struct scatter_walk *walk, { unsigned int nbytes = scatterwalk_clamp(walk, total); - walk->__addr = scatterwalk_map(walk); + scatterwalk_map(walk); return nbytes; } -static inline void scatterwalk_unmap(const void *vaddr) +static inline void scatterwalk_unmap(struct scatter_walk *walk) { if (IS_ENABLED(CONFIG_HIGHMEM)) - kunmap_local(vaddr); + kunmap_local(walk->__addr); } static inline void scatterwalk_advance(struct scatter_walk *walk, @@ -159,7 +164,7 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, static inline void scatterwalk_done_src(struct scatter_walk *walk, unsigned int nbytes) { - scatterwalk_unmap(walk->addr); + scatterwalk_unmap(walk); scatterwalk_advance(walk, nbytes); } @@ -175,7 +180,7 @@ static inline void scatterwalk_done_src(struct scatter_walk *walk, static inline void scatterwalk_done_dst(struct scatter_walk *walk, unsigned int nbytes) { - scatterwalk_unmap(walk->addr); + scatterwalk_unmap(walk); /* * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just * relying on flush_dcache_page() being a no-op when not implemented, -- cgit v1.2.3 From da6f9bf40ac267b5c720694a817beea84fa40f77 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 13 Mar 2025 13:14:56 +0800 Subject: crypto: krb5 - Use SG miter instead of doing it by hand The function crypto_shash_update_sg iterates through an SG by hand. It fails to handle corner cases such as SG entries longer than a page. Fix this by using the SG iterator. Fixes: 348f5669d1f6 ("crypto/krb5: Implement the Kerberos5 rfc3961 get_mic and verify_mic") Signed-off-by: Herbert Xu --- crypto/krb5/rfc3961_simplified.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) (limited to 'crypto') diff --git a/crypto/krb5/rfc3961_simplified.c b/crypto/krb5/rfc3961_simplified.c index c1dcb0dd3a00..79180d28baa9 100644 --- a/crypto/krb5/rfc3961_simplified.c +++ b/crypto/krb5/rfc3961_simplified.c @@ -67,9 +67,9 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include -#include #include #include #include @@ -83,26 +83,21 @@ int crypto_shash_update_sg(struct shash_desc *desc, struct scatterlist *sg, size_t offset, size_t len) { - do { - int ret; - - if (offset < sg->length) { - struct page *page = sg_page(sg); - void *p = kmap_local_page(page); - void *q = p + sg->offset + offset; - size_t seg = min_t(size_t, len, sg->length - offset); - - ret = crypto_shash_update(desc, q, seg); - kunmap_local(p); - if (ret < 0) - return ret; - len -= seg; - offset = 0; - } else { - offset -= sg->length; - } - } while (len > 0 && (sg = sg_next(sg))); - return 0; + struct sg_mapping_iter miter; + size_t i, n; + int ret = 0; + + sg_miter_start(&miter, sg, sg_nents(sg), + SG_MITER_FROM_SG | SG_MITER_LOCAL); + for (i = 0; i < len; i += n) { + sg_miter_next(&miter); + n = min(miter.length, len - i); + ret = crypto_shash_update(desc, miter.addr, n); + if (ret < 0) + break; + } + sg_miter_stop(&miter); + return ret; } static int rfc3961_do_encrypt(struct crypto_sync_skcipher *tfm, void *iv, -- cgit v1.2.3 From 480db5009571aa8d7f39b0357a6fd337fa3caf31 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 13 Mar 2025 13:14:58 +0800 Subject: crypto: hash - Fix test underflow in shash_ahash_digest The test on PAGE_SIZE - offset in shash_ahash_digest can underflow, leading to execution of the fast path even if the data cannot be mapped into a single page. Fix this by splitting the test into four cases: 1) nbytes > sg->length: More than one SG entry, slow path. 2) !IS_ENABLED(CONFIG_HIGHMEM): fast path. 3) nbytes > (unsigned int)PAGE_SIZE - offset: Two highmem pages, slow path. 4) Highmem fast path. Fixes: 5f7082ed4f48 ("crypto: hash - Export shash through hash") Signed-off-by: Herbert Xu --- crypto/ahash.c | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index 9c26175c21a8..1fe594880295 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -201,25 +202,36 @@ int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) unsigned int nbytes = req->nbytes; struct scatterlist *sg; unsigned int offset; + struct page *page; + const u8 *data; int err; - if (ahash_request_isvirt(req)) - return crypto_shash_digest(desc, req->svirt, nbytes, - req->result); - - if (nbytes && - (sg = req->src, offset = sg->offset, - nbytes <= min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset))) { - void *data; - - data = kmap_local_page(sg_page(sg)); - err = crypto_shash_digest(desc, data + offset, nbytes, - req->result); - kunmap_local(data); - } else - err = crypto_shash_init(desc) ?: - shash_ahash_finup(req, desc); + data = req->svirt; + if (!nbytes || ahash_request_isvirt(req)) + return crypto_shash_digest(desc, data, nbytes, req->result); + + sg = req->src; + if (nbytes > sg->length) + return crypto_shash_init(desc) ?: + shash_ahash_finup(req, desc); + + page = sg_page(sg); + offset = sg->offset; + data = lowmem_page_address(page) + offset; + if (!IS_ENABLED(CONFIG_HIGHMEM)) + return crypto_shash_digest(desc, data, nbytes, req->result); + + page += offset >> PAGE_SHIFT; + offset = offset_in_page(offset); + + if (nbytes > (unsigned int)PAGE_SIZE - offset) + return crypto_shash_init(desc) ?: + shash_ahash_finup(req, desc); + data = kmap_local_page(page); + err = crypto_shash_digest(desc, data + offset, nbytes, + req->result); + kunmap_local(data); return err; } EXPORT_SYMBOL_GPL(shash_ahash_digest); -- cgit v1.2.3 From ce3313560c7ea56f0af76853658959d8363a639f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 14 Mar 2025 11:27:23 +0800 Subject: crypto: hash - Use nth_page instead of doing it by hand Use nth_page instead of adding n to the page pointer. Signed-off-by: Herbert Xu --- crypto/ahash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index 1fe594880295..06559e5a715b 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -80,7 +80,7 @@ static int hash_walk_new_entry(struct crypto_hash_walk *walk) sg = walk->sg; walk->offset = sg->offset; - walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); + walk->pg = nth_page(sg_page(walk->sg), (walk->offset >> PAGE_SHIFT)); walk->offset = offset_in_page(walk->offset); walk->entrylen = sg->length; @@ -221,7 +221,7 @@ int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) if (!IS_ENABLED(CONFIG_HIGHMEM)) return crypto_shash_digest(desc, data, nbytes, req->result); - page += offset >> PAGE_SHIFT; + page = nth_page(page, offset >> PAGE_SHIFT); offset = offset_in_page(offset); if (nbytes > (unsigned int)PAGE_SIZE - offset) -- cgit v1.2.3 From 2d3553ecb4e316a74571da253191c37fb90cb815 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:22 +0800 Subject: crypto: scomp - Remove support for some non-trivial SG lists As the only user of acomp/scomp uses a trivial single-page SG list, remove support for everything else in preprataion for the addition of virtual address support. However, keep support for non-trivial source SG lists as that user is currently jumping through hoops in order to linearise the source data. Limit the source SG linearisation buffer to a single page as that user never goes over that. The only other potential user is also unlikely to exceed that (IPComp) and it can easily do its own linearisation if necessary. Also keep the destination SG linearisation for IPComp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 1 - crypto/scompress.c | 127 +++++++++++++++++++++--------------- include/crypto/acompress.h | 17 +---- include/crypto/internal/scompress.h | 2 - 4 files changed, 76 insertions(+), 71 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 45444e99a9db..194a4b36f97f 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -73,7 +73,6 @@ static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) acomp->compress = alg->compress; acomp->decompress = alg->decompress; - acomp->dst_free = alg->dst_free; acomp->reqsize = alg->reqsize; if (alg->exit) diff --git a/crypto/scompress.c b/crypto/scompress.c index a2ce481a10bb..4441c40f541f 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -12,8 +12,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -23,9 +25,14 @@ #include "compress.h" +#define SCOMP_SCRATCH_SIZE 65400 + struct scomp_scratch { spinlock_t lock; - void *src; + union { + void *src; + unsigned long saddr; + }; void *dst; }; @@ -66,7 +73,7 @@ static void crypto_scomp_free_scratches(void) for_each_possible_cpu(i) { scratch = per_cpu_ptr(&scomp_scratch, i); - vfree(scratch->src); + free_page(scratch->saddr); vfree(scratch->dst); scratch->src = NULL; scratch->dst = NULL; @@ -79,14 +86,15 @@ static int crypto_scomp_alloc_scratches(void) int i; for_each_possible_cpu(i) { + struct page *page; void *mem; scratch = per_cpu_ptr(&scomp_scratch, i); - mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); - if (!mem) + page = alloc_pages_node(cpu_to_node(i), GFP_KERNEL, 0); + if (!page) goto error; - scratch->src = mem; + scratch->src = page_address(page); mem = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); if (!mem) goto error; @@ -161,76 +169,88 @@ unlock: static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) { + struct scomp_scratch *scratch = raw_cpu_ptr(&scomp_scratch); struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - void **tfm_ctx = acomp_tfm_ctx(tfm); + struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; struct crypto_acomp_stream *stream; - struct scomp_scratch *scratch; + unsigned int slen = req->slen; + unsigned int dlen = req->dlen; + struct page *spage, *dpage; + unsigned int soff, doff; void *src, *dst; - unsigned int dlen; + unsigned int n; int ret; - if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) + if (!req->src || !slen) return -EINVAL; - if (req->dst && !req->dlen) + if (!req->dst || !dlen) return -EINVAL; - if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) - req->dlen = SCOMP_SCRATCH_SIZE; - - dlen = req->dlen; + soff = req->src->offset; + spage = nth_page(sg_page(req->src), soff / PAGE_SIZE); + soff = offset_in_page(soff); - scratch = raw_cpu_ptr(&scomp_scratch); - spin_lock_bh(&scratch->lock); - - if (sg_nents(req->src) == 1 && !PageHighMem(sg_page(req->src))) { - src = page_to_virt(sg_page(req->src)) + req->src->offset; - } else { - scatterwalk_map_and_copy(scratch->src, req->src, 0, - req->slen, 0); + n = slen / PAGE_SIZE; + n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; + if (slen <= req->src->length && (!PageHighMem(nth_page(spage, n)) || + size_add(soff, slen) <= PAGE_SIZE)) + src = kmap_local_page(spage) + soff; + else src = scratch->src; - } - if (req->dst && sg_nents(req->dst) == 1 && !PageHighMem(sg_page(req->dst))) - dst = page_to_virt(sg_page(req->dst)) + req->dst->offset; - else + doff = req->dst->offset; + dpage = nth_page(sg_page(req->dst), doff / PAGE_SIZE); + doff = offset_in_page(doff); + + n = dlen / PAGE_SIZE; + n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; + if (dlen <= req->dst->length && (!PageHighMem(nth_page(dpage, n)) || + size_add(doff, dlen) <= PAGE_SIZE)) + dst = kmap_local_page(dpage) + doff; + else { + if (dlen > SCOMP_SCRATCH_SIZE) + dlen = SCOMP_SCRATCH_SIZE; dst = scratch->dst; + } + + spin_lock_bh(&scratch->lock); + + if (src == scratch->src) + memcpy_from_sglist(src, req->src, 0, slen); stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); spin_lock(&stream->lock); if (dir) - ret = crypto_scomp_compress(scomp, src, req->slen, - dst, &req->dlen, stream->ctx); + ret = crypto_scomp_compress(scomp, src, slen, + dst, &dlen, stream->ctx); else - ret = crypto_scomp_decompress(scomp, src, req->slen, - dst, &req->dlen, stream->ctx); + ret = crypto_scomp_decompress(scomp, src, slen, + dst, &dlen, stream->ctx); + + if (dst == scratch->dst) + memcpy_to_sglist(req->dst, 0, dst, dlen); + spin_unlock(&stream->lock); - if (!ret) { - if (!req->dst) { - req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL); - if (!req->dst) { - ret = -ENOMEM; - goto out; - } - } else if (req->dlen > dlen) { - ret = -ENOSPC; - goto out; - } - if (dst == scratch->dst) { - scatterwalk_map_and_copy(scratch->dst, req->dst, 0, - req->dlen, 1); - } else { - int nr_pages = DIV_ROUND_UP(req->dst->offset + req->dlen, PAGE_SIZE); - int i; - struct page *dst_page = sg_page(req->dst); - - for (i = 0; i < nr_pages; i++) - flush_dcache_page(dst_page + i); + spin_unlock_bh(&scratch->lock); + + req->dlen = dlen; + + if (dst != scratch->dst) { + kunmap_local(dst); + dlen += doff; + for (;;) { + flush_dcache_page(dpage); + if (dlen <= PAGE_SIZE) + break; + dlen -= PAGE_SIZE; + dpage = nth_page(dpage, 1); } } -out: - spin_unlock_bh(&scratch->lock); + if (src != scratch->src) + kunmap_local(src); + return ret; } @@ -277,7 +297,6 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) crt->compress = scomp_acomp_compress; crt->decompress = scomp_acomp_decompress; - crt->dst_free = sgl_free; return 0; } diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index c4d8a29274c6..53c9e632862b 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -18,8 +18,6 @@ #include #include -#define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 - /* Set this bit if source is virtual address instead of SG list. */ #define CRYPTO_ACOMP_REQ_SRC_VIRT 0x00000002 @@ -84,15 +82,12 @@ struct acomp_req { * * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation - * @dst_free: Frees destination buffer if allocated inside the - * algorithm * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure */ struct crypto_acomp { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); - void (*dst_free)(struct scatterlist *dst); unsigned int reqsize; struct crypto_tfm base; }; @@ -261,9 +256,8 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - u32 keep = CRYPTO_ACOMP_ALLOC_OUTPUT | CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_SRC_NONDMA | CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_DST_NONDMA; + u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.complete = cmpl; req->base.data = data; @@ -297,13 +291,10 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->slen = slen; req->dlen = dlen; - req->base.flags &= ~(CRYPTO_ACOMP_ALLOC_OUTPUT | - CRYPTO_ACOMP_REQ_SRC_VIRT | + req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); - if (!req->dst) - req->base.flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; } /** @@ -403,7 +394,6 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -424,7 +414,6 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 88986ab8ce15..f25aa2ea3b48 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -12,8 +12,6 @@ #include #include -#define SCOMP_SCRATCH_SIZE 131072 - struct acomp_req; struct crypto_scomp { -- cgit v1.2.3 From 2c1808e5fe5afda22cd49f31b24219d147c785fc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:31 +0800 Subject: crypto: scomp - Add chaining and virtual address support Add chaining and virtual address support to all scomp algorithms. Signed-off-by: Herbert Xu --- crypto/scompress.c | 94 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 30 deletions(-) (limited to 'crypto') diff --git a/crypto/scompress.c b/crypto/scompress.c index 4441c40f541f..ba9b22ba53fe 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -178,8 +178,9 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) unsigned int dlen = req->dlen; struct page *spage, *dpage; unsigned int soff, doff; - void *src, *dst; unsigned int n; + const u8 *src; + u8 *dst; int ret; if (!req->src || !slen) @@ -188,37 +189,47 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (!req->dst || !dlen) return -EINVAL; - soff = req->src->offset; - spage = nth_page(sg_page(req->src), soff / PAGE_SIZE); - soff = offset_in_page(soff); - - n = slen / PAGE_SIZE; - n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; - if (slen <= req->src->length && (!PageHighMem(nth_page(spage, n)) || - size_add(soff, slen) <= PAGE_SIZE)) - src = kmap_local_page(spage) + soff; - else - src = scratch->src; - - doff = req->dst->offset; - dpage = nth_page(sg_page(req->dst), doff / PAGE_SIZE); - doff = offset_in_page(doff); + if (acomp_request_src_isvirt(req)) + src = req->svirt; + else { + soff = req->src->offset; + spage = nth_page(sg_page(req->src), soff / PAGE_SIZE); + soff = offset_in_page(soff); + + n = slen / PAGE_SIZE; + n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; + if (slen <= req->src->length && + (!PageHighMem(nth_page(spage, n)) || + size_add(soff, slen) <= PAGE_SIZE)) + src = kmap_local_page(spage) + soff; + else + src = scratch->src; + } - n = dlen / PAGE_SIZE; - n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; - if (dlen <= req->dst->length && (!PageHighMem(nth_page(dpage, n)) || - size_add(doff, dlen) <= PAGE_SIZE)) - dst = kmap_local_page(dpage) + doff; + if (acomp_request_dst_isvirt(req)) + dst = req->dvirt; else { - if (dlen > SCOMP_SCRATCH_SIZE) - dlen = SCOMP_SCRATCH_SIZE; - dst = scratch->dst; + doff = req->dst->offset; + dpage = nth_page(sg_page(req->dst), doff / PAGE_SIZE); + doff = offset_in_page(doff); + + n = dlen / PAGE_SIZE; + n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; + if (dlen <= req->dst->length && + (!PageHighMem(nth_page(dpage, n)) || + size_add(doff, dlen) <= PAGE_SIZE)) + dst = kmap_local_page(dpage) + doff; + else { + if (dlen > SCOMP_SCRATCH_SIZE) + dlen = SCOMP_SCRATCH_SIZE; + dst = scratch->dst; + } } spin_lock_bh(&scratch->lock); if (src == scratch->src) - memcpy_from_sglist(src, req->src, 0, slen); + memcpy_from_sglist(scratch->src, req->src, 0, slen); stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); spin_lock(&stream->lock); @@ -237,7 +248,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) req->dlen = dlen; - if (dst != scratch->dst) { + if (!acomp_request_dst_isvirt(req) && dst != scratch->dst) { kunmap_local(dst); dlen += doff; for (;;) { @@ -248,20 +259,34 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dpage = nth_page(dpage, 1); } } - if (src != scratch->src) + if (!acomp_request_src_isvirt(req) && src != scratch->src) kunmap_local(src); return ret; } +static int scomp_acomp_chain(struct acomp_req *req, int dir) +{ + struct acomp_req *r2; + int err; + + err = scomp_acomp_comp_decomp(req, dir); + req->base.err = err; + + list_for_each_entry(r2, &req->base.list, base.list) + r2->base.err = scomp_acomp_comp_decomp(r2, dir); + + return err; +} + static int scomp_acomp_compress(struct acomp_req *req) { - return scomp_acomp_comp_decomp(req, 1); + return scomp_acomp_chain(req, 1); } static int scomp_acomp_decompress(struct acomp_req *req) { - return scomp_acomp_comp_decomp(req, 0); + return scomp_acomp_chain(req, 0); } static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) @@ -322,12 +347,21 @@ static const struct crypto_type crypto_scomp_type = { .tfmsize = offsetof(struct crypto_scomp, base), }; -int crypto_register_scomp(struct scomp_alg *alg) +static void scomp_prepare_alg(struct scomp_alg *alg) { struct crypto_alg *base = &alg->calg.base; comp_prepare_alg(&alg->calg); + base->cra_flags |= CRYPTO_ALG_REQ_CHAIN; +} + +int crypto_register_scomp(struct scomp_alg *alg) +{ + struct crypto_alg *base = &alg->calg.base; + + scomp_prepare_alg(alg); + base->cra_type = &crypto_scomp_type; base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; -- cgit v1.2.3 From 5416b8a741d6d09369b973cd9d4dacb1887c24df Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:34 +0800 Subject: crypto: acomp - Add ACOMP_REQUEST_ALLOC and acomp_request_alloc_extra Add ACOMP_REQUEST_ALLOC which is a wrapper around acomp_request_alloc that falls back to a synchronous stack reqeust if the allocation fails. Also add ACOMP_REQUEST_ON_STACK which stores the request on the stack only. The request should be freed with acomp_request_free. Finally add acomp_request_alloc_extra which gives the user extra memory to use in conjunction with the request. Signed-off-by: Herbert Xu --- crypto/acompress.c | 38 +++++++++++++++--- include/crypto/acompress.h | 80 +++++++++++++++++++++++++++++++++++-- include/crypto/internal/acompress.h | 6 +++ include/linux/crypto.h | 1 + 4 files changed, 117 insertions(+), 8 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 194a4b36f97f..9da033ded193 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -60,28 +60,56 @@ static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); struct acomp_alg *alg = crypto_acomp_alg(acomp); - alg->exit(acomp); + if (alg->exit) + alg->exit(acomp); + + if (acomp_is_async(acomp)) + crypto_free_acomp(acomp->fb); } static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) { struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); struct acomp_alg *alg = crypto_acomp_alg(acomp); + struct crypto_acomp *fb = NULL; + int err; + + acomp->fb = acomp; if (tfm->__crt_alg->cra_type != &crypto_acomp_type) return crypto_init_scomp_ops_async(tfm); + if (acomp_is_async(acomp)) { + fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0, + CRYPTO_ALG_ASYNC); + if (IS_ERR(fb)) + return PTR_ERR(fb); + + err = -EINVAL; + if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE) + goto out_free_fb; + + acomp->fb = fb; + } + acomp->compress = alg->compress; acomp->decompress = alg->decompress; acomp->reqsize = alg->reqsize; - if (alg->exit) - acomp->base.exit = crypto_acomp_exit_tfm; + acomp->base.exit = crypto_acomp_exit_tfm; - if (alg->init) - return alg->init(acomp); + if (!alg->init) + return 0; + + err = alg->init(acomp); + if (err) + goto out_free_fb; return 0; + +out_free_fb: + crypto_free_acomp(fb); + return err; } static unsigned int crypto_acomp_extsize(struct crypto_alg *alg) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 53c9e632862b..03cb381c2c54 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -10,9 +10,11 @@ #define _CRYPTO_ACOMP_H #include +#include #include #include #include +#include #include #include #include @@ -32,6 +34,14 @@ #define CRYPTO_ACOMP_DST_MAX 131072 +#define MAX_SYNC_COMP_REQSIZE 0 + +#define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_on_stack_init( \ + __##name##_req, (tfm), (gfp), false) + struct acomp_req; struct acomp_req_chain { @@ -83,12 +93,14 @@ struct acomp_req { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @reqsize: Context size for (de)compression requests + * @fb: Synchronous fallback tfm * @base: Common crypto API algorithm data structure */ struct crypto_acomp { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); unsigned int reqsize; + struct crypto_acomp *fb; struct crypto_tfm base; }; @@ -210,24 +222,68 @@ static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) return crypto_has_alg(alg_name, type, mask); } +static inline const char *crypto_acomp_alg_name(struct crypto_acomp *tfm) +{ + return crypto_tfm_alg_name(crypto_acomp_tfm(tfm)); +} + +static inline const char *crypto_acomp_driver_name(struct crypto_acomp *tfm) +{ + return crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); +} + /** * acomp_request_alloc() -- allocates asynchronous (de)compression request * * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * @gfp: gfp to pass to kzalloc (defaults to GFP_KERNEL) * * Return: allocated handle in case of success or NULL in case of an error */ -static inline struct acomp_req *acomp_request_alloc_noprof(struct crypto_acomp *tfm) +static inline struct acomp_req *acomp_request_alloc_extra_noprof( + struct crypto_acomp *tfm, size_t extra, gfp_t gfp) { struct acomp_req *req; + size_t len; + + len = ALIGN(sizeof(*req) + crypto_acomp_reqsize(tfm), CRYPTO_MINALIGN); + if (check_add_overflow(len, extra, &len)) + return NULL; - req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); + req = kzalloc_noprof(len, gfp); if (likely(req)) acomp_request_set_tfm(req, tfm); return req; } +#define acomp_request_alloc_noprof(tfm, ...) \ + CONCATENATE(acomp_request_alloc_noprof_, COUNT_ARGS(__VA_ARGS__))( \ + tfm, ##__VA_ARGS__) +#define acomp_request_alloc_noprof_0(tfm) \ + acomp_request_alloc_noprof_1(tfm, GFP_KERNEL) +#define acomp_request_alloc_noprof_1(tfm, gfp) \ + acomp_request_alloc_extra_noprof(tfm, 0, gfp) #define acomp_request_alloc(...) alloc_hooks(acomp_request_alloc_noprof(__VA_ARGS__)) +/** + * acomp_request_alloc_extra() -- allocate acomp request with extra memory + * + * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * @extra: amount of extra memory + * @gfp: gfp to pass to kzalloc + * + * Return: allocated handle in case of success or NULL in case of an error + */ +#define acomp_request_alloc_extra(...) alloc_hooks(acomp_request_alloc_extra_noprof(__VA_ARGS__)) + +static inline void *acomp_request_extra(struct acomp_req *req) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + size_t len; + + len = ALIGN(sizeof(*req) + crypto_acomp_reqsize(tfm), CRYPTO_MINALIGN); + return (void *)((char *)req + len); +} + /** * acomp_request_free() -- zeroize and free asynchronous (de)compression * request as well as the output buffer if allocated @@ -237,6 +293,8 @@ static inline struct acomp_req *acomp_request_alloc_noprof(struct crypto_acomp * */ static inline void acomp_request_free(struct acomp_req *req) { + if (!req || (req->base.flags & CRYPTO_TFM_REQ_ON_STACK)) + return; kfree_sensitive(req); } @@ -257,7 +315,8 @@ static inline void acomp_request_set_callback(struct acomp_req *req, void *data) { u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA; + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | + CRYPTO_TFM_REQ_ON_STACK; req->base.complete = cmpl; req->base.data = data; @@ -446,4 +505,19 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); +static inline struct acomp_req *acomp_request_on_stack_init( + char *buf, struct crypto_acomp *tfm, gfp_t gfp, bool stackonly) +{ + struct acomp_req *req; + + if (!stackonly && (req = acomp_request_alloc(tfm, gfp))) + return req; + + req = (void *)buf; + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + + return req; +} + #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 575dbcbc0df4..c1ed55a0e3bf 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -12,6 +12,12 @@ #include #include +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_on_stack_init( \ + __##name##_req, (tfm), 0, true) + /** * struct acomp_alg - asynchronous compression algorithm * diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 61ac11226638..ea3b95bdbde3 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -138,6 +138,7 @@ #define CRYPTO_TFM_REQ_FORBID_WEAK_KEYS 0x00000100 #define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200 #define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400 +#define CRYPTO_TFM_REQ_ON_STACK 0x00000800 /* * Miscellaneous stuff. -- cgit v1.2.3 From dfd3bc6977e8b99458169c19cd703d34ffa86acc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:38 +0800 Subject: crypto: acomp - Add async nondma fallback Add support for passing non-DMA virtual addresses to async drivers by passing them along to the fallback software algorithm. Signed-off-by: Herbert Xu --- crypto/acompress.c | 69 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 28 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 9da033ded193..d54abc27330f 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -152,20 +152,6 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); -static bool acomp_request_has_nondma(struct acomp_req *req) -{ - struct acomp_req *r2; - - if (acomp_request_isnondma(req)) - return true; - - list_for_each_entry(r2, &req->base.list, base.list) - if (acomp_request_isnondma(r2)) - return true; - - return false; -} - static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); @@ -234,6 +220,45 @@ static void acomp_virt_to_sg(struct acomp_req *req) } } +static int acomp_do_nondma(struct acomp_req_chain *state, + struct acomp_req *req) +{ + u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA; + ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + int err; + + acomp_request_set_callback(fbreq, req->base.flags, NULL, NULL); + fbreq->base.flags &= ~keep; + fbreq->base.flags |= req->base.flags & keep; + fbreq->src = req->src; + fbreq->dst = req->dst; + fbreq->slen = req->slen; + fbreq->dlen = req->dlen; + + if (state->op == crypto_acomp_reqtfm(req)->compress) + err = crypto_acomp_compress(fbreq); + else + err = crypto_acomp_decompress(fbreq); + + req->dlen = fbreq->dlen; + return err; +} + +static int acomp_do_one_req(struct acomp_req_chain *state, + struct acomp_req *req) +{ + state->cur = req; + + if (acomp_request_isnondma(req)) + return acomp_do_nondma(state, req); + + acomp_virt_to_sg(req); + return state->op(req); +} + static int acomp_reqchain_finish(struct acomp_req_chain *state, int err, u32 mask) { @@ -252,10 +277,8 @@ static int acomp_reqchain_finish(struct acomp_req_chain *state, req->base.flags &= mask; req->base.complete = acomp_reqchain_done; req->base.data = state; - state->cur = req; - acomp_virt_to_sg(req); - err = state->op(req); + err = acomp_do_one_req(state, req); if (err == -EINPROGRESS) { if (!list_empty(&state->head)) @@ -308,27 +331,17 @@ static int acomp_do_req_chain(struct acomp_req *req, (!acomp_request_chained(req) && !acomp_request_isvirt(req))) return op(req); - /* - * There are no in-kernel users that do this. If and ever - * such users come into being then we could add a fall-back - * path. - */ - if (acomp_request_has_nondma(req)) - return -EINVAL; - if (acomp_is_async(tfm)) { acomp_save_req(req, acomp_reqchain_done); state = req->base.data; } state->op = op; - state->cur = req; state->src = NULL; INIT_LIST_HEAD(&state->head); list_splice_init(&req->base.list, &state->head); - acomp_virt_to_sg(req); - err = op(req); + err = acomp_do_one_req(state, req); if (err == -EBUSY || err == -EINPROGRESS) return -EBUSY; -- cgit v1.2.3 From 8a6771cda3f48a4d954647d69ff0094346db6191 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 18:30:40 +0800 Subject: crypto: acomp - Add support for folios For many users, it's easier to supply a folio rather than an SG list since they already have them. Add support for folios to the acomp interface. Signed-off-by: Herbert Xu --- crypto/acompress.c | 40 ++++++++++++++--- crypto/scompress.c | 72 +++++++++++++++++++----------- include/crypto/acompress.h | 89 +++++++++++++++++++++++++++++++++++-- include/crypto/internal/acompress.h | 18 ++++++++ 4 files changed, 184 insertions(+), 35 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index d54abc27330f..6ef335f5bf27 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -189,18 +190,25 @@ static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) req->base.err = err; state = &req->chain; - if (state->src) + if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); - if (state->dst) + else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) + acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); + if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); - state->src = NULL; - state->dst = NULL; + else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) + acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) { struct acomp_req_chain *state = &req->chain; + state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_SRC_FOLIO | + CRYPTO_ACOMP_REQ_DST_FOLIO); + if (acomp_request_src_isvirt(req)) { unsigned int slen = req->slen; const u8 *svirt = req->svirt; @@ -208,6 +216,17 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->src = svirt; sg_init_one(&state->ssg, svirt, slen); acomp_request_set_src_sg(req, &state->ssg, slen); + } else if (acomp_request_src_isfolio(req)) { + struct folio *folio = req->sfolio; + unsigned int slen = req->slen; + size_t off = req->soff; + + state->sfolio = folio; + state->soff = off; + sg_init_table(&state->ssg, 1); + sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), + slen, off % PAGE_SIZE); + acomp_request_set_src_sg(req, &state->ssg, slen); } if (acomp_request_dst_isvirt(req)) { @@ -217,6 +236,17 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->dst = dvirt; sg_init_one(&state->dsg, dvirt, dlen); acomp_request_set_dst_sg(req, &state->dsg, dlen); + } else if (acomp_request_dst_isfolio(req)) { + struct folio *folio = req->dfolio; + unsigned int dlen = req->dlen; + size_t off = req->doff; + + state->dfolio = folio; + state->doff = off; + sg_init_table(&state->dsg, 1); + sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), + dlen, off % PAGE_SIZE); + acomp_request_set_src_sg(req, &state->dsg, dlen); } } @@ -328,7 +358,7 @@ static int acomp_do_req_chain(struct acomp_req *req, int err; if (crypto_acomp_req_chain(tfm) || - (!acomp_request_chained(req) && !acomp_request_isvirt(req))) + (!acomp_request_chained(req) && acomp_request_issg(req))) return op(req); if (acomp_is_async(tfm)) { diff --git a/crypto/scompress.c b/crypto/scompress.c index ba9b22ba53fe..f85cc0d83164 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -177,9 +177,10 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) unsigned int slen = req->slen; unsigned int dlen = req->dlen; struct page *spage, *dpage; - unsigned int soff, doff; unsigned int n; const u8 *src; + size_t soff; + size_t doff; u8 *dst; int ret; @@ -192,38 +193,57 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (acomp_request_src_isvirt(req)) src = req->svirt; else { - soff = req->src->offset; - spage = nth_page(sg_page(req->src), soff / PAGE_SIZE); - soff = offset_in_page(soff); - - n = slen / PAGE_SIZE; - n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; - if (slen <= req->src->length && - (!PageHighMem(nth_page(spage, n)) || - size_add(soff, slen) <= PAGE_SIZE)) + src = scratch->src; + do { + if (acomp_request_src_isfolio(req)) { + spage = folio_page(req->sfolio, 0); + soff = req->soff; + } else if (slen <= req->src->length) { + spage = sg_page(req->src); + soff = req->src->offset; + } else + break; + + spage = nth_page(spage, soff / PAGE_SIZE); + soff = offset_in_page(soff); + + n = slen / PAGE_SIZE; + n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; + if (PageHighMem(nth_page(spage, n)) && + size_add(soff, slen) > PAGE_SIZE) + break; src = kmap_local_page(spage) + soff; - else - src = scratch->src; + } while (0); } if (acomp_request_dst_isvirt(req)) dst = req->dvirt; else { - doff = req->dst->offset; - dpage = nth_page(sg_page(req->dst), doff / PAGE_SIZE); - doff = offset_in_page(doff); - - n = dlen / PAGE_SIZE; - n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; - if (dlen <= req->dst->length && - (!PageHighMem(nth_page(dpage, n)) || - size_add(doff, dlen) <= PAGE_SIZE)) + unsigned int max = SCOMP_SCRATCH_SIZE; + + dst = scratch->dst; + do { + if (acomp_request_dst_isfolio(req)) { + dpage = folio_page(req->dfolio, 0); + doff = req->doff; + } else if (dlen <= req->dst->length) { + dpage = sg_page(req->dst); + doff = req->dst->offset; + } else + break; + + dpage = nth_page(dpage, doff / PAGE_SIZE); + doff = offset_in_page(doff); + + n = dlen / PAGE_SIZE; + n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; + if (PageHighMem(dpage + n) && + size_add(doff, dlen) > PAGE_SIZE) + break; dst = kmap_local_page(dpage) + doff; - else { - if (dlen > SCOMP_SCRATCH_SIZE) - dlen = SCOMP_SCRATCH_SIZE; - dst = scratch->dst; - } + max = dlen; + } while (0); + dlen = min(dlen, max); } spin_lock_bh(&scratch->lock); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 03cb381c2c54..c497c73baf13 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -32,6 +32,12 @@ /* Set this bit for if virtual address destination cannot be used for DMA. */ #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 +/* Set this bit if source is a folio. */ +#define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 + +/* Set this bit if destination is a folio. */ +#define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 + #define CRYPTO_ACOMP_DST_MAX 131072 #define MAX_SYNC_COMP_REQSIZE 0 @@ -43,6 +49,7 @@ __##name##_req, (tfm), (gfp), false) struct acomp_req; +struct folio; struct acomp_req_chain { struct list_head head; @@ -53,16 +60,31 @@ struct acomp_req_chain { void *data; struct scatterlist ssg; struct scatterlist dsg; - const u8 *src; - u8 *dst; + union { + const u8 *src; + struct folio *sfolio; + }; + union { + u8 *dst; + struct folio *dfolio; + }; + size_t soff; + size_t doff; + u32 flags; }; /** * struct acomp_req - asynchronous (de)compression request * * @base: Common attributes for asynchronous crypto requests - * @src: Source Data - * @dst: Destination data + * @src: Source scatterlist + * @dst: Destination scatterlist + * @svirt: Source virtual address + * @dvirt: Destination virtual address + * @sfolio: Source folio + * @soff: Source folio offset + * @dfolio: Destination folio + * @doff: Destination folio offset * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced * @chain: Private API code data, do not use @@ -73,11 +95,15 @@ struct acomp_req { union { struct scatterlist *src; const u8 *svirt; + struct folio *sfolio; }; union { struct scatterlist *dst; u8 *dvirt; + struct folio *dfolio; }; + size_t soff; + size_t doff; unsigned int slen; unsigned int dlen; @@ -316,6 +342,7 @@ static inline void acomp_request_set_callback(struct acomp_req *req, { u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | + CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_TFM_REQ_ON_STACK; req->base.complete = cmpl; @@ -352,6 +379,8 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_SRC_FOLIO | + CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); } @@ -374,6 +403,7 @@ static inline void acomp_request_set_src_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; } /** @@ -393,6 +423,7 @@ static inline void acomp_request_set_src_dma(struct acomp_req *req, req->slen = slen; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -413,10 +444,34 @@ static inline void acomp_request_set_src_nondma(struct acomp_req *req, req->svirt = src; req->slen = slen; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } +/** + * acomp_request_set_src_folio() -- Sets source folio + * + * Sets source folio required by an acomp operation. + * + * @req: asynchronous compress request + * @folio: pointer to input folio + * @off: input folio offset + * @len: size of the input buffer + */ +static inline void acomp_request_set_src_folio(struct acomp_req *req, + struct folio *folio, size_t off, + unsigned int len) +{ + req->sfolio = folio; + req->soff = off; + req->slen = len; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; +} + /** * acomp_request_set_dst_sg() -- Sets destination scatterlist * @@ -435,6 +490,7 @@ static inline void acomp_request_set_dst_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; } /** @@ -454,6 +510,7 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dlen = dlen; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -473,10 +530,34 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } +/** + * acomp_request_set_dst_folio() -- Sets destination folio + * + * Sets destination folio required by an acomp operation. + * + * @req: asynchronous compress request + * @folio: pointer to input folio + * @off: input folio offset + * @len: size of the input buffer + */ +static inline void acomp_request_set_dst_folio(struct acomp_req *req, + struct folio *folio, size_t off, + unsigned int len) +{ + req->dfolio = folio; + req->doff = off; + req->dlen = len; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; +} + static inline void acomp_request_chain(struct acomp_req *req, struct acomp_req *head) { diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index c1ed55a0e3bf..aaf59f3236fa 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -103,6 +103,14 @@ static inline bool acomp_request_chained(struct acomp_req *req) return crypto_request_chained(&req->base); } +static inline bool acomp_request_issg(struct acomp_req *req) +{ + return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_SRC_FOLIO | + CRYPTO_ACOMP_REQ_DST_FOLIO)); +} + static inline bool acomp_request_src_isvirt(struct acomp_req *req) { return req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT; @@ -135,6 +143,16 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } +static inline bool acomp_request_src_isfolio(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; +} + +static inline bool acomp_request_dst_isfolio(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; +} + static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) { return crypto_tfm_req_chain(&tfm->base); -- cgit v1.2.3 From ddd0a42671c0d9742fda97d26068ffbb51dd7c01 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 15 Mar 2025 20:37:36 +0800 Subject: crypto: scompress - Fix scratch allocation failure handling If the scratch allocation fails, all subsequent allocations will silently succeed without actually allocating anything. Fix this by only incrementing users when the allocation succeeds. Fixes: 6a8487a1f29f ("crypto: scompress - defer allocation of scratch buffer to first use") Signed-off-by: Herbert Xu Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/scompress.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'crypto') diff --git a/crypto/scompress.c b/crypto/scompress.c index f85cc0d83164..5e0238af5425 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -159,8 +159,12 @@ static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) if (ret) goto unlock; } - if (!scomp_scratch_users++) + if (!scomp_scratch_users) { ret = crypto_scomp_alloc_scratches(); + if (ret) + goto unlock; + scomp_scratch_users++; + } unlock: mutex_unlock(&scomp_lock); -- cgit v1.2.3 From 2d985ff0072fd4ca2c52a24f6c5c4ffc3c969ee7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:06 +0800 Subject: crypto: 842 - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/842.c | 66 ++---------------------------------------------------------- 1 file changed, 2 insertions(+), 64 deletions(-) (limited to 'crypto') diff --git a/crypto/842.c b/crypto/842.c index 2238478c3493..5fb37a925989 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -18,11 +18,10 @@ * drivers/crypto/nx/nx-842-crypto.c */ +#include #include #include -#include #include -#include struct crypto842_ctx { void *wmem; /* working memory for compress */ @@ -39,38 +38,11 @@ static void *crypto842_alloc_ctx(void) return ctx; } -static int crypto842_init(struct crypto_tfm *tfm) -{ - struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->wmem = crypto842_alloc_ctx(); - if (IS_ERR(ctx->wmem)) - return -ENOMEM; - - return 0; -} - static void crypto842_free_ctx(void *ctx) { kfree(ctx); } -static void crypto842_exit(struct crypto_tfm *tfm) -{ - struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - - crypto842_free_ctx(ctx->wmem); -} - -static int crypto842_compress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - - return sw842_compress(src, slen, dst, dlen, ctx->wmem); -} - static int crypto842_scompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -78,13 +50,6 @@ static int crypto842_scompress(struct crypto_scomp *tfm, return sw842_compress(src, slen, dst, dlen, ctx); } -static int crypto842_decompress(struct crypto_tfm *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - return sw842_decompress(src, slen, dst, dlen); -} - static int crypto842_sdecompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -92,20 +57,6 @@ static int crypto842_sdecompress(struct crypto_scomp *tfm, return sw842_decompress(src, slen, dst, dlen); } -static struct crypto_alg alg = { - .cra_name = "842", - .cra_driver_name = "842-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct crypto842_ctx), - .cra_module = THIS_MODULE, - .cra_init = crypto842_init, - .cra_exit = crypto842_exit, - .cra_u = { .compress = { - .coa_compress = crypto842_compress, - .coa_decompress = crypto842_decompress } } -}; - static struct scomp_alg scomp = { .alloc_ctx = crypto842_alloc_ctx, .free_ctx = crypto842_free_ctx, @@ -121,25 +72,12 @@ static struct scomp_alg scomp = { static int __init crypto842_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } subsys_initcall(crypto842_mod_init); static void __exit crypto842_mod_exit(void) { - crypto_unregister_alg(&alg); crypto_unregister_scomp(&scomp); } module_exit(crypto842_mod_exit); -- cgit v1.2.3 From 0fd486363cc430b46475b44d559466d94f13756c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:08 +0800 Subject: crypto: deflate - drop obsolete 'comp' implementation No users of the obsolete 'comp' crypto compression API remain, so let's drop the software deflate version of it. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/deflate.c | 58 +------------------------------------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) (limited to 'crypto') diff --git a/crypto/deflate.c b/crypto/deflate.c index 1bf7184ad670..5c346c544093 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -130,13 +130,6 @@ static void *deflate_alloc_ctx(void) return ctx; } -static int deflate_init(struct crypto_tfm *tfm) -{ - struct deflate_ctx *ctx = crypto_tfm_ctx(tfm); - - return __deflate_init(ctx); -} - static void __deflate_exit(void *ctx) { deflate_comp_exit(ctx); @@ -149,13 +142,6 @@ static void deflate_free_ctx(void *ctx) kfree_sensitive(ctx); } -static void deflate_exit(struct crypto_tfm *tfm) -{ - struct deflate_ctx *ctx = crypto_tfm_ctx(tfm); - - __deflate_exit(ctx); -} - static int __deflate_compress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -185,14 +171,6 @@ out: return ret; } -static int deflate_compress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct deflate_ctx *dctx = crypto_tfm_ctx(tfm); - - return __deflate_compress(src, slen, dst, dlen, dctx); -} - static int deflate_scompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -241,14 +219,6 @@ out: return ret; } -static int deflate_decompress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct deflate_ctx *dctx = crypto_tfm_ctx(tfm); - - return __deflate_decompress(src, slen, dst, dlen, dctx); -} - static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -256,19 +226,6 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __deflate_decompress(src, slen, dst, dlen, ctx); } -static struct crypto_alg alg = { - .cra_name = "deflate", - .cra_driver_name = "deflate-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct deflate_ctx), - .cra_module = THIS_MODULE, - .cra_init = deflate_init, - .cra_exit = deflate_exit, - .cra_u = { .compress = { - .coa_compress = deflate_compress, - .coa_decompress = deflate_decompress } } -}; - static struct scomp_alg scomp = { .alloc_ctx = deflate_alloc_ctx, .free_ctx = deflate_free_ctx, @@ -283,24 +240,11 @@ static struct scomp_alg scomp = { static int __init deflate_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } static void __exit deflate_mod_fini(void) { - crypto_unregister_alg(&alg); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From 33335afe33c9169223c8d8814e842b1d66b64637 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:10 +0800 Subject: crypto: lz4 - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/lz4.c | 61 +----------------------------------------------------------- 1 file changed, 1 insertion(+), 60 deletions(-) (limited to 'crypto') diff --git a/crypto/lz4.c b/crypto/lz4.c index e66c6d1ba34f..82588607fb2e 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -27,29 +27,11 @@ static void *lz4_alloc_ctx(void) return ctx; } -static int lz4_init(struct crypto_tfm *tfm) -{ - struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->lz4_comp_mem = lz4_alloc_ctx(); - if (IS_ERR(ctx->lz4_comp_mem)) - return -ENOMEM; - - return 0; -} - static void lz4_free_ctx(void *ctx) { vfree(ctx); } -static void lz4_exit(struct crypto_tfm *tfm) -{ - struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - - lz4_free_ctx(ctx->lz4_comp_mem); -} - static int __lz4_compress_crypto(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -70,14 +52,6 @@ static int lz4_scompress(struct crypto_scomp *tfm, const u8 *src, return __lz4_compress_crypto(src, slen, dst, dlen, ctx); } -static int lz4_compress_crypto(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - - return __lz4_compress_crypto(src, slen, dst, dlen, ctx->lz4_comp_mem); -} - static int __lz4_decompress_crypto(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -97,26 +71,6 @@ static int lz4_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __lz4_decompress_crypto(src, slen, dst, dlen, NULL); } -static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, - unsigned int *dlen) -{ - return __lz4_decompress_crypto(src, slen, dst, dlen, NULL); -} - -static struct crypto_alg alg_lz4 = { - .cra_name = "lz4", - .cra_driver_name = "lz4-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct lz4_ctx), - .cra_module = THIS_MODULE, - .cra_init = lz4_init, - .cra_exit = lz4_exit, - .cra_u = { .compress = { - .coa_compress = lz4_compress_crypto, - .coa_decompress = lz4_decompress_crypto } } -}; - static struct scomp_alg scomp = { .alloc_ctx = lz4_alloc_ctx, .free_ctx = lz4_free_ctx, @@ -131,24 +85,11 @@ static struct scomp_alg scomp = { static int __init lz4_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg_lz4); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg_lz4); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } static void __exit lz4_mod_fini(void) { - crypto_unregister_alg(&alg_lz4); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From dbae96559eef6ad17a29bb7e63c9eb8d1994dcb1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:13 +0800 Subject: crypto: lz4hc - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/lz4hc.c | 66 ++-------------------------------------------------------- 1 file changed, 2 insertions(+), 64 deletions(-) (limited to 'crypto') diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index 25a95b65aca5..997e76c0183a 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -4,12 +4,11 @@ * * Copyright (c) 2013 Chanho Min */ +#include #include #include -#include #include #include -#include struct lz4hc_ctx { void *lz4hc_comp_mem; @@ -26,29 +25,11 @@ static void *lz4hc_alloc_ctx(void) return ctx; } -static int lz4hc_init(struct crypto_tfm *tfm) -{ - struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(); - if (IS_ERR(ctx->lz4hc_comp_mem)) - return -ENOMEM; - - return 0; -} - static void lz4hc_free_ctx(void *ctx) { vfree(ctx); } -static void lz4hc_exit(struct crypto_tfm *tfm) -{ - struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - - lz4hc_free_ctx(ctx->lz4hc_comp_mem); -} - static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -69,16 +50,6 @@ static int lz4hc_scompress(struct crypto_scomp *tfm, const u8 *src, return __lz4hc_compress_crypto(src, slen, dst, dlen, ctx); } -static int lz4hc_compress_crypto(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, - unsigned int *dlen) -{ - struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - - return __lz4hc_compress_crypto(src, slen, dst, dlen, - ctx->lz4hc_comp_mem); -} - static int __lz4hc_decompress_crypto(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -98,26 +69,6 @@ static int lz4hc_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __lz4hc_decompress_crypto(src, slen, dst, dlen, NULL); } -static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, - unsigned int *dlen) -{ - return __lz4hc_decompress_crypto(src, slen, dst, dlen, NULL); -} - -static struct crypto_alg alg_lz4hc = { - .cra_name = "lz4hc", - .cra_driver_name = "lz4hc-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct lz4hc_ctx), - .cra_module = THIS_MODULE, - .cra_init = lz4hc_init, - .cra_exit = lz4hc_exit, - .cra_u = { .compress = { - .coa_compress = lz4hc_compress_crypto, - .coa_decompress = lz4hc_decompress_crypto } } -}; - static struct scomp_alg scomp = { .alloc_ctx = lz4hc_alloc_ctx, .free_ctx = lz4hc_free_ctx, @@ -132,24 +83,11 @@ static struct scomp_alg scomp = { static int __init lz4hc_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg_lz4hc); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg_lz4hc); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } static void __exit lz4hc_mod_fini(void) { - crypto_unregister_alg(&alg_lz4hc); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From d32da55c5b0c940c44ef56951e4503ab61e515dc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:15 +0800 Subject: crypto: lzo-rle - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/lzo-rle.c | 68 ++++---------------------------------------------------- 1 file changed, 4 insertions(+), 64 deletions(-) (limited to 'crypto') diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 6c845e7d32f5..b1350ae278b8 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -3,13 +3,11 @@ * Cryptographic API. */ +#include #include -#include -#include -#include -#include #include -#include +#include +#include struct lzorle_ctx { void *lzorle_comp_mem; @@ -26,29 +24,11 @@ static void *lzorle_alloc_ctx(void) return ctx; } -static int lzorle_init(struct crypto_tfm *tfm) -{ - struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->lzorle_comp_mem = lzorle_alloc_ctx(); - if (IS_ERR(ctx->lzorle_comp_mem)) - return -ENOMEM; - - return 0; -} - static void lzorle_free_ctx(void *ctx) { kvfree(ctx); } -static void lzorle_exit(struct crypto_tfm *tfm) -{ - struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - - lzorle_free_ctx(ctx->lzorle_comp_mem); -} - static int __lzorle_compress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -64,14 +44,6 @@ static int __lzorle_compress(const u8 *src, unsigned int slen, return 0; } -static int lzorle_compress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - - return __lzorle_compress(src, slen, dst, dlen, ctx->lzorle_comp_mem); -} - static int lzorle_scompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -94,12 +66,6 @@ static int __lzorle_decompress(const u8 *src, unsigned int slen, return 0; } -static int lzorle_decompress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - return __lzorle_decompress(src, slen, dst, dlen); -} - static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -107,19 +73,6 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __lzorle_decompress(src, slen, dst, dlen); } -static struct crypto_alg alg = { - .cra_name = "lzo-rle", - .cra_driver_name = "lzo-rle-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct lzorle_ctx), - .cra_module = THIS_MODULE, - .cra_init = lzorle_init, - .cra_exit = lzorle_exit, - .cra_u = { .compress = { - .coa_compress = lzorle_compress, - .coa_decompress = lzorle_decompress } } -}; - static struct scomp_alg scomp = { .alloc_ctx = lzorle_alloc_ctx, .free_ctx = lzorle_free_ctx, @@ -134,24 +87,11 @@ static struct scomp_alg scomp = { static int __init lzorle_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } static void __exit lzorle_mod_fini(void) { - crypto_unregister_alg(&alg); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From a3e43a25bad0d3d8e2b26942f91c8eec9ce74ffa Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:17 +0800 Subject: crypto: lzo - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/lzo.c | 68 ++++-------------------------------------------------------- 1 file changed, 4 insertions(+), 64 deletions(-) (limited to 'crypto') diff --git a/crypto/lzo.c b/crypto/lzo.c index 035d62e2afe0..dfe5a07ca35f 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -3,13 +3,11 @@ * Cryptographic API. */ +#include #include -#include -#include -#include -#include #include -#include +#include +#include struct lzo_ctx { void *lzo_comp_mem; @@ -26,29 +24,11 @@ static void *lzo_alloc_ctx(void) return ctx; } -static int lzo_init(struct crypto_tfm *tfm) -{ - struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->lzo_comp_mem = lzo_alloc_ctx(); - if (IS_ERR(ctx->lzo_comp_mem)) - return -ENOMEM; - - return 0; -} - static void lzo_free_ctx(void *ctx) { kvfree(ctx); } -static void lzo_exit(struct crypto_tfm *tfm) -{ - struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - - lzo_free_ctx(ctx->lzo_comp_mem); -} - static int __lzo_compress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -64,14 +44,6 @@ static int __lzo_compress(const u8 *src, unsigned int slen, return 0; } -static int lzo_compress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - - return __lzo_compress(src, slen, dst, dlen, ctx->lzo_comp_mem); -} - static int lzo_scompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -94,12 +66,6 @@ static int __lzo_decompress(const u8 *src, unsigned int slen, return 0; } -static int lzo_decompress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - return __lzo_decompress(src, slen, dst, dlen); -} - static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -107,19 +73,6 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __lzo_decompress(src, slen, dst, dlen); } -static struct crypto_alg alg = { - .cra_name = "lzo", - .cra_driver_name = "lzo-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct lzo_ctx), - .cra_module = THIS_MODULE, - .cra_init = lzo_init, - .cra_exit = lzo_exit, - .cra_u = { .compress = { - .coa_compress = lzo_compress, - .coa_decompress = lzo_decompress } } -}; - static struct scomp_alg scomp = { .alloc_ctx = lzo_alloc_ctx, .free_ctx = lzo_free_ctx, @@ -134,24 +87,11 @@ static struct scomp_alg scomp = { static int __init lzo_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) { - crypto_unregister_alg(&alg); - return ret; - } - - return ret; + return crypto_register_scomp(&scomp); } static void __exit lzo_mod_fini(void) { - crypto_unregister_alg(&alg); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From 8beb40458c213e180fad5ffe0b7207a6bc9dfe61 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:20 +0800 Subject: crypto: zstd - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/zstd.c | 56 +------------------------------------------------------- 1 file changed, 1 insertion(+), 55 deletions(-) (limited to 'crypto') diff --git a/crypto/zstd.c b/crypto/zstd.c index 68a093427944..90bb4f36f846 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -121,13 +121,6 @@ static void *zstd_alloc_ctx(void) return ctx; } -static int zstd_init(struct crypto_tfm *tfm) -{ - struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); - - return __zstd_init(ctx); -} - static void __zstd_exit(void *ctx) { zstd_comp_exit(ctx); @@ -140,13 +133,6 @@ static void zstd_free_ctx(void *ctx) kfree_sensitive(ctx); } -static void zstd_exit(struct crypto_tfm *tfm) -{ - struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); - - __zstd_exit(ctx); -} - static int __zstd_compress(const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) { @@ -161,14 +147,6 @@ static int __zstd_compress(const u8 *src, unsigned int slen, return 0; } -static int zstd_compress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); - - return __zstd_compress(src, slen, dst, dlen, ctx); -} - static int zstd_scompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -189,14 +167,6 @@ static int __zstd_decompress(const u8 *src, unsigned int slen, return 0; } -static int zstd_decompress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - struct zstd_ctx *ctx = crypto_tfm_ctx(tfm); - - return __zstd_decompress(src, slen, dst, dlen, ctx); -} - static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) @@ -204,19 +174,6 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, return __zstd_decompress(src, slen, dst, dlen, ctx); } -static struct crypto_alg alg = { - .cra_name = "zstd", - .cra_driver_name = "zstd-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_ctxsize = sizeof(struct zstd_ctx), - .cra_module = THIS_MODULE, - .cra_init = zstd_init, - .cra_exit = zstd_exit, - .cra_u = { .compress = { - .coa_compress = zstd_compress, - .coa_decompress = zstd_decompress } } -}; - static struct scomp_alg scomp = { .alloc_ctx = zstd_alloc_ctx, .free_ctx = zstd_free_ctx, @@ -231,22 +188,11 @@ static struct scomp_alg scomp = { static int __init zstd_mod_init(void) { - int ret; - - ret = crypto_register_alg(&alg); - if (ret) - return ret; - - ret = crypto_register_scomp(&scomp); - if (ret) - crypto_unregister_alg(&alg); - - return ret; + return crypto_register_scomp(&scomp); } static void __exit zstd_mod_fini(void) { - crypto_unregister_alg(&alg); crypto_unregister_scomp(&scomp); } -- cgit v1.2.3 From be457e4e8da6e0a797dba2344ac34de647a5322e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:24 +0800 Subject: crypto: compress_null - drop obsolete 'comp' implementation The 'comp' API is obsolete and will be removed, so remove this comp implementation. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/crypto_null.c | 31 +++++-------------------------- crypto/testmgr.c | 3 --- 2 files changed, 5 insertions(+), 29 deletions(-) (limited to 'crypto') diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 337867028653..ced90f88ee07 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -24,16 +24,6 @@ static DEFINE_SPINLOCK(crypto_default_null_skcipher_lock); static struct crypto_sync_skcipher *crypto_default_null_skcipher; static int crypto_default_null_skcipher_refcnt; -static int null_compress(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen) -{ - if (slen > *dlen) - return -EINVAL; - memcpy(dst, src, slen); - *dlen = slen; - return 0; -} - static int null_init(struct shash_desc *desc) { return 0; @@ -121,7 +111,7 @@ static struct skcipher_alg skcipher_null = { .decrypt = null_skcipher_crypt, }; -static struct crypto_alg null_algs[] = { { +static struct crypto_alg cipher_null = { .cra_name = "cipher_null", .cra_driver_name = "cipher_null-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, @@ -134,19 +124,8 @@ static struct crypto_alg null_algs[] = { { .cia_setkey = null_setkey, .cia_encrypt = null_crypt, .cia_decrypt = null_crypt } } -}, { - .cra_name = "compress_null", - .cra_driver_name = "compress_null-generic", - .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, - .cra_blocksize = NULL_BLOCK_SIZE, - .cra_ctxsize = 0, - .cra_module = THIS_MODULE, - .cra_u = { .compress = { - .coa_compress = null_compress, - .coa_decompress = null_compress } } -} }; +}; -MODULE_ALIAS_CRYPTO("compress_null"); MODULE_ALIAS_CRYPTO("digest_null"); MODULE_ALIAS_CRYPTO("cipher_null"); @@ -202,7 +181,7 @@ static int __init crypto_null_mod_init(void) { int ret = 0; - ret = crypto_register_algs(null_algs, ARRAY_SIZE(null_algs)); + ret = crypto_register_alg(&cipher_null); if (ret < 0) goto out; @@ -219,14 +198,14 @@ static int __init crypto_null_mod_init(void) out_unregister_shash: crypto_unregister_shash(&digest_null); out_unregister_algs: - crypto_unregister_algs(null_algs, ARRAY_SIZE(null_algs)); + crypto_unregister_alg(&cipher_null); out: return ret; } static void __exit crypto_null_mod_fini(void) { - crypto_unregister_algs(null_algs, ARRAY_SIZE(null_algs)); + crypto_unregister_alg(&cipher_null); crypto_unregister_shash(&digest_null); crypto_unregister_skcipher(&skcipher_null); } diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 140872765dcd..9c5648c45ff0 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4726,9 +4726,6 @@ static const struct alg_test_desc alg_test_descs[] = { .suite = { .hash = __VECS(sm4_cmac128_tv_template) } - }, { - .alg = "compress_null", - .test = alg_test_null, }, { .alg = "crc32", .test = alg_test_hash, -- cgit v1.2.3 From fce8b8d5986b76a4fdd062e3eec1bb6420fee6c5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 16 Mar 2025 09:21:27 +0800 Subject: crypto: remove obsolete 'comp' compression API The 'comp' compression API has been superseded by the acomp API, which is a bit more cumbersome to use, but ultimately more flexible when it comes to hardware implementations. Now that all the users and implementations have been removed, let's remove the core plumbing of the 'comp' API as well. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- Documentation/crypto/architecture.rst | 2 - crypto/Makefile | 2 +- crypto/api.c | 4 - crypto/compress.c | 32 ------- crypto/crypto_user.c | 16 ---- crypto/proc.c | 3 - crypto/testmgr.c | 152 +++------------------------------- include/linux/crypto.h | 76 +---------------- 8 files changed, 15 insertions(+), 272 deletions(-) delete mode 100644 crypto/compress.c (limited to 'crypto') diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst index 15dcd62fd22f..249b54d0849f 100644 --- a/Documentation/crypto/architecture.rst +++ b/Documentation/crypto/architecture.rst @@ -196,8 +196,6 @@ the aforementioned cipher types: - CRYPTO_ALG_TYPE_CIPHER Single block cipher -- CRYPTO_ALG_TYPE_COMPRESS Compression - - CRYPTO_ALG_TYPE_AEAD Authenticated Encryption with Associated Data (MAC) diff --git a/crypto/Makefile b/crypto/Makefile index d1e422249af6..f22ebd6fb221 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -4,7 +4,7 @@ # obj-$(CONFIG_CRYPTO) += crypto.o -crypto-y := api.o cipher.o compress.o +crypto-y := api.o cipher.o obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o obj-$(CONFIG_CRYPTO_FIPS) += fips.o diff --git a/crypto/api.c b/crypto/api.c index 91957bb52f3f..3416e98128a0 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -383,10 +383,6 @@ static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask) case CRYPTO_ALG_TYPE_CIPHER: len += crypto_cipher_ctxsize(alg); break; - - case CRYPTO_ALG_TYPE_COMPRESS: - len += crypto_compress_ctxsize(alg); - break; } return len; diff --git a/crypto/compress.c b/crypto/compress.c deleted file mode 100644 index 9048fe390c46..000000000000 --- a/crypto/compress.c +++ /dev/null @@ -1,32 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * Compression operations. - * - * Copyright (c) 2002 James Morris - */ -#include -#include "internal.h" - -int crypto_comp_compress(struct crypto_comp *comp, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct crypto_tfm *tfm = crypto_comp_tfm(comp); - - return tfm->__crt_alg->cra_compress.coa_compress(tfm, src, slen, dst, - dlen); -} -EXPORT_SYMBOL_GPL(crypto_comp_compress); - -int crypto_comp_decompress(struct crypto_comp *comp, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen) -{ - struct crypto_tfm *tfm = crypto_comp_tfm(comp); - - return tfm->__crt_alg->cra_compress.coa_decompress(tfm, src, slen, dst, - dlen); -} -EXPORT_SYMBOL_GPL(crypto_comp_decompress); diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c index 6c571834e86a..aad429bef03e 100644 --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -84,17 +84,6 @@ static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg) sizeof(rcipher), &rcipher); } -static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg) -{ - struct crypto_report_comp rcomp; - - memset(&rcomp, 0, sizeof(rcomp)); - - strscpy(rcomp.type, "compression", sizeof(rcomp.type)); - - return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, sizeof(rcomp), &rcomp); -} - static int crypto_report_one(struct crypto_alg *alg, struct crypto_user_alg *ualg, struct sk_buff *skb) { @@ -135,11 +124,6 @@ static int crypto_report_one(struct crypto_alg *alg, if (crypto_report_cipher(skb, alg)) goto nla_put_failure; - break; - case CRYPTO_ALG_TYPE_COMPRESS: - if (crypto_report_comp(skb, alg)) - goto nla_put_failure; - break; } diff --git a/crypto/proc.c b/crypto/proc.c index 522b27d90d29..82f15b967e85 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -72,9 +72,6 @@ static int c_show(struct seq_file *m, void *p) seq_printf(m, "max keysize : %u\n", alg->cra_cipher.cia_max_keysize); break; - case CRYPTO_ALG_TYPE_COMPRESS: - seq_printf(m, "type : compression\n"); - break; default: seq_printf(m, "type : unknown\n"); break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 9c5648c45ff0..1b2387291787 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3320,112 +3320,6 @@ out: return err; } -static int test_comp(struct crypto_comp *tfm, - const struct comp_testvec *ctemplate, - const struct comp_testvec *dtemplate, - int ctcount, int dtcount) -{ - const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm)); - char *output, *decomp_output; - unsigned int i; - int ret; - - output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!output) - return -ENOMEM; - - decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!decomp_output) { - kfree(output); - return -ENOMEM; - } - - for (i = 0; i < ctcount; i++) { - int ilen; - unsigned int dlen = COMP_BUF_SIZE; - - memset(output, 0, COMP_BUF_SIZE); - memset(decomp_output, 0, COMP_BUF_SIZE); - - ilen = ctemplate[i].inlen; - ret = crypto_comp_compress(tfm, ctemplate[i].input, - ilen, output, &dlen); - if (ret) { - printk(KERN_ERR "alg: comp: compression failed " - "on test %d for %s: ret=%d\n", i + 1, algo, - -ret); - goto out; - } - - ilen = dlen; - dlen = COMP_BUF_SIZE; - ret = crypto_comp_decompress(tfm, output, - ilen, decomp_output, &dlen); - if (ret) { - pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n", - i + 1, algo, -ret); - goto out; - } - - if (dlen != ctemplate[i].inlen) { - printk(KERN_ERR "alg: comp: Compression test %d " - "failed for %s: output len = %d\n", i + 1, algo, - dlen); - ret = -EINVAL; - goto out; - } - - if (memcmp(decomp_output, ctemplate[i].input, - ctemplate[i].inlen)) { - pr_err("alg: comp: compression failed: output differs: on test %d for %s\n", - i + 1, algo); - hexdump(decomp_output, dlen); - ret = -EINVAL; - goto out; - } - } - - for (i = 0; i < dtcount; i++) { - int ilen; - unsigned int dlen = COMP_BUF_SIZE; - - memset(decomp_output, 0, COMP_BUF_SIZE); - - ilen = dtemplate[i].inlen; - ret = crypto_comp_decompress(tfm, dtemplate[i].input, - ilen, decomp_output, &dlen); - if (ret) { - printk(KERN_ERR "alg: comp: decompression failed " - "on test %d for %s: ret=%d\n", i + 1, algo, - -ret); - goto out; - } - - if (dlen != dtemplate[i].outlen) { - printk(KERN_ERR "alg: comp: Decompression test %d " - "failed for %s: output len = %d\n", i + 1, algo, - dlen); - ret = -EINVAL; - goto out; - } - - if (memcmp(decomp_output, dtemplate[i].output, dlen)) { - printk(KERN_ERR "alg: comp: Decompression test %d " - "failed for %s\n", i + 1, algo); - hexdump(decomp_output, dlen); - ret = -EINVAL; - goto out; - } - } - - ret = 0; - -out: - kfree(decomp_output); - kfree(output); - return ret; -} - static int test_acomp(struct crypto_acomp *tfm, const struct comp_testvec *ctemplate, const struct comp_testvec *dtemplate, @@ -3684,42 +3578,22 @@ static int alg_test_cipher(const struct alg_test_desc *desc, static int alg_test_comp(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask) { - struct crypto_comp *comp; struct crypto_acomp *acomp; int err; - u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK; - if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) { - acomp = crypto_alloc_acomp(driver, type, mask); - if (IS_ERR(acomp)) { - if (PTR_ERR(acomp) == -ENOENT) - return 0; - pr_err("alg: acomp: Failed to load transform for %s: %ld\n", - driver, PTR_ERR(acomp)); - return PTR_ERR(acomp); - } - err = test_acomp(acomp, desc->suite.comp.comp.vecs, - desc->suite.comp.decomp.vecs, - desc->suite.comp.comp.count, - desc->suite.comp.decomp.count); - crypto_free_acomp(acomp); - } else { - comp = crypto_alloc_comp(driver, type, mask); - if (IS_ERR(comp)) { - if (PTR_ERR(comp) == -ENOENT) - return 0; - pr_err("alg: comp: Failed to load transform for %s: %ld\n", - driver, PTR_ERR(comp)); - return PTR_ERR(comp); - } - - err = test_comp(comp, desc->suite.comp.comp.vecs, - desc->suite.comp.decomp.vecs, - desc->suite.comp.comp.count, - desc->suite.comp.decomp.count); - - crypto_free_comp(comp); - } + acomp = crypto_alloc_acomp(driver, type, mask); + if (IS_ERR(acomp)) { + if (PTR_ERR(acomp) == -ENOENT) + return 0; + pr_err("alg: acomp: Failed to load transform for %s: %ld\n", + driver, PTR_ERR(acomp)); + return PTR_ERR(acomp); + } + err = test_acomp(acomp, desc->suite.comp.comp.vecs, + desc->suite.comp.decomp.vecs, + desc->suite.comp.comp.count, + desc->suite.comp.decomp.count); + crypto_free_acomp(acomp); return err; } diff --git a/include/linux/crypto.h b/include/linux/crypto.h index ea3b95bdbde3..1e3809d28abd 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -24,7 +24,6 @@ */ #define CRYPTO_ALG_TYPE_MASK 0x0000000f #define CRYPTO_ALG_TYPE_CIPHER 0x00000001 -#define CRYPTO_ALG_TYPE_COMPRESS 0x00000002 #define CRYPTO_ALG_TYPE_AEAD 0x00000003 #define CRYPTO_ALG_TYPE_LSKCIPHER 0x00000004 #define CRYPTO_ALG_TYPE_SKCIPHER 0x00000005 @@ -246,26 +245,7 @@ struct cipher_alg { void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); }; -/** - * struct compress_alg - compression/decompression algorithm - * @coa_compress: Compress a buffer of specified length, storing the resulting - * data in the specified buffer. Return the length of the - * compressed data in dlen. - * @coa_decompress: Decompress the source buffer, storing the uncompressed - * data in the specified buffer. The length of the data is - * returned in dlen. - * - * All fields are mandatory. - */ -struct compress_alg { - int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen); - int (*coa_decompress)(struct crypto_tfm *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int *dlen); -}; - #define cra_cipher cra_u.cipher -#define cra_compress cra_u.compress /** * struct crypto_alg - definition of a cryptograpic cipher algorithm @@ -316,7 +296,7 @@ struct compress_alg { * transformation types. There are multiple options, such as * &crypto_skcipher_type, &crypto_ahash_type, &crypto_rng_type. * This field might be empty. In that case, there are no common - * callbacks. This is the case for: cipher, compress, shash. + * callbacks. This is the case for: cipher. * @cra_u: Callbacks implementing the transformation. This is a union of * multiple structures. Depending on the type of transformation selected * by @cra_type and @cra_flags above, the associated structure must be @@ -335,8 +315,6 @@ struct compress_alg { * @cra_init. * @cra_u.cipher: Union member which contains a single-block symmetric cipher * definition. See @struct @cipher_alg. - * @cra_u.compress: Union member which contains a (de)compression algorithm. - * See @struct @compress_alg. * @cra_module: Owner of this transformation implementation. Set to THIS_MODULE * @cra_list: internally used * @cra_users: internally used @@ -366,7 +344,6 @@ struct crypto_alg { union { struct cipher_alg cipher; - struct compress_alg compress; } cra_u; int (*cra_init)(struct crypto_tfm *tfm); @@ -440,10 +417,6 @@ struct crypto_tfm { void *__crt_ctx[] CRYPTO_MINALIGN_ATTR; }; -struct crypto_comp { - struct crypto_tfm base; -}; - /* * Transform user interface. */ @@ -500,53 +473,6 @@ static inline unsigned int crypto_tfm_ctx_alignment(void) return __alignof__(tfm->__crt_ctx); } -static inline struct crypto_comp *__crypto_comp_cast(struct crypto_tfm *tfm) -{ - return (struct crypto_comp *)tfm; -} - -static inline struct crypto_comp *crypto_alloc_comp(const char *alg_name, - u32 type, u32 mask) -{ - type &= ~CRYPTO_ALG_TYPE_MASK; - type |= CRYPTO_ALG_TYPE_COMPRESS; - mask |= CRYPTO_ALG_TYPE_MASK; - - return __crypto_comp_cast(crypto_alloc_base(alg_name, type, mask)); -} - -static inline struct crypto_tfm *crypto_comp_tfm(struct crypto_comp *tfm) -{ - return &tfm->base; -} - -static inline void crypto_free_comp(struct crypto_comp *tfm) -{ - crypto_free_tfm(crypto_comp_tfm(tfm)); -} - -static inline int crypto_has_comp(const char *alg_name, u32 type, u32 mask) -{ - type &= ~CRYPTO_ALG_TYPE_MASK; - type |= CRYPTO_ALG_TYPE_COMPRESS; - mask |= CRYPTO_ALG_TYPE_MASK; - - return crypto_has_alg(alg_name, type, mask); -} - -static inline const char *crypto_comp_name(struct crypto_comp *tfm) -{ - return crypto_tfm_alg_name(crypto_comp_tfm(tfm)); -} - -int crypto_comp_compress(struct crypto_comp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); - -int crypto_comp_decompress(struct crypto_comp *tfm, - const u8 *src, unsigned int slen, - u8 *dst, unsigned int *dlen); - static inline void crypto_reqchain_init(struct crypto_async_request *req) { req->err = -EINPROGRESS; -- cgit v1.2.3 From ca17aa664054a5b809dc823ff1c202370ef398ef Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Sat, 15 Mar 2025 21:57:47 -0700 Subject: crypto: lib/chacha - remove unused arch-specific init support All implementations of chacha_init_arch() just call chacha_init_generic(), so it is pointless. Just delete it, and replace chacha_init() with what was previously chacha_init_generic(). Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-glue.c | 10 ++-------- arch/arm64/crypto/chacha-neon-glue.c | 10 ++-------- arch/mips/crypto/chacha-glue.c | 10 ++-------- arch/powerpc/crypto/chacha-p10-glue.c | 10 ++-------- arch/s390/crypto/chacha-glue.c | 8 +------- arch/x86/crypto/chacha_glue.c | 10 ++-------- crypto/chacha_generic.c | 4 ++-- include/crypto/chacha.h | 11 +---------- tools/testing/crypto/chacha20-s390/test-cipher.c | 4 ++-- 9 files changed, 16 insertions(+), 61 deletions(-) (limited to 'crypto') diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index cdde8fd01f8f..50e635512046 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -76,12 +76,6 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { @@ -116,7 +110,7 @@ static int chacha_stream_xor(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -166,7 +160,7 @@ static int do_xchacha(struct skcipher_request *req, bool neon) u32 state[16]; u8 real_iv[16]; - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) { hchacha_block_arm(state, subctx.key, ctx->nrounds); diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index af2bbca38e70..229876acfc58 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -74,12 +74,6 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { @@ -110,7 +104,7 @@ static int chacha_neon_stream_xor(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -151,7 +145,7 @@ static int xchacha_neon(struct skcipher_request *req) u32 state[16]; u8 real_iv[16]; - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); hchacha_block_arch(state, subctx.key, ctx->nrounds); subctx.nrounds = ctx->nrounds; diff --git a/arch/mips/crypto/chacha-glue.c b/arch/mips/crypto/chacha-glue.c index d1fd23e6ef84..f6fc2e1079a1 100644 --- a/arch/mips/crypto/chacha-glue.c +++ b/arch/mips/crypto/chacha-glue.c @@ -20,12 +20,6 @@ EXPORT_SYMBOL(chacha_crypt_arch); asmlinkage void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds); EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - static int chacha_mips_stream_xor(struct skcipher_request *req, const struct chacha_ctx *ctx, const u8 *iv) { @@ -35,7 +29,7 @@ static int chacha_mips_stream_xor(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -67,7 +61,7 @@ static int xchacha_mips(struct skcipher_request *req) u32 state[16]; u8 real_iv[16]; - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); hchacha_block(state, subctx.key, ctx->nrounds); subctx.nrounds = ctx->nrounds; diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c index 7c728755852e..d8796decc1fb 100644 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ b/arch/powerpc/crypto/chacha-p10-glue.c @@ -57,12 +57,6 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { @@ -95,7 +89,7 @@ static int chacha_p10_stream_xor(struct skcipher_request *req, if (err) return err; - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -137,7 +131,7 @@ static int xchacha_p10(struct skcipher_request *req) u32 state[16]; u8 real_iv[16]; - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); hchacha_block_arch(state, subctx.key, ctx->nrounds); subctx.nrounds = ctx->nrounds; diff --git a/arch/s390/crypto/chacha-glue.c b/arch/s390/crypto/chacha-glue.c index f8b0c52e77a4..920e9f0941e7 100644 --- a/arch/s390/crypto/chacha-glue.c +++ b/arch/s390/crypto/chacha-glue.c @@ -41,7 +41,7 @@ static int chacha20_s390(struct skcipher_request *req) int rc; rc = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); while (walk.nbytes > 0) { nbytes = walk.nbytes; @@ -69,12 +69,6 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 7b3a1cf0984b..8bb74a272879 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -133,12 +133,6 @@ void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) } EXPORT_SYMBOL(hchacha_block_arch); -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds) { @@ -169,7 +163,7 @@ static int chacha_simd_stream_xor(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -211,7 +205,7 @@ static int xchacha_simd(struct skcipher_request *req) struct chacha_ctx subctx; u8 real_iv[16]; - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); if (req->cryptlen > CHACHA_BLOCK_SIZE && crypto_simd_usable()) { kernel_fpu_begin(); diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index ba7fcb47f9aa..1fb9fbd302c6 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -21,7 +21,7 @@ static int chacha_stream_xor(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); - chacha_init_generic(state, ctx->key, iv); + chacha_init(state, ctx->key, iv); while (walk.nbytes > 0) { unsigned int nbytes = walk.nbytes; @@ -54,7 +54,7 @@ static int crypto_xchacha_crypt(struct skcipher_request *req) u8 real_iv[16]; /* Compute the subkey given the original key and first 128 nonce bits */ - chacha_init_generic(state, ctx->key, req->iv); + chacha_init(state, ctx->key, req->iv); hchacha_block_generic(state, subctx.key, ctx->nrounds); subctx.nrounds = ctx->nrounds; diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 5bae6a55b333..f8cc073bba41 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -62,8 +62,7 @@ static inline void chacha_init_consts(u32 *state) state[3] = CHACHA_CONSTANT_TE_K; } -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); -static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init(u32 *state, const u32 *key, const u8 *iv) { chacha_init_consts(state); state[4] = key[0]; @@ -80,14 +79,6 @@ static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) state[15] = get_unaligned_le32(iv + 12); } -static inline void chacha_init(u32 *state, const u32 *key, const u8 *iv) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) - chacha_init_arch(state, key, iv); - else - chacha_init_generic(state, key, iv); -} - void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, int nrounds); void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, diff --git a/tools/testing/crypto/chacha20-s390/test-cipher.c b/tools/testing/crypto/chacha20-s390/test-cipher.c index 8141d45df51a..35ea65c54ffa 100644 --- a/tools/testing/crypto/chacha20-s390/test-cipher.c +++ b/tools/testing/crypto/chacha20-s390/test-cipher.c @@ -66,7 +66,7 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) } /* Encrypt */ - chacha_init_arch(chacha_state, (u32*)key, iv); + chacha_init(chacha_state, (u32 *)key, iv); start = ktime_get_ns(); chacha_crypt_arch(chacha_state, cipher, plain, data_size, 20); @@ -81,7 +81,7 @@ static int test_lib_chacha(u8 *revert, u8 *cipher, u8 *plain) pr_info("lib encryption took: %lld nsec", end - start); /* Decrypt */ - chacha_init_arch(chacha_state, (u32 *)key, iv); + chacha_init(chacha_state, (u32 *)key, iv); start = ktime_get_ns(); chacha_crypt_arch(chacha_state, revert, cipher, data_size, 20); -- cgit v1.2.3 From 5a06ef1f8da226b2de587e22c17f88b72cede3be Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Mar 2025 17:50:22 +0800 Subject: crypto: scompress - Fix incorrect stream freeing Fix stream freeing crash by passing the correct pointer. Fixes: 3d72ad46a23a ("crypto: acomp - Move stream management into scomp layer") Signed-off-by: Herbert Xu --- crypto/scompress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'crypto') diff --git a/crypto/scompress.c b/crypto/scompress.c index 5e0238af5425..d435d4b24469 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -117,7 +117,7 @@ static void scomp_free_streams(struct scomp_alg *alg) if (!ps->ctx) break; - alg->free_ctx(ps); + alg->free_ctx(ps->ctx); } free_percpu(stream); -- cgit v1.2.3 From 27b13425349e94ad77b174b032674097cab241c8 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 16 Mar 2025 17:50:24 +0800 Subject: crypto: api - Call crypto_alg_put in crypto_unregister_alg Instead of calling cra_destroy by hand, call it through crypto_alg_put so that the correct unwinding functions are called through crypto_destroy_alg. Fixes: 3d6979bf3bd5 ("crypto: api - Add cra_type->destroy hook") Signed-off-by: Herbert Xu --- crypto/algapi.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/algapi.c b/crypto/algapi.c index e7a9a2ada2cf..ea9ed9580aa8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -464,8 +464,7 @@ void crypto_unregister_alg(struct crypto_alg *alg) if (WARN_ON(refcount_read(&alg->cra_refcnt) != 1)) return; - if (alg->cra_destroy) - alg->cra_destroy(alg); + crypto_alg_put(alg); crypto_remove_final(&list); } -- cgit v1.2.3 From fdd305803bc1eb59c300503fb37911330692ef69 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Sun, 16 Mar 2025 22:15:04 +0100 Subject: crypto: essiv - Replace memcpy() + NUL-termination with strscpy() Use strscpy() to copy the NUL-terminated string 'p' to the destination buffer instead of using memcpy() followed by a manual NUL-termination. No functional changes intended. Signed-off-by: Thorsten Blum Signed-off-by: Herbert Xu --- crypto/essiv.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'crypto') diff --git a/crypto/essiv.c b/crypto/essiv.c index 1c00c3324058..ec0ec8992c2d 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -405,8 +405,7 @@ static bool parse_cipher_name(char *essiv_cipher_name, const char *cra_name) if (len >= CRYPTO_MAX_ALG_NAME) return false; - memcpy(essiv_cipher_name, p, len); - essiv_cipher_name[len] = '\0'; + strscpy(essiv_cipher_name, p, len + 1); return true; } -- cgit v1.2.3 From 108ce629cf73a3df32fa04b17aedc97c1431b2ac Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 21 Mar 2025 16:43:28 +0800 Subject: crypto: hash - Fix synchronous ahash chaining fallback The synchronous ahash fallback code paths are broken because the ahash_restore_req assumes there is always a state object. Fix this by removing the state from ahash_restore_req and localising it to the asynchronous completion callback. Also add a missing synchronous finish call in ahash_def_digest_finish. Fixes: f2ffe5a9183d ("crypto: hash - Add request chaining API") Fixes: 439963cdc3aa ("crypto: ahash - Add virtual address support") Signed-off-by: Herbert Xu --- crypto/ahash.c | 60 +++++++++++++++++++++++++++------------------------------- 1 file changed, 28 insertions(+), 32 deletions(-) (limited to 'crypto') diff --git a/crypto/ahash.c b/crypto/ahash.c index 06559e5a715b..9f57b925b116 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -58,8 +58,9 @@ struct ahash_save_req_state { static void ahash_reqchain_done(void *data, int err); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); -static void ahash_restore_req(struct ahash_save_req_state *state); +static void ahash_restore_req(struct ahash_request *req); static void ahash_def_finup_done1(void *data, int err); +static int ahash_def_finup_finish1(struct ahash_request *req, int err); static int ahash_def_finup(struct ahash_request *req); static int hash_walk_next(struct crypto_hash_walk *walk) @@ -369,14 +370,15 @@ static int ahash_reqchain_virt(struct ahash_save_req_state *state, return err; } -static int ahash_reqchain_finish(struct ahash_save_req_state *state, +static int ahash_reqchain_finish(struct ahash_request *req0, + struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req0 = state->req0; struct ahash_request *req = state->cur; struct crypto_ahash *tfm; struct ahash_request *n; bool update; + u8 *page; err = ahash_reqchain_virt(state, err, mask); if (err == -EINPROGRESS || err == -EBUSY) @@ -430,7 +432,12 @@ static int ahash_reqchain_finish(struct ahash_save_req_state *state, list_add_tail(&req->base.list, &req0->base.list); } - ahash_restore_req(state); + page = state->page; + if (page) { + memset(page, 0, PAGE_SIZE); + free_page((unsigned long)page); + } + ahash_restore_req(req0); out: return err; @@ -449,7 +456,8 @@ static void ahash_reqchain_done(void *data, int err) goto notify; } - err = ahash_reqchain_finish(state, err, CRYPTO_TFM_REQ_MAY_BACKLOG); + err = ahash_reqchain_finish(state->req0, state, err, + CRYPTO_TFM_REQ_MAY_BACKLOG); if (err == -EBUSY) return; @@ -525,13 +533,10 @@ static int ahash_do_req_chain(struct ahash_request *req, if (err == -EBUSY || err == -EINPROGRESS) return -EBUSY; - return ahash_reqchain_finish(state, err, ~0); + return ahash_reqchain_finish(req, state, err, ~0); out_free_page: - if (page) { - memset(page, 0, PAGE_SIZE); - free_page((unsigned long)page); - } + free_page((unsigned long)page); out_set_chain: req->base.err = err; @@ -590,18 +595,15 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) req->base.complete = cplt; req->base.data = state; state->req0 = req; - state->page = NULL; return 0; } -static void ahash_restore_req(struct ahash_save_req_state *state) +static void ahash_restore_req(struct ahash_request *req) { - struct ahash_request *req = state->req0; + struct ahash_save_req_state *state; struct crypto_ahash *tfm; - free_page((unsigned long)state->page); - tfm = crypto_ahash_reqtfm(req); if (!ahash_is_async(tfm)) return; @@ -692,9 +694,8 @@ int crypto_ahash_finup(struct ahash_request *req) } EXPORT_SYMBOL_GPL(crypto_ahash_finup); -static int ahash_def_digest_finish(struct ahash_save_req_state *state, int err) +static int ahash_def_digest_finish(struct ahash_request *req, int err) { - struct ahash_request *req = state->req0; struct crypto_ahash *tfm; if (err) @@ -708,8 +709,10 @@ static int ahash_def_digest_finish(struct ahash_save_req_state *state, int err) if (err == -EINPROGRESS || err == -EBUSY) return err; + return ahash_def_finup_finish1(req, err); + out: - ahash_restore_req(state); + ahash_restore_req(req); return err; } @@ -726,7 +729,7 @@ static void ahash_def_digest_done(void *data, int err) areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - err = ahash_def_digest_finish(state0, err); + err = ahash_def_digest_finish(areq, err); if (err == -EINPROGRESS || err == -EBUSY) return; @@ -736,20 +739,17 @@ out: static int ahash_def_digest(struct ahash_request *req) { - struct ahash_save_req_state *state; int err; err = ahash_save_req(req, ahash_def_digest_done); if (err) return err; - state = req->base.data; - err = crypto_ahash_init(req); if (err == -EINPROGRESS || err == -EBUSY) return err; - return ahash_def_digest_finish(state, err); + return ahash_def_digest_finish(req, err); } int crypto_ahash_digest(struct ahash_request *req) @@ -791,13 +791,12 @@ static void ahash_def_finup_done2(void *data, int err) if (err == -EINPROGRESS) return; - ahash_restore_req(state); + ahash_restore_req(areq); ahash_request_complete(areq, err); } -static int ahash_def_finup_finish1(struct ahash_save_req_state *state, int err) +static int ahash_def_finup_finish1(struct ahash_request *req, int err) { - struct ahash_request *req = state->req0; struct crypto_ahash *tfm; if (err) @@ -812,7 +811,7 @@ static int ahash_def_finup_finish1(struct ahash_save_req_state *state, int err) return err; out: - ahash_restore_req(state); + ahash_restore_req(req); return err; } @@ -829,7 +828,7 @@ static void ahash_def_finup_done1(void *data, int err) areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - err = ahash_def_finup_finish1(state0, err); + err = ahash_def_finup_finish1(areq, err); if (err == -EINPROGRESS || err == -EBUSY) return; @@ -839,20 +838,17 @@ out: static int ahash_def_finup(struct ahash_request *req) { - struct ahash_save_req_state *state; int err; err = ahash_save_req(req, ahash_def_finup_done1); if (err) return err; - state = req->base.data; - err = crypto_ahash_update(req); if (err == -EINPROGRESS || err == -EBUSY) return err; - return ahash_def_finup_finish1(state, err); + return ahash_def_finup_finish1(req, err); } int crypto_ahash_export(struct ahash_request *req, void *out) -- cgit v1.2.3 From 8b54e6a8f4156ed43627f40300b0711dc977fbc1 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 21 Mar 2025 16:43:30 +0800 Subject: crypto: testmgr - Add multibuffer hash testing This is based on a patch by Eric Biggers . Add limited self-test for multibuffer hash code path. This tests only a single request in chain of a random length. The other requests are either all of the same length as the one being tested, or random lengths between 0 and PAGE_SIZE * 2 * XBUFSIZE. Potential extension include testing all requests rather than just the single one. Link: https://lore.kernel.org/all/20241001153718.111665-3-ebiggers@kernel.org/ Signed-off-by: Herbert Xu --- crypto/testmgr.c | 160 ++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 24 deletions(-) (limited to 'crypto') diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 1b2387291787..a4a8aed8181c 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -58,6 +58,9 @@ module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); #endif +/* Multibuffer is unlimited. Set arbitrary limit for testing. */ +#define MAX_MB_MSGS 16 + #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS /* a perfect nop */ @@ -299,6 +302,13 @@ struct test_sg_division { * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to * the @key_offset * @finalization_type: what finalization function to use for hashes + * @multibuffer: test with multibuffer + * @multibuffer_index: random number used to generate the message index to use + * for multibuffer. + * @multibuffer_uneven: test with multibuffer using uneven lengths + * @multibuffer_lens: random lengths to make chained request uneven + * @multibuffer_count: random number used to generate the num_msgs parameter + * for multibuffer * @nosimd: execute with SIMD disabled? Requires !CRYPTO_TFM_REQ_MAY_SLEEP. * This applies to the parts of the operation that aren't controlled * individually by @nosimd_setkey or @src_divs[].nosimd. @@ -318,6 +328,11 @@ struct testvec_config { enum finalization_type finalization_type; bool nosimd; bool nosimd_setkey; + bool multibuffer; + unsigned int multibuffer_index; + unsigned int multibuffer_count; + bool multibuffer_uneven; + unsigned int multibuffer_lens[MAX_MB_MSGS]; }; #define TESTVEC_CONFIG_NAMELEN 192 @@ -557,6 +572,7 @@ struct test_sglist { char *bufs[XBUFSIZE]; struct scatterlist sgl[XBUFSIZE]; struct scatterlist sgl_saved[XBUFSIZE]; + struct scatterlist full_sgl[XBUFSIZE]; struct scatterlist *sgl_ptr; unsigned int nents; }; @@ -670,6 +686,11 @@ static int build_test_sglist(struct test_sglist *tsgl, sg_mark_end(&tsgl->sgl[tsgl->nents - 1]); tsgl->sgl_ptr = tsgl->sgl; memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0])); + + sg_init_table(tsgl->full_sgl, XBUFSIZE); + for (i = 0; i < XBUFSIZE; i++) + sg_set_buf(tsgl->full_sgl, tsgl->bufs[i], PAGE_SIZE * 2); + return 0; } @@ -1146,6 +1167,27 @@ static void generate_random_testvec_config(struct rnd_state *rng, break; } + if (prandom_bool(rng)) { + int i; + + cfg->multibuffer = true; + cfg->multibuffer_count = prandom_u32_state(rng); + cfg->multibuffer_count %= MAX_MB_MSGS; + if (cfg->multibuffer_count++) { + cfg->multibuffer_index = prandom_u32_state(rng); + cfg->multibuffer_index %= cfg->multibuffer_count; + } + + cfg->multibuffer_uneven = prandom_bool(rng); + for (i = 0; i < MAX_MB_MSGS; i++) + cfg->multibuffer_lens[i] = + generate_random_length(rng, PAGE_SIZE * 2 * XBUFSIZE); + + p += scnprintf(p, end - p, " multibuffer(%d/%d%s)", + cfg->multibuffer_index, cfg->multibuffer_count, + cfg->multibuffer_uneven ? "/uneven" : ""); + } + if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) { if (prandom_bool(rng)) { cfg->nosimd = true; @@ -1450,6 +1492,7 @@ static int do_ahash_op(int (*op)(struct ahash_request *req), struct ahash_request *req, struct crypto_wait *wait, bool nosimd) { + struct ahash_request *r2; int err; if (nosimd) @@ -1460,7 +1503,15 @@ static int do_ahash_op(int (*op)(struct ahash_request *req), if (nosimd) crypto_reenable_simd_for_test(); - return crypto_wait_req(err, wait); + err = crypto_wait_req(err, wait); + if (err) + return err; + + list_for_each_entry(r2, &req->base.list, base.list) + if (r2->base.err) + return r2->base.err; + + return 0; } static int check_nonfinal_ahash_op(const char *op, int err, @@ -1481,20 +1532,65 @@ static int check_nonfinal_ahash_op(const char *op, int err, return 0; } +static void setup_ahash_multibuffer( + struct ahash_request *reqs[MAX_MB_MSGS], + const struct testvec_config *cfg, + struct test_sglist *tsgl) +{ + struct scatterlist *sg = tsgl->full_sgl; + static u8 trash[HASH_MAX_DIGESTSIZE]; + struct ahash_request *req = reqs[0]; + unsigned int num_msgs; + unsigned int msg_idx; + int i; + + if (!cfg->multibuffer) + return; + + num_msgs = cfg->multibuffer_count; + if (num_msgs == 1) + return; + + msg_idx = cfg->multibuffer_index; + for (i = 1; i < num_msgs; i++) { + struct ahash_request *r2 = reqs[i]; + unsigned int nbytes = req->nbytes; + + if (cfg->multibuffer_uneven) + nbytes = cfg->multibuffer_lens[i]; + + ahash_request_set_callback(r2, req->base.flags, NULL, NULL); + ahash_request_set_crypt(r2, sg, trash, nbytes); + ahash_request_chain(r2, req); + } + + if (msg_idx) { + reqs[msg_idx]->src = req->src; + reqs[msg_idx]->nbytes = req->nbytes; + reqs[msg_idx]->result = req->result; + req->src = sg; + if (cfg->multibuffer_uneven) + req->nbytes = cfg->multibuffer_lens[0]; + req->result = trash; + } +} + /* Test one hash test vector in one configuration, using the ahash API */ static int test_ahash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, - struct ahash_request *req, + struct ahash_request *reqs[MAX_MB_MSGS], struct test_sglist *tsgl, u8 *hashstate) { + struct ahash_request *req = reqs[0]; struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int digestsize = crypto_ahash_digestsize(tfm); const unsigned int statesize = crypto_ahash_statesize(tfm); const char *driver = crypto_ahash_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; + struct ahash_request *reqi = req; DECLARE_CRYPTO_WAIT(wait); unsigned int i; struct scatterlist *pending_sgl; @@ -1502,6 +1598,9 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; int err; + if (cfg->multibuffer) + reqi = reqs[cfg->multibuffer_index]; + /* Set the key, if specified */ if (vec->ksize) { err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize, @@ -1531,7 +1630,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, /* Do the actual hashing */ - testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); + testmgr_poison(reqi->__ctx, crypto_ahash_reqsize(tfm)); testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || @@ -1540,6 +1639,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize); + setup_ahash_multibuffer(reqs, cfg, tsgl); err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd); if (err) { if (err == vec->digest_error) @@ -1561,6 +1661,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); + setup_ahash_multibuffer(reqs, cfg, tsgl); err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); err = check_nonfinal_ahash_op("init", err, result, digestsize, driver, vec_name, cfg); @@ -1577,6 +1678,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, crypto_req_done, &wait); ahash_request_set_crypt(req, pending_sgl, result, pending_len); + setup_ahash_multibuffer(reqs, cfg, tsgl); err = do_ahash_op(crypto_ahash_update, req, &wait, divs[i]->nosimd); err = check_nonfinal_ahash_op("update", err, @@ -1591,7 +1693,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, /* Test ->export() and ->import() */ testmgr_poison(hashstate + statesize, TESTMGR_POISON_LEN); - err = crypto_ahash_export(req, hashstate); + err = crypto_ahash_export(reqi, hashstate); err = check_nonfinal_ahash_op("export", err, result, digestsize, driver, vec_name, cfg); @@ -1604,8 +1706,8 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, return -EOVERFLOW; } - testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); - err = crypto_ahash_import(req, hashstate); + testmgr_poison(reqi->__ctx, crypto_ahash_reqsize(tfm)); + err = crypto_ahash_import(reqi, hashstate); err = check_nonfinal_ahash_op("import", err, result, digestsize, driver, vec_name, cfg); @@ -1619,6 +1721,7 @@ static int test_ahash_vec_cfg(const struct hash_testvec *vec, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, pending_sgl, result, pending_len); + setup_ahash_multibuffer(reqs, cfg, tsgl); if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); @@ -1650,7 +1753,7 @@ result_ready: static int test_hash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, - struct ahash_request *req, + struct ahash_request *reqs[MAX_MB_MSGS], struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) @@ -1670,11 +1773,12 @@ static int test_hash_vec_cfg(const struct hash_testvec *vec, return err; } - return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate); + return test_ahash_vec_cfg(vec, vec_name, cfg, reqs, tsgl, hashstate); } static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, - struct ahash_request *req, struct shash_desc *desc, + struct ahash_request *reqs[MAX_MB_MSGS], + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { char vec_name[16]; @@ -1686,7 +1790,7 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { err = test_hash_vec_cfg(vec, vec_name, &default_hash_testvec_configs[i], - req, desc, tsgl, hashstate); + reqs, desc, tsgl, hashstate); if (err) return err; } @@ -1703,7 +1807,7 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, generate_random_testvec_config(&rng, &cfg, cfgname, sizeof(cfgname)); err = test_hash_vec_cfg(vec, vec_name, &cfg, - req, desc, tsgl, hashstate); + reqs, desc, tsgl, hashstate); if (err) return err; cond_resched(); @@ -1762,11 +1866,12 @@ done: */ static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, - struct ahash_request *req, + struct ahash_request *reqs[MAX_MB_MSGS], struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { + struct ahash_request *req = reqs[0]; struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int digestsize = crypto_ahash_digestsize(tfm); const unsigned int blocksize = crypto_ahash_blocksize(tfm); @@ -1864,7 +1969,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, sizeof(cfgname)); err = test_hash_vec_cfg(&vec, vec_name, cfg, - req, desc, tsgl, hashstate); + reqs, desc, tsgl, hashstate); if (err) goto out; cond_resched(); @@ -1882,7 +1987,7 @@ out: #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, - struct ahash_request *req, + struct ahash_request *reqs[MAX_MB_MSGS], struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) @@ -1929,8 +2034,8 @@ static int __alg_test_hash(const struct hash_testvec *vecs, u32 type, u32 mask, const char *generic_driver, unsigned int maxkeysize) { + struct ahash_request *reqs[MAX_MB_MSGS] = {}; struct crypto_ahash *atfm = NULL; - struct ahash_request *req = NULL; struct crypto_shash *stfm = NULL; struct shash_desc *desc = NULL; struct test_sglist *tsgl = NULL; @@ -1954,12 +2059,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } driver = crypto_ahash_driver_name(atfm); - req = ahash_request_alloc(atfm, GFP_KERNEL); - if (!req) { - pr_err("alg: hash: failed to allocate request for %s\n", - driver); - err = -ENOMEM; - goto out; + for (i = 0; i < MAX_MB_MSGS; i++) { + reqs[i] = ahash_request_alloc(atfm, GFP_KERNEL); + if (!reqs[i]) { + pr_err("alg: hash: failed to allocate request for %s\n", + driver); + err = -ENOMEM; + goto out; + } } /* @@ -1995,12 +2102,12 @@ static int __alg_test_hash(const struct hash_testvec *vecs, if (fips_enabled && vecs[i].fips_skip) continue; - err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate); + err = test_hash_vec(&vecs[i], i, reqs, desc, tsgl, hashstate); if (err) goto out; cond_resched(); } - err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req, + err = test_hash_vs_generic_impl(generic_driver, maxkeysize, reqs, desc, tsgl, hashstate); out: kfree(hashstate); @@ -2010,7 +2117,12 @@ out: } kfree(desc); crypto_free_shash(stfm); - ahash_request_free(req); + if (reqs[0]) { + ahash_request_set_callback(reqs[0], 0, NULL, NULL); + for (i = 1; i < MAX_MB_MSGS && reqs[i]; i++) + ahash_request_chain(reqs[i], reqs[0]); + ahash_request_free(reqs[0]); + } crypto_free_ahash(atfm); return err; } -- cgit v1.2.3 From 39fc22a8e53e96392f9b2c840e386272affbe6ba Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 21 Mar 2025 16:43:32 +0800 Subject: crypto: acomp - Fix synchronous acomp chaining fallback The synchronous acomp fallback code path is broken because the completion code path assumes that the state object is always set but this is only done for asynchronous algorithms. First of all remove the assumption on the completion code path by passing in req0 instead of the state. However, also remove the conditional setting of the state since it's always in the request object anyway. Fixes: b67a02600372 ("crypto: acomp - Add request chaining and virtual addresses") Reported-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- crypto/acompress.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) (limited to 'crypto') diff --git a/crypto/acompress.c b/crypto/acompress.c index 6ef335f5bf27..f7a3fbe5447e 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -155,12 +155,8 @@ EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) { - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); struct acomp_req_chain *state = &req->chain; - if (!acomp_is_async(tfm)) - return; - state->compl = req->base.complete; state->data = req->base.data; req->base.complete = cplt; @@ -168,14 +164,9 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->req0 = req; } -static void acomp_restore_req(struct acomp_req_chain *state) +static void acomp_restore_req(struct acomp_req *req) { - struct acomp_req *req = state->req0; - struct crypto_acomp *tfm; - - tfm = crypto_acomp_reqtfm(req); - if (!acomp_is_async(tfm)) - return; + struct acomp_req_chain *state = req->base.data; req->base.complete = state->compl; req->base.data = state->data; @@ -289,10 +280,9 @@ static int acomp_do_one_req(struct acomp_req_chain *state, return state->op(req); } -static int acomp_reqchain_finish(struct acomp_req_chain *state, - int err, u32 mask) +static int acomp_reqchain_finish(struct acomp_req *req0, int err, u32 mask) { - struct acomp_req *req0 = state->req0; + struct acomp_req_chain *state = req0->base.data; struct acomp_req *req = state->cur; struct acomp_req *n; @@ -323,7 +313,7 @@ static int acomp_reqchain_finish(struct acomp_req_chain *state, list_add_tail(&req->base.list, &req0->base.list); } - acomp_restore_req(state); + acomp_restore_req(req0); out: return err; @@ -342,7 +332,8 @@ static void acomp_reqchain_done(void *data, int err) goto notify; } - err = acomp_reqchain_finish(state, err, CRYPTO_TFM_REQ_MAY_BACKLOG); + err = acomp_reqchain_finish(state->req0, err, + CRYPTO_TFM_REQ_MAY_BACKLOG); if (err == -EBUSY) return; @@ -354,17 +345,15 @@ static int acomp_do_req_chain(struct acomp_req *req, int (*op)(struct acomp_req *req)) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req_chain *state = &req->chain; + struct acomp_req_chain *state; int err; if (crypto_acomp_req_chain(tfm) || (!acomp_request_chained(req) && acomp_request_issg(req))) return op(req); - if (acomp_is_async(tfm)) { - acomp_save_req(req, acomp_reqchain_done); - state = req->base.data; - } + acomp_save_req(req, acomp_reqchain_done); + state = req->base.data; state->op = op; state->src = NULL; @@ -375,7 +364,7 @@ static int acomp_do_req_chain(struct acomp_req *req, if (err == -EBUSY || err == -EINPROGRESS) return -EBUSY; - return acomp_reqchain_finish(state, err, ~0); + return acomp_reqchain_finish(req, err, ~0); } int crypto_acomp_compress(struct acomp_req *req) -- cgit v1.2.3 From 99585c2192cb1ce212876e82ef01d1c98c7f4699 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 21 Mar 2025 16:43:35 +0800 Subject: crypto: testmgr - Add multibuffer acomp testing Add rudimentary multibuffer acomp testing. Testing coverage is extended to compression vectors only. However, as the compression vectors are compressed and then decompressed, this covers both compression and decompression. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 144 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 64 deletions(-) (limited to 'crypto') diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a4a8aed8181c..cd0f6a778b62 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3438,27 +3438,48 @@ static int test_acomp(struct crypto_acomp *tfm, int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); - unsigned int i; - char *output, *decomp_out; - int ret; - struct scatterlist src, dst; - struct acomp_req *req; + struct scatterlist *src = NULL, *dst = NULL; + struct acomp_req *reqs[MAX_MB_MSGS] = {}; + char *decomp_out[MAX_MB_MSGS] = {}; + char *output[MAX_MB_MSGS] = {}; struct crypto_wait wait; + struct acomp_req *req; + int ret = -ENOMEM; + unsigned int i; - output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!output) - return -ENOMEM; + src = kmalloc_array(MAX_MB_MSGS, sizeof(*src), GFP_KERNEL); + if (!src) + goto out; + dst = kmalloc_array(MAX_MB_MSGS, sizeof(*dst), GFP_KERNEL); + if (!dst) + goto out; - decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!decomp_out) { - kfree(output); - return -ENOMEM; + for (i = 0; i < MAX_MB_MSGS; i++) { + reqs[i] = acomp_request_alloc(tfm); + if (!reqs[i]) + goto out; + + acomp_request_set_callback(reqs[i], + CRYPTO_TFM_REQ_MAY_SLEEP | + CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + if (i) + acomp_request_chain(reqs[i], reqs[0]); + + output[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!output[i]) + goto out; + + decomp_out[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!decomp_out[i]) + goto out; } for (i = 0; i < ctcount; i++) { unsigned int dlen = COMP_BUF_SIZE; int ilen = ctemplate[i].inlen; void *input_vec; + int j; input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL); if (!input_vec) { @@ -3466,70 +3487,61 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(&src, input_vec, ilen); - sg_init_one(&dst, output, dlen); + sg_init_one(src, input_vec, ilen); - req = acomp_request_alloc(tfm); - if (!req) { - pr_err("alg: acomp: request alloc failed for %s\n", - algo); - kfree(input_vec); - ret = -ENOMEM; - goto out; + for (j = 0; j < MAX_MB_MSGS; j++) { + sg_init_one(dst + j, output[j], dlen); + acomp_request_set_params(reqs[j], src, dst + j, ilen, dlen); } - acomp_request_set_params(req, &src, &dst, ilen, dlen); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - + req = reqs[0]; ret = crypto_wait_req(crypto_acomp_compress(req), &wait); if (ret) { pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); kfree(input_vec); - acomp_request_free(req); goto out; } ilen = req->dlen; dlen = COMP_BUF_SIZE; - sg_init_one(&src, output, ilen); - sg_init_one(&dst, decomp_out, dlen); crypto_init_wait(&wait); - acomp_request_set_params(req, &src, &dst, ilen, dlen); - - ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); - if (ret) { - pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", - i + 1, algo, -ret); - kfree(input_vec); - acomp_request_free(req); - goto out; - } + for (j = 0; j < MAX_MB_MSGS; j++) { + sg_init_one(src + j, output[j], ilen); + sg_init_one(dst + j, decomp_out[j], dlen); + acomp_request_set_params(reqs[j], src + j, dst + j, ilen, dlen); + } + + crypto_wait_req(crypto_acomp_decompress(req), &wait); + for (j = 0; j < MAX_MB_MSGS; j++) { + ret = reqs[j]->base.err; + if (ret) { + pr_err("alg: acomp: compression failed on test %d (%d) for %s: ret=%d\n", + i + 1, j, algo, -ret); + kfree(input_vec); + goto out; + } - if (req->dlen != ctemplate[i].inlen) { - pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", - i + 1, algo, req->dlen); - ret = -EINVAL; - kfree(input_vec); - acomp_request_free(req); - goto out; - } + if (reqs[j]->dlen != ctemplate[i].inlen) { + pr_err("alg: acomp: Compression test %d (%d) failed for %s: output len = %d\n", + i + 1, j, algo, reqs[j]->dlen); + ret = -EINVAL; + kfree(input_vec); + goto out; + } - if (memcmp(input_vec, decomp_out, req->dlen)) { - pr_err("alg: acomp: Compression test %d failed for %s\n", - i + 1, algo); - hexdump(output, req->dlen); - ret = -EINVAL; - kfree(input_vec); - acomp_request_free(req); - goto out; + if (memcmp(input_vec, decomp_out[j], reqs[j]->dlen)) { + pr_err("alg: acomp: Compression test %d (%d) failed for %s\n", + i + 1, j, algo); + hexdump(output[j], reqs[j]->dlen); + ret = -EINVAL; + kfree(input_vec); + goto out; + } } kfree(input_vec); - acomp_request_free(req); } for (i = 0; i < dtcount; i++) { @@ -3543,10 +3555,9 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(&src, input_vec, ilen); - sg_init_one(&dst, output, dlen); + sg_init_one(src, input_vec, ilen); + sg_init_one(dst, output[0], dlen); req = acomp_request_alloc(tfm); if (!req) { @@ -3557,7 +3568,7 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - acomp_request_set_params(req, &src, &dst, ilen, dlen); + acomp_request_set_params(req, src, dst, ilen, dlen); acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -3579,10 +3590,10 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - if (memcmp(output, dtemplate[i].output, req->dlen)) { + if (memcmp(output[0], dtemplate[i].output, req->dlen)) { pr_err("alg: acomp: Decompression test %d failed for %s\n", i + 1, algo); - hexdump(output, req->dlen); + hexdump(output[0], req->dlen); ret = -EINVAL; kfree(input_vec); acomp_request_free(req); @@ -3596,8 +3607,13 @@ static int test_acomp(struct crypto_acomp *tfm, ret = 0; out: - kfree(decomp_out); - kfree(output); + acomp_request_free(reqs[0]); + for (i = 0; i < MAX_MB_MSGS; i++) { + kfree(output[i]); + kfree(decomp_out[i]); + } + kfree(dst); + kfree(src); return ret; } -- cgit v1.2.3