diff options
Diffstat (limited to 'include/crypto')
64 files changed, 5037 insertions, 1846 deletions
diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 54937b615239..5d5358dfab73 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -10,30 +10,93 @@ #define _CRYPTO_ACOMP_H #include <linux/atomic.h> +#include <linux/args.h> +#include <linux/compiler_types.h> #include <linux/container_of.h> #include <linux/crypto.h> +#include <linux/err.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/spinlock_types.h> +#include <linux/types.h> + +/* Set this bit if source is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_SRC_VIRT 0x00000002 + +/* Set this bit for if virtual address source cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_SRC_NONDMA 0x00000004 + +/* Set this bit if destination is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_DST_VIRT 0x00000008 + +/* Set this bit for if virtual address destination cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 + +/* Private flags that should not be touched by the user. */ +#define CRYPTO_ACOMP_REQ_PRIVATE \ + (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA) -#define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 #define CRYPTO_ACOMP_DST_MAX 131072 +#define MAX_SYNC_COMP_REQSIZE 0 + +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_on_stack_init( \ + __##name##_req, (tfm)) + +#define ACOMP_REQUEST_CLONE(name, gfp) \ + acomp_request_clone(name, sizeof(__##name##_req), gfp) + +struct acomp_req; +struct folio; + +struct acomp_req_chain { + crypto_completion_t compl; + void *data; + struct scatterlist ssg; + struct scatterlist dsg; + union { + const u8 *src; + struct folio *sfolio; + }; + union { + u8 *dst; + struct folio *dfolio; + }; + u32 flags; +}; + /** * struct acomp_req - asynchronous (de)compression request * * @base: Common attributes for asynchronous crypto requests - * @src: Source Data - * @dst: Destination data + * @src: Source scatterlist + * @dst: Destination scatterlist + * @svirt: Source virtual address + * @dvirt: Destination virtual address * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced - * @flags: Internal flags + * @chain: Private API code data, do not use * @__ctx: Start of private context data */ struct acomp_req { struct crypto_async_request base; - struct scatterlist *src; - struct scatterlist *dst; + union { + struct scatterlist *src; + const u8 *svirt; + }; + union { + struct scatterlist *dst; + u8 *dvirt; + }; unsigned int slen; unsigned int dlen; - u32 flags; + + struct acomp_req_chain chain; + void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -43,15 +106,13 @@ struct acomp_req { * * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation - * @dst_free: Frees destination buffer if allocated inside the - * algorithm * @reqsize: Context size for (de)compression requests + * @fb: Synchronous fallback tfm * @base: Common crypto API algorithm data structure */ struct crypto_acomp { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); - void (*dst_free)(struct scatterlist *dst); unsigned int reqsize; struct crypto_tfm base; }; @@ -133,7 +194,7 @@ static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) static inline void acomp_request_set_tfm(struct acomp_req *req, struct crypto_acomp *tfm) { - req->base.tfm = crypto_acomp_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_acomp_tfm(tfm)); } static inline bool acomp_is_async(struct crypto_acomp *tfm) @@ -168,14 +229,73 @@ static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) return crypto_has_alg(alg_name, type, mask); } +static inline const char *crypto_acomp_alg_name(struct crypto_acomp *tfm) +{ + return crypto_tfm_alg_name(crypto_acomp_tfm(tfm)); +} + +static inline const char *crypto_acomp_driver_name(struct crypto_acomp *tfm) +{ + return crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); +} + +/** + * acomp_request_alloc_extra() -- allocates asynchronous (de)compression request + * + * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * @extra: amount of extra memory + * @gfp: gfp to pass to kzalloc (defaults to GFP_KERNEL) + * + * Return: allocated handle in case of success or NULL in case of an error + */ +static inline struct acomp_req *acomp_request_alloc_extra_noprof( + struct crypto_acomp *tfm, size_t extra, gfp_t gfp) +{ + struct acomp_req *req; + size_t len; + + len = ALIGN(sizeof(*req) + crypto_acomp_reqsize(tfm), CRYPTO_MINALIGN); + if (check_add_overflow(len, extra, &len)) + return NULL; + + req = kzalloc_noprof(len, gfp); + if (likely(req)) + acomp_request_set_tfm(req, tfm); + return req; +} +#define acomp_request_alloc_noprof(tfm, ...) \ + CONCATENATE(acomp_request_alloc_noprof_, COUNT_ARGS(__VA_ARGS__))( \ + tfm, ##__VA_ARGS__) +#define acomp_request_alloc_noprof_0(tfm) \ + acomp_request_alloc_noprof_1(tfm, GFP_KERNEL) +#define acomp_request_alloc_noprof_1(tfm, gfp) \ + acomp_request_alloc_extra_noprof(tfm, 0, gfp) +#define acomp_request_alloc(...) alloc_hooks(acomp_request_alloc_noprof(__VA_ARGS__)) + /** - * acomp_request_alloc() -- allocates asynchronous (de)compression request + * acomp_request_alloc_extra() -- allocate acomp request with extra memory * * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * @extra: amount of extra memory + * @gfp: gfp to pass to kzalloc * * Return: allocated handle in case of success or NULL in case of an error */ -struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); +#define acomp_request_alloc_extra(...) alloc_hooks(acomp_request_alloc_extra_noprof(__VA_ARGS__)) + +static inline void *acomp_request_extra(struct acomp_req *req) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + size_t len; + + len = ALIGN(sizeof(*req) + crypto_acomp_reqsize(tfm), CRYPTO_MINALIGN); + return (void *)((char *)req + len); +} + +static inline bool acomp_req_on_stack(struct acomp_req *req) +{ + return crypto_req_on_stack(&req->base); +} /** * acomp_request_free() -- zeroize and free asynchronous (de)compression @@ -184,7 +304,12 @@ struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); * * @req: request to free */ -void acomp_request_free(struct acomp_req *req); +static inline void acomp_request_free(struct acomp_req *req) +{ + if (!req || acomp_req_on_stack(req)) + return; + kfree_sensitive(req); +} /** * acomp_request_set_callback() -- Sets an asynchronous callback @@ -194,7 +319,7 @@ void acomp_request_free(struct acomp_req *req); * * @req: request that the callback will be set for * @flgs: specify for instance if the operation may backlog - * @cmlp: callback which will be called + * @cmpl: callback which will be called * @data: private data used by the caller */ static inline void acomp_request_set_callback(struct acomp_req *req, @@ -202,10 +327,9 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - req->base.complete = cmpl; - req->base.data = data; - req->base.flags &= CRYPTO_ACOMP_ALLOC_OUTPUT; - req->base.flags |= flgs & ~CRYPTO_ACOMP_ALLOC_OUTPUT; + flgs &= ~CRYPTO_ACOMP_REQ_PRIVATE; + flgs |= req->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flgs, cmpl, data); } /** @@ -232,9 +356,169 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->slen = slen; req->dlen = dlen; - req->flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; - if (!req->dst) - req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA); +} + +/** + * acomp_request_set_src_sg() -- Sets source scatterlist + * + * Sets source scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @src: pointer to input buffer scatterlist + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_sg(struct acomp_req *req, + struct scatterlist *src, + unsigned int slen) +{ + req->src = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_dma() -- Sets DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_dma(struct acomp_req *req, + const u8 *src, unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_nondma() -- Sets non-DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_nondma(struct acomp_req *req, + const u8 *src, + unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_folio() -- Sets source folio + * + * Sets source folio required by an acomp operation. + * + * @req: asynchronous compress request + * @folio: pointer to input folio + * @off: input folio offset + * @len: size of the input buffer + */ +static inline void acomp_request_set_src_folio(struct acomp_req *req, + struct folio *folio, size_t off, + unsigned int len) +{ + sg_init_table(&req->chain.ssg, 1); + sg_set_folio(&req->chain.ssg, folio, len, off); + acomp_request_set_src_sg(req, &req->chain.ssg, len); +} + +/** + * acomp_request_set_dst_sg() -- Sets destination scatterlist + * + * Sets destination scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @dst: pointer to output buffer scatterlist + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_sg(struct acomp_req *req, + struct scatterlist *dst, + unsigned int dlen) +{ + req->dst = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_dma() -- Sets DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_dma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_nondma() -- Sets non-DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_nondma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_folio() -- Sets destination folio + * + * Sets destination folio required by an acomp operation. + * + * @req: asynchronous compress request + * @folio: pointer to input folio + * @off: input folio offset + * @len: size of the input buffer + */ +static inline void acomp_request_set_dst_folio(struct acomp_req *req, + struct folio *folio, size_t off, + unsigned int len) +{ + sg_init_table(&req->chain.dsg, 1); + sg_set_folio(&req->chain.dsg, folio, len, off); + acomp_request_set_dst_sg(req, &req->chain.dsg, len); } /** @@ -246,10 +530,7 @@ static inline void acomp_request_set_params(struct acomp_req *req, * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_compress(struct acomp_req *req) -{ - return crypto_acomp_reqtfm(req)->compress(req); -} +int crypto_acomp_compress(struct acomp_req *req); /** * crypto_acomp_decompress() -- Invoke asynchronous decompress operation @@ -260,9 +541,18 @@ static inline int crypto_acomp_compress(struct acomp_req *req) * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_decompress(struct acomp_req *req) +int crypto_acomp_decompress(struct acomp_req *req); + +static inline struct acomp_req *acomp_request_on_stack_init( + char *buf, struct crypto_acomp *tfm) { - return crypto_acomp_reqtfm(req)->decompress(req); + struct acomp_req *req = (void *)buf; + + crypto_stack_request_init(&req->base, crypto_acomp_tfm(tfm)); + return req; } +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); + #endif diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 0e8a41638678..8e66a1fa9c78 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -159,6 +159,21 @@ struct crypto_aead { struct crypto_tfm base; }; +struct crypto_sync_aead { + struct crypto_aead base; +}; + +#define MAX_SYNC_AEAD_REQSIZE 384 + +#define SYNC_AEAD_REQUEST_ON_STACK(name, _tfm) \ + char __##name##_desc[sizeof(struct aead_request) + \ + MAX_SYNC_AEAD_REQSIZE \ + ] CRYPTO_MINALIGN_ATTR; \ + struct aead_request *name = \ + (((struct aead_request *)__##name##_desc)->base.tfm = \ + crypto_sync_aead_tfm((_tfm)), \ + (void *)__##name##_desc) + static inline struct crypto_aead *__crypto_aead_cast(struct crypto_tfm *tfm) { return container_of(tfm, struct crypto_aead, base); @@ -180,11 +195,18 @@ static inline struct crypto_aead *__crypto_aead_cast(struct crypto_tfm *tfm) */ struct crypto_aead *crypto_alloc_aead(const char *alg_name, u32 type, u32 mask); +struct crypto_sync_aead *crypto_alloc_sync_aead(const char *alg_name, u32 type, u32 mask); + static inline struct crypto_tfm *crypto_aead_tfm(struct crypto_aead *tfm) { return &tfm->base; } +static inline struct crypto_tfm *crypto_sync_aead_tfm(struct crypto_sync_aead *tfm) +{ + return crypto_aead_tfm(&tfm->base); +} + /** * crypto_free_aead() - zeroize and free aead handle * @tfm: cipher handle to be freed @@ -196,6 +218,11 @@ static inline void crypto_free_aead(struct crypto_aead *tfm) crypto_destroy_tfm(tfm, crypto_aead_tfm(tfm)); } +static inline void crypto_free_sync_aead(struct crypto_sync_aead *tfm) +{ + crypto_free_aead(&tfm->base); +} + /** * crypto_has_aead() - Search for the availability of an aead. * @alg_name: is the cra_name / name or cra_driver_name / driver name of the @@ -238,6 +265,11 @@ static inline unsigned int crypto_aead_ivsize(struct crypto_aead *tfm) return crypto_aead_alg_ivsize(crypto_aead_alg(tfm)); } +static inline unsigned int crypto_sync_aead_ivsize(struct crypto_sync_aead *tfm) +{ + return crypto_aead_ivsize(&tfm->base); +} + /** * crypto_aead_authsize() - obtain maximum authentication data size * @tfm: cipher handle @@ -255,6 +287,11 @@ static inline unsigned int crypto_aead_authsize(struct crypto_aead *tfm) return tfm->authsize; } +static inline unsigned int crypto_sync_aead_authsize(struct crypto_sync_aead *tfm) +{ + return crypto_aead_authsize(&tfm->base); +} + static inline unsigned int crypto_aead_alg_maxauthsize(struct aead_alg *alg) { return alg->maxauthsize; @@ -265,6 +302,11 @@ static inline unsigned int crypto_aead_maxauthsize(struct crypto_aead *aead) return crypto_aead_alg_maxauthsize(crypto_aead_alg(aead)); } +static inline unsigned int crypto_sync_aead_maxauthsize(struct crypto_sync_aead *tfm) +{ + return crypto_aead_maxauthsize(&tfm->base); +} + /** * crypto_aead_blocksize() - obtain block size of cipher * @tfm: cipher handle @@ -280,6 +322,11 @@ static inline unsigned int crypto_aead_blocksize(struct crypto_aead *tfm) return crypto_tfm_alg_blocksize(crypto_aead_tfm(tfm)); } +static inline unsigned int crypto_sync_aead_blocksize(struct crypto_sync_aead *tfm) +{ + return crypto_aead_blocksize(&tfm->base); +} + static inline unsigned int crypto_aead_alignmask(struct crypto_aead *tfm) { return crypto_tfm_alg_alignmask(crypto_aead_tfm(tfm)); @@ -300,6 +347,21 @@ static inline void crypto_aead_clear_flags(struct crypto_aead *tfm, u32 flags) crypto_tfm_clear_flags(crypto_aead_tfm(tfm), flags); } +static inline u32 crypto_sync_aead_get_flags(struct crypto_sync_aead *tfm) +{ + return crypto_aead_get_flags(&tfm->base); +} + +static inline void crypto_sync_aead_set_flags(struct crypto_sync_aead *tfm, u32 flags) +{ + crypto_aead_set_flags(&tfm->base, flags); +} + +static inline void crypto_sync_aead_clear_flags(struct crypto_sync_aead *tfm, u32 flags) +{ + crypto_aead_clear_flags(&tfm->base, flags); +} + /** * crypto_aead_setkey() - set key for cipher * @tfm: cipher handle @@ -319,6 +381,12 @@ static inline void crypto_aead_clear_flags(struct crypto_aead *tfm, u32 flags) int crypto_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen); +static inline int crypto_sync_aead_setkey(struct crypto_sync_aead *tfm, + const u8 *key, unsigned int keylen) +{ + return crypto_aead_setkey(&tfm->base, key, keylen); +} + /** * crypto_aead_setauthsize() - set authentication data size * @tfm: cipher handle @@ -331,11 +399,24 @@ int crypto_aead_setkey(struct crypto_aead *tfm, */ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize); +static inline int crypto_sync_aead_setauthsize(struct crypto_sync_aead *tfm, + unsigned int authsize) +{ + return crypto_aead_setauthsize(&tfm->base, authsize); +} + static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) { return __crypto_aead_cast(req->base.tfm); } +static inline struct crypto_sync_aead *crypto_sync_aead_reqtfm(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + + return container_of(tfm, struct crypto_sync_aead, base); +} + /** * crypto_aead_encrypt() - encrypt plaintext * @req: reference to the aead_request handle that holds all information @@ -417,6 +498,12 @@ static inline void aead_request_set_tfm(struct aead_request *req, req->base.tfm = crypto_aead_tfm(tfm); } +static inline void aead_request_set_sync_tfm(struct aead_request *req, + struct crypto_sync_aead *tfm) +{ + aead_request_set_tfm(req, &tfm->base); +} + /** * aead_request_alloc() - allocate request data structure * @tfm: cipher handle to be registered with the request diff --git a/include/crypto/aes-cbc-macs.h b/include/crypto/aes-cbc-macs.h new file mode 100644 index 000000000000..06e8a22f8a0a --- /dev/null +++ b/include/crypto/aes-cbc-macs.h @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Support for AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_MACS_H +#define _CRYPTO_AES_CBC_MACS_H + +#include <crypto/aes.h> +#include <linux/string.h> + +/** + * struct aes_cmac_key - Prepared key for AES-CMAC or AES-XCBC-MAC + * @aes: The AES key for cipher block chaining + * @k_final: Finalization subkeys for the final block. + * k_final[0] (CMAC K1, XCBC-MAC K2) is used if it's a full block. + * k_final[1] (CMAC K2, XCBC-MAC K3) is used if it's a partial block. + */ +struct aes_cmac_key { + struct aes_enckey aes; + union { + u8 b[AES_BLOCK_SIZE]; + __be64 w[2]; + } k_final[2]; +}; + +/** + * aes_cmac_zeroize_key() - Zeroize an aes_cmac_key structure + * @key: The location of the key structure that should be zeroized + * + * Explicitly fills the aes_cmac_key with zeroes. This should be done once + * the key is not required anymore to avoid that its contents are leaked + * on the stack or heap (if not using kfree_sensitive()). + */ +static inline void aes_cmac_zeroize_key(struct aes_cmac_key *key) +{ + memzero_explicit(key, sizeof(*key)); +} + +/** + * struct aes_cmac_ctx - Context for computing an AES-CMAC or AES-XCBC-MAC value + * @key: Pointer to the key struct. A pointer is used rather than a copy of the + * struct, since the key struct size may be large. It is assumed that the + * key lives at least as long as the context. + * @partial_len: Number of bytes that have been XOR'ed into @h since the last + * AES encryption. This is 0 if no data has been processed yet, + * or between 1 and AES_BLOCK_SIZE inclusive otherwise. + * @h: The current chaining value + */ +struct aes_cmac_ctx { + const struct aes_cmac_key *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; + +/** + * aes_cmac_zeroize_ctx() - Zeroize an aes_cmac_ctx structure + * @ctx: The location of the context that should be zeroized + * + * Explicitly fills the aes_cmac_ctx with zeroes. This should be done once + * the context is not required anymore to avoid that its contents are + * leaked on the stack or heap. Only required if not using aes_cmac_final(). + */ +static inline void aes_cmac_zeroize_ctx(struct aes_cmac_ctx *ctx) +{ + memzero_explicit(ctx, sizeof(*ctx)); +} + +/** + * aes_cmac_preparekey() - Prepare a key for AES-CMAC + * @key: (output) The key struct to initialize + * @in_key: The raw AES key + * @key_len: Length of the raw key in bytes. The supported values are + * AES_KEYSIZE_128, AES_KEYSIZE_192, and AES_KEYSIZE_256. + * + * On success, the caller should ensure that the prepared key is zeroized + * at the end of its lifetime, e.g. by calling aes_cmac_zeroize_key() or + * kfree_sensitive(). + * + * Context: Any context. + * Return: 0 on success or -EINVAL if the given key length is invalid. No other + * errors are possible, so callers that always pass a valid key length + * don't need to check for errors. + */ +int aes_cmac_preparekey(struct aes_cmac_key *key, const u8 *in_key, + size_t key_len); + +/** + * aes_xcbcmac_preparekey() - Prepare a key for AES-XCBC-MAC + * @key: (output) The key struct to initialize + * @in_key: The raw key. As per the AES-XCBC-MAC specification (RFC 3566), this + * is 128 bits, matching the internal use of AES-128. + * + * AES-XCBC-MAC and AES-CMAC are the same except for the key preparation. After + * that step, AES-XCBC-MAC is supported via the aes_cmac_* functions. + * + * New users should use AES-CMAC instead of AES-XCBC-MAC. + * + * Context: Any context. + */ +void aes_xcbcmac_preparekey(struct aes_cmac_key *key, + const u8 in_key[at_least AES_KEYSIZE_128]); + +/** + * aes_cmac_init() - Start computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: (output) The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + * + * The caller should ensure that the context is zeroized at the end of its + * lifetime, e.g. by calling aes_cmac_final() or aes_cmac_zeroize_ctx(). + */ +static inline void aes_cmac_init(struct aes_cmac_ctx *ctx, + const struct aes_cmac_key *key) +{ + *ctx = (struct aes_cmac_ctx){ .key = key }; +} + +/** + * aes_cmac_update() - Update an AES-CMAC or AES-XCBC-MAC context with more data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @data_len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void aes_cmac_update(struct aes_cmac_ctx *ctx, const u8 *data, size_t data_len); + +/** + * aes_cmac_final() - Finish computing an AES-CMAC or AES-XCBC-MAC value + * @ctx: The context to finalize; must have been initialized + * @out: (output) The resulting MAC + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void aes_cmac_final(struct aes_cmac_ctx *ctx, u8 out[at_least AES_BLOCK_SIZE]); + +/** + * aes_cmac() - Compute AES-CMAC or AES-XCBC-MAC in one shot + * @key: The key to use + * @data: The message data + * @data_len: The data length in bytes + * @out: (output) The resulting AES-CMAC or AES-XCBC-MAC value + * + * This supports both AES-CMAC and AES-XCBC-MAC. Which one is done depends on + * whether aes_cmac_preparekey() or aes_xcbcmac_preparekey() was called. + * + * Context: Any context. + */ +static inline void aes_cmac(const struct aes_cmac_key *key, const u8 *data, + size_t data_len, u8 out[at_least AES_BLOCK_SIZE]) +{ + struct aes_cmac_ctx ctx; + + aes_cmac_init(&ctx, key); + aes_cmac_update(&ctx, data, data_len); + aes_cmac_final(&ctx, out); +} + +/* + * AES-CBC-MAC support. This is provided only for use by the implementation of + * AES-CCM. It should have no other users. Warning: unlike AES-CMAC and + * AES-XCBC-MAC, AES-CBC-MAC isn't a secure MAC for variable-length messages. + */ +struct aes_cbcmac_ctx { + const struct aes_enckey *key; + size_t partial_len; + u8 h[AES_BLOCK_SIZE]; +}; +static inline void aes_cbcmac_init(struct aes_cbcmac_ctx *ctx, + const struct aes_enckey *key) +{ + *ctx = (struct aes_cbcmac_ctx){ .key = key }; +} +void aes_cbcmac_update(struct aes_cbcmac_ctx *ctx, const u8 *data, + size_t data_len); +void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, + u8 out[at_least AES_BLOCK_SIZE]); + +#endif /* _CRYPTO_AES_CBC_MACS_H */ diff --git a/include/crypto/aes-cbc.h b/include/crypto/aes-cbc.h new file mode 100644 index 000000000000..7bae340935f9 --- /dev/null +++ b/include/crypto/aes-cbc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CBC and AES-CBC-CTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CBC_H +#define _CRYPTO_AES_CBC_H + +#include <crypto/aes.h> + +/** + * aes_cbc_encrypt() - Encrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e. the + * last ciphertext block (or left unchanged if @len == 0). + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This supports incremental encryption. The length of each chunk must be a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each time. + * + * Context: Any context. + */ +void aes_cbc_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_decrypt() - Decrypt data using AES-CBC + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @iv: The initialization vector. It is updated with the next value, i.e. the + * last ciphertext block (or left unchanged if @len == 0). + * @key: The key, already prepared using aes_preparekey() + * + * This supports incremental decryption. The length of each chunk must be a + * multiple of AES_BLOCK_SIZE, and the updated @iv must be passed in each time. + * + * Context: Any context. + */ +void aes_cbc_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], const struct aes_key *key); + +/** + * aes_cbc_cts_encrypt() - Encrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * Context: Any context. + */ +void aes_cbc_cts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_cbc_cts_decrypt() - Decrypt data using AES-CBC-CTS (CS3 variant) + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt, at least AES_BLOCK_SIZE + * @iv: The initialization vector, clobbered by this function + * @key: The key, already prepared using aes_preparekey() + * + * Context: Any context. + */ +void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 iv[at_least AES_BLOCK_SIZE], + const struct aes_key *key); + +#endif /* _CRYPTO_AES_CBC_H */ diff --git a/include/crypto/aes-ccm.h b/include/crypto/aes-ccm.h new file mode 100644 index 000000000000..8b00859ac4d6 --- /dev/null +++ b/include/crypto/aes-ccm.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CCM_H +#define _CRYPTO_AES_CCM_H + +#include <crypto/aes.h> + +/** + * struct aes_ccm_key - A key prepared for AES-CCM encryption and decryption + */ +struct aes_ccm_key { + /* private: */ + struct aes_enckey aes; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_ccm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_ccm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_ccm_key *key; + /* + * The current CBC-MAC chaining value. When not on a block boundary, + * the partial block has been XOR'ed into this. The number of partial + * bytes is 'partial_len'. + */ + u8 mac[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* The current counter, a 128-bit big endian value */ + u8 ctr[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Encrypted counter of 0. This gets XOR'ed with the tag at the end. */ + u8 s0[AES_BLOCK_SIZE] __aligned(__alignof__(__be64)); + /* Number of associated data bytes remaining to be provided */ + u64 ad_remaining; + /* Number of en/decrypted data bytes remaining to be provided */ + u64 data_remaining; + /* Current partial block length, 0 <= partial_len < AES_BLOCK_SIZE */ + u32 partial_len; + /* True if associated data padding has been done */ + bool ad_padded; +}; + +/** + * aes_ccm_preparekey() - Prepare a key for AES-CCM encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-CCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 6, 8, 10, 12, 14, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len); + +/** + * aes_ccm_encrypt() - Encrypt a message with AES-CCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce. All (key, nonce) pairs used MUST be distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_encrypt(u8 *dst, const u8 *src, size_t data_len, + u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_decrypt() - Decrypt a message with AES-CCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). Usually protocols using AES-CCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes + * @nonce: The nonce + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey() + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_init() - Initialize context for incremental AES-CCM encryption or + * decryption + * @ctx: The context to initialize + * @data_len: Length of the en/decrypted data that will be provided in bytes: + * at most 2^(120 - (8 * @nonce_len)) - 1 + * @ad_len: Length of the associated data that will be provided in bytes + * @nonce: The nonce. All (key, nonce) pairs used for encryption MUST be + * distinct. + * @nonce_len: Length of the nonce in bytes: between 7 and 13 inclusive + * @key: The key, already prepared using aes_ccm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * Unlike AES-GCM, AES-CCM requires the total lengths of the associated data and + * the en/decrypted data to be known during initialization. Callers MUST ensure + * that these lengths are correct. + * + * If this function returns success, the context should be zeroized at the end + * of its lifetime. Normally that happens in aes_ccm_encrypt_final() or + * aes_ccm_decrypt_final(), but callers that abandon a context without + * finalizing it should explicitly zeroize it. + * + * IMPORTANT: Callers that are decrypting MUST NOT assume that any decrypted or + * associated data is authentic until the authentication tag has been verified. + * This incremental API is provided solely to support callers that can't + * efficiently use the one-shot functions due to using a nonlinear data layout. + * + * For incremental AES-CCM encryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_encrypt_update() (any number of times) + * 4. aes_ccm_encrypt_final() + * + * For incremental AES-CCM decryption, use: + * + * 1. aes_ccm_init() + * 2. aes_ccm_auth_update() (any number of times) + * 3. aes_ccm_decrypt_update() (any number of times) + * 4. aes_ccm_decrypt_final() + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if @nonce_len is invalid + * * -EOVERFLOW if @data_len is too large for the selected @nonce_len + */ +int __must_check aes_ccm_init(struct aes_ccm_ctx *ctx, u64 data_len, u64 ad_len, + const u8 *nonce, size_t nonce_len, + const struct aes_ccm_key *key); + +/** + * aes_ccm_auth_update() - Incrementally process AES-CCM associated data + * @ctx: An AES-CCM context + * @ad: The associated data + * @len: Length of the associated data in bytes + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * The total length of the associated data (over all calls to this function) + * MUST match the ad_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_auth_update(struct aes_ccm_ctx *ctx, const u8 *ad, size_t len); + +/** + * aes_ccm_encrypt_update() - Incrementally encrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the encrypted data (over all calls to this function) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * Context: Any context. + */ +void aes_ccm_encrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_ccm_decrypt_update() - Incrementally decrypt data with AES-CCM + * @ctx: An AES-CCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt + * + * This can be called only after all associated data has been processed. + * + * The total length of the decrypted data (over all calls to this function) MUST + * match the data_len that was passed to aes_ccm_init(). + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_ccm_decrypt_update(struct aes_ccm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_ccm_encrypt_final() - Finish encrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_ccm_encrypt_final(struct aes_ccm_ctx *ctx, u8 *authtag); + +/** + * aes_ccm_decrypt_final() - Finish decrypting a message with AES-CCM + * @ctx: An AES-CCM context + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_ccm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_ccm_decrypt_final(struct aes_ccm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_CCM_H */ diff --git a/include/crypto/aes-ctr.h b/include/crypto/aes-ctr.h new file mode 100644 index 000000000000..8e214a85716b --- /dev/null +++ b/include/crypto/aes-ctr.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-CTR and AES-XCTR stream ciphers + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_CTR_H +#define _CRYPTO_AES_CTR_H + +#include <crypto/aes.h> + +/** + * aes_ctr() - AES-CTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The counter. It will be incremented by ceil(@len / AES_BLOCK_SIZE). + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This implements AES in counter mode with a 128-bit big endian counter. + * + * This exists only for use by the implementation of modes built on top of CTR + * (e.g., GCM and CCM) and some legacy protocols that use CTR mode directly. + * Callers are expected to know how to use CTR mode appropriately, including + * choosing (key, counter) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be passed in + * each time. + * + * Context: Any context. + */ +void aes_ctr(u8 *dst, const u8 *src, size_t len, + u8 ctr[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +/** + * aes_xctr() - AES-XCTR en/decryption + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to en/decrypt + * @ctr: The block counter (in host endianness). For the first call, set it to + * 1. It will be incremented by ceil(@len / AES_BLOCK_SIZE). + * @iv: The initialization vector + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * This implements AES in XOR Counter mode, as specified in the paper + * "Length-preserving encryption with HCTR2" + * (https://eprint.iacr.org/2021/1441.pdf). + * + * This exists only for use by the implementation of modes built on top of XCTR. + * Callers are expected to know how to use XCTR mode appropriately, including + * choosing (key, IV) pairs appropriately to avoid keystream reuse. + * + * This supports incremental en/decryption. The length of each non-final chunk + * must be a multiple of AES_BLOCK_SIZE, and the updated @ctr must be passed in + * each time. + * + * Context: Any context. + */ +void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr, + const u8 iv[at_least AES_BLOCK_SIZE], aes_encrypt_arg key); + +#endif /* _CRYPTO_AES_CTR_H */ diff --git a/include/crypto/aes-ecb.h b/include/crypto/aes-ecb.h new file mode 100644 index 000000000000..8cac1f8794c7 --- /dev/null +++ b/include/crypto/aes-ecb.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-ECB unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_ECB_H +#define _CRYPTO_AES_ECB_H + +#include <crypto/aes.h> + +/** + * aes_ecb_encrypt() - Encrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() or aes_prepareenckey() + * + * ECB mode is insecure by itself. This function exists only for compatibility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental encryption, but the length of each chunk must be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_encrypt(u8 *dst, const u8 *src, size_t len, aes_encrypt_arg key); + +/** + * aes_ecb_decrypt() - Decrypt data using AES-ECB + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. Must be a multiple of AES_BLOCK_SIZE. + * @key: The key, already prepared using aes_preparekey() + * + * ECB mode is insecure by itself. This function exists only for compatibility + * with legacy protocols and for internal use by other modes. + * + * This supports incremental decryption, but the length of each chunk must be a + * multiple of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len, + const struct aes_key *key); + +#endif /* _CRYPTO_AES_ECB_H */ diff --git a/include/crypto/aes-gcm.h b/include/crypto/aes-gcm.h new file mode 100644 index 000000000000..2aee62f01989 --- /dev/null +++ b/include/crypto/aes-gcm.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-GCM authenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_GCM_H +#define _CRYPTO_AES_GCM_H + +#include <crypto/aes.h> +#include <crypto/gcm.h> +#include <crypto/gf128hash.h> + +/** + * struct aes_gcm_key - A key prepared for AES-GCM encryption and decryption + */ +struct aes_gcm_key { + /* private: */ + struct aes_enckey aes; + struct ghash_key ghash; + size_t authtag_len; /* Length of authentication tags in bytes */ +}; + +/** + * struct aes_gcm_ctx - Context for incrementally en/decrypting a message + */ +struct aes_gcm_ctx { + /* private: */ + /* + * Pointer to the key, which is assumed to live at least as long as this + * struct. + */ + const struct aes_gcm_key *key; + /* The current GHASH context */ + struct ghash_ctx ghash; + /* + * The current counter. This can be viewed as either a 128-bit big + * endian counter, or as a 96-bit nonce followed by a 32-bit big endian + * counter; it doesn't matter, since the last 32-bit word starts at 1, + * and AES-GCM is undefined for messages that would overflow that part. + * In practice this means that code optimized for AES-GCM can just + * increment the last 32-bit word (wrapping at 2^32), but when needed it + * can still call AES-CTR code that does a 128-bit increment. + * + * 'long' alignment is for crypto_xor() to work more efficiently. + */ + union { + u8 ctr[AES_BLOCK_SIZE]; + __be32 ctr32[AES_BLOCK_SIZE / 4]; + } __aligned(__alignof__(long)); + /* Buffered keystream for partial block updates */ + u8 keystream[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Encrypted counter of 1. This gets XOR'ed with the tag at the end. */ + u8 j0_enc[AES_BLOCK_SIZE] __aligned(__alignof__(long)); + /* Number of associated data bytes processed so far */ + u64 ad_len; + /* Number of en/decrypted bytes processed so far */ + u64 data_len; +}; + +/** + * aes_gcm_preparekey() - Prepare a key for AES-GCM encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-GCM key + * @key_len: Length of the raw key in bytes: 16, 24, or 32 + * @authtag_len: Length of the authentication tag in bytes: + * 4, 8, 12, 13, 14, 15, or 16. 16 is recommended. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if either of the lengths is invalid + */ +int __must_check aes_gcm_preparekey(struct aes_gcm_key *key, const u8 *in_key, + size_t key_len, size_t authtag_len); + +/** + * aes_gcm_encrypt() - Encrypt a message with AES-GCM + * @dst: The destination ciphertext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source plaintext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^36 - 32 + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @dst + @data_len and @dst must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce. All (key, nonce) pairs used MUST be distinct. + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=NULL, + * src=NULL, and data_len=0 to generate the AES-GMAC value. + * + * Context: Any context. + */ +void aes_gcm_encrypt(u8 *dst, const u8 *src, size_t data_len, u8 *authtag, + const u8 *ad, size_t ad_len, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_decrypt() - Decrypt a message with AES-GCM + * @dst: The destination plaintext data. Can be in-place or out-of-place. + * For other overlaps the behavior is unspecified. + * @src: The source ciphertext data + * @data_len: Length of plaintext in bytes (and ciphertext excluding the tag): + * at most 2^36 - 32 + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). Usually protocols using AES-GCM + * put the tag at the end of the ciphertext, in which case this should + * be set to @src + @data_len and @src must have room for the tag. + * @ad: The associated data + * @ad_len: Length of associated data in bytes: at most 2^61 - 1 + * @nonce: The 12-byte nonce + * @key: The key, already prepared using aes_gcm_preparekey() + * + * For AES-GMAC (i.e., AES-GCM without any data en/decrypted), use dst=NULL, + * src=NULL, and data_len=0 to verify the AES-GMAC value. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, + const u8 *authtag, const u8 *ad, size_t ad_len, + const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_init() - Initialize context for incremental AES-GCM encryption or + * decryption, or for AES-GMAC computation + * @ctx: The context to initialize + * @nonce: The 12-byte nonce. All (key, nonce) pairs used for encryption or MAC + * generation MUST be distinct. + * @key: The key, already prepared using aes_gcm_preparekey(). Note that a + * pointer to the key is saved in the context, so the key must live at + * least as long as the context. + * + * The context should be zeroized at the end of its lifetime. Normally that + * happens in aes_gcm_encrypt_final() or aes_gcm_decrypt_final(), but callers + * that abandon a context without finalizing it should explicitly zeroize it. + * + * IMPORTANT: Callers that are decrypting data or computing a GMAC value for + * verification MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. This incremental + * API is provided solely to support callers that can't efficiently use the + * one-shot functions due to using a nonlinear data layout. + * + * For incremental AES-GCM encryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_update() (any number of times) + * 4. aes_gcm_encrypt_final() + * + * For incremental AES-GCM decryption, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_decrypt_update() (any number of times) + * 4. aes_gcm_decrypt_final() + * + * AES-GMAC is just AES-GCM with zero bytes en/decrypted. For incremental + * AES-GMAC computation, use: + * + * 1. aes_gcm_init() + * 2. aes_gcm_auth_update() (any number of times) + * 3. aes_gcm_encrypt_final() to return the computed tag to the caller, or + * aes_gcm_decrypt_final() to directly verify the computed tag + * + * Context: Any context. + */ +void aes_gcm_init(struct aes_gcm_ctx *ctx, const u8 nonce[at_least 12], + const struct aes_gcm_key *key); + +/** + * aes_gcm_auth_update() - Incrementally process AES-GCM associated data + * @ctx: An AES-GCM context + * @ad: The associated data + * @len: Number of bytes provided. The caller must ensure that the total + * associated data length doesn't exceed GCM's limit of 2^61 - 1. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_auth_update(struct aes_gcm_ctx *ctx, const u8 *ad, size_t len); + +/** + * aes_gcm_encrypt_update() - Incrementally encrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source plaintext data + * @len: Number of bytes to encrypt. The caller must ensure that the total + * number of bytes encrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * Context: Any context. + */ +void aes_gcm_encrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_gcm_decrypt_update() - Incrementally decrypt data with AES-GCM + * @ctx: An AES-GCM context + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source ciphertext data (not including auth tag) + * @len: Number of bytes to decrypt. The caller must ensure that the total + * number of bytes decrypted doesn't exceed GCM's limit of 2^36 - 32. + * + * This can be called only after all associated data has been processed. + * + * IMPORTANT: Callers MUST NOT assume that any decrypted or associated data is + * authentic until the authentication tag has been verified. + * + * Context: Any context. + */ +void aes_gcm_decrypt_update(struct aes_gcm_ctx *ctx, u8 *dst, const u8 *src, + size_t len); + +/** + * aes_gcm_encrypt_final() - Finish encrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The output authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + */ +void aes_gcm_encrypt_final(struct aes_gcm_ctx *ctx, u8 *authtag); + +/** + * aes_gcm_decrypt_final() - Finish decrypting a message with AES-GCM + * @ctx: An AES-GCM context + * @authtag: The stored authentication tag. Length is the authtag_len that was + * passed to aes_gcm_preparekey(). + * + * This also zeroizes @ctx, so the caller doesn't need to do it. + * + * Context: Any context. + * Return: + * * 0 on success. This is the only case where any decrypted or associated data + * can be used. + * * -EBADMSG if the message is inauthentic + */ +int __must_check aes_gcm_decrypt_final(struct aes_gcm_ctx *ctx, + const u8 *authtag); + +#endif /* _CRYPTO_AES_GCM_H */ diff --git a/include/crypto/aes-xts.h b/include/crypto/aes-xts.h new file mode 100644 index 000000000000..b9e828265e58 --- /dev/null +++ b/include/crypto/aes-xts.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AES-XTS unauthenticated encryption and decryption + * + * Copyright 2026 Google LLC + */ +#ifndef _CRYPTO_AES_XTS_H +#define _CRYPTO_AES_XTS_H + +#include <crypto/aes.h> +#include <crypto/xts.h> + +/** + * struct aes_xts_key - A key prepared for AES-XTS encryption and decryption + * + * Note that (depending on the architecture) this typically is around 768 bytes, + * which makes it a bit too large to allocate on the stack in most cases. + */ +struct aes_xts_key { + /* private: */ + struct aes_key main_key; + struct aes_enckey tweak_key; +}; + +/** + * aes_xts_preparekey() - Prepare a key for AES-XTS encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES-XTS key + * @key_len: Length of the raw key in bytes + * @flags: Optional flag XTS_FORBID_WEAK_KEYS to forbid keys whose two halves + * are the same. + * + * Users should use memzero_explicit() to zeroize the key struct at the end of + * its lifetime. (But if this function fails, zeroization is unnecessary.) + * + * Context: Any context. + * Return: + * * 0 on success + * * -EINVAL if the key is rejected because its length isn't 32, 64, or (when + * FIPS mode isn't enabled) 48; or because its two halves are the same and + * either XTS_FORBID_WEAK_KEYS is given or FIPS mode is enabled. + */ +int __must_check aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key, + size_t key_len, int flags); + +/** + * aes_xts_encrypt() - Encrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to encrypt. On non-final calls it must be a nonzero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn't a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin encrypting a new message (do the tweak encryption); + * %true to continue encrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental encryption. On the first call, + * pass @cont = %false. On any later calls, pass @cont = %true and the updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_encrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +/** + * aes_xts_decrypt() - Decrypt data using AES-XTS + * @dst: The destination buffer. Can be in-place or out-of-place. For other + * overlaps the behavior is unspecified. + * @src: The source data + * @len: Number of bytes to decrypt. On non-final calls it must be a nonzero + * multiple of AES_BLOCK_SIZE. On the final call it can be any value >= + * AES_BLOCK_SIZE, i.e. ciphertext stealing is supported. + * @tweak: The tweak. It is updated with the next value, unless @len isn't a + * multiple of AES_BLOCK_SIZE in which case the value is unspecified. + * @key: The key, already prepared using aes_xts_preparekey() + * @cont: %false to begin decrypting a new message (do the tweak encryption); + * %true to continue decrypting a message (skip tweak encryption) + * + * This supports both one-shot and incremental decryption. On the first call, + * pass @cont = %false. On any later calls, pass @cont = %true and the updated + * @tweak; all earlier @len must have been multiples of AES_BLOCK_SIZE. + * + * Context: Any context. + */ +void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len, + u8 tweak[at_least AES_BLOCK_SIZE], + const struct aes_xts_key *key, bool cont); + +#endif /* _CRYPTO_AES_XTS_H */ diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 9339da7c20a8..3279cfa54608 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -19,6 +19,103 @@ #define AES_MAX_KEYLENGTH_U32 (AES_MAX_KEYLENGTH / sizeof(u32)) /* + * The POWER8 VSX optimized AES assembly code is borrowed from OpenSSL and + * inherits OpenSSL's AES_KEY format, which stores the number of rounds after + * the round keys. That assembly code is difficult to change. So for + * compatibility purposes we reserve space for the extra nrounds field on PPC64. + * + * Note: when prepared for decryption, the round keys are just the reversed + * standard round keys, not the round keys for the Equivalent Inverse Cipher. + */ +struct p8_aes_key { + u32 rndkeys[AES_MAX_KEYLENGTH_U32]; + int nrounds; +}; + +union aes_enckey_arch { + u32 rndkeys[AES_MAX_KEYLENGTH_U32]; +#ifdef CONFIG_CRYPTO_LIB_AES_ARCH +#if defined(CONFIG_PPC) && defined(CONFIG_SPE) + /* Used unconditionally (when SPE AES code is enabled in kconfig) */ + u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8); +#elif defined(CONFIG_PPC) + /* + * Kernels that include the POWER8 VSX optimized AES code use this field + * when that code is usable at key preparation time. Otherwise they + * fall back to rndkeys. In the latter case, p8.nrounds (which doesn't + * overlap rndkeys) is set to 0 to differentiate the two formats. + */ + struct p8_aes_key p8; +#elif defined(CONFIG_S390) + /* Used when the CPU supports CPACF AES for this key's length */ + u8 raw_key[AES_MAX_KEY_SIZE]; +#elif defined(CONFIG_SPARC64) + /* Used when the CPU supports the SPARC64 AES opcodes */ + u64 sparc_rndkeys[AES_MAX_KEYLENGTH / sizeof(u64)]; +#endif +#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */ +}; + +union aes_invkey_arch { + u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32]; +#ifdef CONFIG_CRYPTO_LIB_AES_ARCH +#if defined(CONFIG_PPC) && defined(CONFIG_SPE) + /* Used unconditionally (when SPE AES code is enabled in kconfig) */ + u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8); +#elif defined(CONFIG_PPC) + /* Used conditionally, analogous to aes_enckey_arch::p8 */ + struct p8_aes_key p8; +#endif +#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */ +}; + +/** + * struct aes_enckey - An AES key prepared for encryption + * @len: Key length in bytes: 16 for AES-128, 24 for AES-192, 32 for AES-256. + * @nrounds: Number of rounds: 10 for AES-128, 12 for AES-192, 14 for AES-256. + * This is '6 + @len / 4' and is cached so that AES implementations + * that need it don't have to recompute it for each en/decryption. + * @padding: Padding to make offsetof(@k) be a multiple of 16, so that aligning + * this struct to a 16-byte boundary results in @k also being 16-byte + * aligned. Users aren't required to align this struct to 16 bytes, + * but it may slightly improve performance. + * @k: This typically contains the AES round keys as an array of '@nrounds + 1' + * groups of four u32 words. However, architecture-specific implementations + * of AES may store something else here, e.g. just the raw key if it's all + * they need. + * + * Note that this struct is about half the size of struct aes_key. This is + * separate from struct aes_key so that modes that need only AES encryption + * (e.g. AES-GCM, AES-CTR, AES-CMAC, tweak key in AES-XTS) don't incur the time + * and space overhead of computing and caching the decryption round keys. + * + * Note that there's no decryption-only equivalent (i.e. "struct aes_deckey"), + * since (a) it's rare that modes need decryption-only, and (b) some AES + * implementations use the same @k for both encryption and decryption, either + * always or conditionally; in the latter case both @k and @inv_k are needed. + */ +struct aes_enckey { + u32 len; + u32 nrounds; + u32 padding[2]; + union aes_enckey_arch k; +}; + +/** + * struct aes_key - An AES key prepared for encryption and decryption + * @aes_enckey: Common fields and the key prepared for encryption + * @inv_k: This generally contains the round keys for the AES Equivalent + * Inverse Cipher, as an array of '@nrounds + 1' groups of four u32 + * words. However, architecture-specific implementations of AES may + * store something else here. For example, they may leave this field + * uninitialized if they use @k for both encryption and decryption. + */ +struct aes_key { + struct aes_enckey; /* Include all fields of aes_enckey. */ + union aes_invkey_arch inv_k; +}; + +/* * Please ensure that the first two fields are 16-byte aligned * relative to the start of the structure, i.e., don't move them! */ @@ -28,13 +125,10 @@ struct crypto_aes_ctx { u32 key_length; }; -extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned; -extern const u32 crypto_it_tab[4][256] ____cacheline_aligned; - /* * validate key length for AES algorithms */ -static inline int aes_check_keylen(unsigned int keylen) +static inline int aes_check_keylen(size_t keylen) { switch (keylen) { case AES_KEYSIZE_128: @@ -48,9 +142,6 @@ static inline int aes_check_keylen(unsigned int keylen) return 0; } -int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len); - /** * aes_expandkey - Expands the AES key as described in FIPS-197 * @ctx: The location where the computed key will be stored. @@ -68,28 +159,243 @@ int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); +/* + * The following functions are temporarily exported for use by the AES mode + * implementations in arch/$(SRCARCH)/crypto/. These exports will go away when + * that code is migrated into lib/crypto/. + */ +#ifdef CONFIG_ARM64 +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len); +asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void neon_aes_cbc_cts_encrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_cbc_cts_decrypt(u8 out[], u8 const in[], + u32 const rk[], int rounds, int bytes, + u8 const iv[]); +asmlinkage void neon_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void neon_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void neon_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void neon_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); +asmlinkage void neon_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], + u32 const rk2[]); + +asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 const iv[]); +asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[]); +asmlinkage void ce_aes_xctr_encrypt(u8 out[], u8 const in[], u32 const rk[], + int rounds, int bytes, u8 ctr[], + int byte_ctr); +asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[], + int rounds, int bytes, u32 const rk2[], + u8 iv[], int first); +asmlinkage void ce_aes_essiv_cbc_encrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage void ce_aes_essiv_cbc_decrypt(u8 out[], u8 const in[], + u32 const rk1[], int rounds, + int blocks, u8 iv[], u32 const rk2[]); +asmlinkage void ce_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); +#elif defined(CONFIG_PPC) +void ppc_expand_key_128(u32 *key_enc, const u8 *key); +void ppc_expand_key_192(u32 *key_enc, const u8 *key); +void ppc_expand_key_256(u32 *key_enc, const u8 *key); +void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key_len); +void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes); +void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, + u32 bytes); +void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv); +void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes, + u8 *iv); +void ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv); +void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes, + u8 *iv, u32 *key_twk); +void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes, + u8 *iv, u32 *key_twk); +int aes_p8_set_encrypt_key(const u8 *userKey, const int bits, + struct p8_aes_key *key); +int aes_p8_set_decrypt_key(const u8 *userKey, const int bits, + struct p8_aes_key *key); +void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key); +void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key); +void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len, + const struct p8_aes_key *key, u8 *iv, bool enc); +void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len, + const struct p8_aes_key *key, const u8 *iv); +void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len, + const struct p8_aes_key *key1, + const struct p8_aes_key *key2, u8 *iv); +void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len, + const struct p8_aes_key *key1, + const struct p8_aes_key *key2, u8 *iv); +#elif defined(CONFIG_SPARC64) +void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, + unsigned int key_len); +void aes_sparc64_load_encrypt_keys_128(const u64 *key); +void aes_sparc64_load_encrypt_keys_192(const u64 *key); +void aes_sparc64_load_encrypt_keys_256(const u64 *key); +void aes_sparc64_load_decrypt_keys_128(const u64 *key); +void aes_sparc64_load_decrypt_keys_192(const u64 *key); +void aes_sparc64_load_decrypt_keys_256(const u64 *key); +void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len); +void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +#endif + /** - * aes_encrypt - Encrypt a single AES block - * @ctx: Context struct containing the key schedule - * @out: Buffer to store the ciphertext - * @in: Buffer containing the plaintext + * aes_preparekey() - Prepare an AES key for encryption and decryption + * @key: (output) The key structure to initialize + * @in_key: The raw AES key + * @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128, + * AES_KEYSIZE_192, or AES_KEYSIZE_256. + * + * This prepares an AES key for both the encryption and decryption directions of + * the block cipher. Typically this involves expanding the raw key into both + * the standard round keys and the Equivalent Inverse Cipher round keys, but + * some architecture-specific implementations don't do the full expansion here. + * + * The caller is responsible for zeroizing both the struct aes_key and the raw + * key once they are no longer needed. + * + * If you don't need decryption support, use aes_prepareenckey() instead. + * + * Return: 0 on success or -EINVAL if the given key length is invalid. No other + * errors are possible, so callers that always pass a valid key length + * don't need to check for errors. + * + * Context: Any context. */ -void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); +int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len); /** - * aes_decrypt - Decrypt a single AES block - * @ctx: Context struct containing the key schedule - * @out: Buffer to store the plaintext - * @in: Buffer containing the ciphertext + * aes_prepareenckey() - Prepare an AES key for encryption-only + * @key: (output) The key structure to initialize + * @in_key: The raw AES key + * @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128, + * AES_KEYSIZE_192, or AES_KEYSIZE_256. + * + * This prepares an AES key for only the encryption direction of the block + * cipher. Typically this involves expanding the raw key into only the standard + * round keys, resulting in a struct about half the size of struct aes_key. + * + * The caller is responsible for zeroizing both the struct aes_enckey and the + * raw key once they are no longer needed. + * + * Note that while the resulting prepared key supports only AES encryption, it + * can still be used for decrypting in a mode of operation that uses AES in only + * the encryption (forward) direction, for example counter mode. + * + * Return: 0 on success or -EINVAL if the given key length is invalid. No other + * errors are possible, so callers that always pass a valid key length + * don't need to check for errors. + * + * Context: Any context. + */ +int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len); + +typedef union { + const struct aes_enckey *enc_key; + const struct aes_key *full_key; +} aes_encrypt_arg __attribute__ ((__transparent_union__)); + +/** + * aes_encrypt() - Encrypt a single AES block + * @key: The AES key, as a pointer to either an encryption-only key + * (struct aes_enckey) or a full, bidirectional key (struct aes_key). + * @out: Buffer to store the ciphertext block + * @in: Buffer containing the plaintext block + * + * Context: Any context. + */ +void aes_encrypt(aes_encrypt_arg key, u8 out[at_least AES_BLOCK_SIZE], + const u8 in[at_least AES_BLOCK_SIZE]); + +/** + * aes_decrypt() - Decrypt a single AES block + * @key: The AES key, previously initialized by aes_preparekey() + * @out: Buffer to store the plaintext block + * @in: Buffer containing the ciphertext block + * + * Context: Any context. */ -void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); +void aes_decrypt(const struct aes_key *key, u8 out[at_least AES_BLOCK_SIZE], + const u8 in[at_least AES_BLOCK_SIZE]); extern const u8 crypto_aes_sbox[]; extern const u8 crypto_aes_inv_sbox[]; +extern const u32 aes_enc_tab[256]; +extern const u32 aes_dec_tab[256]; -void aescfb_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src, +void aescfb_encrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, int len, const u8 iv[AES_BLOCK_SIZE]); -void aescfb_decrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src, +void aescfb_decrypt(const struct aes_enckey *key, u8 *dst, const u8 *src, int len, const u8 iv[AES_BLOCK_SIZE]); #endif diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 156de41ca760..05deea9dac5e 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -11,6 +11,7 @@ #include <linux/align.h> #include <linux/cache.h> #include <linux/crypto.h> +#include <linux/list.h> #include <linux/types.h> #include <linux/workqueue.h> @@ -42,8 +43,8 @@ * alias. */ #define MODULE_ALIAS_CRYPTO(name) \ - __MODULE_INFO(alias, alias_userspace, name); \ - __MODULE_INFO(alias, alias_crypto, "crypto-" name) + MODULE_INFO(alias, name); \ + MODULE_INFO(alias, "crypto-" name) struct crypto_aead; struct crypto_instance; @@ -53,20 +54,7 @@ struct rtattr; struct scatterlist; struct seq_file; struct sk_buff; - -struct crypto_type { - unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); - unsigned int (*extsize)(struct crypto_alg *alg); - int (*init_tfm)(struct crypto_tfm *tfm); - void (*show)(struct seq_file *m, struct crypto_alg *alg); - int (*report)(struct sk_buff *skb, struct crypto_alg *alg); - void (*free)(struct crypto_instance *inst); - - unsigned int type; - unsigned int maskclear; - unsigned int maskset; - unsigned int tfmsize; -}; +union crypto_no_such_thing; struct crypto_instance { struct crypto_alg alg; @@ -80,16 +68,17 @@ struct crypto_instance { struct crypto_spawn *spawns; }; - struct work_struct free_work; - void *__ctx[] CRYPTO_MINALIGN_ATTR; }; struct crypto_template { struct list_head list; struct hlist_head instances; + struct hlist_head dead; struct module *module; + struct work_struct free_work; + int (*create)(struct crypto_template *tmpl, struct rtattr **tb); char name[CRYPTO_MAX_ALG_NAME]; @@ -119,6 +108,13 @@ struct crypto_queue { }; struct scatter_walk { + /* Must be the first member, see struct skcipher_walk. */ + union { + void *const addr; + + /* Private API field, do not touch. */ + union crypto_no_such_thing *__addr; + }; struct scatterlist *sg; unsigned int offset; }; @@ -162,8 +158,16 @@ void *crypto_spawn_tfm2(struct crypto_spawn *spawn); struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb); int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret); const char *crypto_attr_alg_name(struct rtattr *rta); -int crypto_inst_setname(struct crypto_instance *inst, const char *name, - struct crypto_alg *alg); +int __crypto_inst_setname(struct crypto_instance *inst, const char *name, + const char *driver, struct crypto_alg *alg); + +#define crypto_inst_setname(inst, name, ...) \ + CONCATENATE(crypto_inst_setname_, COUNT_ARGS(__VA_ARGS__))( \ + inst, name, ##__VA_ARGS__) +#define crypto_inst_setname_1(inst, name, alg) \ + __crypto_inst_setname(inst, name, name, alg) +#define crypto_inst_setname_2(inst, name, driver, alg) \ + __crypto_inst_setname(inst, name, driver, alg) void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, @@ -271,4 +275,14 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } +static inline bool crypto_tfm_req_virt(struct crypto_tfm *tfm) +{ + return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_VIRT; +} + +static inline u32 crypto_request_flags(struct crypto_async_request *req) +{ + return req->flags & ~CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _CRYPTO_ALGAPI_H */ diff --git a/include/crypto/authenc.h b/include/crypto/authenc.h index 5f92a986083c..15a9caa2354a 100644 --- a/include/crypto/authenc.h +++ b/include/crypto/authenc.h @@ -28,5 +28,7 @@ struct crypto_authenc_keys { int crypto_authenc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key, unsigned int keylen); +int crypto_krb5enc_extractkeys(struct crypto_authenc_keys *keys, const u8 *key, + unsigned int keylen); #endif /* _CRYPTO_AUTHENC_H */ diff --git a/include/crypto/blake2b.h b/include/crypto/blake2b.h index 0c0176285349..3bc37fd103a7 100644 --- a/include/crypto/blake2b.h +++ b/include/crypto/blake2b.h @@ -18,7 +18,16 @@ enum blake2b_lengths { BLAKE2B_512_HASH_SIZE = 64, }; -struct blake2b_state { +/** + * struct blake2b_ctx - Context for hashing a message with BLAKE2b + * @h: compression function state + * @t: block counter + * @f: finalization indicator + * @buf: partial block buffer; 'buflen' bytes are valid + * @buflen: number of bytes buffered in @buf + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + */ +struct blake2b_ctx { /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ u64 h[8]; u64 t[2]; @@ -39,28 +48,109 @@ enum blake2b_iv { BLAKE2B_IV7 = 0x5BE0CD19137E2179ULL, }; -static inline void __blake2b_init(struct blake2b_state *state, size_t outlen, +static inline void __blake2b_init(struct blake2b_ctx *ctx, size_t outlen, const void *key, size_t keylen) { - state->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); - state->h[1] = BLAKE2B_IV1; - state->h[2] = BLAKE2B_IV2; - state->h[3] = BLAKE2B_IV3; - state->h[4] = BLAKE2B_IV4; - state->h[5] = BLAKE2B_IV5; - state->h[6] = BLAKE2B_IV6; - state->h[7] = BLAKE2B_IV7; - state->t[0] = 0; - state->t[1] = 0; - state->f[0] = 0; - state->f[1] = 0; - state->buflen = 0; - state->outlen = outlen; + ctx->h[0] = BLAKE2B_IV0 ^ (0x01010000 | keylen << 8 | outlen); + ctx->h[1] = BLAKE2B_IV1; + ctx->h[2] = BLAKE2B_IV2; + ctx->h[3] = BLAKE2B_IV3; + ctx->h[4] = BLAKE2B_IV4; + ctx->h[5] = BLAKE2B_IV5; + ctx->h[6] = BLAKE2B_IV6; + ctx->h[7] = BLAKE2B_IV7; + ctx->t[0] = 0; + ctx->t[1] = 0; + ctx->f[0] = 0; + ctx->f[1] = 0; + ctx->buflen = 0; + ctx->outlen = outlen; if (keylen) { - memcpy(state->buf, key, keylen); - memset(&state->buf[keylen], 0, BLAKE2B_BLOCK_SIZE - keylen); - state->buflen = BLAKE2B_BLOCK_SIZE; + memcpy(ctx->buf, key, keylen); + memset(&ctx->buf[keylen], 0, BLAKE2B_BLOCK_SIZE - keylen); + ctx->buflen = BLAKE2B_BLOCK_SIZE; } } +/** + * blake2b_init() - Initialize a BLAKE2b context for a new message (unkeyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2b_init(struct blake2b_ctx *ctx, size_t outlen) +{ + __blake2b_init(ctx, outlen, NULL, 0); +} + +/** + * blake2b_init_key() - Initialize a BLAKE2b context for a new message (keyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * @key: the key + * @keylen: the key length in bytes, at most BLAKE2B_KEY_SIZE + * + * Context: Any context. + */ +static inline void blake2b_init_key(struct blake2b_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2B_HASH_SIZE || + !key || !keylen || keylen > BLAKE2B_KEY_SIZE)); + + __blake2b_init(ctx, outlen, key, keylen); +} + +/** + * blake2b_update() - Update a BLAKE2b context with message data + * @ctx: the context to update; must have been initialized + * @in: the message data + * @inlen: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void blake2b_update(struct blake2b_ctx *ctx, const u8 *in, size_t inlen); + +/** + * blake2b_final() - Finish computing a BLAKE2b hash + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting BLAKE2b hash. Its length will be equal to the + * @outlen that was passed to blake2b_init() or blake2b_init_key(). + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void blake2b_final(struct blake2b_ctx *ctx, u8 *out); + +/** + * blake2b() - Compute BLAKE2b hash in one shot + * @key: the key, or NULL for an unkeyed hash + * @keylen: the key length in bytes (at most BLAKE2B_KEY_SIZE), or 0 for an + * unkeyed hash + * @in: the message data + * @inlen: the data length in bytes + * @out: (output) the resulting BLAKE2b hash, with length @outlen + * @outlen: length of output hash value in bytes, at most BLAKE2B_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2b(const u8 *key, size_t keylen, + const u8 *in, size_t inlen, + u8 *out, size_t outlen) +{ + struct blake2b_ctx ctx; + + WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || + outlen > BLAKE2B_HASH_SIZE || keylen > BLAKE2B_KEY_SIZE || + (!key && keylen))); + + __blake2b_init(&ctx, outlen, key, keylen); + blake2b_update(&ctx, in, inlen); + blake2b_final(&ctx, out); +} + #endif /* _CRYPTO_BLAKE2B_H */ diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h index f9ffd39194eb..648cb7824358 100644 --- a/include/crypto/blake2s.h +++ b/include/crypto/blake2s.h @@ -22,7 +22,16 @@ enum blake2s_lengths { BLAKE2S_256_HASH_SIZE = 32, }; -struct blake2s_state { +/** + * struct blake2s_ctx - Context for hashing a message with BLAKE2s + * @h: compression function state + * @t: block counter + * @f: finalization indicator + * @buf: partial block buffer; 'buflen' bytes are valid + * @buflen: number of bytes buffered in @buf + * @outlen: length of output hash value in bytes, at most BLAKE2S_HASH_SIZE + */ +struct blake2s_ctx { /* 'h', 't', and 'f' are used in assembly code, so keep them as-is. */ u32 h[8]; u32 t[2]; @@ -43,62 +52,109 @@ enum blake2s_iv { BLAKE2S_IV7 = 0x5BE0CD19UL, }; -static inline void __blake2s_init(struct blake2s_state *state, size_t outlen, +static inline void __blake2s_init(struct blake2s_ctx *ctx, size_t outlen, const void *key, size_t keylen) { - state->h[0] = BLAKE2S_IV0 ^ (0x01010000 | keylen << 8 | outlen); - state->h[1] = BLAKE2S_IV1; - state->h[2] = BLAKE2S_IV2; - state->h[3] = BLAKE2S_IV3; - state->h[4] = BLAKE2S_IV4; - state->h[5] = BLAKE2S_IV5; - state->h[6] = BLAKE2S_IV6; - state->h[7] = BLAKE2S_IV7; - state->t[0] = 0; - state->t[1] = 0; - state->f[0] = 0; - state->f[1] = 0; - state->buflen = 0; - state->outlen = outlen; + ctx->h[0] = BLAKE2S_IV0 ^ (0x01010000 | keylen << 8 | outlen); + ctx->h[1] = BLAKE2S_IV1; + ctx->h[2] = BLAKE2S_IV2; + ctx->h[3] = BLAKE2S_IV3; + ctx->h[4] = BLAKE2S_IV4; + ctx->h[5] = BLAKE2S_IV5; + ctx->h[6] = BLAKE2S_IV6; + ctx->h[7] = BLAKE2S_IV7; + ctx->t[0] = 0; + ctx->t[1] = 0; + ctx->f[0] = 0; + ctx->f[1] = 0; + ctx->buflen = 0; + ctx->outlen = outlen; if (keylen) { - memcpy(state->buf, key, keylen); - memset(&state->buf[keylen], 0, BLAKE2S_BLOCK_SIZE - keylen); - state->buflen = BLAKE2S_BLOCK_SIZE; + memcpy(ctx->buf, key, keylen); + memset(&ctx->buf[keylen], 0, BLAKE2S_BLOCK_SIZE - keylen); + ctx->buflen = BLAKE2S_BLOCK_SIZE; } } -static inline void blake2s_init(struct blake2s_state *state, - const size_t outlen) +/** + * blake2s_init() - Initialize a BLAKE2s context for a new message (unkeyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2S_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2s_init(struct blake2s_ctx *ctx, size_t outlen) { - __blake2s_init(state, outlen, NULL, 0); + __blake2s_init(ctx, outlen, NULL, 0); } -static inline void blake2s_init_key(struct blake2s_state *state, - const size_t outlen, const void *key, - const size_t keylen) +/** + * blake2s_init_key() - Initialize a BLAKE2s context for a new message (keyed) + * @ctx: the context to initialize + * @outlen: length of output hash value in bytes, at most BLAKE2S_HASH_SIZE + * @key: the key + * @keylen: the key length in bytes, at most BLAKE2S_KEY_SIZE + * + * Context: Any context. + */ +static inline void blake2s_init_key(struct blake2s_ctx *ctx, size_t outlen, + const void *key, size_t keylen) { WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE || !key || !keylen || keylen > BLAKE2S_KEY_SIZE)); - __blake2s_init(state, outlen, key, keylen); + __blake2s_init(ctx, outlen, key, keylen); } -void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen); -void blake2s_final(struct blake2s_state *state, u8 *out); +/** + * blake2s_update() - Update a BLAKE2s context with message data + * @ctx: the context to update; must have been initialized + * @in: the message data + * @inlen: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen); -static inline void blake2s(u8 *out, const u8 *in, const u8 *key, - const size_t outlen, const size_t inlen, - const size_t keylen) +/** + * blake2s_final() - Finish computing a BLAKE2s hash + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting BLAKE2s hash. Its length will be equal to the + * @outlen that was passed to blake2s_init() or blake2s_init_key(). + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void blake2s_final(struct blake2s_ctx *ctx, u8 *out); + +/** + * blake2s() - Compute BLAKE2s hash in one shot + * @key: the key, or NULL for an unkeyed hash + * @keylen: the key length in bytes (at most BLAKE2S_KEY_SIZE), or 0 for an + * unkeyed hash + * @in: the message data + * @inlen: the data length in bytes + * @out: (output) the resulting BLAKE2s hash, with length @outlen + * @outlen: length of output hash value in bytes, at most BLAKE2S_HASH_SIZE + * + * Context: Any context. + */ +static inline void blake2s(const u8 *key, size_t keylen, + const u8 *in, size_t inlen, + u8 *out, size_t outlen) { - struct blake2s_state state; + struct blake2s_ctx ctx; WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen || outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE || (!key && keylen))); - __blake2s_init(&state, outlen, key, keylen); - blake2s_update(&state, in, inlen); - blake2s_final(&state, out); + __blake2s_init(&ctx, outlen, key, keylen); + blake2s_update(&ctx, in, inlen); + blake2s_final(&ctx, out); } #endif /* _CRYPTO_BLAKE2S_H */ diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 5bae6a55b333..1cc301a48469 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -16,6 +16,7 @@ #define _CRYPTO_CHACHA_H #include <linux/unaligned.h> +#include <linux/string.h> #include <linux/types.h> /* 32-bit stream position, then 96-bit nonce (RFC7539 convention) */ @@ -25,27 +26,30 @@ #define CHACHA_BLOCK_SIZE 64 #define CHACHAPOLY_IV_SIZE 12 -#define CHACHA_STATE_WORDS (CHACHA_BLOCK_SIZE / sizeof(u32)) +#define CHACHA_KEY_WORDS 8 +#define CHACHA_STATE_WORDS 16 +#define HCHACHA_OUT_WORDS 8 /* 192-bit nonce, then 64-bit stream position */ #define XCHACHA_IV_SIZE 32 -void chacha_block_generic(u32 *state, u8 *stream, int nrounds); -static inline void chacha20_block(u32 *state, u8 *stream) +struct chacha_state { + u32 x[CHACHA_STATE_WORDS]; +}; + +void chacha_block_generic(struct chacha_state *state, + u8 out[at_least CHACHA_BLOCK_SIZE], int nrounds); +static inline void chacha20_block(struct chacha_state *state, + u8 out[at_least CHACHA_BLOCK_SIZE]) { - chacha_block_generic(state, stream, 20); + chacha_block_generic(state, out, 20); } -void hchacha_block_arch(const u32 *state, u32 *out, int nrounds); -void hchacha_block_generic(const u32 *state, u32 *out, int nrounds); +void hchacha_block_generic(const struct chacha_state *state, + u32 out[at_least HCHACHA_OUT_WORDS], int nrounds); -static inline void hchacha_block(const u32 *state, u32 *out, int nrounds) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) - hchacha_block_arch(state, out, nrounds); - else - hchacha_block_generic(state, out, nrounds); -} +void hchacha_block(const struct chacha_state *state, + u32 out[at_least HCHACHA_OUT_WORDS], int nrounds); enum chacha_constants { /* expand 32-byte k */ CHACHA_CONSTANT_EXPA = 0x61707865U, @@ -54,58 +58,45 @@ enum chacha_constants { /* expand 32-byte k */ CHACHA_CONSTANT_TE_K = 0x6b206574U }; -static inline void chacha_init_consts(u32 *state) +static inline void chacha_init_consts(struct chacha_state *state) { - state[0] = CHACHA_CONSTANT_EXPA; - state[1] = CHACHA_CONSTANT_ND_3; - state[2] = CHACHA_CONSTANT_2_BY; - state[3] = CHACHA_CONSTANT_TE_K; + state->x[0] = CHACHA_CONSTANT_EXPA; + state->x[1] = CHACHA_CONSTANT_ND_3; + state->x[2] = CHACHA_CONSTANT_2_BY; + state->x[3] = CHACHA_CONSTANT_TE_K; } -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv); -static inline void chacha_init_generic(u32 *state, const u32 *key, const u8 *iv) +static inline void chacha_init(struct chacha_state *state, + const u32 key[at_least CHACHA_KEY_WORDS], + const u8 iv[at_least CHACHA_IV_SIZE]) { chacha_init_consts(state); - state[4] = key[0]; - state[5] = key[1]; - state[6] = key[2]; - state[7] = key[3]; - state[8] = key[4]; - state[9] = key[5]; - state[10] = key[6]; - state[11] = key[7]; - state[12] = get_unaligned_le32(iv + 0); - state[13] = get_unaligned_le32(iv + 4); - state[14] = get_unaligned_le32(iv + 8); - state[15] = get_unaligned_le32(iv + 12); + state->x[4] = key[0]; + state->x[5] = key[1]; + state->x[6] = key[2]; + state->x[7] = key[3]; + state->x[8] = key[4]; + state->x[9] = key[5]; + state->x[10] = key[6]; + state->x[11] = key[7]; + state->x[12] = get_unaligned_le32(iv + 0); + state->x[13] = get_unaligned_le32(iv + 4); + state->x[14] = get_unaligned_le32(iv + 8); + state->x[15] = get_unaligned_le32(iv + 12); } -static inline void chacha_init(u32 *state, const u32 *key, const u8 *iv) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) - chacha_init_arch(state, key, iv); - else - chacha_init_generic(state, key, iv); -} - -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes, int nrounds); -void chacha_crypt_generic(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes, int nrounds); +void chacha_crypt(struct chacha_state *state, u8 *dst, const u8 *src, + unsigned int bytes, int nrounds); -static inline void chacha_crypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes, int nrounds) +static inline void chacha20_crypt(struct chacha_state *state, + u8 *dst, const u8 *src, unsigned int bytes) { - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CHACHA)) - chacha_crypt_arch(state, dst, src, bytes, nrounds); - else - chacha_crypt_generic(state, dst, src, bytes, nrounds); + chacha_crypt(state, dst, src, bytes, 20); } -static inline void chacha20_crypt(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes) +static inline void chacha_zeroize_state(struct chacha_state *state) { - chacha_crypt(state, dst, src, bytes, 20); + memzero_explicit(state, sizeof(*state)); } #endif /* _CRYPTO_CHACHA_H */ diff --git a/include/crypto/chacha20poly1305.h b/include/crypto/chacha20poly1305.h index d2ac3ff7dc1e..0f6d99170aaf 100644 --- a/include/crypto/chacha20poly1305.h +++ b/include/crypto/chacha20poly1305.h @@ -18,33 +18,32 @@ enum chacha20poly1305_lengths { void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, - const u8 key[CHACHA20POLY1305_KEY_SIZE]); + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); bool __must_check chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, - const u8 key[CHACHA20POLY1305_KEY_SIZE]); + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, const size_t ad_len, - const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], - const u8 key[CHACHA20POLY1305_KEY_SIZE]); + const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); bool __must_check xchacha20poly1305_decrypt( - u8 *dst, const u8 *src, const size_t src_len, const u8 *ad, - const size_t ad_len, const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], - const u8 key[CHACHA20POLY1305_KEY_SIZE]); + u8 *dst, const u8 *src, const size_t src_len, + const u8 *ad, const size_t ad_len, + const u8 nonce[at_least XCHACHA20POLY1305_NONCE_SIZE], + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, - const u8 key[CHACHA20POLY1305_KEY_SIZE]); + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, const u8 *ad, const size_t ad_len, const u64 nonce, - const u8 key[CHACHA20POLY1305_KEY_SIZE]); - -bool chacha20poly1305_selftest(void); + const u8 key[at_least CHACHA20POLY1305_KEY_SIZE]); #endif /* __CHACHA20POLY1305_H */ diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h index 796d986e58e1..29c5878a3609 100644 --- a/include/crypto/cryptd.h +++ b/include/crypto/cryptd.h @@ -16,39 +16,6 @@ #include <linux/types.h> #include <crypto/aead.h> -#include <crypto/hash.h> -#include <crypto/skcipher.h> - -struct cryptd_skcipher { - struct crypto_skcipher base; -}; - -/* alg_name should be algorithm to be cryptd-ed */ -struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, - u32 type, u32 mask); -struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm); -/* Must be called without moving CPUs. */ -bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm); -void cryptd_free_skcipher(struct cryptd_skcipher *tfm); - -struct cryptd_ahash { - struct crypto_ahash base; -}; - -static inline struct cryptd_ahash *__cryptd_ahash_cast( - struct crypto_ahash *tfm) -{ - return (struct cryptd_ahash *)tfm; -} - -/* alg_name should be algorithm to be cryptd-ed */ -struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, - u32 type, u32 mask); -struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm); -struct shash_desc *cryptd_shash_desc(struct ahash_request *req); -/* Must be called without moving CPUs. */ -bool cryptd_ahash_queued(struct cryptd_ahash *tfm); -void cryptd_free_ahash(struct cryptd_ahash *tfm); struct cryptd_aead { struct crypto_aead base; diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index a1c66d1001af..06984a26c8cf 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -8,58 +8,8 @@ #ifndef _CRYPTO_CTR_H #define _CRYPTO_CTR_H -#include <crypto/algapi.h> -#include <crypto/internal/skcipher.h> -#include <linux/string.h> -#include <linux/types.h> - #define CTR_RFC3686_NONCE_SIZE 4 #define CTR_RFC3686_IV_SIZE 8 #define CTR_RFC3686_BLOCK_SIZE 16 -static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, - void (*fn)(struct crypto_skcipher *, - const u8 *, u8 *)) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - int blocksize = crypto_skcipher_chunksize(tfm); - u8 buf[MAX_CIPHER_BLOCKSIZE]; - struct skcipher_walk walk; - int err; - - /* avoid integer division due to variable blocksize parameter */ - if (WARN_ON_ONCE(!is_power_of_2(blocksize))) - return -EINVAL; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes > 0) { - u8 *dst = walk.dst.virt.addr; - u8 *src = walk.src.virt.addr; - int nbytes = walk.nbytes; - int tail = 0; - - if (nbytes < walk.total) { - tail = walk.nbytes & (blocksize - 1); - nbytes -= tail; - } - - do { - int bsize = min(nbytes, blocksize); - - fn(tfm, walk.iv, buf); - - crypto_xor_cpy(dst, src, buf, bsize); - crypto_inc(walk.iv, blocksize); - - dst += bsize; - src += bsize; - nbytes -= bsize; - } while (nbytes > 0); - - err = skcipher_walk_done(&walk, tail); - } - return err; -} - #endif /* _CRYPTO_CTR_H */ diff --git a/include/crypto/curve25519.h b/include/crypto/curve25519.h index ece6a9b5fafc..2362b48f8741 100644 --- a/include/crypto/curve25519.h +++ b/include/crypto/curve25519.h @@ -6,7 +6,6 @@ #ifndef CURVE25519_H #define CURVE25519_H -#include <crypto/algapi.h> // For crypto_memneq. #include <linux/types.h> #include <linux/random.h> @@ -14,57 +13,28 @@ enum curve25519_lengths { CURVE25519_KEY_SIZE = 32 }; -extern const u8 curve25519_null_point[]; -extern const u8 curve25519_base_point[]; +void curve25519_generic(u8 out[at_least CURVE25519_KEY_SIZE], + const u8 scalar[at_least CURVE25519_KEY_SIZE], + const u8 point[at_least CURVE25519_KEY_SIZE]); -void curve25519_generic(u8 out[CURVE25519_KEY_SIZE], - const u8 scalar[CURVE25519_KEY_SIZE], - const u8 point[CURVE25519_KEY_SIZE]); +bool __must_check +curve25519(u8 mypublic[at_least CURVE25519_KEY_SIZE], + const u8 secret[at_least CURVE25519_KEY_SIZE], + const u8 basepoint[at_least CURVE25519_KEY_SIZE]); -void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], - const u8 scalar[CURVE25519_KEY_SIZE], - const u8 point[CURVE25519_KEY_SIZE]); +bool __must_check +curve25519_generate_public(u8 pub[at_least CURVE25519_KEY_SIZE], + const u8 secret[at_least CURVE25519_KEY_SIZE]); -void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], - const u8 secret[CURVE25519_KEY_SIZE]); - -bool curve25519_selftest(void); - -static inline -bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], - const u8 secret[CURVE25519_KEY_SIZE], - const u8 basepoint[CURVE25519_KEY_SIZE]) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) - curve25519_arch(mypublic, secret, basepoint); - else - curve25519_generic(mypublic, secret, basepoint); - return crypto_memneq(mypublic, curve25519_null_point, - CURVE25519_KEY_SIZE); -} - -static inline bool -__must_check curve25519_generate_public(u8 pub[CURVE25519_KEY_SIZE], - const u8 secret[CURVE25519_KEY_SIZE]) -{ - if (unlikely(!crypto_memneq(secret, curve25519_null_point, - CURVE25519_KEY_SIZE))) - return false; - - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_CURVE25519)) - curve25519_base_arch(pub, secret); - else - curve25519_generic(pub, secret, curve25519_base_point); - return crypto_memneq(pub, curve25519_null_point, CURVE25519_KEY_SIZE); -} - -static inline void curve25519_clamp_secret(u8 secret[CURVE25519_KEY_SIZE]) +static inline void +curve25519_clamp_secret(u8 secret[at_least CURVE25519_KEY_SIZE]) { secret[0] &= 248; secret[31] = (secret[31] & 127) | 64; } -static inline void curve25519_generate_secret(u8 secret[CURVE25519_KEY_SIZE]) +static inline void +curve25519_generate_secret(u8 secret[at_least CURVE25519_KEY_SIZE]) { get_random_bytes_wait(secret, CURVE25519_KEY_SIZE); curve25519_clamp_secret(secret); diff --git a/include/crypto/des.h b/include/crypto/des.h index 7812b4331ae4..73eec617f480 100644 --- a/include/crypto/des.h +++ b/include/crypto/des.h @@ -34,9 +34,9 @@ void des3_ede_decrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src); * des_expand_key - Expand a DES input key into a key schedule * @ctx: the key schedule * @key: buffer containing the input key - * @len: size of the buffer contents + * @keylen: size of the buffer contents * - * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if + * Returns: 0 on success, -EINVAL if the input key is rejected and -ENOKEY if * the key is accepted but has been found to be weak. */ int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen); @@ -45,9 +45,9 @@ int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen); * des3_ede_expand_key - Expand a triple DES input key into a key schedule * @ctx: the key schedule * @key: buffer containing the input key - * @len: size of the buffer contents + * @keylen: size of the buffer contents * - * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if + * Returns: 0 on success, -EINVAL if the input key is rejected and -ENOKEY if * the key is accepted but has been found to be weak. Note that weak keys will * be rejected (and -EINVAL will be returned) when running in FIPS mode. */ diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h deleted file mode 100644 index af5ad51d3eef..000000000000 --- a/include/crypto/drbg.h +++ /dev/null @@ -1,286 +0,0 @@ -/* - * DRBG based on NIST SP800-90A - * - * Copyright Stephan Mueller <smueller@chronox.de>, 2014 - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, and the entire permission notice in its entirety, - * including the disclaimer of warranties. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote - * products derived from this software without specific prior - * written permission. - * - * ALTERNATIVELY, this product may be distributed under the terms of - * the GNU General Public License, in which case the provisions of the GPL are - * required INSTEAD OF the above restrictions. (This clause is - * necessary due to a potential bad interaction between the GPL and - * the restrictions contained in a BSD-style copyright.) - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF - * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE - * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - */ - -#ifndef _DRBG_H -#define _DRBG_H - - -#include <linux/random.h> -#include <linux/scatterlist.h> -#include <crypto/hash.h> -#include <crypto/skcipher.h> -#include <linux/module.h> -#include <linux/crypto.h> -#include <linux/slab.h> -#include <crypto/internal/rng.h> -#include <crypto/rng.h> -#include <linux/fips.h> -#include <linux/mutex.h> -#include <linux/list.h> -#include <linux/workqueue.h> - -/* - * Concatenation Helper and string operation helper - * - * SP800-90A requires the concatenation of different data. To avoid copying - * buffers around or allocate additional memory, the following data structure - * is used to point to the original memory with its size. In addition, it - * is used to build a linked list. The linked list defines the concatenation - * of individual buffers. The order of memory block referenced in that - * linked list determines the order of concatenation. - */ -struct drbg_string { - const unsigned char *buf; - size_t len; - struct list_head list; -}; - -static inline void drbg_string_fill(struct drbg_string *string, - const unsigned char *buf, size_t len) -{ - string->buf = buf; - string->len = len; - INIT_LIST_HEAD(&string->list); -} - -struct drbg_state; -typedef uint32_t drbg_flag_t; - -struct drbg_core { - drbg_flag_t flags; /* flags for the cipher */ - __u8 statelen; /* maximum state length */ - __u8 blocklen_bytes; /* block size of output in bytes */ - char cra_name[CRYPTO_MAX_ALG_NAME]; /* mapping to kernel crypto API */ - /* kernel crypto API backend cipher name */ - char backend_cra_name[CRYPTO_MAX_ALG_NAME]; -}; - -struct drbg_state_ops { - int (*update)(struct drbg_state *drbg, struct list_head *seed, - int reseed); - int (*generate)(struct drbg_state *drbg, - unsigned char *buf, unsigned int buflen, - struct list_head *addtl); - int (*crypto_init)(struct drbg_state *drbg); - int (*crypto_fini)(struct drbg_state *drbg); - -}; - -struct drbg_test_data { - struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */ -}; - -enum drbg_seed_state { - DRBG_SEED_STATE_UNSEEDED, - DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */ - DRBG_SEED_STATE_FULL, -}; - -struct drbg_state { - struct mutex drbg_mutex; /* lock around DRBG */ - unsigned char *V; /* internal state 10.1.1.1 1a) */ - unsigned char *Vbuf; - /* hash: static value 10.1.1.1 1b) hmac / ctr: key */ - unsigned char *C; - unsigned char *Cbuf; - /* Number of RNG requests since last reseed -- 10.1.1.1 1c) */ - size_t reseed_ctr; - size_t reseed_threshold; - /* some memory the DRBG can use for its operation */ - unsigned char *scratchpad; - unsigned char *scratchpadbuf; - void *priv_data; /* Cipher handle */ - - struct crypto_skcipher *ctr_handle; /* CTR mode cipher handle */ - struct skcipher_request *ctr_req; /* CTR mode request handle */ - __u8 *outscratchpadbuf; /* CTR mode output scratchpad */ - __u8 *outscratchpad; /* CTR mode aligned outbuf */ - struct crypto_wait ctr_wait; /* CTR mode async wait obj */ - struct scatterlist sg_in, sg_out; /* CTR mode SGLs */ - - enum drbg_seed_state seeded; /* DRBG fully seeded? */ - unsigned long last_seed_time; - bool pr; /* Prediction resistance enabled? */ - bool fips_primed; /* Continuous test primed? */ - unsigned char *prev; /* FIPS 140-2 continuous test value */ - struct crypto_rng *jent; - const struct drbg_state_ops *d_ops; - const struct drbg_core *core; - struct drbg_string test_data; -}; - -static inline __u8 drbg_statelen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return drbg->core->statelen; - return 0; -} - -static inline __u8 drbg_blocklen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return drbg->core->blocklen_bytes; - return 0; -} - -static inline __u8 drbg_keylen(struct drbg_state *drbg) -{ - if (drbg && drbg->core) - return (drbg->core->statelen - drbg->core->blocklen_bytes); - return 0; -} - -static inline size_t drbg_max_request_bytes(struct drbg_state *drbg) -{ - /* SP800-90A requires the limit 2**19 bits, but we return bytes */ - return (1 << 16); -} - -static inline size_t drbg_max_addtl(struct drbg_state *drbg) -{ - /* SP800-90A requires 2**35 bytes additional info str / pers str */ -#if (__BITS_PER_LONG == 32) - /* - * SP800-90A allows smaller maximum numbers to be returned -- we - * return SIZE_MAX - 1 to allow the verification of the enforcement - * of this value in drbg_healthcheck_sanity. - */ - return (SIZE_MAX - 1); -#else - return (1UL<<35); -#endif -} - -static inline size_t drbg_max_requests(struct drbg_state *drbg) -{ - /* SP800-90A requires 2**48 maximum requests before reseeding */ - return (1<<20); -} - -/* - * This is a wrapper to the kernel crypto API function of - * crypto_rng_generate() to allow the caller to provide additional data. - * - * @drng DRBG handle -- see crypto_rng_get_bytes - * @outbuf output buffer -- see crypto_rng_get_bytes - * @outlen length of output buffer -- see crypto_rng_get_bytes - * @addtl_input additional information string input buffer - * @addtllen length of additional information string buffer - * - * return - * see crypto_rng_get_bytes - */ -static inline int crypto_drbg_get_bytes_addtl(struct crypto_rng *drng, - unsigned char *outbuf, unsigned int outlen, - struct drbg_string *addtl) -{ - return crypto_rng_generate(drng, addtl->buf, addtl->len, - outbuf, outlen); -} - -/* - * TEST code - * - * This is a wrapper to the kernel crypto API function of - * crypto_rng_generate() to allow the caller to provide additional data and - * allow furnishing of test_data - * - * @drng DRBG handle -- see crypto_rng_get_bytes - * @outbuf output buffer -- see crypto_rng_get_bytes - * @outlen length of output buffer -- see crypto_rng_get_bytes - * @addtl_input additional information string input buffer - * @addtllen length of additional information string buffer - * @test_data filled test data - * - * return - * see crypto_rng_get_bytes - */ -static inline int crypto_drbg_get_bytes_addtl_test(struct crypto_rng *drng, - unsigned char *outbuf, unsigned int outlen, - struct drbg_string *addtl, - struct drbg_test_data *test_data) -{ - crypto_rng_set_entropy(drng, test_data->testentropy->buf, - test_data->testentropy->len); - return crypto_rng_generate(drng, addtl->buf, addtl->len, - outbuf, outlen); -} - -/* - * TEST code - * - * This is a wrapper to the kernel crypto API function of - * crypto_rng_reset() to allow the caller to provide test_data - * - * @drng DRBG handle -- see crypto_rng_reset - * @pers personalization string input buffer - * @perslen length of additional information string buffer - * @test_data filled test data - * - * return - * see crypto_rng_reset - */ -static inline int crypto_drbg_reset_test(struct crypto_rng *drng, - struct drbg_string *pers, - struct drbg_test_data *test_data) -{ - crypto_rng_set_entropy(drng, test_data->testentropy->buf, - test_data->testentropy->len); - return crypto_rng_reset(drng, pers->buf, pers->len); -} - -/* DRBG type flags */ -#define DRBG_CTR ((drbg_flag_t)1<<0) -#define DRBG_HMAC ((drbg_flag_t)1<<1) -#define DRBG_HASH ((drbg_flag_t)1<<2) -#define DRBG_TYPE_MASK (DRBG_CTR | DRBG_HMAC | DRBG_HASH) -/* DRBG strength flags */ -#define DRBG_STRENGTH128 ((drbg_flag_t)1<<3) -#define DRBG_STRENGTH192 ((drbg_flag_t)1<<4) -#define DRBG_STRENGTH256 ((drbg_flag_t)1<<5) -#define DRBG_STRENGTH_MASK (DRBG_STRENGTH128 | DRBG_STRENGTH192 | \ - DRBG_STRENGTH256) - -enum drbg_prefixes { - DRBG_PREFIX0 = 0x00, - DRBG_PREFIX1, - DRBG_PREFIX2, - DRBG_PREFIX3 -}; - -#endif /* _DRBG_H */ diff --git a/include/crypto/engine.h b/include/crypto/engine.h index 545dbefe3e13..2e60344437da 100644 --- a/include/crypto/engine.h +++ b/include/crypto/engine.h @@ -76,7 +76,6 @@ int crypto_engine_stop(struct crypto_engine *engine); struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt); struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, bool retry_support, - int (*cbk_do_batch)(struct crypto_engine *engine), bool rt, int qlen); void crypto_engine_exit(struct crypto_engine *engine); diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index fd9df607a836..154038ca7343 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -3,9 +3,6 @@ #include <linux/errno.h> -#include <crypto/aes.h> -#include <crypto/gf128mul.h> - #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 #define GCM_RFC4543_IV_SIZE 8 @@ -13,7 +10,7 @@ /* * validate authentication tag for GCM */ -static inline int crypto_gcm_check_authsize(unsigned int authsize) +static inline int crypto_gcm_check_authsize(size_t authsize) { switch (authsize) { case 4: @@ -34,7 +31,7 @@ static inline int crypto_gcm_check_authsize(unsigned int authsize) /* * validate authentication tag for RFC4106 */ -static inline int crypto_rfc4106_check_authsize(unsigned int authsize) +static inline int crypto_rfc4106_check_authsize(size_t authsize) { switch (authsize) { case 8: @@ -64,22 +61,4 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) return 0; } -struct aesgcm_ctx { - be128 ghash_key; - struct crypto_aes_ctx aes_ctx; - unsigned int authsize; -}; - -int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, - unsigned int keysize, unsigned int authsize); - -void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, - int crypt_len, const u8 *assoc, int assoc_len, - const u8 iv[GCM_AES_IV_SIZE], u8 *authtag); - -bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, - const u8 *src, int crypt_len, const u8 *assoc, - int assoc_len, const u8 iv[GCM_AES_IV_SIZE], - const u8 *authtag); - #endif diff --git a/include/crypto/gf128hash.h b/include/crypto/gf128hash.h new file mode 100644 index 000000000000..41c557d55965 --- /dev/null +++ b/include/crypto/gf128hash.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * GF(2^128) polynomial hashing: GHASH and POLYVAL + * + * Copyright 2025 Google LLC + */ + +#ifndef _CRYPTO_GF128HASH_H +#define _CRYPTO_GF128HASH_H + +#include <crypto/ghash.h> +#include <linux/string.h> +#include <linux/types.h> + +#define POLYVAL_BLOCK_SIZE 16 +#define POLYVAL_DIGEST_SIZE 16 + +/** + * struct polyval_elem - An element of the POLYVAL finite field + * @bytes: View of the element as a byte array (unioned with @lo and @hi) + * @lo: The low 64 terms of the element's polynomial + * @hi: The high 64 terms of the element's polynomial + * + * This represents an element of the finite field GF(2^128), using the POLYVAL + * convention: little-endian byte order and natural bit order. + */ +struct polyval_elem { + union { + u8 bytes[POLYVAL_BLOCK_SIZE]; + struct { + __le64 lo; + __le64 hi; + }; + }; +}; + +/** + * struct ghash_key - Prepared key for GHASH + * + * Use ghash_preparekey() to initialize this. + */ +struct ghash_key { +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && defined(CONFIG_PPC64) + /** @htable: GHASH key format used by the POWER8 assembly code */ + u64 htable[4][2]; +#elif defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_RISCV) || defined(CONFIG_S390)) + /** @h_raw: The hash key H, in GHASH format */ + u8 h_raw[GHASH_BLOCK_SIZE]; +#endif + /** @h: The hash key H, in POLYVAL format */ + struct polyval_elem h; +}; + +/** + * struct polyval_key - Prepared key for POLYVAL + * + * This may contain just the raw key H, or it may contain precomputed key + * powers, depending on the platform's POLYVAL implementation. Use + * polyval_preparekey() to initialize this. + * + * By H^i we mean H^(i-1) * H * x^-128, with base case H^1 = H. I.e. the + * exponentiation repeats the POLYVAL dot operation, with its "extra" x^-128. + */ +struct polyval_key { +#if defined(CONFIG_CRYPTO_LIB_GF128HASH_ARCH) && \ + (defined(CONFIG_ARM64) || defined(CONFIG_X86)) + /** @h_powers: Powers of the hash key H^8 through H^1 */ + struct polyval_elem h_powers[8]; +#else + /** @h: The hash key H */ + struct polyval_elem h; +#endif +}; + +/** + * struct ghash_ctx - Context for computing a GHASH value + * @key: Pointer to the prepared GHASH key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator. It is stored in POLYVAL format rather than GHASH + * format, since most implementations want it in POLYVAL format. + * @partial: Number of data bytes processed so far modulo GHASH_BLOCK_SIZE + */ +struct ghash_ctx { + const struct ghash_key *key; + struct polyval_elem acc; + size_t partial; +}; + +/** + * struct polyval_ctx - Context for computing a POLYVAL value + * @key: Pointer to the prepared POLYVAL key. The user of the API is + * responsible for ensuring that the key lives as long as the context. + * @acc: The accumulator + * @partial: Number of data bytes processed so far modulo POLYVAL_BLOCK_SIZE + */ +struct polyval_ctx { + const struct polyval_key *key; + struct polyval_elem acc; + size_t partial; +}; + +/** + * ghash_preparekey() - Prepare a GHASH key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a GHASH key structure from a raw key. + * + * Context: Any context. + */ +void ghash_preparekey(struct ghash_key *key, + const u8 raw_key[GHASH_BLOCK_SIZE]); + +/** + * polyval_preparekey() - Prepare a POLYVAL key + * @key: (output) The key structure to initialize + * @raw_key: The raw hash key + * + * Initialize a POLYVAL key structure from a raw key. This may be a simple + * copy, or it may involve precomputing powers of the key, depending on the + * platform's POLYVAL implementation. + * + * Context: Any context. + */ +void polyval_preparekey(struct polyval_key *key, + const u8 raw_key[POLYVAL_BLOCK_SIZE]); + +/** + * ghash_init() - Initialize a GHASH context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void ghash_init(struct ghash_ctx *ctx, + const struct ghash_key *key) +{ + *ctx = (struct ghash_ctx){ .key = key }; +} + +/** + * polyval_init() - Initialize a POLYVAL context for a new message + * @ctx: The context to initialize + * @key: The key to use. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + */ +static inline void polyval_init(struct polyval_ctx *ctx, + const struct polyval_key *key) +{ + *ctx = (struct polyval_ctx){ .key = key }; +} + +/** + * polyval_import_blkaligned() - Import a POLYVAL accumulator value + * @ctx: The context to initialize + * @key: The key to import. Note that a pointer to the key is saved in the + * context, so the key must live at least as long as the context. + * @acc: The accumulator value to import. + * + * This imports an accumulator that was saved by polyval_export_blkaligned(). + * The same key must be used. + */ +static inline void +polyval_import_blkaligned(struct polyval_ctx *ctx, + const struct polyval_key *key, + const struct polyval_elem *acc) +{ + *ctx = (struct polyval_ctx){ .key = key, .acc = *acc }; +} + +/** + * polyval_export_blkaligned() - Export a POLYVAL accumulator value + * @ctx: The context to export the accumulator value from + * @acc: (output) The exported accumulator value + * + * This exports the accumulator from a POLYVAL context. The number of data + * bytes processed so far must be a multiple of POLYVAL_BLOCK_SIZE. + */ +static inline void polyval_export_blkaligned(const struct polyval_ctx *ctx, + struct polyval_elem *acc) +{ + *acc = ctx->acc; +} + +/** + * ghash_update() - Update a GHASH context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void ghash_update(struct ghash_ctx *ctx, const u8 *data, size_t len); + +/** + * polyval_update() - Update a POLYVAL context with message data + * @ctx: The context to update; must have been initialized + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * + * This can be called any number of times. + * + * Context: Any context. + */ +void polyval_update(struct polyval_ctx *ctx, const u8 *data, size_t len); + +/** + * ghash_final() - Finish computing a GHASH value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of GHASH_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void ghash_final(struct ghash_ctx *ctx, u8 out[GHASH_BLOCK_SIZE]); + +/** + * polyval_final() - Finish computing a POLYVAL value + * @ctx: The context to finalize + * @out: The output value + * + * If the total data length isn't a multiple of POLYVAL_BLOCK_SIZE, then the + * final block is automatically zero-padded. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void polyval_final(struct polyval_ctx *ctx, u8 out[POLYVAL_BLOCK_SIZE]); + +/** + * ghash() - Compute a GHASH value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void ghash(const struct ghash_key *key, const u8 *data, + size_t len, u8 out[GHASH_BLOCK_SIZE]) +{ + struct ghash_ctx ctx; + + ghash_init(&ctx, key); + ghash_update(&ctx, data, len); + ghash_final(&ctx, out); +} + +/** + * polyval() - Compute a POLYVAL value + * @key: The prepared key + * @data: The message data + * @len: The data length in bytes. Doesn't need to be block-aligned. + * @out: The output value + * + * Context: Any context. + */ +static inline void polyval(const struct polyval_key *key, + const u8 *data, size_t len, + u8 out[POLYVAL_BLOCK_SIZE]) +{ + struct polyval_ctx ctx; + + polyval_init(&ctx, key); + polyval_update(&ctx, data, len); + polyval_final(&ctx, out); +} + +#endif /* _CRYPTO_GF128HASH_H */ diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index b0853f7cada0..6ed2a8351902 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -215,25 +215,14 @@ static inline void gf128mul_x_ble(le128 *r, const le128 *x) r->b = cpu_to_le64((b << 1) ^ _tt); } -/* 4k table optimization */ - -struct gf128mul_4k { - be128 t[256]; -}; - -struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); -void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); void gf128mul_x8_ble(le128 *r, const le128 *x); -static inline void gf128mul_free_4k(struct gf128mul_4k *t) -{ - kfree_sensitive(t); -} - /* 64k table optimization, implemented for bbe */ struct gf128mul_64k { - struct gf128mul_4k *t[16]; + struct { + be128 t[256]; + } *t[16]; }; /* First initialize with the constant factor with which you diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index f832c9f2aca3..d187e5af9925 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -6,19 +6,7 @@ #ifndef __CRYPTO_GHASH_H__ #define __CRYPTO_GHASH_H__ -#include <linux/types.h> -#include <crypto/gf128mul.h> - #define GHASH_BLOCK_SIZE 16 #define GHASH_DIGEST_SIZE 16 -struct ghash_ctx { - struct gf128mul_4k *gf128; -}; - -struct ghash_desc_ctx { - u8 buffer[GHASH_BLOCK_SIZE]; - u32 bytes; -}; - #endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 2d5ea9f9ff43..e474f8461ea1 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -8,10 +8,17 @@ #ifndef _CRYPTO_HASH_H #define _CRYPTO_HASH_H -#include <linux/atomic.h> #include <linux/crypto.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> #include <linux/string.h> +/* Set this bit for virtual address instead of SG list. */ +#define CRYPTO_AHASH_REQ_VIRT 0x00000001 + +#define CRYPTO_AHASH_REQ_PRIVATE \ + CRYPTO_AHASH_REQ_VIRT + struct crypto_ahash; /** @@ -52,11 +59,15 @@ struct ahash_request { struct crypto_async_request base; unsigned int nbytes; - struct scatterlist *src; + union { + struct scatterlist *src; + const u8 *svirt; + }; u8 *result; - /* This field may only be used by the ahash API code. */ - void *priv; + struct scatterlist sg_head[2]; + crypto_completion_t saved_complete; + void *saved_data; void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -78,6 +89,8 @@ struct ahash_request { * transformation object. Data processing can happen synchronously * [SHASH] or asynchronously [AHASH] at this point. Driver must not use * req->result. + * For block-only algorithms, @update must return the number + * of bytes to store in the API partial block buffer. * @final: **[mandatory]** Retrieve result from the driver. This function finalizes the * transformation and retrieves the resulting hash from the driver and * pushes it back to upper layers. No data processing happens at this @@ -120,6 +133,10 @@ struct ahash_request { * data so the transformation can continue from this point onward. No * data processing happens at this point. Driver must not use * req->result. + * @export_core: Export partial state without partial block. Only defined + * for algorithms that are not block-only. + * @import_core: Import partial state without partial block. Only defined + * for algorithms that are not block-only. * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation * time, right after the transformation context was @@ -131,7 +148,6 @@ struct ahash_request { * @exit_tfm: Deinitialize the cryptographic transformation object. * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. - * @clone_tfm: Copy transform into new object, may allocate memory. * @halg: see struct hash_alg_common */ struct ahash_alg { @@ -142,11 +158,12 @@ struct ahash_alg { int (*digest)(struct ahash_request *req); int (*export)(struct ahash_request *req, void *out); int (*import)(struct ahash_request *req, const void *in); + int (*export_core)(struct ahash_request *req, void *out); + int (*import_core)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_ahash *tfm); void (*exit_tfm)(struct crypto_ahash *tfm); - int (*clone_tfm)(struct crypto_ahash *dst, struct crypto_ahash *src); struct hash_alg_common halg; }; @@ -159,16 +176,44 @@ struct shash_desc { #define HASH_MAX_DIGESTSIZE 64 /* - * Worst case is hmac(sha3-224-generic). Its context is a nested 'shash_desc' - * containing a 'struct sha3_state'. + * The size of a core hash state and a partial block. The final byte + * is the length of the partial block. + */ +#define HASH_STATE_AND_BLOCK(state, block) ((state) + (block) + 1) + + +/* Worst case is sha3-224. */ +#define HASH_MAX_STATESIZE HASH_STATE_AND_BLOCK(200, 144) + +/* This needs to match arch/s390/crypto/sha.h. */ +#define S390_SHA_CTX_SIZE 216 + +/* + * Worst case is hmac(sha3-224-s390). Its context is a nested 'shash_desc' + * containing a 'struct s390_sha_ctx'. */ -#define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) +#define SHA3_224_S390_DESCSIZE HASH_STATE_AND_BLOCK(S390_SHA_CTX_SIZE, 144) +#define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + \ + SHA3_224_S390_DESCSIZE) +#define MAX_SYNC_HASH_REQSIZE (sizeof(struct ahash_request) + \ + HASH_MAX_DESCSIZE) #define SHASH_DESC_ON_STACK(shash, ctx) \ char __##shash##_desc[sizeof(struct shash_desc) + HASH_MAX_DESCSIZE] \ __aligned(__alignof__(struct shash_desc)); \ struct shash_desc *shash = (struct shash_desc *)__##shash##_desc +#define HASH_REQUEST_ON_STACK(name, _tfm) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = \ + ahash_request_on_stack_init(__##name##_req, (_tfm)) + +#define HASH_REQUEST_CLONE(name, gfp) \ + hash_request_clone(name, sizeof(__##name##_req), gfp) + +#define CRYPTO_HASH_STATESIZE(coresize, blocksize) (coresize + blocksize + 1) + /** * struct shash_alg - synchronous message digest definition * @init: see struct ahash_alg @@ -178,6 +223,8 @@ struct shash_desc { * @digest: see struct ahash_alg * @export: see struct ahash_alg * @import: see struct ahash_alg + * @export_core: see struct ahash_alg + * @import_core: see struct ahash_alg * @setkey: see struct ahash_alg * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation @@ -190,7 +237,6 @@ struct shash_desc { * @exit_tfm: Deinitialize the cryptographic transformation object. * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. - * @clone_tfm: Copy transform into new object, may allocate memory. * @descsize: Size of the operational state for the message digest. This state * size is the memory size that needs to be allocated for * shash_desc.__ctx @@ -208,11 +254,12 @@ struct shash_alg { unsigned int len, u8 *out); int (*export)(struct shash_desc *desc, void *out); int (*import)(struct shash_desc *desc, const void *in); + int (*export_core)(struct shash_desc *desc, void *out); + int (*import_core)(struct shash_desc *desc, const void *in); int (*setkey)(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_shash *tfm); void (*exit_tfm)(struct crypto_shash *tfm); - int (*clone_tfm)(struct crypto_shash *dst, struct crypto_shash *src); unsigned int descsize; @@ -231,7 +278,6 @@ struct crypto_ahash { }; struct crypto_shash { - unsigned int descsize; struct crypto_tfm base; }; @@ -245,6 +291,11 @@ struct crypto_shash { * CRYPTO_ALG_TYPE_SKCIPHER API applies here as well. */ +static inline bool ahash_req_on_stack(struct ahash_request *req) +{ + return crypto_req_on_stack(&req->base); +} + static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { return container_of(tfm, struct crypto_ahash, base); @@ -267,8 +318,6 @@ static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, u32 mask); -struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *tfm); - static inline struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) { return &tfm->base; @@ -452,7 +501,11 @@ int crypto_ahash_finup(struct ahash_request *req); * -EBUSY if queue is full and request should be resubmitted later; * other < 0 if an error occurred */ -int crypto_ahash_final(struct ahash_request *req); +static inline int crypto_ahash_final(struct ahash_request *req) +{ + req->nbytes = 0; + return crypto_ahash_finup(req); +} /** * crypto_ahash_digest() - calculate message digest for a buffer @@ -541,7 +594,7 @@ int crypto_ahash_update(struct ahash_request *req); static inline void ahash_request_set_tfm(struct ahash_request *req, struct crypto_ahash *tfm) { - req->base.tfm = crypto_ahash_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_ahash_tfm(tfm)); } /** @@ -575,10 +628,7 @@ static inline struct ahash_request *ahash_request_alloc_noprof( * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ -static inline void ahash_request_free(struct ahash_request *req) -{ - kfree_sensitive(req); -} +void ahash_request_free(struct ahash_request *req); static inline void ahash_request_zero(struct ahash_request *req) { @@ -622,9 +672,9 @@ static inline void ahash_request_set_callback(struct ahash_request *req, crypto_completion_t compl, void *data) { - req->base.complete = compl; - req->base.data = data; - req->base.flags = flags; + flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + flags |= req->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flags, compl, data); } /** @@ -647,6 +697,30 @@ static inline void ahash_request_set_crypt(struct ahash_request *req, req->src = src; req->nbytes = nbytes; req->result = result; + req->base.flags &= ~CRYPTO_AHASH_REQ_VIRT; +} + +/** + * ahash_request_set_virt() - set virtual address data buffers + * @req: ahash_request handle to be updated + * @src: source virtual address + * @result: buffer that is filled with the message digest -- the caller must + * ensure that the buffer has sufficient space by, for example, calling + * crypto_ahash_digestsize() + * @nbytes: number of bytes to process from the source virtual address + * + * By using this call, the caller references the source virtual address. + * The source virtual address points to the data the message digest is to + * be calculated for. + */ +static inline void ahash_request_set_virt(struct ahash_request *req, + const u8 *src, u8 *result, + unsigned int nbytes) +{ + req->svirt = src; + req->nbytes = nbytes; + req->result = result; + req->base.flags |= CRYPTO_AHASH_REQ_VIRT; } /** @@ -679,8 +753,6 @@ static inline void ahash_request_set_crypt(struct ahash_request *req, struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type, u32 mask); -struct crypto_shash *crypto_clone_shash(struct crypto_shash *tfm); - int crypto_has_shash(const char *alg_name, u32 type, u32 mask); static inline struct crypto_tfm *crypto_shash_tfm(struct crypto_shash *tfm) @@ -784,7 +856,7 @@ static inline void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags) */ static inline unsigned int crypto_shash_descsize(struct crypto_shash *tfm) { - return tfm->descsize; + return crypto_shash_alg(tfm)->descsize; } static inline void *shash_desc_ctx(struct shash_desc *desc) @@ -802,7 +874,7 @@ static inline void *shash_desc_ctx(struct shash_desc *desc) * cipher handle must point to a keyed message digest cipher in order for this * function to succeed. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, @@ -819,7 +891,7 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, * crypto_shash_update and crypto_shash_final. The parameters have the same * meaning as discussed for those separate three functions. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ @@ -839,12 +911,15 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, * directly, and it allocates a hash descriptor on the stack internally. * Note that this stack allocation may be fairly large. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 on success; < 0 if an error occurred. */ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, unsigned int len, u8 *out); +int crypto_hash_digest(struct crypto_ahash *tfm, const u8 *data, + unsigned int len, u8 *out); + /** * crypto_shash_export() - extract operational state for message digest * @desc: reference to the operational state handle whose state is exported @@ -854,7 +929,7 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data, * caller-allocated output buffer out which must have sufficient size (e.g. by * calling crypto_shash_descsize). * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the export creation was successful; < 0 if an error occurred */ int crypto_shash_export(struct shash_desc *desc, void *out); @@ -868,7 +943,7 @@ int crypto_shash_export(struct shash_desc *desc, void *out); * the input buffer. That buffer should have been generated with the * crypto_ahash_export function. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the import was successful; < 0 if an error occurred */ int crypto_shash_import(struct shash_desc *desc, const void *in); @@ -881,19 +956,29 @@ int crypto_shash_import(struct shash_desc *desc, const void *in); * operational state handle. Any potentially existing state created by * previous operations is discarded. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest initialization was successful; < 0 if an * error occurred */ -static inline int crypto_shash_init(struct shash_desc *desc) -{ - struct crypto_shash *tfm = desc->tfm; - - if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - return -ENOKEY; +int crypto_shash_init(struct shash_desc *desc); - return crypto_shash_alg(tfm)->init(desc); -} +/** + * crypto_shash_finup() - calculate message digest of buffer + * @desc: see crypto_shash_final() + * @data: see crypto_shash_update() + * @len: see crypto_shash_update() + * @out: see crypto_shash_final() + * + * This function is a "short-hand" for the function calls of + * crypto_shash_update and crypto_shash_final. The parameters have the same + * meaning as discussed for those separate functions. + * + * Context: Softirq or process context. + * Return: 0 if the message digest creation was successful; < 0 if an error + * occurred + */ +int crypto_shash_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out); /** * crypto_shash_update() - add data to message digest for processing @@ -903,12 +988,15 @@ static inline int crypto_shash_init(struct shash_desc *desc) * * Updates the message digest state of the operational state handle. * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest update was successful; < 0 if an error * occurred */ -int crypto_shash_update(struct shash_desc *desc, const u8 *data, - unsigned int len); +static inline int crypto_shash_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + return crypto_shash_finup(desc, data, len, NULL); +} /** * crypto_shash_final() - calculate message digest @@ -920,29 +1008,14 @@ int crypto_shash_update(struct shash_desc *desc, const u8 *data, * into the output buffer. The caller must ensure that the output buffer is * large enough by using crypto_shash_digestsize. * - * Context: Any context. - * Return: 0 if the message digest creation was successful; < 0 if an error - * occurred - */ -int crypto_shash_final(struct shash_desc *desc, u8 *out); - -/** - * crypto_shash_finup() - calculate message digest of buffer - * @desc: see crypto_shash_final() - * @data: see crypto_shash_update() - * @len: see crypto_shash_update() - * @out: see crypto_shash_final() - * - * This function is a "short-hand" for the function calls of - * crypto_shash_update and crypto_shash_final. The parameters have the same - * meaning as discussed for those separate functions. - * - * Context: Any context. + * Context: Softirq or process context. * Return: 0 if the message digest creation was successful; < 0 if an error * occurred */ -int crypto_shash_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out); +static inline int crypto_shash_final(struct shash_desc *desc, u8 *out) +{ + return crypto_shash_finup(desc, NULL, 0, out); +} static inline void shash_desc_zero(struct shash_desc *desc) { @@ -950,4 +1023,25 @@ static inline void shash_desc_zero(struct shash_desc *desc) sizeof(*desc) + crypto_shash_descsize(desc->tfm)); } +static inline bool ahash_is_async(struct crypto_ahash *tfm) +{ + return crypto_tfm_is_async(&tfm->base); +} + +static inline struct ahash_request *ahash_request_on_stack_init( + char *buf, struct crypto_ahash *tfm) +{ + struct ahash_request *req = (void *)buf; + + crypto_stack_request_init(&req->base, crypto_ahash_tfm(tfm)); + return req; +} + +static inline struct ahash_request *ahash_request_clone( + struct ahash_request *req, size_t total, gfp_t gfp) +{ + return container_of(crypto_request_clone(&req->base, total, gfp), + struct ahash_request, base); +} + #endif /* _CRYPTO_HASH_H */ diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index f7b3b93f3a49..0d51428c1da4 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -8,6 +8,7 @@ #ifndef _CRYPTO_IF_ALG_H #define _CRYPTO_IF_ALG_H +#include <linux/bits.h> #include <linux/compiler.h> #include <linux/completion.h> #include <linux/if_alg.h> @@ -41,7 +42,7 @@ struct af_alg_control { }; struct af_alg_type { - void *(*bind)(const char *name, u32 type, u32 mask); + void *(*bind)(const char *name); void (*release)(void *private); int (*setkey)(void *private, const u8 *key, unsigned int keylen); int (*setentropy)(void *private, sockptr_t entropy, unsigned int len); @@ -80,7 +81,6 @@ struct af_alg_rsgl { /** * struct af_alg_async_req - definition of crypto request - * @iocb: IOCB for AIO operations * @sk: Socket the request is associated with * @first_rsgl: First RX SG * @last_rsgl: Pointer to last RX SG @@ -92,7 +92,6 @@ struct af_alg_rsgl { * @cra_u: Cipher request */ struct af_alg_async_req { - struct kiocb *iocb; struct sock *sk; struct af_alg_rsgl first_rsgl; @@ -123,7 +122,7 @@ struct af_alg_async_req { * @iv: IV for cipher operation * @state: Existing state for continuing operation * @aead_assoclen: Length of AAD for AEAD cipher operations - * @completion: Work queue for synchronous operation + * @wait: For waiting for completion of async crypto ops * @used: TX bytes sent to kernel. This variable is used to * ensure that user space cannot cause the kernel * to allocate too much memory in sendmsg operation. @@ -135,9 +134,10 @@ struct af_alg_async_req { * SG? * @enc: Cryptographic operation to be performed when * recvmsg is invoked. + * @write: True if we are in the middle of a write. * @init: True if metadata has been sent. * @len: Length of memory allocated for this data structure. - * @inflight: Non-zero when AIO requests are in flight. + * @inflight: Non-zero when requests are in flight, for debugging only. */ struct af_alg_ctx { struct list_head tsgl_list; @@ -151,19 +151,31 @@ struct af_alg_ctx { size_t used; atomic_t rcvused; - bool more; - bool merge; - bool enc; - bool init; + bool more:1, + merge:1, + enc:1, + write:1, + init:1; unsigned int len; unsigned int inflight; }; +/* Flags for af_alg_allowlist_entry::flags: */ +#define AF_ALG_UNPRIVILEGED BIT(0) /* Unprivileged use is allowed */ + +struct af_alg_allowlist_entry { + const char *name; + u32 flags; +}; + int af_alg_register_type(const struct af_alg_type *type); int af_alg_unregister_type(const struct af_alg_type *type); +int af_alg_check_restriction(const char *name, + const struct af_alg_allowlist_entry allowlist[]); + int af_alg_release(struct socket *sock); void af_alg_release_parent(struct sock *sk); int af_alg_accept(struct sock *sk, struct socket *newsock, @@ -177,10 +189,11 @@ static inline struct alg_sock *alg_sk(struct sock *sk) } /** - * Size of available buffer for sending data from user space to kernel. + * af_alg_sndbuf - Size of available buffer for sending data from user space to kernel. * - * @sk socket of connection to user space - * @return number of bytes still available + * @sk: socket of connection to user space + * + * Returns: number of bytes still available */ static inline int af_alg_sndbuf(struct sock *sk) { @@ -192,10 +205,11 @@ static inline int af_alg_sndbuf(struct sock *sk) } /** - * Can the send buffer still be written to? + * af_alg_writable - Can the send buffer still be written to? + * + * @sk: socket of connection to user space * - * @sk socket of connection to user space - * @return true => writable, false => not writable + * Returns: true => writable, false => not writable */ static inline bool af_alg_writable(struct sock *sk) { @@ -203,10 +217,11 @@ static inline bool af_alg_writable(struct sock *sk) } /** - * Size of available buffer used by kernel for the RX user space operation. + * af_alg_rcvbuf - Size of available buffer used by kernel for the RX user space operation. + * + * @sk: socket of connection to user space * - * @sk socket of connection to user space - * @return number of bytes still available + * Returns: number of bytes still available */ static inline int af_alg_rcvbuf(struct sock *sk) { @@ -218,25 +233,24 @@ static inline int af_alg_rcvbuf(struct sock *sk) } /** - * Can the RX buffer still be written to? + * af_alg_readable - Can the RX buffer still be read from? * - * @sk socket of connection to user space - * @return true => writable, false => not writable + * @sk: socket of connection to user space + * + * Returns: true => readable, false => not readable */ static inline bool af_alg_readable(struct sock *sk) { return PAGE_SIZE <= af_alg_rcvbuf(sk); } -unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset); -void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, - size_t dst_offset); +unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes); +void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst); void af_alg_wmem_wakeup(struct sock *sk); int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min); int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, unsigned int ivsize); void af_alg_free_resources(struct af_alg_async_req *areq); -void af_alg_async_cb(void *data, int err); __poll_t af_alg_poll(struct file *file, struct socket *sock, poll_table *wait); struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, @@ -245,4 +259,16 @@ int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, struct af_alg_async_req *areq, size_t maxsize, size_t *outlen); +/* + * Mask used to disable unsupported algorithm implementations. + * + * This is the same as FSCRYPT_CRYPTOAPI_MASK in fs/crypto/fscrypt_private.h. + * In additions to the motivations there, this API is exposed to userspace + * that might not be fully trusted. + */ +#define AF_ALG_CRYPTOAPI_MASK \ + (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | \ + CRYPTO_ALG_KERN_DRIVER_ONLY) + + #endif /* _CRYPTO_IF_ALG_H */ diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8831edaafc05..9cd37df32dc4 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -11,13 +11,23 @@ #include <crypto/acompress.h> #include <crypto/algapi.h> +#include <crypto/scatterwalk.h> +#include <linux/compiler_types.h> +#include <linux/cpumask_types.h> +#include <linux/spinlock.h> +#include <linux/workqueue_types.h> + +#define ACOMP_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_fbreq_on_stack_init( \ + __##name##_req, (req)) /** * struct acomp_alg - asynchronous compression algorithm * * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation - * @dst_free: Frees destination buffer if allocated inside the algorithm * @init: Initialize the cryptographic transformation object. * This function is used to initialize the cryptographic * transformation object. This function is called only once at @@ -30,25 +40,68 @@ * counterpart to @init, used to remove various changes set in * @init. * - * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure * @calg: Cmonn algorithm data structure shared with scomp + * @COMP_ALG_COMMON: see struct comp_alg_common */ struct acomp_alg { int (*compress)(struct acomp_req *req); int (*decompress)(struct acomp_req *req); - void (*dst_free)(struct scatterlist *dst); int (*init)(struct crypto_acomp *tfm); void (*exit)(struct crypto_acomp *tfm); - unsigned int reqsize; - union { struct COMP_ALG_COMMON; struct comp_alg_common calg; }; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + +struct crypto_acomp_streams { + /* These must come first because of struct scomp_alg. */ + void *(*alloc_ctx)(void); + void (*free_ctx)(void *); + + struct crypto_acomp_stream __percpu *streams; + struct work_struct stream_work; + cpumask_t stream_want; +}; + +struct acomp_walk { + union { + /* Virtual address of the source. */ + struct { + struct { + const void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + + unsigned int slen; + unsigned int dlen; + + int flags; +}; + /* * Transform internal helpers. */ @@ -68,22 +121,6 @@ static inline void acomp_request_complete(struct acomp_req *req, crypto_request_complete(&req->base, err); } -static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm) -{ - struct acomp_req *req; - - req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); - if (likely(req)) - acomp_request_set_tfm(req, tfm); - return req; -} -#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__)) - -static inline void __acomp_request_free(struct acomp_req *req) -{ - kfree_sensitive(req); -} - /** * crypto_register_acomp() -- Register asynchronous compression algorithm * @@ -109,4 +146,101 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); +static inline bool acomp_request_issg(struct acomp_req *req) +{ + return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT)); +} + +static inline bool acomp_request_src_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +static inline bool acomp_request_dst_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT; +} + +static inline bool acomp_request_isvirt(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT); +} + +static inline bool acomp_request_src_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_NONDMA; +} + +static inline bool acomp_request_dst_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_NONDMA; +} + +static inline bool acomp_request_isnondma(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_NONDMA); +} + +static inline bool crypto_acomp_req_virt(struct crypto_acomp *tfm) +{ + return crypto_tfm_req_virt(&tfm->base); +} + +void crypto_acomp_free_streams(struct crypto_acomp_streams *s); +int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s); + +#define crypto_acomp_lock_stream_bh(...) __acquire_ret(_crypto_acomp_lock_stream_bh(__VA_ARGS__), &__ret->lock); +struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires_ret; + +static inline void crypto_acomp_unlock_stream_bh( + struct crypto_acomp_stream *stream) __releases(&stream->lock) +{ + spin_unlock_bh(&stream->lock); +} + +void acomp_walk_done_src(struct acomp_walk *walk, int used); +void acomp_walk_done_dst(struct acomp_walk *walk, int used); +int acomp_walk_next_src(struct acomp_walk *walk); +int acomp_walk_next_dst(struct acomp_walk *walk); +int acomp_walk_virt(struct acomp_walk *__restrict walk, + struct acomp_req *__restrict req, bool atomic); + +static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) +{ + return walk->slen != cur; +} + +static inline u32 acomp_request_flags(struct acomp_req *req) +{ + return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; +} + +static inline struct crypto_acomp *crypto_acomp_fb(struct crypto_acomp *tfm) +{ + return __crypto_acomp_tfm(crypto_acomp_tfm(tfm)->fb); +} + +static inline struct acomp_req *acomp_fbreq_on_stack_init( + char *buf, struct acomp_req *old) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); + struct acomp_req *req = (void *)buf; + + crypto_stack_request_init(&req->base, + crypto_acomp_tfm(crypto_acomp_fb(tfm))); + acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + req->src = old->src; + req->dst = old->dst; + req->slen = old->slen; + req->dlen = old->dlen; + + return req; +} + #endif diff --git a/include/crypto/internal/blake2b.h b/include/crypto/internal/blake2b.h deleted file mode 100644 index 982fe5e8471c..000000000000 --- a/include/crypto/internal/blake2b.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -/* - * Helper functions for BLAKE2b implementations. - * Keep this in sync with the corresponding BLAKE2s header. - */ - -#ifndef _CRYPTO_INTERNAL_BLAKE2B_H -#define _CRYPTO_INTERNAL_BLAKE2B_H - -#include <crypto/blake2b.h> -#include <crypto/internal/hash.h> -#include <linux/string.h> - -void blake2b_compress_generic(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc); - -static inline void blake2b_set_lastblock(struct blake2b_state *state) -{ - state->f[0] = -1; -} - -typedef void (*blake2b_compress_t)(struct blake2b_state *state, - const u8 *block, size_t nblocks, u32 inc); - -static inline void __blake2b_update(struct blake2b_state *state, - const u8 *in, size_t inlen, - blake2b_compress_t compress) -{ - const size_t fill = BLAKE2B_BLOCK_SIZE - state->buflen; - - if (unlikely(!inlen)) - return; - if (inlen > fill) { - memcpy(state->buf + state->buflen, in, fill); - (*compress)(state, state->buf, 1, BLAKE2B_BLOCK_SIZE); - state->buflen = 0; - in += fill; - inlen -= fill; - } - if (inlen > BLAKE2B_BLOCK_SIZE) { - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2B_BLOCK_SIZE); - /* Hash one less (full) block than strictly possible */ - (*compress)(state, in, nblocks - 1, BLAKE2B_BLOCK_SIZE); - in += BLAKE2B_BLOCK_SIZE * (nblocks - 1); - inlen -= BLAKE2B_BLOCK_SIZE * (nblocks - 1); - } - memcpy(state->buf + state->buflen, in, inlen); - state->buflen += inlen; -} - -static inline void __blake2b_final(struct blake2b_state *state, u8 *out, - blake2b_compress_t compress) -{ - int i; - - blake2b_set_lastblock(state); - memset(state->buf + state->buflen, 0, - BLAKE2B_BLOCK_SIZE - state->buflen); /* Padding */ - (*compress)(state, state->buf, 1, state->buflen); - for (i = 0; i < ARRAY_SIZE(state->h); i++) - __cpu_to_le64s(&state->h[i]); - memcpy(out, state->h, state->outlen); -} - -/* Helper functions for shash implementations of BLAKE2b */ - -struct blake2b_tfm_ctx { - u8 key[BLAKE2B_KEY_SIZE]; - unsigned int keylen; -}; - -static inline int crypto_blake2b_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen) -{ - struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(tfm); - - if (keylen == 0 || keylen > BLAKE2B_KEY_SIZE) - return -EINVAL; - - memcpy(tctx->key, key, keylen); - tctx->keylen = keylen; - - return 0; -} - -static inline int crypto_blake2b_init(struct shash_desc *desc) -{ - const struct blake2b_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); - struct blake2b_state *state = shash_desc_ctx(desc); - unsigned int outlen = crypto_shash_digestsize(desc->tfm); - - __blake2b_init(state, outlen, tctx->key, tctx->keylen); - return 0; -} - -static inline int crypto_blake2b_update(struct shash_desc *desc, - const u8 *in, unsigned int inlen, - blake2b_compress_t compress) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - - __blake2b_update(state, in, inlen, compress); - return 0; -} - -static inline int crypto_blake2b_final(struct shash_desc *desc, u8 *out, - blake2b_compress_t compress) -{ - struct blake2b_state *state = shash_desc_ctx(desc); - - __blake2b_final(state, out, compress); - return 0; -} - -#endif /* _CRYPTO_INTERNAL_BLAKE2B_H */ diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h deleted file mode 100644 index 506d56530ca9..000000000000 --- a/include/crypto/internal/blake2s.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -/* - * Helper functions for BLAKE2s implementations. - * Keep this in sync with the corresponding BLAKE2b header. - */ - -#ifndef _CRYPTO_INTERNAL_BLAKE2S_H -#define _CRYPTO_INTERNAL_BLAKE2S_H - -#include <crypto/blake2s.h> -#include <linux/string.h> - -void blake2s_compress_generic(struct blake2s_state *state, const u8 *block, - size_t nblocks, const u32 inc); - -void blake2s_compress(struct blake2s_state *state, const u8 *block, - size_t nblocks, const u32 inc); - -bool blake2s_selftest(void); - -#endif /* _CRYPTO_INTERNAL_BLAKE2S_H */ diff --git a/include/crypto/internal/chacha.h b/include/crypto/internal/chacha.h deleted file mode 100644 index b085dc1ac151..000000000000 --- a/include/crypto/internal/chacha.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef _CRYPTO_INTERNAL_CHACHA_H -#define _CRYPTO_INTERNAL_CHACHA_H - -#include <crypto/chacha.h> -#include <crypto/internal/skcipher.h> -#include <linux/crypto.h> - -struct chacha_ctx { - u32 key[8]; - int nrounds; -}; - -static inline int chacha_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize, int nrounds) -{ - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - int i; - - if (keysize != CHACHA_KEY_SIZE) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(ctx->key); i++) - ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32)); - - ctx->nrounds = nrounds; - return 0; -} - -static inline int chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 20); -} - -static inline int chacha12_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int keysize) -{ - return chacha_setkey(tfm, key, keysize, 12); -} - -#endif /* _CRYPTO_CHACHA_H */ diff --git a/include/crypto/internal/cipher.h b/include/crypto/internal/cipher.h index 5030f6d2df31..a9174ba90250 100644 --- a/include/crypto/internal/cipher.h +++ b/include/crypto/internal/cipher.h @@ -176,8 +176,6 @@ void crypto_cipher_encrypt_one(struct crypto_cipher *tfm, void crypto_cipher_decrypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src); -struct crypto_cipher *crypto_clone_cipher(struct crypto_cipher *cipher); - struct crypto_cipher_spawn { struct crypto_spawn base; }; diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h index 57cd75242141..d67fe13a543a 100644 --- a/include/crypto/internal/ecc.h +++ b/include/crypto/internal/ecc.h @@ -72,8 +72,8 @@ static inline void ecc_swap_digits(const void *in, u64 *out, unsigned int ndigit /** * ecc_digits_from_bytes() - Create ndigits-sized digits array from byte array * @in: Input byte array - * @nbytes Size of input byte array - * @out Output digits array + * @nbytes: Size of input byte array + * @out: Output digits array * @ndigits: Number of digits to create from byte array * * The first byte in the input byte array is expected to hold the most @@ -90,7 +90,7 @@ void ecc_digits_from_bytes(const u8 *in, unsigned int nbytes, * @private_key: private key to be used for the given curve * @private_key_len: private key length * - * Returns 0 if the key is acceptable, a negative value otherwise + * Returns: 0 if the key is acceptable, a negative value otherwise */ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, const u64 *private_key, unsigned int private_key_len); @@ -104,7 +104,7 @@ int ecc_is_key_valid(unsigned int curve_id, unsigned int ndigits, * @ndigits: curve number of digits * @private_key: buffer for storing the generated private key * - * Returns 0 if the private key was generated successfully, a negative value + * Returns: 0 if the private key was generated successfully, a negative value * if an error occurred. */ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, @@ -118,7 +118,7 @@ int ecc_gen_privkey(unsigned int curve_id, unsigned int ndigits, * @private_key: pregenerated private key for the given curve * @public_key: buffer for storing the generated public key * - * Returns 0 if the public key was generated successfully, a negative value + * Returns: 0 if the public key was generated successfully, a negative value * if an error occurred. */ int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits, @@ -136,7 +136,7 @@ int ecc_make_pub_key(const unsigned int curve_id, unsigned int ndigits, * Note: It is recommended that you hash the result of crypto_ecdh_shared_secret * before using it for symmetric encryption or HMAC. * - * Returns 0 if the shared secret was generated successfully, a negative value + * Returns: 0 if the shared secret was generated successfully, a negative value * if an error occurred. */ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, @@ -149,7 +149,7 @@ int crypto_ecdh_shared_secret(unsigned int curve_id, unsigned int ndigits, * @curve: elliptic curve domain parameters * @pk: public key as a point * - * Valdiate public key according to SP800-56A section 5.6.2.3.4 ECC Partial + * Validate public key according to SP800-56A section 5.6.2.3.4 ECC Partial * Public-Key Validation Routine. * * Note: There is no check that the public key is in the correct elliptic curve @@ -166,7 +166,7 @@ int ecc_is_pubkey_valid_partial(const struct ecc_curve *curve, * @curve: elliptic curve domain parameters * @pk: public key as a point * - * Valdiate public key according to SP800-56A section 5.6.2.3.3 ECC Full + * Validate public key according to SP800-56A section 5.6.2.3.3 ECC Full * Public-Key Validation Routine. * * Return: 0 if validation is successful, -EINVAL if validation is failed. @@ -179,6 +179,8 @@ int ecc_is_pubkey_valid_full(const struct ecc_curve *curve, * * @vli: vli to check. * @ndigits: length of the @vli + * + * Returns: %true if vli == 0, %false otherwise. */ bool vli_is_zero(const u64 *vli, unsigned int ndigits); @@ -189,7 +191,7 @@ bool vli_is_zero(const u64 *vli, unsigned int ndigits); * @right: vli * @ndigits: length of both vlis * - * Returns sign of @left - @right, i.e. -1 if @left < @right, + * Returns: sign of @left - @right, i.e. -1 if @left < @right, * 0 if @left == @right, 1 if @left > @right. */ int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits); @@ -199,7 +201,7 @@ int vli_cmp(const u64 *left, const u64 *right, unsigned int ndigits); * * @result: where to write result * @left: vli - * @right vli + * @right: vli * @ndigits: length of all vlis * * Note: can modify in-place. @@ -263,7 +265,7 @@ void vli_mod_mult_slow(u64 *result, const u64 *left, const u64 *right, unsigned int vli_num_bits(const u64 *vli, unsigned int ndigits); /** - * ecc_aloc_point() - Allocate ECC point. + * ecc_alloc_point() - Allocate ECC point. * * @ndigits: Length of vlis in u64 qwords. * @@ -281,7 +283,7 @@ void ecc_free_point(struct ecc_point *p); /** * ecc_point_is_zero() - Check if point is zero. * - * @p: Point to check for zero. + * @point: Point to check for zero. * * Return: true if point is the point at infinity, false otherwise. */ diff --git a/include/crypto/internal/engine.h b/include/crypto/internal/engine.h index fbf4be56cf12..6a1d27880615 100644 --- a/include/crypto/internal/engine.h +++ b/include/crypto/internal/engine.h @@ -21,24 +21,15 @@ struct device; /* * struct crypto_engine - crypto hardware engine * @name: the engine name - * @idling: the engine is entering idle state * @busy: request pump is busy * @running: the engine is on working * @retry_support: indication that the hardware allows re-execution * of a failed backlog request * crypto-engine, in head position to keep order + * @rt: whether this queue is set to run as a realtime task * @list: link with the global crypto engine list * @queue_lock: spinlock to synchronise access to request queue * @queue: the crypto queue of the engine - * @rt: whether this queue is set to run as a realtime task - * @prepare_crypt_hardware: a request will soon arrive from the queue - * so the subsystem requests the driver to prepare the hardware - * by issuing this call - * @unprepare_crypt_hardware: there are currently no more requests on the - * queue so the subsystem notifies the driver that it may relax the - * hardware by issuing this call - * @do_batch_requests: execute a batch of requests. Depends on multiple - * requests support. * @kworker: kthread worker struct for request pump * @pump_requests: work struct for scheduling work to the request pump * @priv_data: the engine private data @@ -46,24 +37,17 @@ struct device; */ struct crypto_engine { char name[ENGINE_NAME_LEN]; - bool idling; bool busy; bool running; bool retry_support; + bool rt; struct list_head list; spinlock_t queue_lock; - struct crypto_queue queue; + struct crypto_queue queue __guarded_by(&queue_lock); struct device *dev; - bool rt; - - int (*prepare_crypt_hardware)(struct crypto_engine *engine); - int (*unprepare_crypt_hardware)(struct crypto_engine *engine); - int (*do_batch_requests)(struct crypto_engine *engine); - - struct kthread_worker *kworker; struct kthread_work pump_requests; diff --git a/include/crypto/internal/geniv.h b/include/crypto/internal/geniv.h index 7fd7126f593a..e38d9f0487ec 100644 --- a/include/crypto/internal/geniv.h +++ b/include/crypto/internal/geniv.h @@ -9,13 +9,10 @@ #define _CRYPTO_INTERNAL_GENIV_H #include <crypto/internal/aead.h> -#include <linux/spinlock.h> #include <linux/types.h> struct aead_geniv_ctx { - spinlock_t lock; struct crypto_aead *child; - struct crypto_sync_skcipher *sknull; u8 salt[] __attribute__ ((aligned(__alignof__(u32)))); }; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 84da3424decc..6ec5f2f37ccb 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -11,7 +11,39 @@ #include <crypto/algapi.h> #include <crypto/hash.h> +/* Set this bit to handle partial blocks in the API. */ +#define CRYPTO_AHASH_ALG_BLOCK_ONLY 0x01000000 + +/* Set this bit if final requires at least one byte. */ +#define CRYPTO_AHASH_ALG_FINAL_NONZERO 0x02000000 + +/* Set this bit if finup can deal with multiple blocks. */ +#define CRYPTO_AHASH_ALG_FINUP_MAX 0x04000000 + +/* This bit is set by the Crypto API if export_core is not supported. */ +#define CRYPTO_AHASH_ALG_NO_EXPORT_CORE 0x08000000 + +#define HASH_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct ahash_request) + \ + MAX_SYNC_HASH_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct ahash_request *name = ahash_fbreq_on_stack_init( \ + __##name##_req, (req)) + struct ahash_request; +struct scatterlist; + +struct crypto_hash_walk { + const char *data; + + unsigned int offset; + unsigned int flags; + + struct page *pg; + unsigned int entrylen; + + unsigned int total; + struct scatterlist *sg; +}; struct ahash_instance { void (*free)(struct ahash_instance *inst); @@ -43,12 +75,22 @@ struct crypto_shash_spawn { struct crypto_spawn base; }; +int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err); +int crypto_hash_walk_first(struct ahash_request *req, + struct crypto_hash_walk *walk); + +static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) +{ + return !(walk->entrylen | walk->total); +} + int crypto_register_ahash(struct ahash_alg *alg); void crypto_unregister_ahash(struct ahash_alg *alg); int crypto_register_ahashes(struct ahash_alg *algs, int count); void crypto_unregister_ahashes(struct ahash_alg *algs, int count); int ahash_register_instance(struct crypto_template *tmpl, struct ahash_instance *inst); +void ahash_free_singlespawn_instance(struct ahash_instance *inst); int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); @@ -58,12 +100,26 @@ static inline bool crypto_shash_alg_has_setkey(struct shash_alg *alg) return alg->setkey != shash_no_setkey; } +bool crypto_hash_alg_has_setkey(struct hash_alg_common *halg); + static inline bool crypto_shash_alg_needs_key(struct shash_alg *alg) { return crypto_shash_alg_has_setkey(alg) && !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); } +static inline bool crypto_hash_alg_needs_key(struct hash_alg_common *alg) +{ + return crypto_hash_alg_has_setkey(alg) && + !(alg->base.cra_flags & CRYPTO_ALG_OPTIONAL_KEY); +} + +static inline bool crypto_hash_no_export_core(struct crypto_ahash *tfm) +{ + return crypto_hash_alg_common(tfm)->base.cra_flags & + CRYPTO_AHASH_ALG_NO_EXPORT_CORE; +} + int crypto_grab_ahash(struct crypto_ahash_spawn *spawn, struct crypto_instance *inst, const char *name, u32 type, u32 mask); @@ -140,6 +196,13 @@ static inline void crypto_ahash_set_reqsize(struct crypto_ahash *tfm, tfm->reqsize = reqsize; } +static inline bool crypto_ahash_tested(struct crypto_ahash *tfm) +{ + struct crypto_tfm *tfm_base = crypto_ahash_tfm(tfm); + + return tfm_base->__crt_alg->cra_flags & CRYPTO_ALG_TESTED; +} + static inline void crypto_ahash_set_reqsize_dma(struct crypto_ahash *ahash, unsigned int reqsize) { @@ -187,7 +250,7 @@ static inline void ahash_request_complete(struct ahash_request *req, int err) static inline u32 ahash_request_flags(struct ahash_request *req) { - return req->base.flags; + return crypto_request_flags(&req->base) & ~CRYPTO_AHASH_REQ_PRIVATE; } static inline struct crypto_ahash *crypto_spawn_ahash( @@ -247,5 +310,96 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) return container_of(tfm, struct crypto_shash, base); } +static inline bool ahash_request_isvirt(struct ahash_request *req) +{ + return req->base.flags & CRYPTO_AHASH_REQ_VIRT; +} + +static inline bool crypto_ahash_req_virt(struct crypto_ahash *tfm) +{ + return crypto_tfm_req_virt(&tfm->base); +} + +static inline struct crypto_ahash *crypto_ahash_fb(struct crypto_ahash *tfm) +{ + return __crypto_ahash_cast(crypto_ahash_tfm(tfm)->fb); +} + +static inline struct ahash_request *ahash_fbreq_on_stack_init( + char *buf, struct ahash_request *old) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(old); + struct ahash_request *req = (void *)buf; + + crypto_stack_request_init(&req->base, + crypto_ahash_tfm(crypto_ahash_fb(tfm))); + ahash_request_set_callback(req, ahash_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_AHASH_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_AHASH_REQ_PRIVATE; + req->src = old->src; + req->result = old->result; + req->nbytes = old->nbytes; + + return req; +} + +/* Return the state size without partial block for block-only algorithms. */ +static inline unsigned int crypto_shash_coresize(struct crypto_shash *tfm) +{ + return crypto_shash_statesize(tfm) - crypto_shash_blocksize(tfm) - 1; +} + +/* This can only be used if the request was never cloned. */ +#define HASH_REQUEST_ZERO(name) \ + memzero_explicit(__##name##_req, sizeof(__##name##_req)) + +/** + * crypto_ahash_export_core() - extract core state for message digest + * @req: reference to the ahash_request handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_ahash_export_core(struct ahash_request *req, void *out); + +/** + * crypto_ahash_import_core() - import core state + * @req: reference to ahash_request handle the state is imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_ahash_import_core(struct ahash_request *req, const void *in); + +/** + * crypto_shash_export_core() - extract core state for message digest + * @desc: reference to the operational state handle whose state is exported + * @out: output buffer of sufficient size that can hold the hash state + * + * Export the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the export creation was successful; < 0 if an error occurred + */ +int crypto_shash_export_core(struct shash_desc *desc, void *out); + +/** + * crypto_shash_import_core() - import core state + * @desc: reference to the operational state handle the state imported into + * @in: buffer holding the state + * + * Import the hash state without the partial block buffer. + * + * Context: Softirq or process context. + * Return: 0 if the import was successful; < 0 if an error occurred + */ +int crypto_shash_import_core(struct shash_desc *desc, const void *in); + #endif /* _CRYPTO_INTERNAL_HASH_H */ diff --git a/include/crypto/internal/poly1305.h b/include/crypto/internal/poly1305.h index e614594f88c1..a72fff409ab8 100644 --- a/include/crypto/internal/poly1305.h +++ b/include/crypto/internal/poly1305.h @@ -6,9 +6,8 @@ #ifndef _CRYPTO_INTERNAL_POLY1305_H #define _CRYPTO_INTERNAL_POLY1305_H -#include <linux/unaligned.h> -#include <linux/types.h> #include <crypto/poly1305.h> +#include <linux/types.h> /* * Poly1305 core functions. These only accept whole blocks; the caller must @@ -31,4 +30,27 @@ void poly1305_core_blocks(struct poly1305_state *state, void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4], void *dst); +static inline void +poly1305_block_init_generic(struct poly1305_block_state *desc, + const u8 raw_key[POLY1305_BLOCK_SIZE]) +{ + poly1305_core_init(&desc->h); + poly1305_core_setkey(&desc->core_r, raw_key); +} + +static inline void poly1305_blocks_generic(struct poly1305_block_state *state, + const u8 *src, unsigned int len, + u32 padbit) +{ + poly1305_core_blocks(&state->h, &state->core_r, src, + len / POLY1305_BLOCK_SIZE, padbit); +} + +static inline void poly1305_emit_generic(const struct poly1305_state *state, + u8 digest[POLY1305_DIGEST_SIZE], + const u32 nonce[4]) +{ + poly1305_core_emit(state, nonce, digest); +} + #endif diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 07a10fd2d321..13a0851a995b 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -9,12 +9,7 @@ #ifndef _CRYPTO_SCOMP_INT_H #define _CRYPTO_SCOMP_INT_H -#include <crypto/acompress.h> -#include <crypto/algapi.h> - -#define SCOMP_SCRATCH_SIZE 131072 - -struct acomp_req; +#include <crypto/internal/acompress.h> struct crypto_scomp { struct crypto_tfm base; @@ -23,16 +18,13 @@ struct crypto_scomp { /** * struct scomp_alg - synchronous compression algorithm * - * @alloc_ctx: Function allocates algorithm specific context - * @free_ctx: Function frees context allocated with alloc_ctx * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation - * @base: Common crypto API algorithm data structure + * @streams: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp + * @COMP_ALG_COMMON: see struct comp_alg_common */ struct scomp_alg { - void *(*alloc_ctx)(struct crypto_scomp *tfm); - void (*free_ctx)(struct crypto_scomp *tfm, void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -40,6 +32,8 @@ struct scomp_alg { unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); + struct crypto_acomp_streams streams; + union { struct COMP_ALG_COMMON; struct comp_alg_common calg; @@ -71,17 +65,6 @@ static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg); } -static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) -{ - return crypto_scomp_alg(tfm)->alloc_ctx(tfm); -} - -static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, - void *ctx) -{ - return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx); -} - static inline int crypto_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index be97b97a75dd..f5e5d7b63951 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -6,28 +6,10 @@ #ifndef _CRYPTO_INTERNAL_SIMD_H #define _CRYPTO_INTERNAL_SIMD_H +#include <asm/simd.h> #include <linux/percpu.h> #include <linux/types.h> -/* skcipher support */ - -struct simd_skcipher_alg; -struct skcipher_alg; - -struct simd_skcipher_alg *simd_skcipher_create_compat(struct skcipher_alg *ialg, - const char *algname, - const char *drvname, - const char *basename); -void simd_skcipher_free(struct simd_skcipher_alg *alg); - -int simd_register_skciphers_compat(struct skcipher_alg *algs, int count, - struct simd_skcipher_alg **simd_algs); - -void simd_unregister_skciphers(struct skcipher_alg *algs, int count, - struct simd_skcipher_alg **simd_algs); - -/* AEAD support */ - struct simd_aead_alg; struct aead_alg; @@ -44,13 +26,10 @@ void simd_unregister_aeads(struct aead_alg *algs, int count, * This delegates to may_use_simd(), except that this also returns false if SIMD * in crypto code has been temporarily disabled on this CPU by the crypto * self-tests, in order to test the no-SIMD fallback code. This override is - * currently limited to configurations where the extra self-tests are enabled, - * because it might be a bit too invasive to be part of the regular self-tests. - * - * This is a macro so that <asm/simd.h>, which some architectures don't have, - * doesn't have to be included directly here. + * currently limited to configurations where the "full" self-tests are enabled, + * because it might be a bit too invasive to be part of the "fast" self-tests. */ -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS_FULL DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); #define crypto_simd_usable() \ (may_use_simd() && !this_cpu_read(crypto_simd_disabled_for_test)) diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index 4f49621d3eb6..a965b6aabf61 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -56,15 +56,30 @@ struct crypto_lskcipher_spawn { struct skcipher_walk { union { + /* Virtual address of the source. */ struct { - void *addr; - } virt; - } src, dst; + struct { + const void *const addr; + } virt; + } src; - struct scatter_walk in; - unsigned int nbytes; + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; + + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; - struct scatter_walk out; + unsigned int nbytes; unsigned int total; u8 *page; @@ -197,13 +212,15 @@ int lskcipher_register_instance(struct crypto_template *tmpl, struct lskcipher_instance *inst); int skcipher_walk_done(struct skcipher_walk *walk, int res); -int skcipher_walk_virt(struct skcipher_walk *walk, - struct skcipher_request *req, +int skcipher_walk_virt(struct skcipher_walk *__restrict walk, + struct skcipher_request *__restrict req, bool atomic); -int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic); -int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, - struct aead_request *req, bool atomic); +int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic); +int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk, + struct aead_request *__restrict req, + bool atomic); static inline void skcipher_walk_abort(struct skcipher_walk *walk) { @@ -225,6 +242,13 @@ static inline void *crypto_skcipher_ctx_dma(struct crypto_skcipher *tfm) return crypto_tfm_ctx_dma(&tfm->base); } +static inline bool crypto_skcipher_tested(struct crypto_skcipher *tfm) +{ + struct crypto_tfm *tfm_base = crypto_skcipher_tfm(tfm); + + return tfm_base->__crt_alg->cra_flags & CRYPTO_ALG_TESTED; +} + static inline void *skcipher_request_ctx(struct skcipher_request *req) { return req->__ctx; diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h new file mode 100644 index 000000000000..aac3ecf88467 --- /dev/null +++ b/include/crypto/krb5.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Kerberos 5 crypto + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef _CRYPTO_KRB5_H +#define _CRYPTO_KRB5_H + +#include <linux/crypto.h> +#include <crypto/aead.h> +#include <crypto/hash.h> + +struct crypto_shash; +struct scatterlist; + +/* + * Per Kerberos v5 protocol spec crypto types from the wire. These get mapped + * to linux kernel crypto routines. + */ +#define KRB5_ENCTYPE_NULL 0x0000 +#define KRB5_ENCTYPE_DES_CBC_CRC 0x0001 /* DES cbc mode with CRC-32 */ +#define KRB5_ENCTYPE_DES_CBC_MD4 0x0002 /* DES cbc mode with RSA-MD4 */ +#define KRB5_ENCTYPE_DES_CBC_MD5 0x0003 /* DES cbc mode with RSA-MD5 */ +#define KRB5_ENCTYPE_DES_CBC_RAW 0x0004 /* DES cbc mode raw */ +/* XXX deprecated? */ +#define KRB5_ENCTYPE_DES3_CBC_SHA 0x0005 /* DES-3 cbc mode with NIST-SHA */ +#define KRB5_ENCTYPE_DES3_CBC_RAW 0x0006 /* DES-3 cbc mode raw */ +#define KRB5_ENCTYPE_DES_HMAC_SHA1 0x0008 +#define KRB5_ENCTYPE_DES3_CBC_SHA1 0x0010 +#define KRB5_ENCTYPE_AES128_CTS_HMAC_SHA1_96 0x0011 +#define KRB5_ENCTYPE_AES256_CTS_HMAC_SHA1_96 0x0012 +#define KRB5_ENCTYPE_AES128_CTS_HMAC_SHA256_128 0x0013 +#define KRB5_ENCTYPE_AES256_CTS_HMAC_SHA384_192 0x0014 +#define KRB5_ENCTYPE_ARCFOUR_HMAC 0x0017 +#define KRB5_ENCTYPE_ARCFOUR_HMAC_EXP 0x0018 +#define KRB5_ENCTYPE_CAMELLIA128_CTS_CMAC 0x0019 +#define KRB5_ENCTYPE_CAMELLIA256_CTS_CMAC 0x001a +#define KRB5_ENCTYPE_UNKNOWN 0x01ff + +#define KRB5_CKSUMTYPE_CRC32 0x0001 +#define KRB5_CKSUMTYPE_RSA_MD4 0x0002 +#define KRB5_CKSUMTYPE_RSA_MD4_DES 0x0003 +#define KRB5_CKSUMTYPE_DESCBC 0x0004 +#define KRB5_CKSUMTYPE_RSA_MD5 0x0007 +#define KRB5_CKSUMTYPE_RSA_MD5_DES 0x0008 +#define KRB5_CKSUMTYPE_NIST_SHA 0x0009 +#define KRB5_CKSUMTYPE_HMAC_SHA1_DES3 0x000c +#define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES128 0x000f +#define KRB5_CKSUMTYPE_HMAC_SHA1_96_AES256 0x0010 +#define KRB5_CKSUMTYPE_CMAC_CAMELLIA128 0x0011 +#define KRB5_CKSUMTYPE_CMAC_CAMELLIA256 0x0012 +#define KRB5_CKSUMTYPE_HMAC_SHA256_128_AES128 0x0013 +#define KRB5_CKSUMTYPE_HMAC_SHA384_192_AES256 0x0014 +#define KRB5_CKSUMTYPE_HMAC_MD5_ARCFOUR -138 /* Microsoft md5 hmac cksumtype */ + +/* + * Constants used for key derivation + */ +/* from rfc3961 */ +#define KEY_USAGE_SEED_CHECKSUM (0x99) +#define KEY_USAGE_SEED_ENCRYPTION (0xAA) +#define KEY_USAGE_SEED_INTEGRITY (0x55) + +/* + * Standard Kerberos error codes. + */ +#define KRB5_PROG_KEYTYPE_NOSUPP -1765328233 + +/* + * Mode of operation. + */ +enum krb5_crypto_mode { + KRB5_CHECKSUM_MODE, /* Checksum only */ + KRB5_ENCRYPT_MODE, /* Fully encrypted, possibly with integrity checksum */ +}; + +struct krb5_buffer { + unsigned int len; + void *data; +}; + +/* + * Kerberos encoding type definition. + */ +struct krb5_enctype { + int etype; /* Encryption (key) type */ + int ctype; /* Checksum type */ + const char *name; /* "Friendly" name */ + const char *encrypt_name; /* Crypto encrypt+checksum name */ + const char *cksum_name; /* Crypto checksum name */ + const char *hash_name; /* Crypto hash name */ + const char *derivation_enc; /* Cipher used in key derivation */ + u16 block_len; /* Length of encryption block */ + u16 conf_len; /* Length of confounder (normally == block_len) */ + u16 cksum_len; /* Length of checksum */ + u16 key_bytes; /* Length of raw key, in bytes */ + u16 key_len; /* Length of final key, in bytes */ + u16 hash_len; /* Length of hash in bytes */ + u16 prf_len; /* Length of PRF() result in bytes */ + u16 Kc_len; /* Length of Kc in bytes */ + u16 Ke_len; /* Length of Ke in bytes */ + u16 Ki_len; /* Length of Ki in bytes */ + bool keyed_cksum; /* T if a keyed cksum */ + + const struct krb5_crypto_profile *profile; + + int (*random_to_key)(const struct krb5_enctype *krb5, + const struct krb5_buffer *in, + struct krb5_buffer *out); /* complete key generation */ +}; + +/* + * krb5_api.c + */ +const struct krb5_enctype *crypto_krb5_find_enctype(u32 enctype); +size_t crypto_krb5_how_much_buffer(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t data_size, size_t *_offset); +size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_buffer_size, size_t *_offset); +int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t *_offset, size_t *_len); +int crypto_krb5_check_data_len(const struct krb5_enctype *krb5, + enum krb5_crypto_mode mode, + size_t len, size_t min_content); +struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp); +struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb5, + const struct krb5_buffer *TK, + u32 usage, gfp_t gfp); +ssize_t crypto_krb5_encrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len, + bool preconfounded); +int crypto_krb5_decrypt(const struct krb5_enctype *krb5, + struct crypto_aead *aead, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); +ssize_t crypto_krb5_get_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t sg_len, + size_t data_offset, size_t data_len); +int crypto_krb5_verify_mic(const struct krb5_enctype *krb5, + struct crypto_shash *shash, + const struct krb5_buffer *metadata, + struct scatterlist *sg, unsigned int nr_sg, + size_t *_offset, size_t *_len); + +/* + * krb5_kdf.c + */ +int crypto_krb5_calc_PRFplus(const struct krb5_enctype *krb5, + const struct krb5_buffer *K, + unsigned int L, + const struct krb5_buffer *S, + struct krb5_buffer *result, + gfp_t gfp); + +#endif /* _CRYPTO_KRB5_H */ diff --git a/include/crypto/md5.h b/include/crypto/md5.h index cf9e9dec3d21..c47aedfe67ec 100644 --- a/include/crypto/md5.h +++ b/include/crypto/md5.h @@ -2,24 +2,209 @@ #ifndef _CRYPTO_MD5_H #define _CRYPTO_MD5_H +#include <crypto/hash.h> #include <linux/types.h> #define MD5_DIGEST_SIZE 16 #define MD5_HMAC_BLOCK_SIZE 64 +#define MD5_BLOCK_SIZE 64 #define MD5_BLOCK_WORDS 16 #define MD5_HASH_WORDS 4 +#define MD5_STATE_SIZE 24 #define MD5_H0 0x67452301UL #define MD5_H1 0xefcdab89UL #define MD5_H2 0x98badcfeUL #define MD5_H3 0x10325476UL +#define CRYPTO_MD5_STATESIZE \ + CRYPTO_HASH_STATESIZE(MD5_STATE_SIZE, MD5_HMAC_BLOCK_SIZE) + extern const u8 md5_zero_message_hash[MD5_DIGEST_SIZE]; struct md5_state { u32 hash[MD5_HASH_WORDS]; - u32 block[MD5_BLOCK_WORDS]; u64 byte_count; + u32 block[MD5_BLOCK_WORDS]; +}; + +/* State for the MD5 compression function */ +struct md5_block_state { + u32 h[MD5_HASH_WORDS]; +}; + +/** + * struct md5_ctx - Context for hashing a message with MD5 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % MD5_BLOCK_SIZE bytes are valid + */ +struct md5_ctx { + struct md5_block_state state; + u64 bytecount; + u8 buf[MD5_BLOCK_SIZE] __aligned(__alignof__(__le64)); +}; + +/** + * md5_init() - Initialize an MD5 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider md5() instead. + * + * Context: Any context. + */ +void md5_init(struct md5_ctx *ctx); + +/** + * md5_update() - Update an MD5 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void md5_update(struct md5_ctx *ctx, const u8 *data, size_t len); + +/** + * md5_final() - Finish computing an MD5 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting MD5 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void md5_final(struct md5_ctx *ctx, u8 out[at_least MD5_DIGEST_SIZE]); + +/** + * md5() - Compute MD5 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting MD5 message digest + * + * Context: Any context. + */ +void md5(const u8 *data, size_t len, u8 out[at_least MD5_DIGEST_SIZE]); + +/** + * struct hmac_md5_key - Prepared key for HMAC-MD5 + * @istate: private + * @ostate: private + */ +struct hmac_md5_key { + struct md5_block_state istate; + struct md5_block_state ostate; +}; + +/** + * struct hmac_md5_ctx - Context for computing HMAC-MD5 of a message + * @hash_ctx: private + * @ostate: private + */ +struct hmac_md5_ctx { + struct md5_ctx hash_ctx; + struct md5_block_state ostate; }; -#endif +/** + * hmac_md5_preparekey() - Prepare a key for HMAC-MD5 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-MD5 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_md5_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_md5_preparekey(struct hmac_md5_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_md5_init() - Initialize an HMAC-MD5 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_md5() instead. + * + * Context: Any context. + */ +void hmac_md5_init(struct hmac_md5_ctx *ctx, const struct hmac_md5_key *key); + +/** + * hmac_md5_init_usingrawkey() - Initialize an HMAC-MD5 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-MD5 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_md5_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_md5_init_usingrawkey(struct hmac_md5_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_md5_update() - Update an HMAC-MD5 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_md5_update(struct hmac_md5_ctx *ctx, + const u8 *data, size_t data_len) +{ + md5_update(&ctx->hash_ctx, data, data_len); +} + +/** + * hmac_md5_final() - Finish computing an HMAC-MD5 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-MD5 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_md5_final(struct hmac_md5_ctx *ctx, u8 out[at_least MD5_DIGEST_SIZE]); + +/** + * hmac_md5() - Compute HMAC-MD5 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-MD5 value + * + * If you're using the key only once, consider using hmac_md5_usingrawkey(). + * + * Context: Any context. + */ +void hmac_md5(const struct hmac_md5_key *key, + const u8 *data, size_t data_len, + u8 out[at_least MD5_DIGEST_SIZE]); + +/** + * hmac_md5_usingrawkey() - Compute HMAC-MD5 in one shot, using a raw key + * @raw_key: the raw HMAC-MD5 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-MD5 value + * + * If you're using the key multiple times, prefer to use hmac_md5_preparekey() + * followed by multiple calls to hmac_md5() instead. + * + * Context: Any context. + */ +void hmac_md5_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least MD5_DIGEST_SIZE]); + +#endif /* _CRYPTO_MD5_H */ diff --git a/include/crypto/mldsa.h b/include/crypto/mldsa.h new file mode 100644 index 000000000000..3ef2676787c9 --- /dev/null +++ b/include/crypto/mldsa.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Support for verifying ML-DSA signatures + * + * Copyright 2025 Google LLC + */ +#ifndef _CRYPTO_MLDSA_H +#define _CRYPTO_MLDSA_H + +#include <linux/types.h> + +/* Identifier for an ML-DSA parameter set */ +enum mldsa_alg { + MLDSA44, /* ML-DSA-44 */ + MLDSA65, /* ML-DSA-65 */ + MLDSA87, /* ML-DSA-87 */ +}; + +/* Lengths of ML-DSA public keys and signatures in bytes */ +#define MLDSA44_PUBLIC_KEY_SIZE 1312 +#define MLDSA65_PUBLIC_KEY_SIZE 1952 +#define MLDSA87_PUBLIC_KEY_SIZE 2592 +#define MLDSA44_SIGNATURE_SIZE 2420 +#define MLDSA65_SIGNATURE_SIZE 3309 +#define MLDSA87_SIGNATURE_SIZE 4627 + +/** + * mldsa_verify() - Verify an ML-DSA signature + * @alg: The ML-DSA parameter set to use + * @sig: The signature + * @sig_len: Length of the signature in bytes. Should match the + * MLDSA*_SIGNATURE_SIZE constant associated with @alg, + * otherwise -EBADMSG will be returned. + * @msg: The message + * @msg_len: Length of the message in bytes + * @pk: The public key + * @pk_len: Length of the public key in bytes. Should match the + * MLDSA*_PUBLIC_KEY_SIZE constant associated with @alg, + * otherwise -EBADMSG will be returned. + * + * This verifies a signature using pure ML-DSA with the specified parameter set. + * The context string is assumed to be empty. This corresponds to FIPS 204 + * Algorithm 3 "ML-DSA.Verify" with the ctx parameter set to the empty string + * and the lengths of the signature and key given explicitly by the caller. + * + * Context: Might sleep + * + * Return: + * * 0 if the signature is valid + * * -EBADMSG if the signature and/or public key is malformed + * * -EKEYREJECTED if the signature is invalid but otherwise well-formed + * * -ENOMEM if out of memory so the validity of the signature is unknown + */ +int mldsa_verify(enum mldsa_alg alg, const u8 *sig, size_t sig_len, + const u8 *msg, size_t msg_len, const u8 *pk, size_t pk_len); + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_MLDSA_KUNIT_TEST) +/* Internal function, exposed only for unit testing */ +s32 mldsa_use_hint(u8 h, s32 r, s32 gamma2); +#endif + +#endif /* _CRYPTO_MLDSA_H */ diff --git a/include/crypto/nh.h b/include/crypto/nh.h new file mode 100644 index 000000000000..465e85bf203f --- /dev/null +++ b/include/crypto/nh.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NH hash function for Adiantum + */ + +#ifndef _CRYPTO_NH_H +#define _CRYPTO_NH_H + +#include <linux/types.h> + +/* NH parameterization: */ + +/* Endianness: little */ +/* Word size: 32 bits (works well on NEON, SSE2, AVX2) */ + +/* Stride: 2 words (optimal on ARM32 NEON; works okay on other CPUs too) */ +#define NH_PAIR_STRIDE 2 +#define NH_MESSAGE_UNIT (NH_PAIR_STRIDE * 2 * sizeof(u32)) + +/* Num passes (Toeplitz iteration count): 4, to give ε = 2^{-128} */ +#define NH_NUM_PASSES 4 +#define NH_HASH_BYTES (NH_NUM_PASSES * sizeof(u64)) + +/* Max message size: 1024 bytes (32x compression factor) */ +#define NH_NUM_STRIDES 64 +#define NH_MESSAGE_WORDS (NH_PAIR_STRIDE * 2 * NH_NUM_STRIDES) +#define NH_MESSAGE_BYTES (NH_MESSAGE_WORDS * sizeof(u32)) +#define NH_KEY_WORDS (NH_MESSAGE_WORDS + \ + NH_PAIR_STRIDE * 2 * (NH_NUM_PASSES - 1)) +#define NH_KEY_BYTES (NH_KEY_WORDS * sizeof(u32)) + +/** + * nh() - NH hash function for Adiantum + * @key: The key. @message_len + 48 bytes of it are used. This is NH_KEY_BYTES + * if @message_len has its maximum length of NH_MESSAGE_BYTES. + * @message: The message + * @message_len: The message length in bytes. Must be a multiple of 16 + * (NH_MESSAGE_UNIT) and at most 1024 (NH_MESSAGE_BYTES). + * @hash: (output) The resulting hash value + * + * Note: the pseudocode for NH in the Adiantum paper iterates over 1024-byte + * segments of the message, computes a 32-byte hash for each, and returns all + * the hashes concatenated together. In contrast, this function just hashes one + * segment and returns one hash. It's the caller's responsibility to call this + * function for each 1024-byte segment and collect all the hashes. + * + * Context: Any context. + */ +void nh(const u32 *key, const u8 *message, size_t message_len, + __le64 hash[NH_NUM_PASSES]); + +#endif /* _CRYPTO_NH_H */ diff --git a/include/crypto/nhpoly1305.h b/include/crypto/nhpoly1305.h deleted file mode 100644 index 306925fea190..000000000000 --- a/include/crypto/nhpoly1305.h +++ /dev/null @@ -1,74 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common values and helper functions for the NHPoly1305 hash function. - */ - -#ifndef _NHPOLY1305_H -#define _NHPOLY1305_H - -#include <crypto/hash.h> -#include <crypto/internal/poly1305.h> - -/* NH parameterization: */ - -/* Endianness: little */ -/* Word size: 32 bits (works well on NEON, SSE2, AVX2) */ - -/* Stride: 2 words (optimal on ARM32 NEON; works okay on other CPUs too) */ -#define NH_PAIR_STRIDE 2 -#define NH_MESSAGE_UNIT (NH_PAIR_STRIDE * 2 * sizeof(u32)) - -/* Num passes (Toeplitz iteration count): 4, to give ε = 2^{-128} */ -#define NH_NUM_PASSES 4 -#define NH_HASH_BYTES (NH_NUM_PASSES * sizeof(u64)) - -/* Max message size: 1024 bytes (32x compression factor) */ -#define NH_NUM_STRIDES 64 -#define NH_MESSAGE_WORDS (NH_PAIR_STRIDE * 2 * NH_NUM_STRIDES) -#define NH_MESSAGE_BYTES (NH_MESSAGE_WORDS * sizeof(u32)) -#define NH_KEY_WORDS (NH_MESSAGE_WORDS + \ - NH_PAIR_STRIDE * 2 * (NH_NUM_PASSES - 1)) -#define NH_KEY_BYTES (NH_KEY_WORDS * sizeof(u32)) - -#define NHPOLY1305_KEY_SIZE (POLY1305_BLOCK_SIZE + NH_KEY_BYTES) - -struct nhpoly1305_key { - struct poly1305_core_key poly_key; - u32 nh_key[NH_KEY_WORDS]; -}; - -struct nhpoly1305_state { - - /* Running total of polynomial evaluation */ - struct poly1305_state poly_state; - - /* Partial block buffer */ - u8 buffer[NH_MESSAGE_UNIT]; - unsigned int buflen; - - /* - * Number of bytes remaining until the current NH message reaches - * NH_MESSAGE_BYTES. When nonzero, 'nh_hash' holds the partial NH hash. - */ - unsigned int nh_remaining; - - __le64 nh_hash[NH_NUM_PASSES]; -}; - -typedef void (*nh_t)(const u32 *key, const u8 *message, size_t message_len, - __le64 hash[NH_NUM_PASSES]); - -int crypto_nhpoly1305_setkey(struct crypto_shash *tfm, - const u8 *key, unsigned int keylen); - -int crypto_nhpoly1305_init(struct shash_desc *desc); -int crypto_nhpoly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen); -int crypto_nhpoly1305_update_helper(struct shash_desc *desc, - const u8 *src, unsigned int srclen, - nh_t nh_fn); -int crypto_nhpoly1305_final(struct shash_desc *desc, u8 *dst); -int crypto_nhpoly1305_final_helper(struct shash_desc *desc, u8 *dst, - nh_t nh_fn); - -#endif /* _NHPOLY1305_H */ diff --git a/include/crypto/null.h b/include/crypto/null.h index 0ef577cc00e3..1c66abf9de3b 100644 --- a/include/crypto/null.h +++ b/include/crypto/null.h @@ -9,7 +9,4 @@ #define NULL_DIGEST_SIZE 0 #define NULL_IV_SIZE 0 -struct crypto_sync_skcipher *crypto_get_default_null_skcipher(void); -void crypto_put_default_null_skcipher(void); - #endif diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 090692ec3bc7..190beb427c6d 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -7,7 +7,6 @@ #define _CRYPTO_POLY1305_H #include <linux/types.h> -#include <linux/crypto.h> #define POLY1305_BLOCK_SIZE 16 #define POLY1305_KEY_SIZE 32 @@ -38,17 +37,8 @@ struct poly1305_state { }; }; -struct poly1305_desc_ctx { - /* partial buffer */ - u8 buf[POLY1305_BLOCK_SIZE]; - /* bytes used in partial buffer */ - unsigned int buflen; - /* how many keys have been set in r[] */ - unsigned short rset; - /* whether s[] has been set */ - bool sset; - /* finalize key */ - u32 s[4]; +/* Combined state for block function. */ +struct poly1305_block_state { /* accumulator */ struct poly1305_state h; /* key */ @@ -58,42 +48,20 @@ struct poly1305_desc_ctx { }; }; -void poly1305_init_arch(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); -void poly1305_init_generic(struct poly1305_desc_ctx *desc, - const u8 key[POLY1305_KEY_SIZE]); - -static inline void poly1305_init(struct poly1305_desc_ctx *desc, const u8 *key) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_init_arch(desc, key); - else - poly1305_init_generic(desc, key); -} - -void poly1305_update_arch(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); -void poly1305_update_generic(struct poly1305_desc_ctx *desc, const u8 *src, - unsigned int nbytes); - -static inline void poly1305_update(struct poly1305_desc_ctx *desc, - const u8 *src, unsigned int nbytes) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_update_arch(desc, src, nbytes); - else - poly1305_update_generic(desc, src, nbytes); -} - -void poly1305_final_arch(struct poly1305_desc_ctx *desc, u8 *digest); -void poly1305_final_generic(struct poly1305_desc_ctx *desc, u8 *digest); +struct poly1305_desc_ctx { + /* partial buffer */ + u8 buf[POLY1305_BLOCK_SIZE]; + /* bytes used in partial buffer */ + unsigned int buflen; + /* finalize key */ + u32 s[4]; + struct poly1305_block_state state; +}; -static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) -{ - if (IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305)) - poly1305_final_arch(desc, digest); - else - poly1305_final_generic(desc, digest); -} +void poly1305_init(struct poly1305_desc_ctx *desc, + const u8 key[at_least POLY1305_KEY_SIZE]); +void poly1305_update(struct poly1305_desc_ctx *desc, + const u8 *src, unsigned int nbytes); +void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest); #endif diff --git a/include/crypto/polyval.h b/include/crypto/polyval.h deleted file mode 100644 index 1d630f371f77..000000000000 --- a/include/crypto/polyval.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Common values for the Polyval hash algorithm - * - * Copyright 2021 Google LLC - */ - -#ifndef _CRYPTO_POLYVAL_H -#define _CRYPTO_POLYVAL_H - -#include <linux/types.h> -#include <linux/crypto.h> - -#define POLYVAL_BLOCK_SIZE 16 -#define POLYVAL_DIGEST_SIZE 16 - -void polyval_mul_non4k(u8 *op1, const u8 *op2); - -void polyval_update_non4k(const u8 *key, const u8 *in, - size_t nblocks, u8 *accumulator); - -#endif diff --git a/include/crypto/public_key.h b/include/crypto/public_key.h index 81098e00c08f..4c5199b20338 100644 --- a/include/crypto/public_key.h +++ b/include/crypto/public_key.h @@ -43,9 +43,11 @@ extern void public_key_free(struct public_key *key); struct public_key_signature { struct asymmetric_key_id *auth_ids[3]; u8 *s; /* Signature */ - u8 *digest; + u8 *m; /* Message data to pass to verifier */ u32 s_size; /* Number of bytes in signature */ - u32 digest_size; /* Number of bytes in digest */ + u32 m_size; /* Number of bytes in ->m */ + bool m_free; /* T if ->m needs freeing */ + bool algo_takes_data; /* T if public key algo operates on data, not a hash */ const char *pkey_algo; const char *hash_algo; const char *encoding; diff --git a/include/crypto/rng.h b/include/crypto/rng.h index 5ac4388f50e1..07f494b2c881 100644 --- a/include/crypto/rng.h +++ b/include/crypto/rng.h @@ -12,6 +12,8 @@ #include <linux/atomic.h> #include <linux/container_of.h> #include <linux/crypto.h> +#include <linux/fips.h> +#include <linux/random.h> struct crypto_rng; @@ -57,10 +59,27 @@ struct crypto_rng { struct crypto_tfm base; }; -extern struct crypto_rng *crypto_default_rng; +int __crypto_stdrng_get_bytes(void *buf, unsigned int len); -int crypto_get_default_rng(void); -void crypto_put_default_rng(void); +/** + * crypto_stdrng_get_bytes() - get cryptographically secure random bytes + * @buf: output buffer holding the random numbers + * @len: length of the output buffer + * + * This function fills the caller-allocated buffer with random numbers using the + * normal Linux RNG if fips_enabled=0, or the highest-priority "stdrng" + * algorithm in the crypto_rng subsystem if fips_enabled=1. + * + * Context: May sleep + * Return: 0 function was successful; < 0 if an error occurred + */ +static inline int crypto_stdrng_get_bytes(void *buf, unsigned int len) +{ + might_sleep(); + if (fips_enabled) + return __crypto_stdrng_get_bytes(buf, len); + return get_random_bytes_wait(buf, len); +} /** * DOC: Random number generator API @@ -102,12 +121,10 @@ static inline struct rng_alg *__crypto_rng_alg(struct crypto_alg *alg) } /** - * crypto_rng_alg - obtain name of RNG - * @tfm: cipher handle - * - * Return the generic name (cra_name) of the initialized random number generator + * crypto_rng_alg() - obtain 'struct rng_alg' pointer from RNG handle + * @tfm: RNG handle * - * Return: generic name string + * Return: Pointer to 'struct rng_alg', derived from @tfm RNG handle */ static inline struct rng_alg *crypto_rng_alg(struct crypto_rng *tfm) { @@ -171,12 +188,11 @@ static inline int crypto_rng_get_bytes(struct crypto_rng *tfm, * * The reset function completely re-initializes the random number generator * referenced by the cipher handle by clearing the current state. The new state - * is initialized with the caller provided seed or automatically, depending - * on the random number generator type (the ANSI X9.31 RNG requires - * caller-provided seed, the SP800-90A DRBGs perform an automatic seeding). - * The seed is provided as a parameter to this function call. The provided seed - * should have the length of the seed size defined for the random number - * generator as defined by crypto_rng_seedsize. + * is initialized with the caller provided seed or automatically, depending on + * the random number generator type. (The SP800-90A DRBGs perform an automatic + * seeding.) The seed is provided as a parameter to this function call. The + * provided seed should have the length of the seed size defined for the random + * number generator as defined by crypto_rng_seedsize. * * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 32fc4473175b..624fab589c2c 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -26,76 +26,228 @@ static inline void scatterwalk_crypto_chain(struct scatterlist *head, sg_mark_end(head); } -static inline unsigned int scatterwalk_pagelen(struct scatter_walk *walk) +static inline void scatterwalk_start(struct scatter_walk *walk, + struct scatterlist *sg) { - unsigned int len = walk->sg->offset + walk->sg->length - walk->offset; - unsigned int len_this_page = offset_in_page(~walk->offset) + 1; - return len_this_page > len ? len : len_this_page; + walk->sg = sg; + walk->offset = sg->offset; +} + +/* + * This is equivalent to scatterwalk_start(walk, sg) followed by + * scatterwalk_skip(walk, pos). + */ +static inline void scatterwalk_start_at_pos(struct scatter_walk *walk, + struct scatterlist *sg, + unsigned int pos) +{ + while (pos > sg->length) { + pos -= sg->length; + sg = sg_next(sg); + } + walk->sg = sg; + walk->offset = sg->offset + pos; } static inline unsigned int scatterwalk_clamp(struct scatter_walk *walk, unsigned int nbytes) { - unsigned int len_this_page = scatterwalk_pagelen(walk); - return nbytes > len_this_page ? len_this_page : nbytes; + unsigned int len_this_sg; + unsigned int limit; + + if (walk->offset >= walk->sg->offset + walk->sg->length) + scatterwalk_start(walk, sg_next(walk->sg)); + len_this_sg = walk->sg->offset + walk->sg->length - walk->offset; + + /* + * HIGHMEM case: the page may have to be mapped into memory. To avoid + * the complexity of having to map multiple pages at once per sg entry, + * clamp the returned length to not cross a page boundary. + * + * !HIGHMEM case: no mapping is needed; all pages of the sg entry are + * already mapped contiguously in the kernel's direct map. For improved + * performance, allow the walker to return data segments that cross a + * page boundary. Do still cap the length to PAGE_SIZE, since some + * users rely on that to avoid disabling preemption for too long when + * using SIMD. It's also needed for when skcipher_walk uses a bounce + * page due to the data not being aligned to the algorithm's alignmask. + */ + if (IS_ENABLED(CONFIG_HIGHMEM)) + limit = PAGE_SIZE - offset_in_page(walk->offset); + else + limit = PAGE_SIZE; + + return min3(nbytes, len_this_sg, limit); +} + +/* + * Create a scatterlist that represents the remaining data in a walk. Uses + * chaining to reference the original scatterlist, so this uses at most two + * entries in @sg_out regardless of the number of entries in the original list. + * Assumes that sg_init_table() was already done. + */ +static inline void scatterwalk_get_sglist(struct scatter_walk *walk, + struct scatterlist sg_out[2]) +{ + if (walk->offset >= walk->sg->offset + walk->sg->length) + scatterwalk_start(walk, sg_next(walk->sg)); + sg_set_page(sg_out, sg_page(walk->sg), + walk->sg->offset + walk->sg->length - walk->offset, + walk->offset); + scatterwalk_crypto_chain(sg_out, sg_next(walk->sg), 2); } -static inline void scatterwalk_advance(struct scatter_walk *walk, - unsigned int nbytes) +static inline void scatterwalk_map(struct scatter_walk *walk) { - walk->offset += nbytes; + struct page *base_page = sg_page(walk->sg); + unsigned int offset = walk->offset; + void *addr; + + if (IS_ENABLED(CONFIG_HIGHMEM)) { + struct page *page; + + page = base_page + (offset >> PAGE_SHIFT); + offset = offset_in_page(offset); + addr = kmap_local_page(page) + offset; + } else { + /* + * When !HIGHMEM we allow the walker to return segments that + * span a page boundary; see scatterwalk_clamp(). To make it + * clear that in this case we're working in the linear buffer of + * the whole sg entry in the kernel's direct map rather than + * within the mapped buffer of a single page, compute the + * address as an offset from the page_address() of the first + * page of the sg entry. Either way the result is the address + * in the direct map, but this makes it clearer what is really + * going on. + */ + addr = page_address(base_page) + offset; + } + + walk->__addr = addr; } -static inline struct page *scatterwalk_page(struct scatter_walk *walk) +/** + * scatterwalk_next() - Get the next data buffer in a scatterlist walk + * @walk: the scatter_walk + * @total: the total number of bytes remaining, > 0 + * + * A virtual address for the next segment of data from the scatterlist will + * be placed into @walk->addr. The caller must call scatterwalk_done_src() + * or scatterwalk_done_dst() when it is done using this virtual address. + * + * Returns: the next number of bytes available, <= @total + */ +static inline unsigned int scatterwalk_next(struct scatter_walk *walk, + unsigned int total) { - return sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); + unsigned int nbytes = scatterwalk_clamp(walk, total); + + scatterwalk_map(walk); + return nbytes; } -static inline void scatterwalk_unmap(void *vaddr) +static inline void scatterwalk_unmap(struct scatter_walk *walk) { - kunmap_local(vaddr); + if (IS_ENABLED(CONFIG_HIGHMEM)) + kunmap_local(walk->__addr); } -static inline void scatterwalk_start(struct scatter_walk *walk, - struct scatterlist *sg) +static inline void scatterwalk_advance(struct scatter_walk *walk, + unsigned int nbytes) { - walk->sg = sg; - walk->offset = sg->offset; + walk->offset += nbytes; } -static inline void *scatterwalk_map(struct scatter_walk *walk) +/** + * scatterwalk_done_src() - Finish one step of a walk of source scatterlist + * @walk: the scatter_walk + * @nbytes: the number of bytes processed this step, less than or equal to the + * number of bytes that scatterwalk_next() returned. + * + * Use this if the mapped address was not written to, i.e. it is source data. + */ +static inline void scatterwalk_done_src(struct scatter_walk *walk, + unsigned int nbytes) { - return kmap_local_page(scatterwalk_page(walk)) + - offset_in_page(walk->offset); + scatterwalk_unmap(walk); + scatterwalk_advance(walk, nbytes); } -static inline void scatterwalk_pagedone(struct scatter_walk *walk, int out, - unsigned int more) +/* + * Flush the dcache of any pages that overlap the region + * [offset, offset + nbytes) relative to base_page. + * + * This should be called only when ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE, to ensure + * that all relevant code (including the call to sg_page() in the caller, if + * applicable) gets fully optimized out when !ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE. + */ +static inline void __scatterwalk_flush_dcache_pages(struct page *base_page, + unsigned int offset, + unsigned int nbytes) { - if (out) { - struct page *page; + unsigned int num_pages; - page = sg_page(walk->sg) + ((walk->offset - 1) >> PAGE_SHIFT); - flush_dcache_page(page); - } + base_page += offset / PAGE_SIZE; + offset %= PAGE_SIZE; - if (more && walk->offset >= walk->sg->offset + walk->sg->length) - scatterwalk_start(walk, sg_next(walk->sg)); + /* + * This is an overflow-safe version of + * num_pages = DIV_ROUND_UP(offset + nbytes, PAGE_SIZE). + */ + num_pages = nbytes / PAGE_SIZE; + num_pages += DIV_ROUND_UP(offset + (nbytes % PAGE_SIZE), PAGE_SIZE); + + for (unsigned int i = 0; i < num_pages; i++) + flush_dcache_page(base_page + i); } -static inline void scatterwalk_done(struct scatter_walk *walk, int out, - int more) +/** + * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist + * @walk: the scatter_walk + * @nbytes: the number of bytes processed this step, less than or equal to the + * number of bytes that scatterwalk_next() returned. + * + * Use this if the mapped address may have been written to, i.e. it is + * destination data. + */ +static inline void scatterwalk_done_dst(struct scatter_walk *walk, + unsigned int nbytes) { - if (!more || walk->offset >= walk->sg->offset + walk->sg->length || - !(walk->offset & (PAGE_SIZE - 1))) - scatterwalk_pagedone(walk, out, more); + scatterwalk_unmap(walk); + if (ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE) + __scatterwalk_flush_dcache_pages(sg_page(walk->sg), + walk->offset, nbytes); + scatterwalk_advance(walk, nbytes); } -void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, - size_t nbytes, int out); +void scatterwalk_skip(struct scatter_walk *walk, unsigned int nbytes); + +void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, + unsigned int nbytes); -void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, - unsigned int start, unsigned int nbytes, int out); +void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, + unsigned int nbytes); + +void memcpy_from_sglist(void *buf, struct scatterlist *sg, + unsigned int start, unsigned int nbytes); + +void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, + const void *buf, unsigned int nbytes); + +void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes); + +/* In new code, please use memcpy_{from,to}_sglist() directly instead. */ +static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, + unsigned int start, + unsigned int nbytes, int out) +{ + if (out) + memcpy_to_sglist(sg, start, buf, nbytes); + else + memcpy_from_sglist(buf, sg, start, nbytes); +} struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h index 044ecea60ac8..4d973e016cd6 100644 --- a/include/crypto/sha1.h +++ b/include/crypto/sha1.h @@ -10,6 +10,7 @@ #define SHA1_DIGEST_SIZE 20 #define SHA1_BLOCK_SIZE 64 +#define SHA1_STATE_SIZE offsetof(struct sha1_state, buffer) #define SHA1_H0 0x67452301UL #define SHA1_H1 0xefcdab89UL @@ -25,22 +26,184 @@ struct sha1_state { u8 buffer[SHA1_BLOCK_SIZE]; }; -struct shash_desc; +/* State for the SHA-1 compression function */ +struct sha1_block_state { + u32 h[SHA1_DIGEST_SIZE / 4]; +}; + +/** + * struct sha1_ctx - Context for hashing a message with SHA-1 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % SHA1_BLOCK_SIZE bytes are valid + */ +struct sha1_ctx { + struct sha1_block_state state; + u64 bytecount; + u8 buf[SHA1_BLOCK_SIZE]; +}; -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); +/** + * sha1_init() - Initialize a SHA-1 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha1() instead. + * + * Context: Any context. + */ +void sha1_init(struct sha1_ctx *ctx); -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); +/** + * sha1_update() - Update a SHA-1 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +void sha1_update(struct sha1_ctx *ctx, const u8 *data, size_t len); -/* - * An implementation of SHA-1's compression function. Don't use in new code! - * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't - * the correct way to hash something with SHA-1 (use crypto_shash instead). - */ -#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) -#define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); -void sha1_transform(__u32 *digest, const char *data, __u32 *W); +/** + * sha1_final() - Finish computing a SHA-1 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-1 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha1_final(struct sha1_ctx *ctx, u8 out[at_least SHA1_DIGEST_SIZE]); + +/** + * sha1() - Compute SHA-1 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-1 message digest + * + * Context: Any context. + */ +void sha1(const u8 *data, size_t len, u8 out[at_least SHA1_DIGEST_SIZE]); + +/** + * struct hmac_sha1_key - Prepared key for HMAC-SHA1 + * @istate: private + * @ostate: private + */ +struct hmac_sha1_key { + struct sha1_block_state istate; + struct sha1_block_state ostate; +}; + +/** + * struct hmac_sha1_ctx - Context for computing HMAC-SHA1 of a message + * @sha_ctx: private + * @ostate: private + */ +struct hmac_sha1_ctx { + struct sha1_ctx sha_ctx; + struct sha1_block_state ostate; +}; + +/** + * hmac_sha1_preparekey() - Prepare a key for HMAC-SHA1 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha1_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha1_preparekey(struct hmac_sha1_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha1_init() - Initialize an HMAC-SHA1 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha1() instead. + * + * Context: Any context. + */ +void hmac_sha1_init(struct hmac_sha1_ctx *ctx, const struct hmac_sha1_key *key); + +/** + * hmac_sha1_init_usingrawkey() - Initialize an HMAC-SHA1 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha1_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha1_init_usingrawkey(struct hmac_sha1_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha1_update() - Update an HMAC-SHA1 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha1_update(struct hmac_sha1_ctx *ctx, + const u8 *data, size_t data_len) +{ + sha1_update(&ctx->sha_ctx, data, data_len); +} + +/** + * hmac_sha1_final() - Finish computing an HMAC-SHA1 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA1 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha1_final(struct hmac_sha1_ctx *ctx, + u8 out[at_least SHA1_DIGEST_SIZE]); + +/** + * hmac_sha1() - Compute HMAC-SHA1 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA1 value + * + * If you're using the key only once, consider using hmac_sha1_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha1(const struct hmac_sha1_key *key, + const u8 *data, size_t data_len, + u8 out[at_least SHA1_DIGEST_SIZE]); + +/** + * hmac_sha1_usingrawkey() - Compute HMAC-SHA1 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA1 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA1 value + * + * If you're using the key multiple times, prefer to use hmac_sha1_preparekey() + * followed by multiple calls to hmac_sha1() instead. + * + * Context: Any context. + */ +void hmac_sha1_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least SHA1_DIGEST_SIZE]); #endif /* _CRYPTO_SHA1_H */ diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h deleted file mode 100644 index 0c342ed0d038..000000000000 --- a/include/crypto/sha1_base.h +++ /dev/null @@ -1,109 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha1_base.h - core logic for SHA-1 implementations - * - * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#ifndef _CRYPTO_SHA1_BASE_H -#define _CRYPTO_SHA1_BASE_H - -#include <crypto/internal/hash.h> -#include <crypto/sha1.h> -#include <linux/crypto.h> -#include <linux/module.h> -#include <linux/string.h> - -#include <linux/unaligned.h> - -typedef void (sha1_block_fn)(struct sha1_state *sst, u8 const *src, int blocks); - -static inline int sha1_base_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA1_H0; - sctx->state[1] = SHA1_H1; - sctx->state[2] = SHA1_H2; - sctx->state[3] = SHA1_H3; - sctx->state[4] = SHA1_H4; - sctx->count = 0; - - return 0; -} - -static inline int sha1_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha1_block_fn *block_fn) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SHA1_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA1_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buffer, 1); - } - - blocks = len / SHA1_BLOCK_SIZE; - len %= SHA1_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA1_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; -} - -static inline int sha1_base_do_finalize(struct shash_desc *desc, - sha1_block_fn *block_fn) -{ - const int bit_offset = SHA1_BLOCK_SIZE - sizeof(__be64); - struct sha1_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SHA1_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); - - return 0; -} - -static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); - return 0; -} - -#endif /* _CRYPTO_SHA1_BASE_H */ diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index b9e9281d76c9..7bb8fe169daf 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -13,12 +13,14 @@ #define SHA256_DIGEST_SIZE 32 #define SHA256_BLOCK_SIZE 64 +#define SHA256_STATE_WORDS 8 #define SHA384_DIGEST_SIZE 48 #define SHA384_BLOCK_SIZE 128 #define SHA512_DIGEST_SIZE 64 #define SHA512_BLOCK_SIZE 128 +#define SHA512_STATE_SIZE 80 #define SHA224_H0 0xc1059ed8UL #define SHA224_H1 0x367cd507UL @@ -64,9 +66,45 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; -struct sha256_state { - u32 state[SHA256_DIGEST_SIZE / 4]; +struct crypto_sha256_state { + u32 state[SHA256_STATE_WORDS]; u64 count; +}; + +static inline void sha224_block_init(struct crypto_sha256_state *sctx) +{ + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; +} + +static inline void sha256_block_init(struct crypto_sha256_state *sctx) +{ + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; +} + +struct sha256_state { + union { + struct crypto_sha256_state ctx; + struct { + u32 state[SHA256_STATE_WORDS]; + u64 count; + }; + }; u8 buf[SHA256_BLOCK_SIZE]; }; @@ -76,59 +114,800 @@ struct sha512_state { u8 buf[SHA512_BLOCK_SIZE]; }; -struct shash_desc; +/* State for the SHA-256 (and SHA-224) compression function */ +struct sha256_block_state { + u32 h[SHA256_STATE_WORDS]; +}; + +/* + * Context structure, shared by SHA-224 and SHA-256. The sha224_ctx and + * sha256_ctx structs wrap this one so that the API has proper typing and + * doesn't allow mixing the SHA-224 and SHA-256 functions arbitrarily. + */ +struct __sha256_ctx { + struct sha256_block_state state; + u64 bytecount; + u8 buf[SHA256_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; +void __sha256_update(struct __sha256_ctx *ctx, const u8 *data, size_t len); + +/* + * HMAC key and message context structs, shared by HMAC-SHA224 and HMAC-SHA256. + * The hmac_sha224_* and hmac_sha256_* structs wrap this one so that the API has + * proper typing and doesn't allow mixing the functions arbitrarily. + */ +struct __hmac_sha256_key { + struct sha256_block_state istate; + struct sha256_block_state ostate; +}; +struct __hmac_sha256_ctx { + struct __sha256_ctx sha_ctx; + struct sha256_block_state ostate; +}; +void __hmac_sha256_init(struct __hmac_sha256_ctx *ctx, + const struct __hmac_sha256_key *key); + +/** + * struct sha224_ctx - Context for hashing a message with SHA-224 + * @ctx: private + */ +struct sha224_ctx { + struct __sha256_ctx ctx; +}; + +/** + * sha224_init() - Initialize a SHA-224 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha224() instead. + * + * Context: Any context. + */ +void sha224_init(struct sha224_ctx *ctx); + +/** + * sha224_update() - Update a SHA-224 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha224_update(struct sha224_ctx *ctx, + const u8 *data, size_t len) +{ + __sha256_update(&ctx->ctx, data, len); +} + +/** + * sha224_final() - Finish computing a SHA-224 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-224 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha224_final(struct sha224_ctx *ctx, u8 out[at_least SHA224_DIGEST_SIZE]); + +/** + * sha224() - Compute SHA-224 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-224 message digest + * + * Context: Any context. + */ +void sha224(const u8 *data, size_t len, u8 out[at_least SHA224_DIGEST_SIZE]); + +/** + * struct hmac_sha224_key - Prepared key for HMAC-SHA224 + * @key: private + */ +struct hmac_sha224_key { + struct __hmac_sha256_key key; +}; + +/** + * struct hmac_sha224_ctx - Context for computing HMAC-SHA224 of a message + * @ctx: private + */ +struct hmac_sha224_ctx { + struct __hmac_sha256_ctx ctx; +}; + +/** + * hmac_sha224_preparekey() - Prepare a key for HMAC-SHA224 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha224_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha224_preparekey(struct hmac_sha224_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha224_init() - Initialize an HMAC-SHA224 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha224() instead. + * + * Context: Any context. + */ +static inline void hmac_sha224_init(struct hmac_sha224_ctx *ctx, + const struct hmac_sha224_key *key) +{ + __hmac_sha256_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha224_init_usingrawkey() - Initialize an HMAC-SHA224 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha224_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha224_init_usingrawkey(struct hmac_sha224_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha224_update() - Update an HMAC-SHA224 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha224_update(struct hmac_sha224_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha256_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha224_final() - Finish computing an HMAC-SHA224 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA224 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha224_final(struct hmac_sha224_ctx *ctx, + u8 out[at_least SHA224_DIGEST_SIZE]); + +/** + * hmac_sha224() - Compute HMAC-SHA224 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA224 value + * + * If you're using the key only once, consider using hmac_sha224_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha224(const struct hmac_sha224_key *key, + const u8 *data, size_t data_len, + u8 out[at_least SHA224_DIGEST_SIZE]); + +/** + * hmac_sha224_usingrawkey() - Compute HMAC-SHA224 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA224 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA224 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha224_preparekey() followed by multiple calls to hmac_sha224() instead. + * + * Context: Any context. + */ +void hmac_sha224_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least SHA224_DIGEST_SIZE]); + +/** + * struct sha256_ctx - Context for hashing a message with SHA-256 + * @ctx: private + */ +struct sha256_ctx { + struct __sha256_ctx ctx; +}; + +/** + * sha256_init() - Initialize a SHA-256 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha256() instead. + * + * Context: Any context. + */ +void sha256_init(struct sha256_ctx *ctx); -extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len); +/** + * sha256_update() - Update a SHA-256 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha256_update(struct sha256_ctx *ctx, + const u8 *data, size_t len) +{ + __sha256_update(&ctx->ctx, data, len); +} -extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); +/** + * sha256_final() - Finish computing a SHA-256 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-256 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha256_final(struct sha256_ctx *ctx, u8 out[at_least SHA256_DIGEST_SIZE]); + +/** + * sha256() - Compute SHA-256 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-256 message digest + * + * Context: Any context. + */ +void sha256(const u8 *data, size_t len, u8 out[at_least SHA256_DIGEST_SIZE]); + +/** + * sha256_finup_2x() - Compute two SHA-256 digests from a common initial + * context. On some CPUs, this is faster than sequentially + * computing each digest. + * @ctx: an optional initial context, which may have already processed data. If + * NULL, a default initial context is used (equivalent to sha256_init()). + * @data1: data for the first message + * @data2: data for the second message + * @len: the length of each of @data1 and @data2, in bytes + * @out1: (output) the first SHA-256 message digest + * @out2: (output) the second SHA-256 message digest + * + * Context: Any context. + */ +void sha256_finup_2x(const struct sha256_ctx *ctx, const u8 *data1, + const u8 *data2, size_t len, + u8 out1[at_least SHA256_DIGEST_SIZE], + u8 out2[at_least SHA256_DIGEST_SIZE]); + +/** + * sha256_finup_2x_is_optimized() - Check if sha256_finup_2x() is using a real + * interleaved implementation, as opposed to a + * sequential fallback + * @return: true if optimized + * + * Context: Any context. + */ +bool sha256_finup_2x_is_optimized(void); + +/** + * struct hmac_sha256_key - Prepared key for HMAC-SHA256 + * @key: private + */ +struct hmac_sha256_key { + struct __hmac_sha256_key key; +}; + +/** + * struct hmac_sha256_ctx - Context for computing HMAC-SHA256 of a message + * @ctx: private + */ +struct hmac_sha256_ctx { + struct __hmac_sha256_ctx ctx; +}; + +/** + * hmac_sha256_preparekey() - Prepare a key for HMAC-SHA256 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha256_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha256_preparekey(struct hmac_sha256_key *key, + const u8 *raw_key, size_t raw_key_len); -extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, - unsigned int len); +/** + * hmac_sha256_init() - Initialize an HMAC-SHA256 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha256() instead. + * + * Context: Any context. + */ +static inline void hmac_sha256_init(struct hmac_sha256_ctx *ctx, + const struct hmac_sha256_key *key) +{ + __hmac_sha256_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha256_init_usingrawkey() - Initialize an HMAC-SHA256 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha256_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha256_init_usingrawkey(struct hmac_sha256_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); -extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); +/** + * hmac_sha256_update() - Update an HMAC-SHA256 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha256_update(struct hmac_sha256_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha256_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha256_final() - Finish computing an HMAC-SHA256 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA256 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha256_final(struct hmac_sha256_ctx *ctx, + u8 out[at_least SHA256_DIGEST_SIZE]); + +/** + * hmac_sha256() - Compute HMAC-SHA256 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA256 value + * + * If you're using the key only once, consider using hmac_sha256_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha256(const struct hmac_sha256_key *key, + const u8 *data, size_t data_len, + u8 out[at_least SHA256_DIGEST_SIZE]); + +/** + * hmac_sha256_usingrawkey() - Compute HMAC-SHA256 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA256 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA256 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha256_preparekey() followed by multiple calls to hmac_sha256() instead. + * + * Context: Any context. + */ +void hmac_sha256_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least SHA256_DIGEST_SIZE]); + +/* State for the SHA-512 (and SHA-384) compression function */ +struct sha512_block_state { + u64 h[8]; +}; /* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. + * Context structure, shared by SHA-384 and SHA-512. The sha384_ctx and + * sha512_ctx structs wrap this one so that the API has proper typing and + * doesn't allow mixing the SHA-384 and SHA-512 functions arbitrarily. + */ +struct __sha512_ctx { + struct sha512_block_state state; + u64 bytecount_lo; + u64 bytecount_hi; + u8 buf[SHA512_BLOCK_SIZE] __aligned(__alignof__(__be64)); +}; +void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len); + +/* + * HMAC key and message context structs, shared by HMAC-SHA384 and HMAC-SHA512. + * The hmac_sha384_* and hmac_sha512_* structs wrap this one so that the API has + * proper typing and doesn't allow mixing the functions arbitrarily. + */ +struct __hmac_sha512_key { + struct sha512_block_state istate; + struct sha512_block_state ostate; +}; +struct __hmac_sha512_ctx { + struct __sha512_ctx sha_ctx; + struct sha512_block_state ostate; +}; +void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx, + const struct __hmac_sha512_key *key); + +/** + * struct sha384_ctx - Context for hashing a message with SHA-384 + * @ctx: private + */ +struct sha384_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha384_init() - Initialize a SHA-384 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha384() instead. * - * For details see lib/crypto/sha256.c + * Context: Any context. */ +void sha384_init(struct sha384_ctx *ctx); -static inline void sha256_init(struct sha256_state *sctx) +/** + * sha384_update() - Update a SHA-384 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha384_update(struct sha384_ctx *ctx, + const u8 *data, size_t len) { - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; + __sha512_update(&ctx->ctx, data, len); } -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha256_final(struct sha256_state *sctx, u8 *out); -void sha256(const u8 *data, unsigned int len, u8 *out); -static inline void sha224_init(struct sha256_state *sctx) +/** + * sha384_final() - Finish computing a SHA-384 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-384 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha384_final(struct sha384_ctx *ctx, u8 out[at_least SHA384_DIGEST_SIZE]); + +/** + * sha384() - Compute SHA-384 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-384 message digest + * + * Context: Any context. + */ +void sha384(const u8 *data, size_t len, u8 out[at_least SHA384_DIGEST_SIZE]); + +/** + * struct hmac_sha384_key - Prepared key for HMAC-SHA384 + * @key: private + */ +struct hmac_sha384_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha384_ctx - Context for computing HMAC-SHA384 of a message + * @ctx: private + */ +struct hmac_sha384_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha384_preparekey() - Prepare a key for HMAC-SHA384 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha384_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha384_preparekey(struct hmac_sha384_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha384_init() - Initialize an HMAC-SHA384 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha384() instead. + * + * Context: Any context. + */ +static inline void hmac_sha384_init(struct hmac_sha384_ctx *ctx, + const struct hmac_sha384_key *key) { - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; + __hmac_sha512_init(&ctx->ctx, &key->key); } -/* Simply use sha256_update as it is equivalent to sha224_update. */ -void sha224_final(struct sha256_state *sctx, u8 *out); + +/** + * hmac_sha384_init_usingrawkey() - Initialize an HMAC-SHA384 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha384_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha384_update() - Update an HMAC-SHA384 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha384_update(struct hmac_sha384_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha384_final() - Finish computing an HMAC-SHA384 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA384 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha384_final(struct hmac_sha384_ctx *ctx, + u8 out[at_least SHA384_DIGEST_SIZE]); + +/** + * hmac_sha384() - Compute HMAC-SHA384 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key only once, consider using hmac_sha384_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha384(const struct hmac_sha384_key *key, + const u8 *data, size_t data_len, + u8 out[at_least SHA384_DIGEST_SIZE]); + +/** + * hmac_sha384_usingrawkey() - Compute HMAC-SHA384 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA384 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA384 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha384_preparekey() followed by multiple calls to hmac_sha384() instead. + * + * Context: Any context. + */ +void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least SHA384_DIGEST_SIZE]); + +/** + * struct sha512_ctx - Context for hashing a message with SHA-512 + * @ctx: private + */ +struct sha512_ctx { + struct __sha512_ctx ctx; +}; + +/** + * sha512_init() - Initialize a SHA-512 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sha512() instead. + * + * Context: Any context. + */ +void sha512_init(struct sha512_ctx *ctx); + +/** + * sha512_update() - Update a SHA-512 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void sha512_update(struct sha512_ctx *ctx, + const u8 *data, size_t len) +{ + __sha512_update(&ctx->ctx, data, len); +} + +/** + * sha512_final() - Finish computing a SHA-512 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SHA-512 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha512_final(struct sha512_ctx *ctx, u8 out[at_least SHA512_DIGEST_SIZE]); + +/** + * sha512() - Compute SHA-512 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SHA-512 message digest + * + * Context: Any context. + */ +void sha512(const u8 *data, size_t len, u8 out[at_least SHA512_DIGEST_SIZE]); + +/** + * struct hmac_sha512_key - Prepared key for HMAC-SHA512 + * @key: private + */ +struct hmac_sha512_key { + struct __hmac_sha512_key key; +}; + +/** + * struct hmac_sha512_ctx - Context for computing HMAC-SHA512 of a message + * @ctx: private + */ +struct hmac_sha512_ctx { + struct __hmac_sha512_ctx ctx; +}; + +/** + * hmac_sha512_preparekey() - Prepare a key for HMAC-SHA512 + * @key: (output) the key structure to initialize + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * Note: the caller is responsible for zeroizing both the struct hmac_sha512_key + * and the raw key once they are no longer needed. + * + * Context: Any context. + */ +void hmac_sha512_preparekey(struct hmac_sha512_key *key, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha512_init() - Initialize an HMAC-SHA512 context for a new message + * @ctx: (output) the HMAC context to initialize + * @key: the prepared HMAC key + * + * If you don't need incremental computation, consider hmac_sha512() instead. + * + * Context: Any context. + */ +static inline void hmac_sha512_init(struct hmac_sha512_ctx *ctx, + const struct hmac_sha512_key *key) +{ + __hmac_sha512_init(&ctx->ctx, &key->key); +} + +/** + * hmac_sha512_init_usingrawkey() - Initialize an HMAC-SHA512 context for a new + * message, using a raw key + * @ctx: (output) the HMAC context to initialize + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * + * If you don't need incremental computation, consider hmac_sha512_usingrawkey() + * instead. + * + * Context: Any context. + */ +void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx, + const u8 *raw_key, size_t raw_key_len); + +/** + * hmac_sha512_update() - Update an HMAC-SHA512 context with message data + * @ctx: the HMAC context to update; must have been initialized + * @data: the message data + * @data_len: the data length in bytes + * + * This can be called any number of times. + * + * Context: Any context. + */ +static inline void hmac_sha512_update(struct hmac_sha512_ctx *ctx, + const u8 *data, size_t data_len) +{ + __sha512_update(&ctx->ctx.sha_ctx, data, data_len); +} + +/** + * hmac_sha512_final() - Finish computing an HMAC-SHA512 value + * @ctx: the HMAC context to finalize; must have been initialized + * @out: (output) the resulting HMAC-SHA512 value + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void hmac_sha512_final(struct hmac_sha512_ctx *ctx, + u8 out[at_least SHA512_DIGEST_SIZE]); + +/** + * hmac_sha512() - Compute HMAC-SHA512 in one shot, using a prepared key + * @key: the prepared HMAC key + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key only once, consider using hmac_sha512_usingrawkey(). + * + * Context: Any context. + */ +void hmac_sha512(const struct hmac_sha512_key *key, + const u8 *data, size_t data_len, + u8 out[at_least SHA512_DIGEST_SIZE]); + +/** + * hmac_sha512_usingrawkey() - Compute HMAC-SHA512 in one shot, using a raw key + * @raw_key: the raw HMAC-SHA512 key + * @raw_key_len: the key length in bytes. All key lengths are supported. + * @data: the message data + * @data_len: the data length in bytes + * @out: (output) the resulting HMAC-SHA512 value + * + * If you're using the key multiple times, prefer to use + * hmac_sha512_preparekey() followed by multiple calls to hmac_sha512() instead. + * + * Context: Any context. + */ +void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len, + const u8 *data, size_t data_len, + u8 out[at_least SHA512_DIGEST_SIZE]); #endif /* _CRYPTO_SHA2_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h deleted file mode 100644 index e0418818d63c..000000000000 --- a/include/crypto/sha256_base.h +++ /dev/null @@ -1,135 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha256_base.h - core logic for SHA-256 implementations - * - * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#ifndef _CRYPTO_SHA256_BASE_H -#define _CRYPTO_SHA256_BASE_H - -#include <asm/byteorder.h> -#include <linux/unaligned.h> -#include <crypto/internal/hash.h> -#include <crypto/sha2.h> -#include <linux/string.h> -#include <linux/types.h> - -typedef void (sha256_block_fn)(struct sha256_state *sst, u8 const *src, - int blocks); - -static inline int sha224_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha224_init(sctx); - return 0; -} - -static inline int sha256_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_init(sctx); - return 0; -} - -static inline int lib_sha256_base_do_update(struct sha256_state *sctx, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SHA256_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA256_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buf, 1); - } - - blocks = len / SHA256_BLOCK_SIZE; - len %= SHA256_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA256_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - - return 0; -} - -static inline int sha256_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - return lib_sha256_base_do_update(sctx, data, len, block_fn); -} - -static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, - sha256_block_fn *block_fn) -{ - const int bit_offset = SHA256_BLOCK_SIZE - sizeof(__be64); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA256_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buf, 1); - } - - memset(sctx->buf + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buf, 1); - - return 0; -} - -static inline int sha256_base_do_finalize(struct shash_desc *desc, - sha256_block_fn *block_fn) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - return lib_sha256_base_do_finalize(sctx, block_fn); -} - -static inline int lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) -{ - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) - put_unaligned_be32(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); - return 0; -} - -static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct sha256_state *sctx = shash_desc_ctx(desc); - - return lib_sha256_base_finish(sctx, out, digest_size); -} - -#endif /* _CRYPTO_SHA256_BASE_H */ diff --git a/include/crypto/sha3.h b/include/crypto/sha3.h index 080f60c2e6b1..c9e4182ff74f 100644 --- a/include/crypto/sha3.h +++ b/include/crypto/sha3.h @@ -1,34 +1,346 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Common values for SHA-3 algorithms + * + * See also Documentation/crypto/sha3.rst */ #ifndef __CRYPTO_SHA3_H__ #define __CRYPTO_SHA3_H__ +#include <linux/types.h> +#include <linux/string.h> + #define SHA3_224_DIGEST_SIZE (224 / 8) #define SHA3_224_BLOCK_SIZE (200 - 2 * SHA3_224_DIGEST_SIZE) +#define SHA3_224_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_224_BLOCK_SIZE + 1 #define SHA3_256_DIGEST_SIZE (256 / 8) #define SHA3_256_BLOCK_SIZE (200 - 2 * SHA3_256_DIGEST_SIZE) +#define SHA3_256_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_256_BLOCK_SIZE + 1 #define SHA3_384_DIGEST_SIZE (384 / 8) #define SHA3_384_BLOCK_SIZE (200 - 2 * SHA3_384_DIGEST_SIZE) +#define SHA3_384_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_384_BLOCK_SIZE + 1 #define SHA3_512_DIGEST_SIZE (512 / 8) #define SHA3_512_BLOCK_SIZE (200 - 2 * SHA3_512_DIGEST_SIZE) +#define SHA3_512_EXPORT_SIZE SHA3_STATE_SIZE + SHA3_512_BLOCK_SIZE + 1 + +/* + * SHAKE128 and SHAKE256 actually have variable output size, but this is used to + * calculate the block size (rate) analogously to the above. + */ +#define SHAKE128_DEFAULT_SIZE (128 / 8) +#define SHAKE128_BLOCK_SIZE (200 - 2 * SHAKE128_DEFAULT_SIZE) +#define SHAKE256_DEFAULT_SIZE (256 / 8) +#define SHAKE256_BLOCK_SIZE (200 - 2 * SHAKE256_DEFAULT_SIZE) +#define SHA3_STATE_SIZE 200 + +/* + * State for the Keccak-f[1600] permutation: 25 64-bit words. + * + * We usually keep the state words as little-endian, to make absorbing and + * squeezing easier. (It means that absorbing and squeezing can just treat the + * state as a byte array.) The state words are converted to native-endian only + * temporarily by implementations of the permutation that need native-endian + * words. Of course, that conversion is a no-op on little-endian machines. + */ struct sha3_state { - u64 st[25]; - unsigned int rsiz; - unsigned int rsizw; + union { + __le64 words[SHA3_STATE_SIZE / 8]; + u8 bytes[SHA3_STATE_SIZE]; + + u64 native_words[SHA3_STATE_SIZE / 8]; /* see comment above */ + }; +}; + +/* Internal context, shared by the digests (SHA3-*) and the XOFs (SHAKE*) */ +struct __sha3_ctx { + struct sha3_state state; + u8 digest_size; /* Digests only: the digest size in bytes */ + u8 block_size; /* Block size in bytes */ + u8 absorb_offset; /* Index of next state byte to absorb into */ + u8 squeeze_offset; /* XOFs only: index of next state byte to extract */ +}; + +void __sha3_update(struct __sha3_ctx *ctx, const u8 *in, size_t in_len); + +/** + * struct sha3_ctx - Context for SHA3-224, SHA3-256, SHA3-384, or SHA3-512 + * @ctx: private + */ +struct sha3_ctx { + struct __sha3_ctx ctx; +}; + +/** + * sha3_zeroize_ctx() - Zeroize a SHA-3 context + * @ctx: The context to zeroize + * + * This is already called by sha3_final(). Call this explicitly when abandoning + * a context without calling sha3_final(). + */ +static inline void sha3_zeroize_ctx(struct sha3_ctx *ctx) +{ + memzero_explicit(ctx, sizeof(*ctx)); +} - unsigned int partial; - u8 buf[SHA3_224_BLOCK_SIZE]; +/** + * struct shake_ctx - Context for SHAKE128 or SHAKE256 + * @ctx: private + */ +struct shake_ctx { + struct __sha3_ctx ctx; }; -int crypto_sha3_init(struct shash_desc *desc); -int crypto_sha3_update(struct shash_desc *desc, const u8 *data, - unsigned int len); -int crypto_sha3_final(struct shash_desc *desc, u8 *out); +/** + * shake_zeroize_ctx() - Zeroize a SHAKE context + * @ctx: The context to zeroize + * + * Call this after the last squeeze. + */ +static inline void shake_zeroize_ctx(struct shake_ctx *ctx) +{ + memzero_explicit(ctx, sizeof(*ctx)); +} + +/** + * sha3_224_init() - Initialize a context for SHA3-224 + * @ctx: The context to initialize + * + * This begins a new SHA3-224 message digest computation. + * + * Context: Any context. + */ +static inline void sha3_224_init(struct sha3_ctx *ctx) +{ + *ctx = (struct sha3_ctx){ + .ctx.digest_size = SHA3_224_DIGEST_SIZE, + .ctx.block_size = SHA3_224_BLOCK_SIZE, + }; +} + +/** + * sha3_256_init() - Initialize a context for SHA3-256 + * @ctx: The context to initialize + * + * This begins a new SHA3-256 message digest computation. + * + * Context: Any context. + */ +static inline void sha3_256_init(struct sha3_ctx *ctx) +{ + *ctx = (struct sha3_ctx){ + .ctx.digest_size = SHA3_256_DIGEST_SIZE, + .ctx.block_size = SHA3_256_BLOCK_SIZE, + }; +} + +/** + * sha3_384_init() - Initialize a context for SHA3-384 + * @ctx: The context to initialize + * + * This begins a new SHA3-384 message digest computation. + * + * Context: Any context. + */ +static inline void sha3_384_init(struct sha3_ctx *ctx) +{ + *ctx = (struct sha3_ctx){ + .ctx.digest_size = SHA3_384_DIGEST_SIZE, + .ctx.block_size = SHA3_384_BLOCK_SIZE, + }; +} + +/** + * sha3_512_init() - Initialize a context for SHA3-512 + * @ctx: The context to initialize + * + * This begins a new SHA3-512 message digest computation. + * + * Context: Any context. + */ +static inline void sha3_512_init(struct sha3_ctx *ctx) +{ + *ctx = (struct sha3_ctx){ + .ctx.digest_size = SHA3_512_DIGEST_SIZE, + .ctx.block_size = SHA3_512_BLOCK_SIZE, + }; +} + +/** + * sha3_update() - Update a SHA-3 digest context with input data + * @ctx: The context to update; must have been initialized + * @in: The input data + * @in_len: Length of the input data in bytes + * + * This can be called any number of times to add data to a SHA3-224, SHA3-256, + * SHA3-384, or SHA3-512 digest (depending on which init function was called). + * + * Context: Any context. + */ +static inline void sha3_update(struct sha3_ctx *ctx, + const u8 *in, size_t in_len) +{ + __sha3_update(&ctx->ctx, in, in_len); +} + +/** + * sha3_final() - Finish computing a SHA-3 message digest + * @ctx: The context to finalize; must have been initialized + * @out: (output) The resulting SHA3-224, SHA3-256, SHA3-384, or SHA3-512 + * message digest, matching the init function that was called. Note that + * the size differs for each one; see SHA3_*_DIGEST_SIZE. + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sha3_final(struct sha3_ctx *ctx, u8 *out); + +/** + * shake128_init() - Initialize a context for SHAKE128 + * @ctx: The context to initialize + * + * This begins a new SHAKE128 extendable-output function (XOF) computation. + * + * Context: Any context. + */ +static inline void shake128_init(struct shake_ctx *ctx) +{ + *ctx = (struct shake_ctx){ + .ctx.block_size = SHAKE128_BLOCK_SIZE, + }; +} + +/** + * shake256_init() - Initialize a context for SHAKE256 + * @ctx: The context to initialize + * + * This begins a new SHAKE256 extendable-output function (XOF) computation. + * + * Context: Any context. + */ +static inline void shake256_init(struct shake_ctx *ctx) +{ + *ctx = (struct shake_ctx){ + .ctx.block_size = SHAKE256_BLOCK_SIZE, + }; +} + +/** + * shake_update() - Update a SHAKE context with input data + * @ctx: The context to update; must have been initialized + * @in: The input data + * @in_len: Length of the input data in bytes + * + * This can be called any number of times to add more input data to SHAKE128 or + * SHAKE256. This cannot be called after squeezing has begun. + * + * Context: Any context. + */ +static inline void shake_update(struct shake_ctx *ctx, + const u8 *in, size_t in_len) +{ + __sha3_update(&ctx->ctx, in, in_len); +} + +/** + * shake_squeeze() - Generate output from SHAKE128 or SHAKE256 + * @ctx: The context to squeeze; must have been initialized + * @out: Where to write the resulting output data + * @out_len: The amount of data to extract to @out in bytes + * + * This may be called multiple times. A number of consecutive squeezes laid + * end-to-end will yield the same output as one big squeeze generating the same + * total amount of output. More input cannot be provided after squeezing has + * begun. After the last squeeze, call shake_zeroize_ctx(). + * + * Context: Any context. + */ +void shake_squeeze(struct shake_ctx *ctx, u8 *out, size_t out_len); + +/** + * sha3_224() - Compute SHA3-224 digest in one shot + * @in: The input data to be digested + * @in_len: Length of the input data in bytes + * @out: The buffer into which the digest will be stored + * + * Convenience function that computes a SHA3-224 digest. Use this instead of + * the incremental API if you're able to provide all the input at once. + * + * Context: Any context. + */ +void sha3_224(const u8 *in, size_t in_len, u8 out[SHA3_224_DIGEST_SIZE]); + +/** + * sha3_256() - Compute SHA3-256 digest in one shot + * @in: The input data to be digested + * @in_len: Length of the input data in bytes + * @out: The buffer into which the digest will be stored + * + * Convenience function that computes a SHA3-256 digest. Use this instead of + * the incremental API if you're able to provide all the input at once. + * + * Context: Any context. + */ +void sha3_256(const u8 *in, size_t in_len, u8 out[SHA3_256_DIGEST_SIZE]); + +/** + * sha3_384() - Compute SHA3-384 digest in one shot + * @in: The input data to be digested + * @in_len: Length of the input data in bytes + * @out: The buffer into which the digest will be stored + * + * Convenience function that computes a SHA3-384 digest. Use this instead of + * the incremental API if you're able to provide all the input at once. + * + * Context: Any context. + */ +void sha3_384(const u8 *in, size_t in_len, u8 out[SHA3_384_DIGEST_SIZE]); + +/** + * sha3_512() - Compute SHA3-512 digest in one shot + * @in: The input data to be digested + * @in_len: Length of the input data in bytes + * @out: The buffer into which the digest will be stored + * + * Convenience function that computes a SHA3-512 digest. Use this instead of + * the incremental API if you're able to provide all the input at once. + * + * Context: Any context. + */ +void sha3_512(const u8 *in, size_t in_len, u8 out[SHA3_512_DIGEST_SIZE]); + +/** + * shake128() - Compute SHAKE128 in one shot + * @in: The input data to be used + * @in_len: Length of the input data in bytes + * @out: The buffer into which the output will be stored + * @out_len: Length of the output to produce in bytes + * + * Convenience function that computes SHAKE128 in one shot. Use this instead of + * the incremental API if you're able to provide all the input at once as well + * as receive all the output at once. All output lengths are supported. + * + * Context: Any context. + */ +void shake128(const u8 *in, size_t in_len, u8 *out, size_t out_len); + +/** + * shake256() - Compute SHAKE256 in one shot + * @in: The input data to be used + * @in_len: Length of the input data in bytes + * @out: The buffer into which the output will be stored + * @out_len: Length of the output to produce in bytes + * + * Convenience function that computes SHAKE256 in one shot. Use this instead of + * the incremental API if you're able to provide all the input at once as well + * as receive all the output at once. All output lengths are supported. + * + * Context: Any context. + */ +void shake256(const u8 *in, size_t in_len, u8 *out, size_t out_len); -#endif +#endif /* __CRYPTO_SHA3_H__ */ diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h deleted file mode 100644 index 679916a84cb2..000000000000 --- a/include/crypto/sha512_base.h +++ /dev/null @@ -1,134 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha512_base.h - core logic for SHA-512 implementations - * - * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> - */ - -#ifndef _CRYPTO_SHA512_BASE_H -#define _CRYPTO_SHA512_BASE_H - -#include <crypto/internal/hash.h> -#include <crypto/sha2.h> -#include <linux/crypto.h> -#include <linux/module.h> -#include <linux/string.h> - -#include <linux/unaligned.h> - -typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src, - int blocks); - -static inline int sha384_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static inline int sha512_base_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static inline int sha512_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sha512_block_fn *block_fn) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - sctx->count[0] += len; - if (sctx->count[0] < len) - sctx->count[1]++; - - if (unlikely((partial + len) >= SHA512_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SHA512_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buf, 1); - } - - blocks = len / SHA512_BLOCK_SIZE; - len %= SHA512_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SHA512_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - - return 0; -} - -static inline int sha512_base_do_finalize(struct shash_desc *desc, - sha512_block_fn *block_fn) -{ - const int bit_offset = SHA512_BLOCK_SIZE - sizeof(__be64[2]); - struct sha512_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buf + bit_offset); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - - sctx->buf[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buf + partial, 0x0, SHA512_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buf, 1); - } - - memset(sctx->buf + partial, 0x0, bit_offset - partial); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - bits[1] = cpu_to_be64(sctx->count[0] << 3); - block_fn(sctx, sctx->buf, 1); - - return 0; -} - -static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct sha512_state *sctx = shash_desc_ctx(desc); - __be64 *digest = (__be64 *)out; - int i; - - for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) - put_unaligned_be64(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); - return 0; -} - -#endif /* _CRYPTO_SHA512_BASE_H */ diff --git a/include/crypto/sig.h b/include/crypto/sig.h index cff41ad93824..fa6dafafab3f 100644 --- a/include/crypto/sig.h +++ b/include/crypto/sig.h @@ -23,7 +23,8 @@ struct crypto_sig { * struct sig_alg - generic public key signature algorithm * * @sign: Function performs a sign operation as defined by public key - * algorithm. Optional. + * algorithm. On success, the signature size is returned. + * Optional. * @verify: Function performs a complete verify operation as defined by * public key algorithm, returning verification status. Optional. * @set_pub_key: Function invokes the algorithm specific set public key @@ -127,7 +128,7 @@ static inline void crypto_free_sig(struct crypto_sig *tfm) /** * crypto_sig_keysize() - Get key size * - * Function returns the key size in bytes. + * Function returns the key size in bits. * Function assumes that the key is already set in the transformation. If this * function is called without a setkey or with a failed setkey, you may end up * in a NULL dereference. @@ -186,7 +187,7 @@ static inline unsigned int crypto_sig_maxsize(struct crypto_sig *tfm) * @dst: destination obuffer * @dlen: destination length * - * Return: zero on success; error code in case of error + * Return: signature size on success; error code in case of error */ static inline int crypto_sig_sign(struct crypto_sig *tfm, const void *src, unsigned int slen, diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index 18a86e0af016..4efe2ca8c4d1 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -145,6 +145,7 @@ struct skcipher_alg_common SKCIPHER_ALG_COMMON; * considerably more efficient if it can operate on multiple chunks * in parallel. Should be a multiple of chunksize. * @co: see struct skcipher_alg_common + * @SKCIPHER_ALG_COMMON: see struct skcipher_alg_common * * All fields except @ivsize are mandatory and must be filled. */ @@ -214,16 +215,17 @@ struct lskcipher_alg { #define MAX_SYNC_SKCIPHER_REQSIZE 384 /* - * This performs a type-check against the "tfm" argument to make sure + * This performs a type-check against the "_tfm" argument to make sure * all users have the correct skcipher tfm for doing on-stack requests. */ -#define SYNC_SKCIPHER_REQUEST_ON_STACK(name, tfm) \ +#define SYNC_SKCIPHER_REQUEST_ON_STACK(name, _tfm) \ char __##name##_desc[sizeof(struct skcipher_request) + \ - MAX_SYNC_SKCIPHER_REQSIZE + \ - (!(sizeof((struct crypto_sync_skcipher *)1 == \ - (typeof(tfm))1))) \ + MAX_SYNC_SKCIPHER_REQSIZE \ ] CRYPTO_MINALIGN_ATTR; \ - struct skcipher_request *name = (void *)__##name##_desc + struct skcipher_request *name = \ + (((struct skcipher_request *)__##name##_desc)->base.tfm = \ + crypto_sync_skcipher_tfm((_tfm)), \ + (void *)__##name##_desc) /** * DOC: Symmetric Key Cipher API @@ -311,6 +313,12 @@ static inline struct crypto_tfm *crypto_lskcipher_tfm( return &tfm->base; } +static inline struct crypto_tfm *crypto_sync_skcipher_tfm( + struct crypto_sync_skcipher *tfm) +{ + return crypto_skcipher_tfm(&tfm->base); +} + /** * crypto_free_skcipher() - zeroize and free cipher handle * @tfm: cipher handle to be freed diff --git a/include/crypto/sm3.h b/include/crypto/sm3.h index 1f021ad0533f..371e8a661705 100644 --- a/include/crypto/sm3.h +++ b/include/crypto/sm3.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Common values for SM3 algorithm + * SM3 hash algorithm * * Copyright (C) 2017 ARM Limited or its affiliates. * Copyright (C) 2017 Gilad Ben-Yossef <gilad@benyossef.com> @@ -15,9 +15,6 @@ #define SM3_DIGEST_SIZE 32 #define SM3_BLOCK_SIZE 64 -#define SM3_T1 0x79CC4519 -#define SM3_T2 0x7A879D8A - #define SM3_IVA 0x7380166f #define SM3_IVB 0x4914b2b9 #define SM3_IVC 0x172442d7 @@ -27,38 +24,64 @@ #define SM3_IVG 0xe38dee4d #define SM3_IVH 0xb0fb0e4e -extern const u8 sm3_zero_message_hash[SM3_DIGEST_SIZE]; +/* State for the SM3 compression function */ +struct sm3_block_state { + u32 h[SM3_DIGEST_SIZE / 4]; +}; -struct sm3_state { - u32 state[SM3_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SM3_BLOCK_SIZE]; +/** + * struct sm3_ctx - Context for hashing a message with SM3 + * @state: the compression function state + * @bytecount: number of bytes processed so far + * @buf: partial block buffer; bytecount % SM3_BLOCK_SIZE bytes are valid + */ +struct sm3_ctx { + struct sm3_block_state state; + u64 bytecount; + u8 buf[SM3_BLOCK_SIZE] __aligned(__alignof__(__be64)); }; -/* - * Stand-alone implementation of the SM3 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. +/** + * sm3_init() - Initialize an SM3 context for a new message + * @ctx: the context to initialize + * + * If you don't need incremental computation, consider sm3() instead. + * + * Context: Any context. + */ +void sm3_init(struct sm3_ctx *ctx); + +/** + * sm3_update() - Update an SM3 context with message data + * @ctx: the context to update; must have been initialized + * @data: the message data + * @len: the data length in bytes + * + * This can be called any number of times. * - * For details see lib/crypto/sm3.c + * Context: Any context. */ +void sm3_update(struct sm3_ctx *ctx, const u8 *data, size_t len); -static inline void sm3_init(struct sm3_state *sctx) -{ - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; -} +/** + * sm3_final() - Finish computing an SM3 message digest + * @ctx: the context to finalize; must have been initialized + * @out: (output) the resulting SM3 message digest + * + * After finishing, this zeroizes @ctx. So the caller does not need to do it. + * + * Context: Any context. + */ +void sm3_final(struct sm3_ctx *ctx, u8 out[at_least SM3_DIGEST_SIZE]); -void sm3_update(struct sm3_state *sctx, const u8 *data, unsigned int len); -void sm3_final(struct sm3_state *sctx, u8 *out); +/** + * sm3() - Compute SM3 message digest in one shot + * @data: the message data + * @len: the data length in bytes + * @out: (output) the resulting SM3 message digest + * + * Context: Any context. + */ +void sm3(const u8 *data, size_t len, u8 out[at_least SM3_DIGEST_SIZE]); -#endif +#endif /* _CRYPTO_SM3_H */ diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h deleted file mode 100644 index b33ed39c2bce..000000000000 --- a/include/crypto/sm3_base.h +++ /dev/null @@ -1,112 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sm3_base.h - core logic for SM3 implementations - * - * Copyright (C) 2017 ARM Limited or its affiliates. - * Written by Gilad Ben-Yossef <gilad@benyossef.com> - */ - -#ifndef _CRYPTO_SM3_BASE_H -#define _CRYPTO_SM3_BASE_H - -#include <crypto/internal/hash.h> -#include <crypto/sm3.h> -#include <linux/crypto.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/unaligned.h> - -typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); - -static inline int sm3_base_init(struct shash_desc *desc) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SM3_IVA; - sctx->state[1] = SM3_IVB; - sctx->state[2] = SM3_IVC; - sctx->state[3] = SM3_IVD; - sctx->state[4] = SM3_IVE; - sctx->state[5] = SM3_IVF; - sctx->state[6] = SM3_IVG; - sctx->state[7] = SM3_IVH; - sctx->count = 0; - - return 0; -} - -static inline int sm3_base_do_update(struct shash_desc *desc, - const u8 *data, - unsigned int len, - sm3_block_fn *block_fn) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->count += len; - - if (unlikely((partial + len) >= SM3_BLOCK_SIZE)) { - int blocks; - - if (partial) { - int p = SM3_BLOCK_SIZE - partial; - - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - - block_fn(sctx, sctx->buffer, 1); - } - - blocks = len / SM3_BLOCK_SIZE; - len %= SM3_BLOCK_SIZE; - - if (blocks) { - block_fn(sctx, data, blocks); - data += blocks * SM3_BLOCK_SIZE; - } - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); - - return 0; -} - -static inline int sm3_base_do_finalize(struct shash_desc *desc, - sm3_block_fn *block_fn) -{ - const int bit_offset = SM3_BLOCK_SIZE - sizeof(__be64); - struct sm3_state *sctx = shash_desc_ctx(desc); - __be64 *bits = (__be64 *)(sctx->buffer + bit_offset); - unsigned int partial = sctx->count % SM3_BLOCK_SIZE; - - sctx->buffer[partial++] = 0x80; - if (partial > bit_offset) { - memset(sctx->buffer + partial, 0x0, SM3_BLOCK_SIZE - partial); - partial = 0; - - block_fn(sctx, sctx->buffer, 1); - } - - memset(sctx->buffer + partial, 0x0, bit_offset - partial); - *bits = cpu_to_be64(sctx->count << 3); - block_fn(sctx, sctx->buffer, 1); - - return 0; -} - -static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) -{ - struct sm3_state *sctx = shash_desc_ctx(desc); - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], digest++); - - memzero_explicit(sctx, sizeof(*sctx)); - return 0; -} - -#endif /* _CRYPTO_SM3_BASE_H */ diff --git a/include/crypto/streebog.h b/include/crypto/streebog.h index cae1b4a01971..570f720a113b 100644 --- a/include/crypto/streebog.h +++ b/include/crypto/streebog.h @@ -23,15 +23,10 @@ struct streebog_uint512 { }; struct streebog_state { - union { - u8 buffer[STREEBOG_BLOCK_SIZE]; - struct streebog_uint512 m; - }; struct streebog_uint512 hash; struct streebog_uint512 h; struct streebog_uint512 N; struct streebog_uint512 Sigma; - size_t fillsize; }; #endif /* !_CRYPTO_STREEBOG_H_ */ diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 15b16c4853d8..16aef89f021f 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -7,9 +7,9 @@ #include <linux/fips.h> #define XTS_BLOCK_SIZE 16 +#define XTS_FORBID_WEAK_KEYS (1 << 0) -static inline int xts_verify_key(struct crypto_skcipher *tfm, - const u8 *key, unsigned int keylen) +static inline int __xts_verify_key(const u8 *key, size_t keylen, int flags) { /* * key consists of keys of equal size concatenated, therefore @@ -29,12 +29,22 @@ static inline int xts_verify_key(struct crypto_skcipher *tfm, * Ensure that the AES and tweak key are not identical when * in FIPS mode or the FORBID_WEAK_KEYS flag is set. */ - if ((fips_enabled || (crypto_skcipher_get_flags(tfm) & - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) && + if ((fips_enabled || (flags & XTS_FORBID_WEAK_KEYS)) && !crypto_memneq(key, key + (keylen / 2), keylen / 2)) return -EINVAL; return 0; } +static inline int xts_verify_key(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + int flags = (crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) ? + XTS_FORBID_WEAK_KEYS : + 0; + + return __xts_verify_key(key, keylen, flags); +} + #endif /* _CRYPTO_XTS_H */ |