path: root/lib/crypto/riscv/aes.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AES using the RISC-V vector crypto extensions
 *
 * Copyright (C) 2023 VRULL GmbH
 * Copyright (C) 2023 SiFive, Inc.
 * Copyright 2024 Google LLC
 */

#include <asm/simd.h>
#include <asm/vector.h>

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_zvkned_zvkb);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_xts);

/* The assembly code assumes the following offsets. */
static_assert(offsetof(struct aes_enckey, len) == 0);
static_assert(offsetof(struct aes_enckey, k.rndkeys) == 16);
static_assert(offsetof(struct aes_key, len) == 0);
static_assert(offsetof(struct aes_key, k.rndkeys) == 16);

void aes_encrypt_zvkned(const struct aes_enckey *key, u8 out[AES_BLOCK_SIZE],
			const u8 in[AES_BLOCK_SIZE]);
void aes_decrypt_zvkned(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
			const u8 in[AES_BLOCK_SIZE]);

static void aes_preparekey_arch(union aes_enckey_arch *k,
				union aes_invkey_arch *inv_k,
				const u8 *in_key, int key_len, int nrounds)
{
	aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
			      in_key, key_len);
}

static void aes_encrypt_arch(const struct aes_enckey *key,
			     u8 out[AES_BLOCK_SIZE],
			     const u8 in[AES_BLOCK_SIZE])
{
	if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
		kernel_vector_begin();
		aes_encrypt_zvkned(key, out, in);
		kernel_vector_end();
	} else {
		aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
	}
}

static void aes_decrypt_arch(const struct aes_key *key,
			     u8 out[AES_BLOCK_SIZE],
			     const u8 in[AES_BLOCK_SIZE])
{
	/*
	 * Note that the Zvkned code uses the standard round keys, while the
	 * fallback uses the inverse round keys.  Thus both must be present.
	 */
	if (static_branch_likely(&have_zvkned) && likely(may_use_simd())) {
		kernel_vector_begin();
		aes_decrypt_zvkned(key, out, in);
		kernel_vector_end();
	} else {
		aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
				    out, in);
	}
}

#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
			    const struct aes_enckey *key);
void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
			    const struct aes_key *key);

/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
				 const struct aes_enckey *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	aes_ecb_encrypt_zvkned(dst, src, len, key);
	kernel_vector_end();
	return true;
}

/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
				 const struct aes_key *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	aes_ecb_decrypt_zvkned(dst, src, len, key);
	kernel_vector_end();
	return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */

#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
void aes_cbc_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
			    u8 iv[AES_BLOCK_SIZE], const struct aes_enckey *key);
void aes_cbc_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
			    u8 iv[AES_BLOCK_SIZE], const struct aes_key *key);
void aes_cbc_cts_crypt_zvkned(u8 *dst, const u8 *src, size_t len,
			      const u8 iv[AES_BLOCK_SIZE],
			      aes_encrypt_arg key, bool enc);

/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_cbc_encrypt_arch aes_cbc_encrypt_arch
static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
				 u8 iv[AES_BLOCK_SIZE],
				 const struct aes_enckey *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	aes_cbc_encrypt_zvkned(dst, src, len, iv, key);
	kernel_vector_end();
	return true;
}

/* len is always a positive multiple of AES_BLOCK_SIZE here. */
#define aes_cbc_decrypt_arch aes_cbc_decrypt_arch
static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
				 u8 iv[AES_BLOCK_SIZE],
				 const struct aes_key *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	aes_cbc_decrypt_zvkned(dst, src, len, iv, key);
	kernel_vector_end();
	return true;
}

/* len can be any value greater than AES_BLOCK_SIZE here. */
#define aes_cbc_cts_encrypt_arch aes_cbc_cts_encrypt_arch
static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
				     u8 iv[AES_BLOCK_SIZE],
				     const struct aes_enckey *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;

	kernel_vector_begin();
	aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, true);
	kernel_vector_end();
	return true;
}

/* len can be any value greater than AES_BLOCK_SIZE here. */
#define aes_cbc_cts_decrypt_arch aes_cbc_cts_decrypt_arch
static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
				     u8 iv[AES_BLOCK_SIZE],
				     const struct aes_key *key)
{
	if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
		return false;

	kernel_vector_begin();
	aes_cbc_cts_crypt_zvkned(dst, src, len, iv, key, false);
	kernel_vector_end();
	return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC */

#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
void aes_ctr32_crypt_zvkned_zvkb(u8 *dst, const u8 *src, size_t len, u8 iv[16],
				 const struct aes_enckey *key);

static void aes_ctr_riscv(u8 *dst, const u8 *src, size_t len,
			  u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
{
	u32 ctr32 = get_unaligned_be32(&ctr[12]);
	u32 nblocks;
	size_t part1_len;

	/* This assumes len <= U32_MAX * AES_BLOCK_SIZE. */
	nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
	ctr32 += nblocks;

	if (likely(ctr32 >= nblocks)) {
		/* The low 32 bits of the counter won't overflow. */
		aes_ctr32_crypt_zvkned_zvkb(dst, src, len, ctr, key);
	} else {
		/*
		 * The low 32 bits of the counter will overflow.  The
		 * assembly doesn't handle this case, so split the
		 * operation into two at the point where the overflow
		 * will occur.  After the first part, add the carry bit.
		 */
		part1_len =
			min(len, (size_t)(nblocks - ctr32) * AES_BLOCK_SIZE);
		aes_ctr32_crypt_zvkned_zvkb(dst, src, part1_len, ctr, key);
		for (int i = AES_BLOCK_SIZE - 5; i >= 0; i--) {
			if (++ctr[i] != 0)
				break;
		}
		if (part1_len < len)
			aes_ctr32_crypt_zvkned_zvkb(dst + part1_len,
						    src + part1_len,
						    len - part1_len, ctr, key);
	}
}

#define aes_ctr_arch aes_ctr_arch
static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
			 u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
{
	if (!static_branch_likely(&have_zvkned_zvkb) ||
	    unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	while (len) {
		/*
		 * Split super long messages into chunks that each need at most
		 * 1 carry bit out of the low 32-bit word of the counter.  Up to
		 * U32_MAX * AES_BLOCK_SIZE bytes could be allowed per chunk,
		 * but use a lower limit to guard against other errors.
		 */
		size_t n = min(len, round_down(S32_MAX, 32 * AES_BLOCK_SIZE));

		aes_ctr_riscv(dst, src, n, ctr, key);
		dst += n;
		src += n;
		len -= n;
	}
	kernel_vector_end();
	return true;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CTR */

#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
void aes_xts_encrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len,
				      u8 tweak[AES_BLOCK_SIZE],
				      const struct aes_key *key);
void aes_xts_decrypt_zvkned_zvbb_zvkg(u8 *dst, const u8 *src, size_t len,
				      u8 tweak[AES_BLOCK_SIZE],
				      const struct aes_key *key);

/* len is always a positive multiple of AES_BLOCK_SIZE here. */
static __always_inline bool
aes_xts_crypt_riscv(u8 *dst, const u8 *src, size_t len,
		    u8 tweak[AES_BLOCK_SIZE],
		    const struct aes_xts_key *key, bool cont, bool enc)
{
	if (!static_branch_likely(&have_xts) || unlikely(!may_use_simd()))
		return false;
	kernel_vector_begin();
	if (!cont)
		aes_encrypt_zvkned(&key->tweak_key, tweak, tweak);
	if (enc)
		aes_xts_encrypt_zvkned_zvbb_zvkg(dst, src, len, tweak,
						 &key->main_key);
	else
		aes_xts_decrypt_zvkned_zvbb_zvkg(dst, src, len, tweak,
						 &key->main_key);
	kernel_vector_end();
	return true;
}

#define aes_xts_encrypt_arch aes_xts_encrypt_arch
static bool aes_xts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
				 u8 tweak[AES_BLOCK_SIZE],
				 const struct aes_xts_key *key, bool cont)
{
	return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, true);
}

#define aes_xts_decrypt_arch aes_xts_decrypt_arch
static bool aes_xts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
				 u8 tweak[AES_BLOCK_SIZE],
				 const struct aes_xts_key *key, bool cont)
{
	return aes_xts_crypt_riscv(dst, src, len, tweak, key, cont, false);
}
#endif /* CONFIG_CRYPTO_LIB_AES_XTS */

#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
	if (riscv_isa_extension_available(NULL, ZVKNED) &&
	    riscv_vector_vlen() >= 128) {
		static_branch_enable(&have_zvkned);
		if (riscv_isa_extension_available(NULL, ZVKB))
			static_branch_enable(&have_zvkned_zvkb);
		if (riscv_isa_extension_available(NULL, ZVBB) &&
		    riscv_isa_extension_available(NULL, ZVKG) &&
		    riscv_vector_vlen() < 2048 /* Implementation limitation */)
			static_branch_enable(&have_xts);
	}
}