path: root/arch/arm64/include/asm/kvm_pkvm.h

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2020 - Google LLC
 * Author: Quentin Perret <qperret@google.com>
 */
#ifndef __ARM64_KVM_PKVM_H__
#define __ARM64_KVM_PKVM_H__

#include <linux/arm_ffa.h>
#include <linux/kvm_host.h>
#include <linux/memblock.h>
#include <linux/scatterlist.h>
#include <asm/kvm_pgtable.h>

/* Maximum number of VMs that can co-exist under pKVM. */
#define KVM_MAX_PVMS 255

#define HYP_MEMBLOCK_REGIONS 128

int pkvm_init_host_vm(struct kvm *kvm);
int pkvm_create_hyp_vm(struct kvm *kvm);
bool pkvm_hyp_vm_is_created(struct kvm *kvm);
void pkvm_destroy_hyp_vm(struct kvm *kvm);
int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu);

/*
 * Check whether the specific capability is allowed in pKVM.
 *
 * Certain features are allowed only for non-protected VMs in pKVM, which is why
 * this takes the VM (kvm) as a parameter.
 */
static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
{
	switch (ext) {
	case KVM_CAP_IRQCHIP:
	case KVM_CAP_ARM_PSCI:
	case KVM_CAP_ARM_PSCI_0_2:
	case KVM_CAP_NR_VCPUS:
	case KVM_CAP_MAX_VCPUS:
	case KVM_CAP_MAX_VCPU_ID:
	case KVM_CAP_MSI_DEVID:
	case KVM_CAP_ARM_VM_IPA_SIZE:
	case KVM_CAP_ARM_PTRAUTH_ADDRESS:
	case KVM_CAP_ARM_PTRAUTH_GENERIC:
	case KVM_CAP_ONE_REG:
	case KVM_CAP_MP_STATE:
	case KVM_CAP_VCPU_EVENTS:
	case KVM_CAP_VCPU_ATTRIBUTES:
	case KVM_CAP_IMMEDIATE_EXIT:
	case KVM_CAP_IOEVENTFD:
	case KVM_CAP_IRQFD_RESAMPLE:
	case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
	case KVM_CAP_ARM_INJECT_SERROR_ESR:
		return true;
	case KVM_CAP_ARM_MTE:
	case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
	case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
	case KVM_CAP_PRE_FAULT_MEMORY:
		return false;
	default:
		return !kvm || (kvm->arch.vm_flavor == VM_PKVM);
	}
}

/*
 * The vCPU features a protected VM may use: checked by the host at
 * KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created.
 */
static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm,
						  unsigned long *allowed)
{
	bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES);

	set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed);

	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3))
		set_bit(KVM_ARM_VCPU_PMU_V3, allowed);

	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS))
		set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed);

	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC))
		set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed);

	if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE))
		set_bit(KVM_ARM_VCPU_SVE, allowed);
}

/*
 * Check whether the KVM VM IOCTL is allowed in pKVM.
 *
 * Certain features are allowed only for non-protected VMs in pKVM, which is why
 * this takes the VM (kvm) as a parameter.
 */
static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl)
{
	long ext;
	int r;

	r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext);
	if (r < 0)
		return false;

	return kvm_pkvm_ext_allowed(kvm, ext);
}

extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);

static inline unsigned long
hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size)
{
	unsigned long nr_pages = reg->size >> PAGE_SHIFT;
	unsigned long start, end;

	start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size;
	end = start + nr_pages * vmemmap_entry_size;
	start = ALIGN_DOWN(start, PAGE_SIZE);
	end = ALIGN(end, PAGE_SIZE);

	return end - start;
}

static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size)
{
	unsigned long res = 0, i;

	for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
		res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i],
						 vmemmap_entry_size);
	}

	return res >> PAGE_SHIFT;
}

static inline unsigned long hyp_vm_table_pages(void)
{
	return PAGE_ALIGN(KVM_MAX_PVMS * sizeof(void *)) >> PAGE_SHIFT;
}

static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
{
	unsigned long total = 0;
	int i;

	/* Provision the worst case scenario */
	for (i = KVM_PGTABLE_FIRST_LEVEL; i <= KVM_PGTABLE_LAST_LEVEL; i++) {
		nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
		total += nr_pages;
	}

	return total;
}

static inline unsigned long __hyp_pgtable_total_pages(void)
{
	unsigned long res = 0, i;

	/* Cover all of memory with page-granularity */
	for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
		struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
		res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
	}

	return res;
}

static inline unsigned long hyp_s1_pgtable_pages(void)
{
	unsigned long res;

	res = __hyp_pgtable_total_pages();

	/* Allow 1 GiB for private mappings */
	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);

	return res;
}

static inline unsigned long host_s2_pgtable_pages(void)
{
	unsigned long res;

	/*
	 * Include an extra 16 pages to safely upper-bound the worst case of
	 * concatenated pgds.
	 */
	res = __hyp_pgtable_total_pages() + 16;

	/* Allow 1 GiB for MMIO mappings */
	res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);

	return res;
}

#ifdef CONFIG_NVHE_EL2_DEBUG
static inline unsigned long pkvm_selftest_pages(void) { return 32; }
#else
static inline unsigned long pkvm_selftest_pages(void) { return 0; }
#endif

#define KVM_FFA_MBOX_NR_PAGES	1

static inline unsigned long hyp_ffa_proxy_pages(void)
{
	size_t desc_max;

	/*
	 * The hypervisor FFA proxy needs enough memory to buffer a fragmented
	 * descriptor returned from EL3 in response to a RETRIEVE_REQ call.
	 */
	desc_max = sizeof(struct ffa_mem_region) +
		   sizeof(struct ffa_mem_region_attributes) +
		   sizeof(struct ffa_composite_mem_region) +
		   SG_MAX_SEGMENTS * sizeof(struct ffa_mem_region_addr_range);

	/* Plus a page each for the hypervisor's RX and TX mailboxes. */
	return (2 * KVM_FFA_MBOX_NR_PAGES) + DIV_ROUND_UP(desc_max, PAGE_SIZE);
}

static inline size_t pkvm_host_sve_state_size(void)
{
	if (!system_supports_sve())
		return 0;

	return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl));
}

struct pkvm_mapping {
	struct rb_node node;
	u64 gfn;
	u64 pfn;
	struct {
		u64 nr_pages:48;
		u64 nc:1;
	};
	u64 __subtree_last;	/* Internal member for interval tree */
};

enum pkvm_hyp_req_type {
	PKVM_HYP_NO_REQ = 0,
	PKVM_HYP_REQ_HYP_ALLOC,
	PKVM_HYP_REQ_HYP_ALLOC_SELFTEST,
	__PKVM_HYP_REQ_TYPE_MAX,
};

#define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \
	(sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1)

struct pkvm_hyp_req {
	u8 type;
	union {
		struct {
			u32	nr_pages;
		} mem;
		struct {
			/* Helper for SMCCC encoding/decoding */
			u8	args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX];
		} args;
	};
};

static inline size_t pkvm_hyp_req_arg_size(u8 type)
{
	struct pkvm_hyp_req *req;

	switch (type) {
	case PKVM_HYP_NO_REQ:
		return 0;
	case PKVM_HYP_REQ_HYP_ALLOC:
	case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST:
		return sizeof(req->mem);
	default:
		WARN_ON(1);
	}

	return 0;
}

/* Encode the pending pkvm_hyp_req type into the SMCCC args */
static inline void
pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req)
{
	u8 *dst, type = req->type;
	size_t size;

	if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) {
		host_ctxt->regs.regs[2] = 0;
		return;
	}

	size = pkvm_hyp_req_arg_size(type);
	if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX))
		return;

	dst = (u8 *)&host_ctxt->regs.regs[2];
	*dst = type;

	memcpy(dst + 1, &req->args, size);
}

/* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */
static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res)
{
	u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2);
	u8 type = *src;

	if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX)
		return false;

	req->type = type;
	memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type));

	return true;
}

int __pkvm_handle_smccc_req(struct arm_smccc_res *res);

/**
 * pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests
 * @f: Hypervisor function symbol to call.
 * @...: Arguments to pass to the hypercall.
 *
 * Re-issue an HVC and process any pending hypervisor request until completion
 * or error.
 *
 * Only use this helper for HVCs whose hypervisor handlers format their return
 * registers with pkvm_hyp_req_to_smccc().
 *
 * Return: Result of the hypercall or a negative error if the hyp request
 * handling failed.
 */
#define pkvm_call_hyp_req(f, ...)								\
({												\
	struct arm_smccc_res __res;								\
	int __ret;										\
	do {											\
		__ret = -1;									\
		arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res);		\
		if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS))					\
			break;									\
		__ret = __res.a1;								\
		if (!__ret)									\
			break;									\
		__ret = __pkvm_handle_smccc_req(&__res);					\
	} while (!__ret);									\
	__ret;											\
})

int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu,
			     struct kvm_pgtable_mm_ops *mm_ops);
void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt,
					u64 addr, u64 size);
void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt);
int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
			    enum kvm_pgtable_prot prot, void *mc,
			    enum kvm_pgtable_walk_flags flags);
int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size);
int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size);
int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);
bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold);
int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot,
				    enum kvm_pgtable_walk_flags flags);
void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,
				 enum kvm_pgtable_walk_flags flags);
int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size,
			      struct kvm_mmu_memory_cache *mc);
void pkvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level);
kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level,
					       enum kvm_pgtable_prot prot, void *mc,
					       bool force_pte);
#endif	/* __ARM64_KVM_PKVM_H__ */