summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/arch/x86/tdx.rst21
-rw-r--r--arch/x86/coco/tdx/tdx.c6
-rw-r--r--arch/x86/include/asm/kvm-x86-ops.h1
-rw-r--r--arch/x86/include/asm/kvm_host.h1
-rw-r--r--arch/x86/include/asm/shared/tdx.h5
-rw-r--r--arch/x86/include/asm/tdx.h23
-rw-r--r--arch/x86/include/asm/tdx_global_metadata.h9
-rw-r--r--arch/x86/kernel/asm-offsets.c1
-rw-r--r--arch/x86/kvm/mmu/mmu.c4
-rw-r--r--arch/x86/kvm/vmx/tdx.c99
-rw-r--r--arch/x86/kvm/vmx/tdx.h2
-rw-r--r--arch/x86/virt/vmx/tdx/seamcall_internal.h19
-rw-r--r--arch/x86/virt/vmx/tdx/tdx.c437
-rw-r--r--arch/x86/virt/vmx/tdx/tdx.h10
-rw-r--r--arch/x86/virt/vmx/tdx/tdx_global_metadata.c23
-rw-r--r--arch/x86/virt/vmx/tdx/tdxcall.S10
-rw-r--r--drivers/virt/coco/tdx-guest/tdx-guest.c6
17 files changed, 565 insertions, 112 deletions
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst
index 3303499ad4c6..a36ea2bd4301 100644
--- a/Documentation/arch/x86/tdx.rst
+++ b/Documentation/arch/x86/tdx.rst
@@ -200,6 +200,27 @@ reflects the TCB of the currently running TDX module and therefore
changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD
launch time and is not affected.
+Dynamic PAMT
+------------
+
+The Physical Address Metadata Table (PAMT) is metadata in which the TDX
+module keeps data about each physical page (think struct page). Space
+for it is allocated by the VMM, consumes up to about 0.4% of system
+memory and needs to be supplied to the TDX module when the TDX module is
+first loaded.
+
+Dynamic PAMT is an add-on feature that allows a VMM to dynamically
+allocate the part of the PAMT which tracks 4KB pages. This reduces the
+amount of memory that TDX consumes while TDs are not in use.
+
+When Dynamic PAMT is in use, dmesg shows it like::
+
+ [..] virt/tdx: Enable Dynamic PAMT
+ [..] virt/tdx: 10092 KB allocated for PAMT
+ [..] virt/tdx: TDX-Module initialized
+
+Dynamic PAMT is enabled automatically if supported.
+
TDX Interaction to Other Kernel Components
------------------------------------------
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index a9a16d0fb5c2..ad0131813a30 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -139,7 +139,7 @@ int tdx_mcall_get_report0(u8 *reportdata, u8 *tdreport)
return 0;
}
-EXPORT_SYMBOL_GPL(tdx_mcall_get_report0);
+EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_get_report0, "tdx-guest");
/**
* tdx_mcall_extend_rtmr() - Wrapper to extend RTMR registers using
@@ -175,7 +175,7 @@ int tdx_mcall_extend_rtmr(u8 index, u8 *data)
return 0;
}
-EXPORT_SYMBOL_GPL(tdx_mcall_extend_rtmr);
+EXPORT_SYMBOL_FOR_MODULES(tdx_mcall_extend_rtmr, "tdx-guest");
/**
* tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
@@ -196,7 +196,7 @@ u64 tdx_hcall_get_quote(u8 *buf, size_t size)
/* Since buf is a shared memory, set the shared (decrypted) bits */
return _tdx_hypercall(TDVMCALL_GET_QUOTE, cc_mkdec(virt_to_phys(buf)), size, 0, 0);
}
-EXPORT_SYMBOL_GPL(tdx_hcall_get_quote);
+EXPORT_SYMBOL_FOR_MODULES(tdx_hcall_get_quote, "tdx-guest");
/*
* The kernel cannot handle #VEs when accessing normal kernel memory. Ensure
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index e213c9ae3e30..5c358c40eae8 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -99,6 +99,7 @@ KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)
KVM_X86_OP(load_mmu_pgd)
KVM_X86_OP_OPTIONAL_RET0(set_external_spte)
KVM_X86_OP_OPTIONAL(free_external_spt)
+KVM_X86_OP_OPTIONAL_RET0(topup_external_cache)
KVM_X86_OP(has_wbinvd_exit)
KVM_X86_OP(get_l2_tsc_offset)
KVM_X86_OP(get_l2_tsc_multiplier)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 5e8aecfc795f..20b9db3f5203 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1635,6 +1635,7 @@ struct kvm_x86_ops {
/* Update external page tables for page table about to be freed. */
void (*free_external_spt)(struct kvm *kvm, struct kvm_mmu_page *sp);
+ int (*topup_external_cache)(struct kvm_vcpu *vcpu, int min_nr_spts);
bool (*has_wbinvd_exit)(void);
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index e5785258e547..bf000f1fb42e 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -143,6 +143,11 @@ struct tdx_module_args {
u64 rbx;
u64 rdi;
u64 rsi;
+ /*
+ * Leaf ABI version. Note that it gets encoded into RAX along with the
+ * leaf number.
+ */
+ u8 version;
};
/* Used to communicate with the TDX module */
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index 89e97d5761d8..e186dfe5bf88 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -36,6 +36,7 @@
/* Bit definitions of TDX_FEATURES0 metadata field */
#define TDX_FEATURES0_TD_PRESERVING BIT_ULL(1)
#define TDX_FEATURES0_NO_RBP_MOD BIT_ULL(18)
+#define TDX_FEATURES0_DYNAMIC_PAMT BIT_ULL(36)
#ifndef __ASSEMBLER__
@@ -118,12 +119,34 @@ static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinf
return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING;
}
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo);
+
+/* Simple structure for pre-allocating DPAMT pages outside of spinlocks. */
+struct tdx_pamt_cache {
+ struct list_head page_list;
+ int cnt;
+};
+
+static inline void tdx_init_pamt_cache(struct tdx_pamt_cache *cache)
+{
+ INIT_LIST_HEAD(&cache->page_list);
+ cache->cnt = 0;
+}
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache);
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages);
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache);
+void tdx_pamt_put(kvm_pfn_t pfn);
+
int tdx_guest_keyid_alloc(void);
u32 tdx_get_nr_guest_keyids(void);
void tdx_guest_keyid_free(unsigned int keyid);
void tdx_quirk_reset_paddr(unsigned long base, unsigned long size);
+struct page *tdx_alloc_control_page(void);
+void tdx_free_control_page(struct page *page);
+
struct tdx_td {
/* TD root structure: */
struct page *tdr_page;
diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h
index 41150d546589..8a3cc1a2a41e 100644
--- a/arch/x86/include/asm/tdx_global_metadata.h
+++ b/arch/x86/include/asm/tdx_global_metadata.h
@@ -1,7 +1,7 @@
/* SPDX-License-Identifier: GPL-2.0 */
-/* Automatically generated TDX global metadata structures. */
-#ifndef _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
-#define _X86_VIRT_TDX_AUTO_GENERATED_TDX_GLOBAL_METADATA_H
+/* TDX global metadata structures. */
+#ifndef _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H
+#define _X86_VIRT_TDX_TDX_GLOBAL_METADATA_H
#include <linux/types.h>
@@ -21,6 +21,9 @@ struct tdx_sys_info_tdmr {
u16 pamt_4k_entry_size;
u16 pamt_2m_entry_size;
u16 pamt_1g_entry_size;
+
+ /* Optional metadata, if DPAMT is supported */
+ u8 pamt_page_bitmap_entry_bits;
};
struct tdx_sys_info_td_ctrl {
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 081816888f7a..b3c00ff4d819 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -95,6 +95,7 @@ static void __used common(void)
OFFSET(TDX_MODULE_rbx, tdx_module_args, rbx);
OFFSET(TDX_MODULE_rdi, tdx_module_args, rdi);
OFFSET(TDX_MODULE_rsi, tdx_module_args, rsi);
+ OFFSET(TDX_MODULE_version, tdx_module_args, version);
BLANK();
OFFSET(BP_scratch, boot_params, scratch);
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 8e62476e477b..58d455f8e23c 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -617,6 +617,10 @@ static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
PT64_ROOT_MAX_LEVEL);
if (r)
return r;
+
+ r = kvm_x86_call(topup_external_cache)(vcpu, PT64_ROOT_MAX_LEVEL);
+ if (r)
+ return r;
}
r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
PT64_ROOT_MAX_LEVEL);
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 6c842e9191a5..6746fc8ad8e7 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -362,7 +362,7 @@ static void tdx_reclaim_control_page(struct page *ctrl_page)
if (tdx_reclaim_page(ctrl_page))
return;
- __free_page(ctrl_page);
+ tdx_free_control_page(ctrl_page);
}
struct tdx_flush_vp_arg {
@@ -589,7 +589,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm)
tdx_quirk_reset_paddr(page_to_phys(kvm_tdx->td.tdr_page), PAGE_SIZE);
- __free_page(kvm_tdx->td.tdr_page);
+ tdx_free_control_page(kvm_tdx->td.tdr_page);
kvm_tdx->td.tdr_page = NULL;
}
@@ -681,6 +681,8 @@ int tdx_vcpu_create(struct kvm_vcpu *vcpu)
if (!irqchip_split(vcpu->kvm))
return -EINVAL;
+ tdx_init_pamt_cache(&tdx->pamt_cache);
+
fpstate_set_confidential(&vcpu->arch.guest_fpu);
vcpu->arch.apic->guest_apic_protected = true;
INIT_LIST_HEAD(&tdx->vt.pi_wakeup_list);
@@ -866,6 +868,8 @@ void tdx_vcpu_free(struct kvm_vcpu *vcpu)
struct vcpu_tdx *tdx = to_tdx(vcpu);
int i;
+ tdx_free_pamt_cache(&tdx->pamt_cache);
+
if (vcpu->cpu != -1) {
KVM_BUG_ON(tdx->state == VCPU_TD_STATE_INITIALIZED, vcpu->kvm);
tdx_flush_vp_on_cpu(vcpu);
@@ -1618,6 +1622,17 @@ void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa);
}
+static int tdx_topup_external_pamt_cache(struct kvm_vcpu *vcpu, int min_nr_spts)
+{
+ /*
+ * Minus one page to exclude the root SPT, but plus one page for a
+ * possible 4KB private mapping.
+ */
+ min_nr_spts += -1 + 1;
+
+ return tdx_topup_pamt_cache(&to_tdx(vcpu)->pamt_cache, min_nr_spts);
+}
+
static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
kvm_pfn_t pfn)
{
@@ -1676,16 +1691,28 @@ static struct page *tdx_spte_to_sept_pt(struct kvm *kvm, gfn_t gfn,
static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
enum pg_level level, u64 new_spte)
{
+ struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
gpa_t gpa = gfn_to_gpa(gfn);
u64 err, entry, level_state;
struct page *sept_pt;
+ int ret;
+
+ if (KVM_BUG_ON(!vcpu, kvm))
+ return -EIO;
sept_pt = tdx_spte_to_sept_pt(kvm, gfn, new_spte, level);
if (!sept_pt)
return -EIO;
+ ret = tdx_pamt_get(page_to_pfn(sept_pt), &to_tdx(vcpu)->pamt_cache);
+ if (KVM_BUG_ON(ret, kvm))
+ return ret;
+
err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, sept_pt,
&entry, &level_state);
+ if (err)
+ tdx_pamt_put(page_to_pfn(sept_pt));
+
if (unlikely(tdx_operand_busy(err)))
return -EBUSY;
@@ -1698,8 +1725,13 @@ static int tdx_sept_map_nonleaf_spte(struct kvm *kvm, gfn_t gfn,
static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level level,
u64 new_spte)
{
+ struct kvm_vcpu *vcpu = kvm_get_running_vcpu();
struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
kvm_pfn_t pfn = spte_to_pfn(new_spte);
+ int ret;
+
+ if (KVM_BUG_ON(!vcpu, kvm))
+ return -EIO;
/* TODO: handle large pages. */
if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
@@ -1707,6 +1739,10 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
WARN_ON_ONCE((new_spte & VMX_EPT_RWX_MASK) != VMX_EPT_RWX_MASK);
+ ret = tdx_pamt_get(pfn, &to_tdx(vcpu)->pamt_cache);
+ if (KVM_BUG_ON(ret, kvm))
+ return ret;
+
/*
* Ensure pre_fault_allowed is read by kvm_arch_vcpu_pre_fault_memory()
* before kvm_tdx->state. Userspace must not be allowed to pre-fault
@@ -1719,10 +1755,15 @@ static int tdx_sept_map_leaf_spte(struct kvm *kvm, gfn_t gfn, enum pg_level leve
* If the TD isn't finalized/runnable, then userspace is initializing
* the VM image via KVM_TDX_INIT_MEM_REGION; ADD the page to the TD.
*/
- if (unlikely(kvm_tdx->state != TD_STATE_RUNNABLE))
- return tdx_mem_page_add(kvm, gfn, level, pfn);
+ if (likely(kvm_tdx->state == TD_STATE_RUNNABLE))
+ ret = tdx_mem_page_aug(kvm, gfn, level, pfn);
+ else
+ ret = tdx_mem_page_add(kvm, gfn, level, pfn);
+
+ if (ret)
+ tdx_pamt_put(pfn);
- return tdx_mem_page_aug(kvm, gfn, level, pfn);
+ return ret;
}
/*
@@ -1819,6 +1860,7 @@ static int tdx_sept_remove_leaf_spte(struct kvm *kvm, gfn_t gfn,
return -EIO;
tdx_quirk_reset_paddr(PFN_PHYS(pfn), PAGE_SIZE);
+ tdx_pamt_put(pfn);
return 0;
}
@@ -1862,6 +1904,8 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn, u64 old_spte,
*/
static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
{
+ struct page *sept_pt = virt_to_page(sp->external_spt);
+
/*
* KVM doesn't (yet) zap page table pages in mirror page table while
* TD is active, though guest pages mapped in mirror page table could be
@@ -1875,15 +1919,15 @@ static void tdx_sept_free_private_spt(struct kvm *kvm, struct kvm_mmu_page *sp)
* the page to prevent the kernel from accessing the encrypted page.
*/
if (KVM_BUG_ON(is_hkid_assigned(to_kvm_tdx(kvm)), kvm) ||
- tdx_reclaim_page(virt_to_page(sp->external_spt)))
+ tdx_reclaim_page(sept_pt))
goto out;
/*
- * Immediately free the S-EPT page because RCU-time free is unnecessary
- * after TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding
- * readers.
+ * Immediately free the S-EPT page as the TDX subsystem doesn't support
+ * freeing pages from RCU callbacks, and more importantly because
+ * TDH.PHYMEM.PAGE.RECLAIM ensures there are no outstanding readers.
*/
- free_page((unsigned long)sp->external_spt);
+ tdx_free_control_page(sept_pt);
out:
sp->external_spt = NULL;
}
@@ -2456,7 +2500,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
ret = -ENOMEM;
- tdr_page = alloc_page(GFP_KERNEL_ACCOUNT);
+ tdr_page = tdx_alloc_control_page();
if (!tdr_page)
goto free_hkid;
@@ -2469,7 +2513,7 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
goto free_tdr;
for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
- tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);
+ tdcs_pages[i] = tdx_alloc_control_page();
if (!tdcs_pages[i])
goto free_tdcs;
}
@@ -2587,10 +2631,8 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
teardown:
/* Only free pages not yet added, so start at 'i' */
for (; i < kvm_tdx->td.tdcs_nr_pages; i++) {
- if (tdcs_pages[i]) {
- __free_page(tdcs_pages[i]);
- tdcs_pages[i] = NULL;
- }
+ tdx_free_control_page(tdcs_pages[i]);
+ tdcs_pages[i] = NULL;
}
if (!kvm_tdx->td.tdcs_pages)
kfree(tdcs_pages);
@@ -2605,16 +2647,13 @@ free_packages:
free_cpumask_var(packages);
free_tdcs:
- for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
- if (tdcs_pages[i])
- __free_page(tdcs_pages[i]);
- }
+ for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++)
+ tdx_free_control_page(tdcs_pages[i]);
kfree(tdcs_pages);
kvm_tdx->td.tdcs_pages = NULL;
free_tdr:
- if (tdr_page)
- __free_page(tdr_page);
+ tdx_free_control_page(tdr_page);
kvm_tdx->td.tdr_page = NULL;
free_hkid:
@@ -2948,7 +2987,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
int ret, i;
u64 err;
- page = alloc_page(GFP_KERNEL_ACCOUNT);
+ page = tdx_alloc_control_page();
if (!page)
return -ENOMEM;
tdx->vp.tdvpr_page = page;
@@ -2968,7 +3007,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
}
for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
- page = alloc_page(GFP_KERNEL_ACCOUNT);
+ page = tdx_alloc_control_page();
if (!page) {
ret = -ENOMEM;
goto free_tdcx;
@@ -2990,7 +3029,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
* method, but the rest are freed here.
*/
for (; i < kvm_tdx->td.tdcx_nr_pages; i++) {
- __free_page(tdx->vp.tdcx_pages[i]);
+ tdx_free_control_page(tdx->vp.tdcx_pages[i]);
tdx->vp.tdcx_pages[i] = NULL;
}
return -EIO;
@@ -3018,16 +3057,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
free_tdcx:
for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
- if (tdx->vp.tdcx_pages[i])
- __free_page(tdx->vp.tdcx_pages[i]);
+ tdx_free_control_page(tdx->vp.tdcx_pages[i]);
tdx->vp.tdcx_pages[i] = NULL;
}
kfree(tdx->vp.tdcx_pages);
tdx->vp.tdcx_pages = NULL;
free_tdvpr:
- if (tdx->vp.tdvpr_page)
- __free_page(tdx->vp.tdvpr_page);
+ tdx_free_control_page(tdx->vp.tdvpr_page);
tdx->vp.tdvpr_page = NULL;
tdx->vp.tdvpr_pa = 0;
@@ -3487,6 +3524,10 @@ int __init tdx_hardware_setup(void)
vt_x86_ops.set_external_spte = tdx_sept_set_private_spte;
vt_x86_ops.free_external_spt = tdx_sept_free_private_spt;
+
+ if (tdx_supports_dynamic_pamt(tdx_sysinfo))
+ vt_x86_ops.topup_external_cache = tdx_topup_external_pamt_cache;
+
vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt;
return 0;
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index ac8323a68b16..fd368e3ee060 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -72,6 +72,8 @@ struct vcpu_tdx {
u64 map_gpa_next;
u64 map_gpa_end;
+
+ struct tdx_pamt_cache pamt_cache;
};
void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err);
diff --git a/arch/x86/virt/vmx/tdx/seamcall_internal.h b/arch/x86/virt/vmx/tdx/seamcall_internal.h
index be5f446467df..051ad2d45cab 100644
--- a/arch/x86/virt/vmx/tdx/seamcall_internal.h
+++ b/arch/x86/virt/vmx/tdx/seamcall_internal.h
@@ -11,6 +11,7 @@
#ifndef _X86_VIRT_SEAMCALL_INTERNAL_H
#define _X86_VIRT_SEAMCALL_INTERNAL_H
+#include <linux/bitfield.h>
#include <linux/printk.h>
#include <linux/types.h>
#include <asm/archrandom.h>
@@ -23,9 +24,27 @@ u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args);
typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args);
+/*
+ * SEAMCALL leaf:
+ *
+ * Bit 15:0 Leaf number
+ * Bit 23:16 Leaf ABI version number
+ * Bit 24 Pending interrupts detection mode
+ * Bit 63 1 for P-SEAMLDR leaf, 0 for TDX module leaf
+ */
+#define SEAMCALL_LEAF_MASK GENMASK_U64(15, 0)
+#define SEAMCALL_SEAMLDR_MASK BIT_U64(63)
+
static __always_inline u64 __seamcall_dirty_cache(sc_func_t func, u64 fn,
struct tdx_module_args *args)
{
+ /*
+ * fn contains leaf number for TDX module calls and P-SEAMLDR calls.
+ * Other fields in SEAMCALL leaf like leaf ABI version number are in
+ * struct tdx_module_args.
+ */
+ BUILD_BUG_ON(fn & ~(SEAMCALL_LEAF_MASK | SEAMCALL_SEAMLDR_MASK));
+
lockdep_assert_preemption_disabled();
/*
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index 1b9ff749dd8e..96ced0494b68 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -30,6 +30,7 @@
#include <linux/suspend.h>
#include <linux/syscore_ops.h>
#include <linux/idr.h>
+#include <linux/vmalloc.h>
#include <asm/page.h>
#include <asm/special_insns.h>
#include <asm/msr-index.h>
@@ -46,6 +47,9 @@
#include "seamcall_internal.h"
#include "tdx.h"
+/* Number of DPAMT pages to be provided to TDX module per 2MB region of PA */
+#define TDX_DPAMT_ENTRY_PAGE_CNT 2
+
struct tdx_module_state {
bool initialized;
bool sysinit_done;
@@ -63,6 +67,14 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized);
static struct tdmr_info_list tdx_tdmr_list;
+/*
+ * On a machine with DPAMT, the kernel maintains a reference counter
+ * for every 2MB range. The counter indicates how many users there are for
+ * the DPAMT at the 2MB range. The kernel allocates DPAMT refcounts at
+ * initialization.
+ */
+static atomic_t *dpamt_refcounts;
+
/* All TDX-usable memory regions. Protected by mem_hotplug_lock. */
static LIST_HEAD(tdx_memlist);
@@ -253,6 +265,42 @@ static struct syscore tdx_syscore = {
};
/*
+ * Allocate DPAMT reference counters for all physical memory.
+ *
+ * It consumes 2MB for every 1TB of physical memory.
+ */
+static __init int init_dpamt_refcounts(void)
+{
+ size_t size = DIV_ROUND_UP(max_pfn, PTRS_PER_PTE) * sizeof(*dpamt_refcounts);
+
+ if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+ return 0;
+
+ dpamt_refcounts = vzalloc(size);
+ if (!dpamt_refcounts)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static __init void free_dpamt_refcounts(void)
+{
+ if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+ return;
+
+ vfree(dpamt_refcounts);
+ dpamt_refcounts = NULL;
+}
+
+static atomic_t *tdx_find_dpamt_refcount(unsigned long pfn)
+{
+ /* Find which PMD a PFN is in. */
+ unsigned long index = pfn >> (PMD_SHIFT - PAGE_SHIFT);
+
+ return &dpamt_refcounts[index];
+}
+
+/*
* Add a memory region as a TDX memory block. The caller must make sure
* all memory regions are added in address ascending order and don't
* overlap.
@@ -510,35 +558,37 @@ static __init int fill_out_tdmrs(struct list_head *tmb_list,
return 0;
}
+static __init unsigned long tdmr_get_pamt_bitmap_sz(struct tdmr_info *tdmr)
+{
+ unsigned long pamt_sz, nr_pamt_entries;
+ int bits_per_entry;
+
+ bits_per_entry = tdx_sysinfo.tdmr.pamt_page_bitmap_entry_bits;
+ nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
+ pamt_sz = DIV_ROUND_UP(nr_pamt_entries * bits_per_entry, BITS_PER_BYTE);
+
+ return PAGE_ALIGN(pamt_sz);
+}
+
/*
* Calculate PAMT size given a TDMR and a page size. The returned
* PAMT size is always aligned up to 4K page boundary.
*/
-static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz,
- u16 pamt_entry_size)
+static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz)
{
unsigned long pamt_sz, nr_pamt_entries;
+ const int tdx_pg_size_shift[TDX_PS_NR] = { PAGE_SHIFT, PMD_SHIFT, PUD_SHIFT };
+ const u16 pamt_entry_size[TDX_PS_NR] = {
+ tdx_sysinfo.tdmr.pamt_4k_entry_size,
+ tdx_sysinfo.tdmr.pamt_2m_entry_size,
+ tdx_sysinfo.tdmr.pamt_1g_entry_size,
+ };
- switch (pgsz) {
- case TDX_PS_4K:
- nr_pamt_entries = tdmr->size >> PAGE_SHIFT;
- break;
- case TDX_PS_2M:
- nr_pamt_entries = tdmr->size >> PMD_SHIFT;
- break;
- case TDX_PS_1G:
- nr_pamt_entries = tdmr->size >> PUD_SHIFT;
- break;
- default:
- WARN_ON_ONCE(1);
- return 0;
- }
+ nr_pamt_entries = tdmr->size >> tdx_pg_size_shift[pgsz];
+ pamt_sz = nr_pamt_entries * pamt_entry_size[pgsz];
- pamt_sz = nr_pamt_entries * pamt_entry_size;
/* TDX requires PAMT size must be 4K aligned */
- pamt_sz = ALIGN(pamt_sz, PAGE_SIZE);
-
- return pamt_sz;
+ return PAGE_ALIGN(pamt_sz);
}
/*
@@ -576,15 +626,11 @@ static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_lis
* within @tdmr, and set up PAMTs for @tdmr.
*/
static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
- struct list_head *tmb_list,
- u16 pamt_entry_size[])
+ struct list_head *tmb_list)
{
- unsigned long pamt_base[TDX_PS_NR];
- unsigned long pamt_size[TDX_PS_NR];
- unsigned long tdmr_pamt_base;
unsigned long tdmr_pamt_size;
struct page *pamt;
- int pgsz, nid;
+ int nid;
nid = tdmr_get_nid(tdmr, tmb_list);
@@ -592,13 +638,18 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
* Calculate the PAMT size for each TDX supported page size
* and the total PAMT size.
*/
- tdmr_pamt_size = 0;
- for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
- pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz,
- pamt_entry_size[pgsz]);
- tdmr_pamt_size += pamt_size[pgsz];
+ tdmr->pamt_1g_size = tdmr_get_pamt_sz(tdmr, TDX_PS_1G);
+ tdmr->pamt_2m_size = tdmr_get_pamt_sz(tdmr, TDX_PS_2M);
+
+ if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+ /* With DPAMT, PAMT_4K is replaced with a bitmap */
+ tdmr->pamt_4k_size = tdmr_get_pamt_bitmap_sz(tdmr);
+ } else {
+ tdmr->pamt_4k_size = tdmr_get_pamt_sz(tdmr, TDX_PS_4K);
}
+ tdmr_pamt_size = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size;
+
/*
* Allocate one chunk of physically contiguous memory for all
* PAMTs. This helps minimize the PAMT's use of reserved areas
@@ -606,25 +657,17 @@ static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr,
*/
pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL,
nid, &node_online_map);
- if (!pamt)
- return -ENOMEM;
/*
- * Break the contiguous allocation back up into the
- * individual PAMTs for each page size.
+ * tdmr->pamt_4k_base is still zero so the error
+ * path of the caller will skip freeing the PAMT.
*/
- tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT;
- for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) {
- pamt_base[pgsz] = tdmr_pamt_base;
- tdmr_pamt_base += pamt_size[pgsz];
- }
+ if (!pamt)
+ return -ENOMEM;
- tdmr->pamt_4k_base = pamt_base[TDX_PS_4K];
- tdmr->pamt_4k_size = pamt_size[TDX_PS_4K];
- tdmr->pamt_2m_base = pamt_base[TDX_PS_2M];
- tdmr->pamt_2m_size = pamt_size[TDX_PS_2M];
- tdmr->pamt_1g_base = pamt_base[TDX_PS_1G];
- tdmr->pamt_1g_size = pamt_size[TDX_PS_1G];
+ tdmr->pamt_4k_base = page_to_phys(pamt);
+ tdmr->pamt_2m_base = tdmr->pamt_4k_base + tdmr->pamt_4k_size;
+ tdmr->pamt_1g_base = tdmr->pamt_2m_base + tdmr->pamt_2m_size;
return 0;
}
@@ -655,10 +698,7 @@ static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr,
tdmr_get_pamt(tdmr, &pamt_base, &pamt_size);
/* Do nothing if PAMT hasn't been allocated for this TDMR */
- if (!pamt_size)
- return;
-
- if (WARN_ON_ONCE(!pamt_base))
+ if (!pamt_base)
return;
pamt_func(pamt_base, pamt_size);
@@ -684,14 +724,12 @@ static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list)
/* Allocate and set up PAMTs for all TDMRs */
static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list,
- struct list_head *tmb_list,
- u16 pamt_entry_size[])
+ struct list_head *tmb_list)
{
int i, ret = 0;
for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) {
- ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list,
- pamt_entry_size);
+ ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list);
if (ret)
goto err;
}
@@ -968,18 +1006,13 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
struct tdmr_info_list *tdmr_list,
struct tdx_sys_info_tdmr *sysinfo_tdmr)
{
- u16 pamt_entry_size[TDX_PS_NR] = {
- sysinfo_tdmr->pamt_4k_entry_size,
- sysinfo_tdmr->pamt_2m_entry_size,
- sysinfo_tdmr->pamt_1g_entry_size,
- };
int ret;
ret = fill_out_tdmrs(tmb_list, tdmr_list);
if (ret)
return ret;
- ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size);
+ ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list);
if (ret)
return ret;
@@ -998,6 +1031,8 @@ static __init int construct_tdmrs(struct list_head *tmb_list,
return ret;
}
+#define TDX_SYS_CONFIG_DYNAMIC_PAMT BIT(16)
+
static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
u64 global_keyid)
{
@@ -1026,6 +1061,12 @@ static __init int config_tdx_module(struct tdmr_info_list *tdmr_list,
args.rcx = __pa(tdmr_pa_array);
args.rdx = tdmr_list->nr_consumed_tdmrs;
args.r8 = global_keyid;
+
+ if (tdx_supports_dynamic_pamt(&tdx_sysinfo)) {
+ pr_info("Enable Dynamic PAMT\n");
+ args.r8 |= TDX_SYS_CONFIG_DYNAMIC_PAMT;
+ }
+
ret = seamcall_prerr(TDH_SYS_CONFIG, &args);
/* Free the array as it is not required anymore. */
@@ -1167,10 +1208,14 @@ static __init int init_tdx_module(void)
*/
get_online_mems();
- ret = build_tdx_memlist(&tdx_memlist);
+ ret = init_dpamt_refcounts();
if (ret)
goto out_put_tdxmem;
+ ret = build_tdx_memlist(&tdx_memlist);
+ if (ret)
+ goto err_free_dpamt_refcounts;
+
/* Allocate enough space for constructing TDMRs */
ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr);
if (ret)
@@ -1220,6 +1265,8 @@ err_free_tdmrs:
free_tdmr_list(&tdx_tdmr_list);
err_free_tdxmem:
free_tdx_memlist(&tdx_memlist);
+err_free_dpamt_refcounts:
+ free_dpamt_refcounts();
goto out_put_tdxmem;
}
@@ -1912,10 +1959,11 @@ u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid)
.rcx = vp->tdvpr_pa,
.rdx = initial_rcx,
.r8 = x2apicid,
+ /* apicid requires version == 1. */
+ .version = 1,
};
- /* apicid requires version == 1. */
- return seamcall(TDH_VP_INIT | (1ULL << TDX_VERSION_SHIFT), &args);
+ return seamcall(TDH_VP_INIT, &args);
}
EXPORT_SYMBOL_FOR_KVM(tdh_vp_init);
@@ -2005,6 +2053,269 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn)
}
EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);
+bool tdx_supports_dynamic_pamt(const struct tdx_sys_info *sysinfo)
+{
+ return sysinfo->features.tdx_features0 & TDX_FEATURES0_DYNAMIC_PAMT;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_supports_dynamic_pamt);
+
+static struct page *tdx_alloc_page_pamt_cache(struct tdx_pamt_cache *cache)
+{
+ struct page *page;
+
+ page = list_first_entry_or_null(&cache->page_list, struct page, lru);
+ if (page) {
+ list_del(&page->lru);
+ cache->cnt--;
+ }
+
+ return page;
+}
+
+static struct page *alloc_dpamt_page(struct tdx_pamt_cache *cache)
+{
+ if (cache)
+ return tdx_alloc_page_pamt_cache(cache);
+
+ return alloc_page(GFP_KERNEL_ACCOUNT);
+}
+
+static int alloc_pamt_array(struct page **pamt_pages, struct tdx_pamt_cache *cache)
+{
+ int i, j;
+
+ for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+ pamt_pages[i] = alloc_dpamt_page(cache);
+ if (!pamt_pages[i])
+ goto err;
+ }
+
+ return 0;
+
+err:
+ for (j = 0; j < i; j++)
+ __free_page(pamt_pages[j]);
+
+ return -ENOMEM;
+}
+
+static void free_pamt_array(struct page **pamt_pages)
+{
+ int i;
+
+ for (i = 0; i < TDX_DPAMT_ENTRY_PAGE_CNT; i++) {
+ /*
+ * Reset pages unconditionally to cover cases
+ * where they were passed to the TDX module.
+ */
+ tdx_quirk_reset_paddr(page_to_phys(pamt_pages[i]), PAGE_SIZE);
+
+ __free_page(pamt_pages[i]);
+ }
+}
+
+/* Helper for building DPAMT seamcall() arguments. */
+static u64 pamt_2mb_arg(kvm_pfn_t pfn)
+{
+ /* Find the 2MB-wide DPAMT region for 'pfn': */
+ unsigned long hpa_2mb = ALIGN_DOWN(pfn << PAGE_SHIFT, PMD_SIZE);
+
+ /*
+ * TDX ABI requires specifying the page level the installed DPAMT
+ * backing will cover, even though today only 2MB is supported.
+ */
+ return hpa_2mb | TDX_PS_2M;
+}
+
+/* Add DPAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_add(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+ struct tdx_module_args args = {
+ .rcx = pamt_2mb_arg(pfn),
+ .rdx = page_to_phys(pamt_pages[0]),
+ .r8 = page_to_phys(pamt_pages[1]),
+ };
+
+ return seamcall(TDH_PHYMEM_PAMT_ADD, &args);
+}
+
+/* Remove DPAMT backing for the 2MB region surrounding the given pfn. */
+static u64 tdh_phymem_pamt_remove(kvm_pfn_t pfn, struct page **pamt_pages)
+{
+ struct tdx_module_args args = {
+ .rcx = pamt_2mb_arg(pfn),
+ };
+ u64 ret;
+
+ ret = seamcall_ret(TDH_PHYMEM_PAMT_REMOVE, &args);
+ if (ret)
+ return ret;
+
+ /* Copy PAMT pages out of the struct per the TDX ABI */
+ pamt_pages[0] = phys_to_page(args.rdx);
+ pamt_pages[1] = phys_to_page(args.r8);
+
+ return 0;
+}
+
+/* Serializes adding/removing DPAMT memory */
+static DEFINE_SPINLOCK(dpamt_lock);
+
+/* Bump DPAMT refcount for the given pfn and allocate DPAMT backing if needed. */
+int tdx_pamt_get(kvm_pfn_t pfn, struct tdx_pamt_cache *cache)
+{
+ struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT];
+ atomic_t *dpamt_refcount;
+ u64 tdx_status;
+ int ret;
+
+ if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+ return 0;
+
+ ret = alloc_pamt_array(pamt_pages, cache);
+ if (ret)
+ return ret;
+
+ dpamt_refcount = tdx_find_dpamt_refcount(pfn);
+
+ spin_lock(&dpamt_lock);
+
+ /*
+ * If the DPAMT entry is already added (i.e. refcount >= 1),
+ * then just increment the refcount.
+ */
+ if (atomic_inc_not_zero(dpamt_refcount))
+ goto out_free;
+
+ /* Try to add the PAMT page and take the refcount 0->1. */
+ tdx_status = tdh_phymem_pamt_add(pfn, pamt_pages);
+ if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS)) {
+ ret = -EIO;
+ goto out_free;
+ }
+
+ atomic_set(dpamt_refcount, 1);
+ spin_unlock(&dpamt_lock);
+ return 0;
+
+out_free:
+ spin_unlock(&dpamt_lock);
+ free_pamt_array(pamt_pages);
+
+ return ret;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_get);
+
+/* Drop DPAMT refcount for the given pfn and free DPAMT backing if needed. */
+void tdx_pamt_put(kvm_pfn_t pfn)
+{
+ struct page *pamt_pages[TDX_DPAMT_ENTRY_PAGE_CNT] = {};
+ atomic_t *dpamt_refcount;
+ u64 tdx_status;
+
+ if (!tdx_supports_dynamic_pamt(&tdx_sysinfo))
+ return;
+
+ dpamt_refcount = tdx_find_dpamt_refcount(pfn);
+
+ spin_lock(&dpamt_lock);
+ /*
+ * If there is more than 1 reference on the DPAMT entry, don't
+ * remove it yet. Just decrement the refcount.
+ */
+ if (atomic_read(dpamt_refcount) > 1) {
+ atomic_dec(dpamt_refcount);
+ goto out_unlock;
+ }
+
+ /* Try to remove the pamt page and take the refcount 1->0. */
+ tdx_status = tdh_phymem_pamt_remove(pfn, pamt_pages);
+
+ /*
+ * Don't free pamt_pages as it could hold garbage when
+ * tdh_phymem_pamt_remove() fails. Don't panic/BUG_ON(), as
+ * there is no risk of data corruption, but do yell loudly as
+ * failure indicates a kernel bug, memory is being leaked, and
+ * the dangling DPAMT entry may cause future operations to fail.
+ */
+ if (WARN_ON_ONCE(tdx_status != TDX_SUCCESS))
+ goto out_unlock;
+
+ atomic_set(dpamt_refcount, 0);
+ spin_unlock(&dpamt_lock);
+ free_pamt_array(pamt_pages);
+ return;
+out_unlock:
+ spin_unlock(&dpamt_lock);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_pamt_put);
+
+void tdx_free_pamt_cache(struct tdx_pamt_cache *cache)
+{
+ struct page *page;
+
+ while ((page = tdx_alloc_page_pamt_cache(cache)))
+ __free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_pamt_cache);
+
+int tdx_topup_pamt_cache(struct tdx_pamt_cache *cache, unsigned long npages)
+{
+ if (WARN_ON_ONCE(!tdx_supports_dynamic_pamt(&tdx_sysinfo)))
+ return 0;
+
+ npages *= TDX_DPAMT_ENTRY_PAGE_CNT;
+
+ while (cache->cnt < npages) {
+ struct page *page = alloc_page(GFP_KERNEL_ACCOUNT);
+
+ if (!page)
+ return -ENOMEM;
+
+ list_add(&page->lru, &cache->page_list);
+ cache->cnt++;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_topup_pamt_cache);
+
+/*
+ * Return a page that can be gifted to the TDX module for use as a "control"
+ * page, i.e. pages that are used for control structures for a given TDX
+ * guest, and thus obtain TDX protections, including DPAMT tracking.
+ */
+struct page *tdx_alloc_control_page(void)
+{
+ struct page *page;
+
+ page = alloc_page(GFP_KERNEL_ACCOUNT);
+ if (!page)
+ return NULL;
+
+ if (tdx_pamt_get(page_to_pfn(page), NULL)) {
+ __free_page(page);
+ return NULL;
+ }
+
+ return page;
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_alloc_control_page);
+
+/*
+ * Free a page that was gifted to the TDX module for use as a control
+ * page. After this, the page is no longer protected by TDX.
+ */
+void tdx_free_control_page(struct page *page)
+{
+ if (!page)
+ return;
+
+ tdx_pamt_put(page_to_pfn(page));
+ __free_page(page);
+}
+EXPORT_SYMBOL_FOR_KVM(tdx_free_control_page);
+
void tdx_sys_disable(void)
{
struct tdx_module_args args = {};
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index bdfd0e1e337a..db209541d3cd 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -48,16 +48,10 @@
#define TDH_SYS_CONFIG 45
#define TDH_SYS_SHUTDOWN 52
#define TDH_SYS_UPDATE 53
+#define TDH_PHYMEM_PAMT_ADD 58
+#define TDH_PHYMEM_PAMT_REMOVE 59
#define TDH_SYS_DISABLE 69
-/*
- * SEAMCALL leaf:
- *
- * Bit 15:0 Leaf number
- * Bit 23:16 Version number
- */
-#define TDX_VERSION_SHIFT 16
-
/* TDX page types */
#define PT_NDA 0x0
#define PT_RSVD 0x1
diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
index e49c300f23d4..98ebf17aab1c 100644
--- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Automatically generated functions to read TDX global metadata.
+ * Functions to read TDX global metadata.
*
* This file doesn't compile on its own as it lacks of inclusion
* of SEAMCALL wrapper primitive which reads global metadata.
@@ -33,6 +33,18 @@ static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinf
return ret;
}
+static __init int get_tdx_sys_info_tdmr_dpamt(struct tdx_sys_info_tdmr *sysinfo_tdmr)
+{
+ int ret;
+ u64 val;
+
+ ret = read_sys_metadata_field(0x9100000000000013, &val);
+ if (!ret)
+ sysinfo_tdmr->pamt_page_bitmap_entry_bits = val;
+
+ return ret;
+}
+
static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr)
{
int ret = 0;
@@ -129,5 +141,14 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo)
ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl);
ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf);
+ /*
+ * The kernel supports using TDX without DPAMT, so
+ * avoid reporting failure if it's not supported. Don't
+ * try to support buggy TDX modules that advertise
+ * DPAMT but don't expose the metadata.
+ */
+ if (!ret && tdx_supports_dynamic_pamt(sysinfo))
+ ret = get_tdx_sys_info_tdmr_dpamt(&sysinfo->tdmr);
+
return ret;
}
diff --git a/arch/x86/virt/vmx/tdx/tdxcall.S b/arch/x86/virt/vmx/tdx/tdxcall.S
index 016a2a1ec1d6..a194e83613e7 100644
--- a/arch/x86/virt/vmx/tdx/tdxcall.S
+++ b/arch/x86/virt/vmx/tdx/tdxcall.S
@@ -45,8 +45,14 @@
.macro TDX_MODULE_CALL host:req ret=0 saved=0
FRAME_BEGIN
- /* Move Leaf ID to RAX */
- mov %rdi, %rax
+ /* Leaf ABI version -> RAX[23:16]. Zero rest of RAX. */
+ movzbl TDX_MODULE_version(%rsi), %eax
+ shl $16, %eax
+ /*
+ * Combine leaf number arg and leaf ABI version into RAX, they don't
+ * overlap.
+ */
+ or %rdi, %rax
/* Move other input regs from 'struct tdx_module_args' */
movq TDX_MODULE_rcx(%rsi), %rcx
diff --git a/drivers/virt/coco/tdx-guest/tdx-guest.c b/drivers/virt/coco/tdx-guest/tdx-guest.c
index d0303e31e816..a21bd0376b74 100644
--- a/drivers/virt/coco/tdx-guest/tdx-guest.c
+++ b/drivers/virt/coco/tdx-guest/tdx-guest.c
@@ -265,7 +265,7 @@ static int wait_for_quote_completion(struct tdx_quote_buf *quote_buf, u32 timeou
return (i == timeout) ? -ETIMEDOUT : 0;
}
-static int tdx_report_new_locked(struct tsm_report *report, void *data)
+static int tdx_report_new_locked(struct tsm_report *report)
{
u8 *buf;
struct tdx_quote_buf *quote_buf = quote_data;
@@ -333,10 +333,10 @@ static int tdx_report_new_locked(struct tsm_report *report, void *data)
return ret;
}
-static int tdx_report_new(struct tsm_report *report, void *data)
+static int tdx_report_new(struct tsm_report *report, void *unused)
{
scoped_cond_guard(mutex_intr, return -EINTR, &quote_lock)
- return tdx_report_new_locked(report, data);
+ return tdx_report_new_locked(report);
}
static bool tdx_report_attr_visible(int n)