// SPDX-License-Identifier: GPL-2.0 //! Common page table types shared between MMU v2 and v3. //! //! This module provides foundational types used by both MMU versions: //! - Page table level hierarchy //! - Memory aperture types for PDEs and PTEs #![expect(dead_code)] pub(super) mod map; pub(super) mod ver2; pub(super) mod ver3; pub(super) mod walk; use kernel::{ io::Io, num::Bounded, prelude::*, // }; use crate::{ gpu::Architecture, mm::{ pramin, Pfn, VirtualAddress, VramAddress, // }, }; /// Extracts the page table index at a given level from a virtual address. pub(super) trait VaLevelIndex { /// Return the page table index at `level` for this virtual address. fn level_index(&self, level: u64) -> u64; } /// MMU version enumeration. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum MmuVersion { /// MMU v2 for Turing/Ampere/Ada. V2, /// MMU v3 for Hopper and later. V3, } impl From for MmuVersion { fn from(arch: Architecture) -> Self { match arch { Architecture::Turing | Architecture::Ampere | Architecture::Ada => Self::V2, Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { Self::V3 } } } } /// Page Table Level hierarchy for MMU v2/v3. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum PageTableLevel { /// Level 0 - Page Directory Base (root). Pdb, /// Level 1 - Intermediate page directory. L1, /// Level 2 - Intermediate page directory. L2, /// Level 3 - Intermediate page directory or dual PDE (version-dependent). L3, /// Level 4 - PTE level for v2, intermediate page directory for v3. L4, /// Level 5 - PTE level used for MMU v3 only. L5, } impl PageTableLevel { /// Number of entries per page table (512 for 4KB pages). pub(super) const ENTRIES_PER_TABLE: usize = 512; /// Get the next level in the hierarchy. pub(super) const fn next(&self) -> Option { match self { Self::Pdb => Some(Self::L1), Self::L1 => Some(Self::L2), Self::L2 => Some(Self::L3), Self::L3 => Some(Self::L4), Self::L4 => Some(Self::L5), Self::L5 => None, } } /// Convert level to index. pub(super) const fn as_index(&self) -> u64 { match self { Self::Pdb => 0, Self::L1 => 1, Self::L2 => 2, Self::L3 => 3, Self::L4 => 4, Self::L5 => 5, } } } // Trait abstractions for page table operations. /// Operations on Page Table Entries (`PTE`s). pub(super) trait PteOps: Copy + core::fmt::Debug + Into { /// Create a `PTE` from a raw `u64` value. fn from_raw(val: u64) -> Self; /// Create an invalid `PTE`. fn invalid() -> Self; /// Create a valid `PTE` for the given memory aperture. fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self; /// Check if this `PTE` is valid. fn is_valid(&self) -> bool; /// Get the physical frame number. fn frame_number(&self) -> Pfn; /// Read a `PTE` from VRAM. fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { let val = pramin.window_at::(addr)?.view().read_val(); Ok(Self::from_raw(val)) } /// Write this `PTE` to VRAM. fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { pramin .window_at::(addr)? .view() .write_val((*self).into()); Ok(()) } } /// Operations on Page Directory Entries (`PDE`s). pub(super) trait PdeOps: Copy + core::fmt::Debug + Into { /// Create a `PDE` from a raw `u64` value. fn from_raw(val: u64) -> Self; /// Create a valid `PDE` pointing to a page table in the given aperture. fn new(aperture: AperturePde, table_pfn: Pfn) -> Self; /// Create an invalid `PDE`. fn invalid() -> Self; /// Check if this `PDE` is valid. fn is_valid(&self) -> bool; /// Get the memory aperture of this `PDE`. fn aperture(&self) -> AperturePde; /// Get the VRAM address of the page table. fn table_vram_address(&self) -> VramAddress; /// Read a `PDE` from VRAM. fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { let val = pramin.window_at::(addr)?.view().read_val(); Ok(Self::from_raw(val)) } /// Write this `PDE` to VRAM. fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { pramin .window_at::(addr)? .view() .write_val((*self).into()); Ok(()) } /// Check if this `PDE` is valid and points to video memory. fn is_valid_vram(&self) -> bool { self.is_valid() && self.aperture() == AperturePde::VideoMemory } } /// Operations on Dual Page Directory Entries (128-bit `DualPde`s). pub(super) trait DualPdeOps: Copy + core::fmt::Debug { /// Create a `DualPde` from raw 128-bit value (two `u64`s). fn from_raw(big: u64, small: u64) -> Self; /// Create a `DualPde` with only the small page table pointer set. fn new_small(table_pfn: Pfn) -> Self; /// Check if the small page table pointer is valid. fn has_small(&self) -> bool; /// Get the small page table VRAM address. fn small_vram_address(&self) -> VramAddress; /// Get the raw `u64` value of the big PDE. fn big_raw_u64(&self) -> u64; /// Get the raw `u64` value of the small PDE. fn small_raw_u64(&self) -> u64; /// Read a dual PDE (128-bit) from VRAM. fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { let lo = pramin.window_at::(addr)?.view().read_val(); let hi = pramin.window_at::(addr + 8)?.view().read_val(); Ok(Self::from_raw(lo, hi)) } /// Write this dual PDE (128-bit) to VRAM. fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { pramin .window_at::(addr)? .view() .write_val(self.big_raw_u64()); pramin .window_at::(addr + 8)? .view() .write_val(self.small_raw_u64()); Ok(()) } } /// MMU configuration trait -- encodes version-specific constants and types. pub(super) trait MmuConfig: 'static { /// Page Table Entry type. type Pte: PteOps; /// Page Directory Entry type. type Pde: PdeOps; /// Dual Page Directory Entry type (128-bit). type DualPde: DualPdeOps; /// PDE levels (excluding PTE level) for page table walking. const PDE_LEVELS: &'static [PageTableLevel]; /// PTE level for this MMU version. const PTE_LEVEL: PageTableLevel; /// Dual PDE level (128-bit entries) for this MMU version. const DUAL_PDE_LEVEL: PageTableLevel; /// Get the number of entries per page table page for a given level. fn entries_per_page(level: PageTableLevel) -> usize; /// Extract the page table index at `level` from `va`. fn level_index(va: VirtualAddress, level: u64) -> u64; /// Get the entry size in bytes for a given level. fn entry_size(level: PageTableLevel) -> usize { if level == Self::DUAL_PDE_LEVEL { 16 // 128-bit dual PDE } else { 8 // 64-bit PDE/PTE } } /// Compute upper bound on page table pages needed for `num_virt_pages`. /// /// Walks from PTE level up through PDE levels, accumulating the tree. fn pt_pages_upper_bound(num_virt_pages: usize) -> usize { let mut total = 0; // PTE pages at the leaf level. let pte_epp = Self::entries_per_page(Self::PTE_LEVEL); let mut pages_at_level = num_virt_pages.div_ceil(pte_epp); total += pages_at_level; // Walk PDE levels bottom-up (reverse of PDE_LEVELS). for &level in Self::PDE_LEVELS.iter().rev() { let epp = Self::entries_per_page(level); // How many pages at this level do we need to point to // the previous pages_at_level? pages_at_level = pages_at_level.div_ceil(epp); total += pages_at_level; } total } } /// Marker struct for MMU v2 (Turing/Ampere/Ada). pub(super) struct MmuV2; impl MmuConfig for MmuV2 { type Pte = ver2::Pte; type Pde = ver2::Pde; type DualPde = ver2::DualPde; const PDE_LEVELS: &'static [PageTableLevel] = ver2::PDE_LEVELS; const PTE_LEVEL: PageTableLevel = ver2::PTE_LEVEL; const DUAL_PDE_LEVEL: PageTableLevel = ver2::DUAL_PDE_LEVEL; fn entries_per_page(level: PageTableLevel) -> usize { // TODO: Calculate these values from the bitfield dynamically // instead of hardcoding them. match level { PageTableLevel::Pdb => 4, // PD3 root: bits [48:47] = 2 bits PageTableLevel::L3 => 256, // PD0 dual: bits [28:21] = 8 bits _ => 512, // PD2, PD1, PT: 9 bits each } } fn level_index(va: VirtualAddress, level: u64) -> u64 { ver2::VirtualAddressV2::new(va).level_index(level) } } /// Marker struct for MMU v3 (Hopper and later). pub(super) struct MmuV3; impl MmuConfig for MmuV3 { type Pte = ver3::Pte; type Pde = ver3::Pde; type DualPde = ver3::DualPde; const PDE_LEVELS: &'static [PageTableLevel] = ver3::PDE_LEVELS; const PTE_LEVEL: PageTableLevel = ver3::PTE_LEVEL; const DUAL_PDE_LEVEL: PageTableLevel = ver3::DUAL_PDE_LEVEL; fn entries_per_page(level: PageTableLevel) -> usize { match level { PageTableLevel::Pdb => 2, // PDE4 root: bit [56] = 1 bit, 2 entries PageTableLevel::L4 => 256, // PDE0 dual: bits [28:21] = 8 bits _ => 512, // PDE3, PDE2, PDE1, PT: 9 bits each } } fn level_index(va: VirtualAddress, level: u64) -> u64 { ver3::VirtualAddressV3::new(va).level_index(level) } } /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub(super) enum AperturePte { /// Local video memory (VRAM). #[default] VideoMemory = 0, /// Peer GPU's video memory. PeerMemory = 1, /// System memory with cache coherence. SystemCoherent = 2, /// System memory without cache coherence. SystemNonCoherent = 3, } // TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. impl From> for AperturePte { fn from(val: Bounded) -> Self { match *val { 0 => Self::VideoMemory, 1 => Self::PeerMemory, 2 => Self::SystemCoherent, 3 => Self::SystemNonCoherent, _ => Self::VideoMemory, } } } // TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. impl From for Bounded { fn from(val: AperturePte) -> Self { Bounded::from_expr(val as u64 & 0x3) } } /// Memory aperture for Page Directory Entries (`PDE`s). /// /// Note: For `PDE`s, `Invalid` (0) means the entry is not valid. #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub(super) enum AperturePde { /// Invalid/unused entry. #[default] Invalid = 0, /// Page table is in video memory. VideoMemory = 1, /// Page table is in system memory with coherence. SystemCoherent = 2, /// Page table is in system memory without coherence. SystemNonCoherent = 3, } // TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. impl From> for AperturePde { fn from(val: Bounded) -> Self { match *val { 1 => Self::VideoMemory, 2 => Self::SystemCoherent, 3 => Self::SystemNonCoherent, _ => Self::Invalid, } } } // TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. impl From for Bounded { fn from(val: AperturePde) -> Self { Bounded::from_expr(val as u64 & 0x3) } } /// Check if the PDB has valid, VRAM-backed page tables. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] fn check_pdb_inner(pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress) -> Result { let raw = pramin.window_at::(pdb_addr)?.view().read_val(); if !M::Pde::from_raw(raw).is_valid_vram() { return Err(ENOENT); } Ok(()) } /// Check if the PDB has valid, VRAM-backed page tables, dispatching by MMU version. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(super) fn check_pdb_valid( pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress, chipset: crate::gpu::Chipset, ) -> Result { match MmuVersion::from(chipset.arch()) { MmuVersion::V2 => check_pdb_inner::(pramin, pdb_addr), MmuVersion::V3 => check_pdb_inner::(pramin, pdb_addr), } }