path: root/drivers/gpu/nova-core/mm/pagetable/ver3.rs

// SPDX-License-Identifier: GPL-2.0

//! MMU v3 page table types for Hopper and later GPUs.
//!
//! This module defines MMU version 3 specific types (Hopper and later GPUs).
//!
//! Key differences from MMU v2:
//! - Unified 40-bit address field for all apertures (v2 had separate sys/vid fields).
//! - PCF (Page Classification Field) replaces separate privilege/RO/atomic/cache bits.
//! - KIND field is 4 bits (not 8).
//! - IS_PTE bit in PDE to support large pages directly.
//! - No COMPTAGLINE field (compression handled differently in v3).
//! - No separate ENCRYPTED bit.
//!
//! Bit field layouts derived from the NVIDIA OpenRM documentation:
//! `open-gpu-kernel-modules/src/common/inc/swref/published/hopper/gh100/dev_mmu.h`

#![allow(dead_code)]

use kernel::{
    bitfield,
    num::Bounded,
    prelude::*, //
};

use pin_init::Zeroable;

use super::{
    AperturePde,
    AperturePte,
    DualPdeOps,
    PageTableLevel,
    PdeOps,
    PteOps,
    VaLevelIndex, //
};

use crate::mm::{
    Pfn,
    VirtualAddress,
    VramAddress, //
};

// Bounded to version 3 Pfn conversion.
impl_pfn_bounded!(40);

bitfield! {
    /// MMU v3 57-bit virtual address layout.
    pub(super) struct VirtualAddressV3(u64) {
        /// Page offset [11:0].
        11:0    offset;
        /// PT index [20:12].
        20:12   pt_idx;
        /// PDE0 index [28:21].
        28:21   pde0_idx;
        /// PDE1 index [37:29].
        37:29   pde1_idx;
        /// PDE2 index [46:38].
        46:38   pde2_idx;
        /// PDE3 index [55:47].
        55:47   pde3_idx;
        /// PDE4 index [56].
        56:56   pde4_idx;
    }
}

impl VirtualAddressV3 {
    /// Create a [`VirtualAddressV3`] from a [`VirtualAddress`].
    pub(super) fn new(va: VirtualAddress) -> Self {
        Self::from_raw(va.into_raw())
    }
}

impl VaLevelIndex for VirtualAddressV3 {
    fn level_index(&self, level: u64) -> u64 {
        match level {
            0 => *self.pde4_idx(),
            1 => *self.pde3_idx(),
            2 => *self.pde2_idx(),
            3 => *self.pde1_idx(),
            4 => *self.pde0_idx(),
            5 => *self.pt_idx(),
            _ => 0,
        }
    }
}

/// PDE levels for MMU v3 (6-level hierarchy).
pub(super) const PDE_LEVELS: &[PageTableLevel] = &[
    PageTableLevel::Pdb,
    PageTableLevel::L1,
    PageTableLevel::L2,
    PageTableLevel::L3,
    PageTableLevel::L4,
];

/// PTE level for MMU v3.
pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L5;

/// Dual PDE level for MMU v3 (128-bit entries).
pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L4;

bitfield! {
    /// Page Classification Field for PTEs (5 bits) in MMU v3.
    pub(in crate::mm) struct PtePcf(u8) {
        /// Bypass L2 cache (0=cached, 1=bypass).
        0:0     uncached;
        /// Access counting disabled (0=enabled, 1=disabled).
        1:1     acd;
        /// Read-only access (0=read-write, 1=read-only).
        2:2     read_only;
        /// Atomics disabled (0=enabled, 1=disabled).
        3:3     no_atomic;
        /// Privileged access only (0=regular, 1=privileged).
        4:4     privileged;
    }
}

impl PtePcf {
    /// Create PCF for read-write mapping (cached, no atomics, regular mode).
    fn rw() -> Self {
        Self::zeroed().with_no_atomic(true)
    }

    /// Create PCF for read-only mapping (cached, no atomics, regular mode).
    fn ro() -> Self {
        Self::zeroed().with_read_only(true).with_no_atomic(true)
    }

    /// Get the raw `u8` value.
    fn raw_u8(&self) -> u8 {
        self.into_raw()
    }
}

impl From<Bounded<u64, 5>> for PtePcf {
    fn from(val: Bounded<u64, 5>) -> Self {
        Self::from_raw(u8::from(val))
    }
}

impl From<PtePcf> for Bounded<u64, 5> {
    fn from(pcf: PtePcf) -> Self {
        Bounded::from_expr(u64::from(pcf.into_raw()) & 0x1F)
    }
}

bitfield! {
    /// Page Classification Field for PDEs (3 bits) in MMU v3.
    ///
    /// Controls Address Translation Services (ATS) and caching.
    pub(in crate::mm) struct PdePcf(u8) {
        /// Bypass L2 cache (0=cached, 1=bypass).
        0:0     uncached;
        /// ATS disabled (0=enabled, 1=disabled).
        1:1     no_ats;
    }
}

impl PdePcf {
    /// Create PCF for cached mapping with ATS enabled (default).
    fn cached() -> Self {
        Self::zeroed()
    }

    /// Get the raw `u8` value.
    fn raw_u8(&self) -> u8 {
        self.into_raw()
    }
}

impl From<Bounded<u64, 3>> for PdePcf {
    fn from(val: Bounded<u64, 3>) -> Self {
        Self::from_raw(u8::from(val))
    }
}

impl From<PdePcf> for Bounded<u64, 3> {
    fn from(pcf: PdePcf) -> Self {
        Bounded::from_expr(u64::from(pcf.into_raw()) & 0x7)
    }
}

bitfield! {
    /// Page Table Entry for MMU v3.
    pub(in crate::mm) struct Pte(u64) {
        /// Entry is valid.
        0:0     valid;
        /// Memory aperture type.
        2:1     aperture => AperturePte;
        /// Page Classification Field.
        7:3     pcf => PtePcf;
        /// Surface kind (4 bits, 0x0=pitch, 0xF=invalid).
        11:8    kind;
        /// Physical frame number (for all apertures).
        51:12   frame_number => Pfn;
        /// Peer GPU ID for peer memory (0-7).
        63:61   peer_id;
    }
}

impl PteOps for Pte {
    fn from_raw(val: u64) -> Self {
        Self::from_raw(val)
    }

    fn invalid() -> Self {
        Self::zeroed()
    }

    fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self {
        let pcf = match (aperture, writable) {
            (AperturePte::VideoMemory, true) => PtePcf::rw(),
            (AperturePte::VideoMemory, false) => PtePcf::ro(),
            // Sysmem PTEs use uncached+no_atomic PCF for cache coherency.
            (AperturePte::SystemCoherent, true) => {
                PtePcf::zeroed().with_uncached(true).with_no_atomic(true)
            }
            (AperturePte::SystemCoherent, false) => PtePcf::zeroed()
                .with_uncached(true)
                .with_no_atomic(true)
                .with_read_only(true),
            (AperturePte::PeerMemory | AperturePte::SystemNonCoherent, _) => {
                kernel::pr_warn!("MMU v3 PTE aperture {:?} not supported\n", aperture);
                return Self::invalid();
            }
        };
        Self::zeroed()
            .with_valid(true)
            .with_aperture(aperture)
            .with_pcf(pcf)
            .with_frame_number(pfn)
    }

    fn is_valid(&self) -> bool {
        self.valid().into_bool()
    }

    fn frame_number(&self) -> Pfn {
        Pte::frame_number(*self)
    }
}

bitfield! {
    /// Page Directory Entry for MMU v3 (Hopper+).
    ///
    /// ## Note
    ///
    /// v3 uses a unified 40-bit address field (v2 had separate sys/vid address fields).
    pub(in crate::mm) struct Pde(u64) {
        /// Entry is a PTE (0=PDE, 1=large page PTE).
        0:0     is_pte;
        /// Memory aperture type.
        2:1     aperture => AperturePde;
        /// Page Classification Field (3 bits for PDE).
        5:3     pcf => PdePcf;
        /// Table frame number (40-bit unified address).
        51:12   table_frame => Pfn;
    }
}

impl PdeOps for Pde {
    fn from_raw(val: u64) -> Self {
        Self::from_raw(val)
    }

    fn new(aperture: AperturePde, table_pfn: Pfn) -> Self {
        match aperture {
            AperturePde::VideoMemory => Self::zeroed()
                .with_is_pte(false)
                .with_aperture(aperture)
                .with_table_frame(table_pfn),
            AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => {
                kernel::pr_warn!("MMU v3 PDE aperture {:?} not supported\n", aperture);
                Self::invalid()
            }
        }
    }

    fn invalid() -> Self {
        Self::zeroed().with_aperture(AperturePde::Invalid)
    }

    fn is_valid(&self) -> bool {
        Pde::aperture(*self) != AperturePde::Invalid
    }

    fn aperture(&self) -> AperturePde {
        Pde::aperture(*self)
    }

    fn table_vram_address(&self) -> VramAddress {
        debug_assert!(
            Pde::aperture(*self) == AperturePde::VideoMemory,
            "table_vram_address called on non-VRAM PDE (aperture: {:?})",
            Pde::aperture(*self)
        );
        VramAddress::from(self.table_frame())
    }
}

bitfield! {
    /// Big Page Table pointer in Dual PDE (MMU v3).
    ///
    /// 64-bit lower word of the 128-bit Dual PDE.
    pub(super) struct DualPdeBig(u64) {
        /// Entry is a PTE (for large pages).
        0:0     is_pte;
        /// Memory aperture type.
        2:1     aperture => AperturePde;
        /// Page Classification Field.
        5:3     pcf => PdePcf;
        /// Table frame (table address 256-byte aligned).
        51:8    table_frame;
    }
}

impl DualPdeBig {
    /// Create an invalid big page table pointer.
    fn invalid() -> Self {
        Self::zeroed().with_aperture(AperturePde::Invalid)
    }

    /// Create a valid big PDE pointing to a page table in the given aperture.
    fn new(aperture: AperturePde, table_addr: VramAddress) -> Result<Self> {
        // Big page table addresses must be 256-byte aligned (shift 8).
        if table_addr.into_raw() & 0xFF != 0 {
            return Err(EINVAL);
        }
        let table_frame = Bounded::from_expr(table_addr.into_raw() >> 8);
        match aperture {
            AperturePde::VideoMemory => Ok(Self::zeroed()
                .with_is_pte(false)
                .with_aperture(aperture)
                .with_table_frame(table_frame)),
            AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => {
                kernel::pr_warn!("MMU v3 DualPdeBig aperture {:?} not supported\n", aperture);
                Ok(Self::invalid())
            }
        }
    }

    /// Check if this big PDE is valid.
    fn is_valid(&self) -> bool {
        self.aperture() != AperturePde::Invalid
    }

    /// Get the VRAM address of the big page table.
    fn table_vram_address(&self) -> VramAddress {
        debug_assert!(
            self.aperture() == AperturePde::VideoMemory,
            "table_vram_address called on non-VRAM DualPdeBig (aperture: {:?})",
            self.aperture()
        );
        VramAddress::from_raw(*self.table_frame() << 8)
    }
}

/// Dual PDE at Level 4 for MMU v3 - 128-bit entry.
///
/// Contains both big (64KB) and small (4KB) page table pointers:
/// - Lower 64 bits: Big Page Table pointer.
/// - Upper 64 bits: Small Page Table pointer.
///
/// ## Note
///
/// The big and small page table pointers have different address layouts:
/// - Big address = field value << 8 (256-byte alignment).
/// - Small address = field value << 12 (4KB alignment).
///
/// This is why `DualPdeBig` is a separate type from `Pde`.
#[repr(C)]
#[derive(Debug, Clone, Copy)]
pub(in crate::mm) struct DualPde {
    /// Big Page Table pointer.
    pub(super) big: DualPdeBig,
    /// Small Page Table pointer.
    pub(super) small: Pde,
}

// SAFETY: Both `DualPdeBig` and `Pde` fields are `Zeroable` (bitfield types are Zeroable).
unsafe impl Zeroable for DualPde {}

impl DualPde {
    /// Check if the big page table pointer is valid.
    fn has_big(&self) -> bool {
        self.big.is_valid()
    }
}

impl DualPdeOps for DualPde {
    fn from_raw(big: u64, small: u64) -> Self {
        Self {
            big: DualPdeBig::from_raw(big),
            small: PdeOps::from_raw(small),
        }
    }

    fn new_small(table_pfn: Pfn) -> Self {
        Self {
            big: DualPdeBig::invalid(),
            small: PdeOps::new(AperturePde::VideoMemory, table_pfn),
        }
    }

    fn has_small(&self) -> bool {
        PdeOps::is_valid(&self.small)
    }

    fn small_vram_address(&self) -> VramAddress {
        PdeOps::table_vram_address(&self.small)
    }

    fn big_raw_u64(&self) -> u64 {
        self.big.into_raw()
    }

    fn small_raw_u64(&self) -> u64 {
        self.small.into_raw()
    }
}