path: root/drivers/gpu/nova-core/mm/pagetable/ver2.rs

// SPDX-License-Identifier: GPL-2.0

//! MMU v2 page table types for Turing, Ampere and Ada GPUs.
//!
//! This module defines MMU version 2 specific types (Turing, Ampere and Ada GPUs).
//!
//! Bit field layouts derived from the NVIDIA OpenRM documentation:
//! `open-gpu-kernel-modules/src/common/inc/swref/published/turing/tu102/dev_mmu.h`

#![allow(dead_code)]

use kernel::{
    bitfield,
    num::Bounded, //
};

use pin_init::Zeroable;

use super::{
    AperturePde,
    AperturePte,
    DualPdeOps,
    PageTableLevel,
    PdeOps,
    PteOps,
    VaLevelIndex, //
};

use crate::mm::{
    Pfn,
    VirtualAddress,
    VramAddress, //
};

// Bounded to version 2 Pfn bitfield conversions:
// 25 bits for video memory frame numbers (bits 32:8).
impl_pfn_bounded!(25);
// 46 bits for system memory frame numbers (bits 53:8).
impl_pfn_bounded!(46);

bitfield! {
    /// MMU v2 49-bit virtual address layout.
    pub(super) struct VirtualAddressV2(u64) {
        /// Page offset [11:0].
        11:0    offset;
        /// PT index [20:12].
        20:12   pt_idx;
        /// PDE0 index [28:21].
        28:21   pde0_idx;
        /// PDE1 index [37:29].
        37:29   pde1_idx;
        /// PDE2 index [46:38].
        46:38   pde2_idx;
        /// PDE3 index [48:47].
        48:47   pde3_idx;
    }
}

impl VirtualAddressV2 {
    /// Create a [`VirtualAddressV2`] from a [`VirtualAddress`].
    pub(super) fn new(va: VirtualAddress) -> Self {
        Self::from_raw(va.into_raw())
    }
}

impl VaLevelIndex for VirtualAddressV2 {
    fn level_index(&self, level: u64) -> u64 {
        match level {
            0 => *self.pde3_idx(),
            1 => *self.pde2_idx(),
            2 => *self.pde1_idx(),
            3 => *self.pde0_idx(),
            4 => *self.pt_idx(),
            _ => 0,
        }
    }
}

/// `PDE` levels for MMU v2 (5-level hierarchy: `PDB` -> `L1` -> `L2` -> `L3` -> `L4`).
pub(super) const PDE_LEVELS: &[PageTableLevel] = &[
    PageTableLevel::Pdb,
    PageTableLevel::L1,
    PageTableLevel::L2,
    PageTableLevel::L3,
];

/// `PTE` level for MMU v2.
pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L4;

/// Dual `PDE` level for MMU v2 (128-bit entries).
pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L3;

// Page Table Entry (PTE) for MMU v2 - 64-bit entry at level 4.
bitfield! {
    /// Page Table Entry for MMU v2.
    pub(in crate::mm) struct Pte(u64) {
        /// Entry is valid.
        0:0     valid;
        /// Memory aperture type.
        2:1     aperture => AperturePte;
        /// Volatile (bypass L2 cache).
        3:3     volatile;
        /// Encryption enabled (Confidential Computing).
        4:4     encrypted;
        /// Privileged access only.
        5:5     privilege;
        /// Write protection.
        6:6     read_only;
        /// Atomic operations disabled.
        7:7     atomic_disable;
        /// Frame number for system memory.
        53:8    frame_number_sys => Pfn;
        /// Frame number for video memory.
        32:8    frame_number_vid => Pfn;
        /// Peer GPU ID for peer memory (0-7).
        35:33   peer_id;
        /// Compression tag line bits.
        53:36   comptagline;
        /// Surface kind/format.
        63:56   kind;
    }
}

impl PteOps for Pte {
    fn from_raw(val: u64) -> Self {
        Self::from_raw(val)
    }

    fn invalid() -> Self {
        Self::zeroed()
    }

    fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self {
        let base = Self::zeroed()
            .with_valid(true)
            .with_aperture(aperture)
            .with_read_only(!writable);
        match aperture {
            AperturePte::VideoMemory => base.with_frame_number_vid(pfn),
            // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency.
            AperturePte::SystemCoherent => base.with_frame_number_sys(pfn).with_volatile(true),
            AperturePte::PeerMemory | AperturePte::SystemNonCoherent => {
                kernel::pr_warn!("MMU v2 PTE aperture {:?} not supported\n", aperture);
                Self::invalid()
            }
        }
    }

    fn is_valid(&self) -> bool {
        self.valid().into_bool()
    }

    fn frame_number(&self) -> Pfn {
        match self.aperture() {
            AperturePte::VideoMemory => self.frame_number_vid(),
            _ => self.frame_number_sys(),
        }
    }
}

// Page Directory Entry (PDE) for MMU v2 - 64-bit entry at levels 0-2.
bitfield! {
    /// Page Directory Entry for MMU v2.
    pub(in crate::mm) struct Pde(u64) {
        /// Valid bit (inverted logic).
        0:0     valid_inverted;
        /// Memory aperture type.
        2:1     aperture => AperturePde;
        /// Volatile (bypass L2 cache).
        3:3     volatile;
        /// Disable Address Translation Services.
        5:5     no_ats;
        /// Table frame number for system memory.
        53:8    table_frame_sys => Pfn;
        /// Table frame number for video memory.
        32:8    table_frame_vid => Pfn;
        /// Peer GPU ID (0-7).
        35:33   peer_id;
    }
}

impl PdeOps for Pde {
    fn from_raw(val: u64) -> Self {
        Self::from_raw(val)
    }

    fn new(aperture: AperturePde, table_pfn: Pfn) -> Self {
        let base = Self::zeroed()
            .with_valid_inverted(false) // 0 = valid
            .with_aperture(aperture);
        match aperture {
            AperturePde::VideoMemory => base.with_table_frame_vid(table_pfn),
            // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency.
            AperturePde::SystemCoherent => base.with_table_frame_sys(table_pfn).with_volatile(true),
            AperturePde::Invalid | AperturePde::SystemNonCoherent => {
                kernel::pr_warn!("MMU v2 PDE aperture {:?} not supported\n", aperture);
                Self::invalid()
            }
        }
    }

    fn invalid() -> Self {
        Self::zeroed()
            .with_valid_inverted(true)
            .with_aperture(AperturePde::Invalid)
    }

    fn is_valid(&self) -> bool {
        !self.valid_inverted().into_bool() && self.aperture() != AperturePde::Invalid
    }

    fn aperture(&self) -> AperturePde {
        Pde::aperture(*self)
    }

    fn table_vram_address(&self) -> VramAddress {
        debug_assert!(
            Pde::aperture(*self) == AperturePde::VideoMemory,
            "table_vram_address called on non-VRAM PDE (aperture: {:?})",
            Pde::aperture(*self)
        );
        VramAddress::from(self.table_frame_vid())
    }
}

/// Dual `PDE` at Level 3 - 128-bit entry of Large/Small Page Table pointers.
///
/// The dual `PDE` supports both large (64KB) and small (4KB) page tables.
#[repr(C)]
#[derive(Debug, Clone, Copy)]
pub(in crate::mm) struct DualPde {
    /// Large/Big Page Table pointer (lower 64 bits).
    pub(super) big: Pde,
    /// Small Page Table pointer (upper 64 bits).
    pub(super) small: Pde,
}

impl DualPde {
    /// Check if the big page table pointer is valid.
    fn has_big(&self) -> bool {
        PdeOps::is_valid(&self.big)
    }
}

impl DualPdeOps for DualPde {
    fn from_raw(big: u64, small: u64) -> Self {
        Self {
            big: PdeOps::from_raw(big),
            small: PdeOps::from_raw(small),
        }
    }

    fn new_small(table_pfn: Pfn) -> Self {
        Self {
            big: PdeOps::from_raw(0),
            small: PdeOps::new(AperturePde::VideoMemory, table_pfn),
        }
    }

    fn has_small(&self) -> bool {
        PdeOps::is_valid(&self.small)
    }

    fn small_vram_address(&self) -> VramAddress {
        PdeOps::table_vram_address(&self.small)
    }

    fn big_raw_u64(&self) -> u64 {
        self.big.into_raw()
    }

    fn small_raw_u64(&self) -> u64 {
        self.small.into_raw()
    }
}