From 6556f7556d622dbbc5c936df9d39c2f33600adfb Mon Sep 17 00:00:00 2001 From: Timur Tabi Date: Mon, 10 Aug 2026 13:12:43 -0500 Subject: gpu: nova-core: write initial RPCs before booting GSP-RM Make sure that the two setup RPCs, SetSystemInfo and SetRegistry, are in the command queue before booting GSP-RM. These message are expected to already be present, because GSP-RM will process the queue early in boot but will silently ignore the error if absent. GSP-RM then continues booting, but since the registry is not set up, it fails to initialize. Fixes: 0e42ec83d46a ("gpu: nova-core: gsp: shuffle boot code a bit to keep chipset-specific parts close") Signed-off-by: Timur Tabi Reviewed-by: Alexandre Courbot Reviewed-by: John Hubbard Link: https://patch.msgid.link/20260810181243.3036669-1-ttabi@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gsp/boot.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e03700ee7bea..874f6e9499f0 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -44,6 +44,11 @@ impl super::Gsp { let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; + self.cmdq + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; + self.cmdq + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); @@ -73,11 +78,6 @@ impl super::Gsp { dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),); - self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; - self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; - hal.post_boot(&self, ctx, &gsp_fw)?; // Wait until GSP is fully initialized. -- cgit v1.2.3 From d53bc149eea6213e9c4a4604df5521db7b7296ca Mon Sep 17 00:00:00 2001 From: Antonin Malzieu Ridolfi Date: Mon, 17 Aug 2026 18:51:08 +0200 Subject: gpu: nova-core: Extract PMC registers definitions Move PMC register definitions (NV_PMC_BOOT_0, NV_PMC_BOOT_42) and their associated implementations from the root regs.rs file into the gpu module that own them, in the new gpu/regs.rs file. This follows the same pattern established by previous commits for GSP, PDISP, PFB, and PBUS registers: register definitions move to the module that owns them, visibility changes to pub(super), and cross-module access is provided via pub(crate) helper functions. Since falcon.rs (outside the gpu module) also reads NV_PMC_BOOT_0 for its raw value, a pub(crate) helper function boot_0_raw() is added in gpu.rs to provide that access without exposing the register type directly. Suggested-by: Alexandre Courbot Suggested-by: Danilo Krummrich Signed-off-by: Antonin Malzieu Ridolfi Link: https://patch.msgid.link/20260817-b4-extract-pmc-registers-to-gpu-mod-v2-1-9dd1d599deb4@nanonej.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/falcon.rs | 2 +- drivers/gpu/nova-core/gpu.rs | 7 +++- drivers/gpu/nova-core/gpu/regs.rs | 82 +++++++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/regs.rs | 76 ------------------------------------ 4 files changed, 89 insertions(+), 78 deletions(-) create mode 100644 drivers/gpu/nova-core/gpu/regs.rs diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..11c1008b7d1e 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -394,7 +394,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.bar.write( WithBase::of::(), - regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), + regs::NV_PFALCON_FALCON_RM::from(crate::gpu::boot_0_raw(self.bar)), ); Ok(()) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 42a4cd7971fa..9e4232645a7e 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -29,11 +29,11 @@ use crate::{ Gsp, GspBootContext, // }, - regs, vgpu::VgpuManager, // }; mod hal; +mod regs; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => @@ -414,3 +414,8 @@ impl<'gpu> Gpu<'gpu> { }) } } + +/// Reads the boot0 register and returns its raw value. +pub(crate) fn boot_0_raw(bar: Bar0<'_>) -> u32 { + bar.read(regs::NV_PMC_BOOT_0).into_raw() +} diff --git a/drivers/gpu/nova-core/gpu/regs.rs b/drivers/gpu/nova-core/gpu/regs.rs new file mode 100644 index 000000000000..1c4db9625250 --- /dev/null +++ b/drivers/gpu/nova-core/gpu/regs.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::{ + io::register, + prelude::*, // +}; + +use super::{ + Architecture, + Chipset, // +}; + +// PMC + +register! { + /// Basic revision information about the GPU. + pub(super) NV_PMC_BOOT_0(u32) @ 0x00000000 { + /// Lower bits of the architecture. + 28:24 architecture_0; + /// Implementation version of the architecture. + 23:20 implementation; + /// MSB of the architecture. + 8:8 architecture_1; + /// Major revision of the chip. + 7:4 major_revision; + /// Minor revision of the chip. + 3:0 minor_revision; + } + + /// Extended architecture information. + pub(super) NV_PMC_BOOT_42(u32) @ 0x00000a00 { + /// Architecture value. + 29:24 architecture ?=> Architecture; + /// Implementation version of the architecture. + 23:20 implementation; + /// Major revision of the chip. + 19:16 major_revision; + /// Minor revision of the chip. + 15:12 minor_revision; + } +} + +impl NV_PMC_BOOT_0 { + pub(super) fn is_older_than_fermi(self) -> bool { + // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : + const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; + + // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than + // GF100, means "older than Fermi". + self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 + } +} + +impl NV_PMC_BOOT_42 { + /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. + pub(super) fn chipset(self) -> Result { + self.architecture() + .map(|arch| { + ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) + | u32::from(self.implementation()) + }) + .and_then(Chipset::try_from) + } + + /// Returns the raw architecture value from the register. + fn architecture_raw(self) -> u8 { + ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) + & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 + } +} + +impl kernel::fmt::Display for NV_PMC_BOOT_42 { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!( + f, + "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", + self.inner, + self.architecture_raw(), + self.implementation() + ) + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index caeef4d85874..3422b49df7a7 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -7,7 +7,6 @@ use kernel::{ register::WithBase, Io, // }, - prelude::*, sizes::SizeConstants, time, // }; @@ -28,83 +27,8 @@ use crate::{ PFalconBase, PeregrineCoreSelect, // }, - gpu::{ - Architecture, - Chipset, // - }, }; -// PMC - -register! { - /// Basic revision information about the GPU. - pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 { - /// Lower bits of the architecture. - 28:24 architecture_0; - /// Implementation version of the architecture. - 23:20 implementation; - /// MSB of the architecture. - 8:8 architecture_1; - /// Major revision of the chip. - 7:4 major_revision; - /// Minor revision of the chip. - 3:0 minor_revision; - } - - /// Extended architecture information. - pub(crate) NV_PMC_BOOT_42(u32) @ 0x00000a00 { - /// Architecture value. - 29:24 architecture ?=> Architecture; - /// Implementation version of the architecture. - 23:20 implementation; - /// Major revision of the chip. - 19:16 major_revision; - /// Minor revision of the chip. - 15:12 minor_revision; - } -} - -impl NV_PMC_BOOT_0 { - pub(crate) fn is_older_than_fermi(self) -> bool { - // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : - const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; - - // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than - // GF100, means "older than Fermi". - self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 - } -} - -impl NV_PMC_BOOT_42 { - /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. - pub(crate) fn chipset(self) -> Result { - self.architecture() - .map(|arch| { - ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) - | u32::from(self.implementation()) - }) - .and_then(Chipset::try_from) - } - - /// Returns the raw architecture value from the register. - fn architecture_raw(self) -> u8 { - ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) - & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 - } -} - -impl kernel::fmt::Display for NV_PMC_BOOT_42 { - fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - write!( - f, - "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", - self.inner, - self.architecture_raw(), - self.implementation() - ) - } -} - // PBUS register! { -- cgit v1.2.3 From b705c185105762676aa6ec16cf976101df87cc35 Mon Sep 17 00:00:00 2001 From: Alexandre Courbot Date: Thu, 23 Jul 2026 22:54:37 +0900 Subject: gpu: nova-core: fix incorrect naming/framing of GSP-FMC firmware The GSP-FMC firmware code lived in `firmware/fsp.rs` and described itself as running on the FSP. This is incorrect: GSP-FMC is loaded by the FSP, but runs on the GSP as the loader and verifier of GSP-RM. FSP secure-boots on its own before the driver even starts. Correct the naming and location of the GSP-FMC code by: - Renaming `firmware/fsp.rs` to `firmware/gsp_fmc.rs` (as GSP-FMC is not the FSP firmware), - Changing the prefix of some declarations in that file to `FMC` or `GspFmc` (interestingly some were already correct), - Rewording the `fsp.rs` module documentation to describe the actual boot flow. Reviewed-by: Danilo Krummrich Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260825-gsp-fmc-rename-v1-1-f82526f01d7d@nvidia.com [acourbot: drop the documentation change to be picked up by Zhi, reword commit message a bit.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/firmware.rs | 2 +- drivers/gpu/nova-core/firmware/fsp.rs | 122 ---------------------------- drivers/gpu/nova-core/firmware/gsp_fmc.rs | 127 ++++++++++++++++++++++++++++++ drivers/gpu/nova-core/fsp.rs | 29 ++++--- 4 files changed, 144 insertions(+), 136 deletions(-) delete mode 100644 drivers/gpu/nova-core/firmware/fsp.rs create mode 100644 drivers/gpu/nova-core/firmware/gsp_fmc.rs diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index b49613a90bf0..c16fee6e2b2a 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -23,9 +23,9 @@ use crate::{ }; pub(crate) mod booter; -pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; +pub(crate) mod gsp_fmc; pub(crate) mod riscv; pub(crate) mod tlv; diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs deleted file mode 100644 index 5462e318410a..000000000000 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ /dev/null @@ -1,122 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -//! FSP is a hardware unit that runs FMC firmware. - -use kernel::{ - device, - dma::Coherent, - prelude::*, // -}; - -use crate::{ - firmware::tlv::{ - request_tlv, // - Tlv, - }, - gpu::Chipset, // -}; - -/// Size of the FSP SHA-384 hash, in bytes. -const FSP_HASH_SIZE: usize = 48; -/// Maximum size of the FSP public key (RSA-3072), in bytes. -/// -/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_PKEY_SIZE: usize = 384; -/// Maximum size of the FSP signature (RSA-3072), in bytes. -/// -/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_SIG_SIZE: usize = 384; - -/// Structure to hold FMC signatures. -/// -/// C representation is used because this type is used for communication with the FSP. -#[derive(Debug, Clone, Copy, Zeroable)] -#[repr(C)] -pub(crate) struct FmcSignatures { - pub(crate) hash384: [u8; FSP_HASH_SIZE], - pub(crate) public_key: [u8; FSP_PKEY_SIZE], - pub(crate) signature: [u8; FSP_SIG_SIZE], -} - -pub(crate) struct FspFirmware { - /// FMC firmware image data - pub(crate) fmc_image: Coherent<[u8]>, - /// FMC firmware signatures. - pub(crate) fmc_sigs: KBox, -} - -impl FspFirmware { - pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { - let fw = request_tlv(dev, chipset, "fmc")?; - let tlv = Tlv::new(fw.data())?; - dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); - - let fmc_image_data = tlv.get_bytes(b"BLOB")?; - let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; - - Ok(Self { - fmc_image, - fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?, - }) - } - - /// Extract FMC firmware signatures for Chain of Trust verification. - /// - /// Extracts real cryptographic signatures from FMC TLV firmware tags. - /// Returns signatures in a heap-allocated structure to prevent stack overflow. - fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result> { - let hash_section = tlv.get_bytes(b"HASH")?; - let pkey_section = tlv.get_bytes(b"PKEY")?; - let sig_section = tlv.get_bytes(b"SIGN")?; - - // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. - if hash_section.len() != FSP_HASH_SIZE { - dev_err!( - dev, - "FMC hash section size {} != expected {}\n", - hash_section.len(), - FSP_HASH_SIZE - ); - return Err(EINVAL); - } - - // The key and signature sections are zero-padded to a fixed maximum, so they may be - // shorter, but must not exceed the destination buffers. - if pkey_section.len() > FSP_PKEY_SIZE { - dev_err!( - dev, - "FMC public key section size {} > maximum {}\n", - pkey_section.len(), - FSP_PKEY_SIZE - ); - return Err(EINVAL); - } - if sig_section.len() > FSP_SIG_SIZE { - dev_err!( - dev, - "FMC signature section size {} > maximum {}\n", - sig_section.len(), - FSP_SIG_SIZE - ); - return Err(EINVAL); - } - - // Initialize the signatures in place to avoid building the large `FmcSignatures` on the - // stack, then fill each section from the firmware. - let signatures = KBox::init( - pin_init::init_zeroed::().chain(|sigs| { - // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). - sigs.hash384.copy_from_slice(hash_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above). - sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above). - sigs.signature[..sig_section.len()].copy_from_slice(sig_section); - Ok(()) - }), - GFP_KERNEL, - )?; - - Ok(signatures) - } -} diff --git a/drivers/gpu/nova-core/firmware/gsp_fmc.rs b/drivers/gpu/nova-core/firmware/gsp_fmc.rs new file mode 100644 index 000000000000..fbb0af133f24 --- /dev/null +++ b/drivers/gpu/nova-core/firmware/gsp_fmc.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! GSP-FMC (First Mutable Code) is loaded by FSP into GSP to serve as the loader and verifier of +//! GSP-RM. + +use kernel::{ + device, + dma::Coherent, + prelude::*, // +}; + +use crate::{ + firmware::tlv::{ + request_tlv, // + Tlv, + }, + gpu::Chipset, // +}; + +/// Size of the GSP-FMC SHA-384 hash, in bytes. +const FMC_HASH_SIZE: usize = 48; +/// Maximum size of the GSP-FMC public key (RSA-3072), in bytes. +/// +/// The `PKEY` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_PKEY_SIZE: usize = 384; +/// Maximum size of the GSP-FMC signature (RSA-3072), in bytes. +/// +/// The `SIGN` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_SIG_SIZE: usize = 384; + +/// Structure to hold FMC signatures. +/// +/// C representation is used because this type is used for communication with the FSP. +#[derive(Debug, Clone, Copy, Zeroable)] +#[repr(C)] +pub(crate) struct FmcSignatures { + pub(crate) hash384: [u8; FMC_HASH_SIZE], + pub(crate) public_key: [u8; FMC_PKEY_SIZE], + pub(crate) signature: [u8; FMC_SIG_SIZE], +} + +pub(crate) struct GspFmcFirmware { + /// FMC firmware image data + pub(crate) fmc_image: Coherent<[u8]>, + /// FMC firmware signatures. + pub(crate) fmc_sigs: KBox, +} + +impl GspFmcFirmware { + pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { + let fw = request_tlv(dev, chipset, "fmc")?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded GSP-FMC firmware v{}\n", + tlv.get_string(b"VERS")? + ); + + let fmc_image_data = tlv.get_bytes(b"BLOB")?; + let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; + + Ok(Self { + fmc_image, + fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?, + }) + } + + /// Extract FMC firmware signatures for Chain of Trust verification. + /// + /// Extracts real cryptographic signatures from FMC TLV firmware tags. + /// Returns signatures in a heap-allocated structure to prevent stack overflow. + fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result> { + let hash_section = tlv.get_bytes(b"HASH")?; + let pkey_section = tlv.get_bytes(b"PKEY")?; + let sig_section = tlv.get_bytes(b"SIGN")?; + + // The hash section is a SHA-384 output: it must be exactly `FMC_HASH_SIZE` bytes. + if hash_section.len() != FMC_HASH_SIZE { + dev_err!( + dev, + "FMC hash section size {} != expected {}\n", + hash_section.len(), + FMC_HASH_SIZE + ); + return Err(EINVAL); + } + + // The key and signature sections are zero-padded to a fixed maximum, so they may be + // shorter, but must not exceed the destination buffers. + if pkey_section.len() > FMC_PKEY_SIZE { + dev_err!( + dev, + "FMC public key section size {} > maximum {}\n", + pkey_section.len(), + FMC_PKEY_SIZE + ); + return Err(EINVAL); + } + if sig_section.len() > FMC_SIG_SIZE { + dev_err!( + dev, + "FMC signature section size {} > maximum {}\n", + sig_section.len(), + FMC_SIG_SIZE + ); + return Err(EINVAL); + } + + // Initialize the signatures in place to avoid building the large `FmcSignatures` on the + // stack, then fill each section from the firmware. + let signatures = KBox::init( + pin_init::init_zeroed::().chain(|sigs| { + // PANIC: src and dst lengths are both `FMC_HASH_SIZE` (verified above). + sigs.hash384.copy_from_slice(hash_section); + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_PKEY_SIZE` (verified above). + sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_SIG_SIZE` (verified above). + sigs.signature[..sig_section.len()].copy_from_slice(sig_section); + Ok(()) + }), + GFP_KERNEL, + )?; + + Ok(signatures) + } +} diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ab685fb4168f..d2be19b8bff9 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -3,9 +3,12 @@ //! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs. //! -//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP. +//! Hopper/Blackwell use a simplified firmware boot sequence: FSP secure-boots independently before +//! the driver starts. The driver then sends FSP a Chain-of-Trust request containing the GSP-FMC +//! image. FSP authenticates the image and launches GSP-FMC on the GSP RISC-V core; GSP-FMC +//! subsequently authenticates and boots GSP-RM. +//! //! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage. -//! FSP handles secure boot directly using FMC firmware and Chain of Trust. use kernel::{ device, @@ -32,9 +35,9 @@ use crate::{ Falcon, // }, fb::FbSizes, - firmware::fsp::{ + firmware::gsp_fmc::{ FmcSignatures, - FspFirmware, // + GspFmcFirmware, // }, gpu::Chipset, gsp::{ @@ -267,7 +270,7 @@ impl FspCotMessage { /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_info: &FbSizes, - fsp_fw: &'a FspFirmware, + fmc_fw: &'a GspFmcFirmware, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; @@ -296,13 +299,13 @@ impl FspCotMessage { .chain(move |msg| { msg.cot.version = version; msg.cot.size = size; - msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address(); + msg.cot.gsp_fmc_sysmem_offset = fmc_fw.fmc_image.dma_address(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address(); - msg.cot.sigs = *fsp_fw.fmc_sigs; + msg.cot.sigs = *fmc_fw.fmc_sigs; Ok(()) })) @@ -381,12 +384,12 @@ impl<'a> FmcBootArgs<'a> { /// FSP interface for Hopper/Blackwell GPUs. /// -/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot -/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent +/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot has +/// completed. It owns the FSP falcon and the GSP-FMC firmware, which are used for the subsequent /// Chain of Trust boot. pub(crate) struct Fsp<'a> { falcon: Falcon<'a, FspEngine>, - fsp_fw: FspFirmware, + fmc_fw: GspFmcFirmware, } impl<'a> Fsp<'a> { @@ -422,7 +425,7 @@ impl<'a> Fsp<'a> { const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let falcon = Falcon::::new(dev, chipset, bar)?; - let fsp_fw = FspFirmware::new(dev, chipset)?; + let fmc_fw = GspFmcFirmware::new(dev, chipset)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), @@ -434,7 +437,7 @@ impl<'a> Fsp<'a> { dev_err!(dev, "FSP secure boot completion error: {:?}\n", e); })?; - Ok(Fsp { falcon, fsp_fw }) + Ok(Fsp { falcon, fmc_fw }) } /// Sends a message to FSP and waits for the response. @@ -540,7 +543,7 @@ impl<'a> Fsp<'a> { ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init(FspCotMessage::new(fb_info, &self.fmc_fw, args)?, GFP_KERNEL)?; let _response_buf = self.send_sync_fsp(dev, &*msg)?; -- cgit v1.2.3 From 9e270c4fd863918883977615d7fc54c9b9d018c8 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 21:59:32 +0100 Subject: gpu: nova-core: fix barrier usage in CPU->GSP messaging path In the CPU->GSP messaging path, the code reads the read pointer from GSP, writes the command, advances the write pointer, and then notifies the GSP. A LOAD->STORE ordering is needed after reading the read pointer from GSP and writing the command. Control dependency exists here which provide the needed ordering, but it's best to avoid depending on it. A STORE->STORE ordering is needed after the command write and before the write pointer advance. This is currently incorrectly done after the write pointer advance (and before GSP notification), but this can cause issue if GSP is still processing ring buffer, as it may observe the write pointer advance before command write. Thus move this barrier to be before the write pointer advance. Note that barriers are not needed between write pointer advance and GSP notification, as MMIO accessors already carries the required barrier. Signed-off-by: Gary Guo Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260901-rust-barrier-v4-1-94427f445310@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 6da728201281..bfd61e678802 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -27,6 +27,11 @@ use kernel::{ ptr, sync::{ aref::ARef, + barrier::{ + dma_mb, + Full, + Write, // + }, Mutex, // }, time::Delta, @@ -413,7 +418,12 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES + let ptr = MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->STORE ordering needed to order `gsp_read_ptr` read before data write. + dma_mb(Full); + + ptr } // Returns the index of the memory page the CPU can read the next message from. @@ -447,12 +457,12 @@ impl DmaGspMem { // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { + // ORDERING: STORE->STORE ordering needed to order `cpu_write_ptr` write after data write. + dma_mb(Write); + let tx = io_project!(self.0, .cpuq.tx); let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; MsgqTxHeader::set_write_ptr(tx, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); } } -- cgit v1.2.3 From 6cb331644c441ff4101a4f8726283a6ed0d5947e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 21:59:33 +0100 Subject: gpu: nova-core: fix barrier usage in GSP->CPU messaging path In the GSP->CPU messaging path, the code reads the write pointer from GSP, reads the response and advances the read pointer. A LOAD->LOAD ordering is required after the write pointer read and the data read. Add it as this is currently missing. A LOAD->STORE ordering is required after the data read and the advance of read pointer. Currently a Rust `SeqCst` barrier is used, which roughly maps to `smp_mb(Full)`; this however does not order DMA operations (notably on ARM, the generate barrier orders inner shareable and not outer shareable, which is ordered by `dma_mb`). This ordering does not need to be in between read pointer read and write, because it's for ordering between the ring buffer data and the pointer; the RMW operation does not internally need a barrier (nor it has to be atomic, as CPU pointers are updated by CPU only), so move it to before the RMW sequence for clarity. Signed-off-by: Gary Guo Reviewed-by: Eliot Courtney Link: https://patch.msgid.link/20260901-rust-barrier-v4-2-94427f445310@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index bfd61e678802..14a711307654 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,13 +2,7 @@ mod continuation; -use core::{ - mem, - sync::atomic::{ - fence, - Ordering, // - }, -}; +use core::mem; use kernel::{ device, @@ -30,6 +24,7 @@ use kernel::{ barrier::{ dma_mb, Full, + Read, Write, // }, Mutex, // @@ -409,7 +404,12 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_write_ptr(&self) -> u32 { - MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES + let ptr = MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->LOAD ordering needed to order `gsp_write_ptr` read before data read. + dma_mb(Read); + + ptr } // Returns the index of the memory page the GSP will read the next command from. @@ -437,12 +437,11 @@ impl DmaGspMem { // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { + // ORDERING: LOAD->STORE ordering needed to order `cpu_read_ptr` write after data read. + dma_mb(Full); + let rx = io_project!(self.0, .cpuq.rx); let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - MsgqRxHeader::set_read_ptr(rx, rptr) } -- cgit v1.2.3 From 63b2578517df761ec8aed607b29e24c0e2655410 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:25 +0100 Subject: rust: io: register: reimplement as proc macro The existing `register!` macro is implemented as a declarative macro. Reimplement it as proc macro instead, with no functional changes intended. The old implementation produces unhelpful diagnostics when things go wrong. For example, for code like register! { pub(crate) TESTREG(u32) { 31:0 data; } } which misses out the "@ offset" part of the specification, and the following error is produced: error: no rules expected `{` --> test.rs:42:5 | 42 | / register! { 43 | | pub(crate) TESTREG(u32) { 44 | | 31:0 data; ... | 100 | | } | |_____^ no rules expected this token in macro call which isn't very helpful. With the proc macro implementation, the following error is produced: error: expected `@` or `=>` --> tests.rs:43:33 | 43 | pub(crate) TESTREG(u32) { | ^ which is much more helpful. Apart from diagnostics, proc macro also has a benefit of not having follow-set restrictions, which makes syntax like register!(name: ty @ offset); possible; declarative macro will reject this as `@` is not in the follow-set of "ty" metavariable kind. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-1-5552b1d59525@garyguo.net [ Fix typo in module-level doc comment and fix ArrayDef syntax description to match the actual macro syntax. - Danilo ] Signed-off-by: Danilo Krummrich --- MAINTAINERS | 1 + rust/kernel/io/register.rs | 224 +----------------------------------- rust/macros/io/mod.rs | 3 + rust/macros/io/register.rs | 276 +++++++++++++++++++++++++++++++++++++++++++++ rust/macros/lib.rs | 9 ++ 5 files changed, 291 insertions(+), 222 deletions(-) create mode 100644 rust/macros/io/mod.rs create mode 100644 rust/macros/io/register.rs diff --git a/MAINTAINERS b/MAINTAINERS index 3a19da74d00c..d92c7d834693 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7494,6 +7494,7 @@ F: rust/kernel/io.rs F: rust/kernel/io/ F: rust/kernel/irq.rs F: rust/kernel/irq/ +F: rust/macros/io/ DEVICE RESOURCE MANAGEMENT HELPERS M: Hans de Goede diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 03dfd2ff48c7..6a19552ffb95 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -801,227 +801,7 @@ where /// ``` #[macro_export] macro_rules! register { - // Entry point for the macro, allowing multiple registers to be defined in one call. - // It matches all possible register declaration patterns to dispatch them to corresponding - // `@reg` rule that defines a single register. - // - // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O - // projections. - ( - $( - $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - $([ $size:expr $(, stride = $stride:expr)? ])? - $(@ $($base:ident +)? $offset:literal)? - $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )? - { $($fields:tt)* } - )* - ) => { - $( - $crate::register!( - @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])? - $(@ $($base +)? $offset)? - $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )? - { $($fields)* } - ); - )* - }; - - // All the rules below are private helpers. - - // Creates a register at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an alias register of fixed offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates a register at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an alias register of relative offset register `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident - { $($fields:tt)* } - ) => { - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Creates an array of registers at a fixed offset of the MMIO space. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_array $name [ $size, stride = $stride ]); - }; - - // Shortcut for contiguous array of registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal - { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of array of registers `alias` with its own fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ] - { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET - + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_fixed $(#[$attr])* $vis $name); - }; - - // Creates an array of registers at a relative offset from a base address provider. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - [ $size:expr, stride = $stride:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!(@io_base $name($storage) @ $offset); - $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base); - }; - - // Shortcut for contiguous array of relative registers (stride == size of element). - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] - @ $base:ident + $offset:literal { $($fields:tt)* } - ) => { - $crate::register!( - @reg $(#[$attr])* $vis $name($storage) - [ $size, stride = ::core::mem::size_of::<$storage>() ] - @ $base + $offset { $($fields)* } - ); - }; - - // Creates an alias of register `idx` of relative array of registers `alias` with its own - // fields. - ( - @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) - => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* } - ) => { - $crate::build_assert::static_assert!( - $idx < <$alias as $crate::io::register::RegisterArray>::SIZE - ); - - $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* }); - $crate::register!( - @io_base $name($storage) @ - <$alias as $crate::io::register::Register>::OFFSET + - $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE - ); - $crate::register!(@io_relative $name @ $base); - }; - - // Generates the bitfield for the register. - // - // `#[allow(non_camel_case_types)]` is added since register names typically use - // `SCREAMING_CASE`. - ( - @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* } - ) => { - $crate::bitfield!( - #[allow(non_camel_case_types)] - $(#[$attr])* $vis struct $name($storage) { $($fields)* } - ); - }; - - // Implementations shared by all registers types. - (@io_base $name:ident($storage:ty) @ $offset:expr) => { - impl $crate::io::register::Register for $name { - type Storage = $storage; - - const OFFSET: usize = $offset; - } - }; - - // Implementations of fixed registers. - (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => { - impl $crate::io::register::FixedRegister for $name {} - - $(#[$attr])* - $vis const $name: $crate::io::register::FixedRegisterLoc<$name> = - $crate::io::register::FixedRegisterLoc::<$name>::new(); - }; - - // Implementations of relative registers. - (@io_relative $name:ident @ $base:ident) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RelativeRegister for $name {} - }; - - // Implementations of register arrays. - (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => { - impl $crate::io::register::Array for $name {} - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - }; - - // Implementations of relative array registers. - ( - @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident - ) => { - impl $crate::io::register::WithBase for $name { - type BaseFamily = $base; - } - - impl $crate::io::register::RegisterArray for $name { - const SIZE: usize = $size; - const STRIDE: usize = $stride; - } - - impl $crate::io::register::RelativeRegisterArray for $name {} + ($($tt:tt)*) => { + $crate::macros::register!($($tt)*); }; } diff --git a/rust/macros/io/mod.rs b/rust/macros/io/mod.rs new file mode 100644 index 000000000000..87f7742f4619 --- /dev/null +++ b/rust/macros/io/mod.rs @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 + +pub(crate) mod register; diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs new file mode 100644 index 000000000000..cc5bcad31ae8 --- /dev/null +++ b/rust/macros/io/register.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Documentation and usage example of the macro can be found at `rust/kernel/io/register.rs`. + +use proc_macro2::{ + Group, + Literal, + Span, + TokenStream, // +}; +use quote::{ + quote, + quote_spanned, // +}; +use syn::{ + bracketed, + parenthesized, + parse::Parse, + spanned::Spanned, + token, + Attribute, + Expr, + Ident, + Path, + Result, + Token, + Type, + Visibility, // +}; + +mod kw { + syn::custom_keyword!(stride); +} + +/// Definition of a register array. +/// +/// Specify a size, and optionally a stride. Syntax is of form `[EXPR $(, stride = EXPR)?]`. +struct RegArrayDef { + size: Expr, + stride: Option, +} + +/// Offset of a register. +/// +/// Can be either of form +/// * `@ offset` for fixed offset +/// * `=> alias` for alias of register `alias`. +/// * `=> alias[idx]` for alias of register array `alias`'s `idx`-th element. +enum RegOffset { + /// Register is located at fixed address. + Fixed { offset: Literal }, + /// Register is an alias of a fixed register. + Alias { alias: Path }, + /// Register is an alias of an element of a register array. + ElementAlias { alias: Path, idx: Expr }, +} + +/// Definition of a single register. +struct Reg { + attrs: Vec, + vis: Visibility, + name: Ident, + storage: Type, + array: Option, + relative_base: Option, + offset: RegOffset, + bitfield_args: Group, +} + +impl Parse for Reg { + fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis = input.parse()?; + let name = input.parse()?; + let storage = { + let content; + parenthesized!(content in input); + content.parse()? + }; + let array = if input.peek(token::Bracket) { + let content; + bracketed!(content in input); + let size = content.parse()?; + let stride = if content.peek(Token![,]) { + let _: Token![,] = content.parse()?; + let _: kw::stride = content.parse()?; + let _: Token![=] = content.parse()?; + Some(content.parse()?) + } else { + None + }; + Some(RegArrayDef { size, stride }) + } else { + None + }; + + // Parse offset and the base it's relative to. + let lh = input.lookahead1(); + let mut relative_base = None; + let offset = if lh.peek(Token![@]) { + let _: Token![@] = input.parse()?; + + if input.peek(Ident) { + relative_base = Some(input.parse()?); + let _: Token![+] = input.parse()?; + } + + RegOffset::Fixed { + offset: input.parse()?, + } + } else if lh.peek(Token![=>]) { + let _: Token![=>] = input.parse()?; + let mut alias: Path = input.parse()?; + if input.peek(Token![+]) { + let _: Token![+] = input.parse()?; + relative_base = Some(alias); + alias = input.parse()?; + } + + if input.peek(token::Bracket) { + let content; + bracketed!(content in input); + RegOffset::ElementAlias { + alias, + idx: content.parse()?, + } + } else { + RegOffset::Alias { alias } + } + } else { + Err(lh.error())? + }; + + let lh = input.lookahead1(); + let bitfield_args = if lh.peek(token::Brace) { + input.parse()? + } else { + Err(lh.error())? + }; + + Ok(Self { + attrs, + vis, + name, + storage, + array, + relative_base, + offset, + bitfield_args, + }) + } +} + +pub(crate) struct RegDef { + regs: Vec, +} + +impl Parse for RegDef { + fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let mut regs = Vec::new(); + while !input.is_empty() { + regs.push(input.parse()?); + } + Ok(RegDef { regs }) + } +} + +pub(crate) fn register(def: RegDef) -> Result { + let mut outputs = TokenStream::new(); + + for reg in def.regs { + let Reg { + attrs, + vis, + name, + storage, + array, + relative_base, + offset, + bitfield_args, + } = reg; + + // Use register name's span for generated code, so error messages (if any) can point to it + // instead of the entire register allocation. + let span = name.span().resolved_at(Span::mixed_site()); + + let offset = match offset { + RegOffset::Fixed { offset } => quote!(#offset), + RegOffset::Alias { alias } => { + quote_spanned!(alias.span().resolved_at(span) => + <#alias as ::kernel::io::register::Register>::OFFSET + ) + } + RegOffset::ElementAlias { alias, idx } => { + outputs.extend(quote_spanned!(idx.span().resolved_at(span) => + ::kernel::build_assert::static_assert!( + (#idx) < <#alias as ::kernel::io::register::RegisterArray>::SIZE + ); + )); + quote_spanned!(alias.span().resolved_at(span) => + <#alias as ::kernel::io::register::Register>::OFFSET + + (#idx) * <#alias as ::kernel::io::register::RegisterArray>::STRIDE + ) + } + }; + + outputs.extend(quote_spanned!(span => + ::kernel::bitfield!( + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + #[allow(non_camel_case_types)] + #(#attrs)* #vis struct #name(#storage) #bitfield_args + ); + + impl ::kernel::io::register::Register for #name { + type Storage = #storage; + + const OFFSET: usize = #offset; + } + )); + + match array { + None => match relative_base { + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::FixedRegister for #name {} + + #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = + ::kernel::io::register::FixedRegisterLoc::<#name>::new(); + )), + Some(relative_base) => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::WithBase for #name { + type BaseFamily = #relative_base; + } + + impl ::kernel::io::register::RelativeRegister for #name {} + )), + }, + + Some(def) => { + let size = &def.size; + let stride = if let Some(stride) = &def.stride { + outputs.extend(quote_spanned!(stride.span().resolved_at(span) => + ::kernel::build_assert::static_assert!( + ::core::mem::size_of::<#storage>() <= #stride + ); + )); + quote!(#stride) + } else { + quote_spanned!(span => ::core::mem::size_of::<#storage>()) + }; + + outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::RegisterArray for #name { + const SIZE: usize = #size; + const STRIDE: usize = #stride; + } + )); + + match relative_base { + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::Array for #name {} + )), + Some(relative_base) => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::WithBase for #name { + type BaseFamily = #relative_base; + } + + impl ::kernel::io::register::RelativeRegisterArray for #name {} + )), + } + } + }; + } + + Ok(outputs) +} diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs index 24f96feaeb34..9b76efe1476f 100644 --- a/rust/macros/lib.rs +++ b/rust/macros/lib.rs @@ -19,6 +19,7 @@ mod export; mod fmt; mod for_lt; mod helpers; +mod io; mod kunit; mod module; mod paste; @@ -481,6 +482,14 @@ pub fn paste(input: TokenStream) -> TokenStream { .into() } +#[doc(hidden)] // Documented in `kernel` crate. +#[proc_macro] +pub fn register(input: TokenStream) -> TokenStream { + io::register::register(parse_macro_input!(input)) + .unwrap_or_else(|e| e.into_compile_error()) + .into() +} + /// Registers a KUnit test suite and its test cases using a user-space like syntax. /// /// This macro should be used on modules. If `CONFIG_KUNIT` (in `.config`) is `n`, the target module -- cgit v1.2.3 From 9dc6cedcd971d084a64a9bd11d9c822e1125b798 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 14:42:54 +0200 Subject: rust: debugfs: drop 'static bound from ScopedDir file creation methods Drop the T: 'static bound from ScopedDir's file creation methods (read_binary_file(), read_only_file(), etc.) to support registering debugfs files backed by types that contain non-'static references, such as dma::Coherent<'a, T>. The previous 'static bound existed because ScopedDir::create_file() took &'static FileOps, and &'static requires T: 'static for well- formedness. However, this was overly conservative: the file_operations pointer passed to the C debugfs API just needs to be 'static, not the entire FileOps. Store &'static bindings::file_operations in FileOps instead of the file_operations by value. In each trait impl, take a reference to the file_operations struct within the const block; since bindings::file_operations does not mention T, the reference is promoted to 'static regardless of T's lifetime parameters. Replace the Deref impl with an explicit fops() method that returns the stored &'static reference. Link: https://lore.kernel.org/driver-core/20260830193824.471089-2-dakr@kernel.org Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-2-dakr@kernel.org [ Remove unrelated change from Self to T. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/debugfs.rs | 26 +++++++------------------- rust/kernel/debugfs/entry.rs | 4 ++-- rust/kernel/debugfs/file_ops.rs | 29 +++++++++++++---------------- 3 files changed, 22 insertions(+), 37 deletions(-) diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs index d7b8014a6474..2beb55d444ca 100644 --- a/rust/kernel/debugfs.rs +++ b/rust/kernel/debugfs.rs @@ -538,7 +538,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { } } - fn create_file(&self, name: &CStr, data: &'data T, vtable: &'static FileOps) { + fn create_file(&self, name: &CStr, data: &'data T, vtable: &FileOps) { #[cfg(CONFIG_DEBUG_FS)] core::mem::forget(Entry::file(name, &self.entry, data, vtable)); } @@ -550,7 +550,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_only_file(&self, name: &CStr, data: &'data T) { + pub fn read_only_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -560,11 +560,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_binary_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_binary_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } @@ -596,11 +592,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn read_write_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn read_write_file(&self, name: &CStr, data: &'data T) { let vtable = &>::FILE_OPS; self.create_file(name, data, vtable) } @@ -612,7 +604,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn read_write_binary_file( + pub fn read_write_binary_file( &self, name: &CStr, data: &'data T, @@ -655,7 +647,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// This function does not produce an owning handle to the file. The created /// file is removed when the [`Scope`] that this directory belongs /// to is dropped. - pub fn write_only_file(&self, name: &CStr, data: &'data T) { + pub fn write_only_file(&self, name: &CStr, data: &'data T) { let vtable = &>::FILE_OPS; self.create_file(name, data, vtable) } @@ -666,11 +658,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> { /// /// This function does not produce an owning handle to the file. The created file is removed /// when the [`Scope`] that this directory belongs to is dropped. - pub fn write_binary_file( - &self, - name: &CStr, - data: &'data T, - ) { + pub fn write_binary_file(&self, name: &CStr, data: &'data T) { self.create_file(name, data, &T::FILE_OPS) } diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs index 46aad64896ec..88a870d8c295 100644 --- a/rust/kernel/debugfs/entry.rs +++ b/rust/kernel/debugfs/entry.rs @@ -74,7 +74,7 @@ impl Entry<'static> { parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; @@ -127,7 +127,7 @@ impl<'a> Entry<'a> { parent.as_ptr(), core::ptr::from_ref(data) as *mut c_void, core::ptr::null(), - &**file_ops, + file_ops.fops(), ) }; diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs index f15908f71c4a..47acc4851d47 100644 --- a/rust/kernel/debugfs/file_ops.rs +++ b/rust/kernel/debugfs/file_ops.rs @@ -20,9 +20,6 @@ use crate::{ use core::marker::PhantomData; -#[cfg(CONFIG_DEBUG_FS)] -use core::ops::Deref; - /// # Invariant /// /// `FileOps` will always contain an `operations` which is safe to use for a file backed @@ -30,7 +27,7 @@ use core::ops::Deref; /// into a reference. pub(super) struct FileOps { #[cfg(CONFIG_DEBUG_FS)] - operations: bindings::file_operations, + operations: &'static bindings::file_operations, #[cfg(CONFIG_DEBUG_FS)] mode: u16, _phantom: PhantomData, @@ -41,7 +38,7 @@ impl FileOps { /// /// The caller asserts that the provided `operations` is safe to use for a file whose /// inode has a pointer to `T` in its private data that is safe to convert into a reference. - const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self { + const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self { Self { #[cfg(CONFIG_DEBUG_FS)] operations, @@ -65,11 +62,11 @@ impl FileOps { } #[cfg(CONFIG_DEBUG_FS)] -impl Deref for FileOps { - type Target = bindings::file_operations; - - fn deref(&self) -> &Self::Target { - &self.operations +impl FileOps { + /// Returns a `'static` reference to the inner `file_operations`. + #[inline] + pub(crate) fn fops(&self) -> &'static bindings::file_operations { + self.operations } } @@ -130,7 +127,7 @@ pub(crate) trait ReadFile { impl ReadFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(bindings::seq_read), llseek: Some(bindings::seq_lseek), release: Some(bindings::single_release), @@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile { impl ReadWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(writer_open::), read: Some(bindings::seq_read), write: Some(write::), @@ -238,7 +235,7 @@ pub(crate) trait WriteFile { impl WriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { open: Some(write_only_open), write: Some(write_only_write::), llseek: Some(bindings::noop_llseek), @@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile { impl BinaryReadFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile { impl BinaryWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { write: Some(blob_write::), llseek: Some(bindings::default_llseek), open: Some(bindings::simple_open), @@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile { impl BinaryReadWriteFile for T { const FILE_OPS: FileOps = { - let operations = bindings::file_operations { + let operations = &bindings::file_operations { read: Some(blob_read::), write: Some(blob_write::), llseek: Some(bindings::default_llseek), -- cgit v1.2.3 From f6b56da8e1b8217e5516290ef9b09728a8f96a10 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 14:54:14 +0200 Subject: rust: dma: tie CoherentHandle to the device's bound lifetime Add a lifetime parameter to CoherentHandle that ties the DMA allocation to the device's bound scope, ensuring it is freed before the device is unbound. DMA allocations carry device resources (e.g. IOMMU mappings) that must not outlive the device's bound lifetime. Without a lifetime parameter, there was no compile-time enforcement that a CoherentHandle is dropped before the device is unbound. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-3-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb.rs | 2 +- rust/kernel/dma.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 1576399389b1..9ef232a73dee 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -49,7 +49,7 @@ pub(crate) struct SysmemFlush<'sys> { device: &'sys device::Device, bar: Bar0<'sys>, /// Keep the page alive as long as we need it. - page: CoherentHandle, + page: CoherentHandle<'sys>, } impl<'sys> SysmemFlush<'sys> { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 2ce09f8e90c6..79f453e9ec0b 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -996,15 +996,15 @@ impl debugfs::BinaryWriter for Coherent { /// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`. /// - `dma_attrs` contains the attributes used for the allocation, always including /// `DMA_ATTR_NO_KERNEL_MAPPING`. -pub struct CoherentHandle { - dev: ARef, +pub struct CoherentHandle<'a> { + dev: &'a device::Device, dma_addr: DmaAddress, cpu_handle: NonNull, size: usize, dma_attrs: Attrs, } -impl CoherentHandle { +impl<'a> CoherentHandle<'a> { /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. /// /// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is @@ -1012,7 +1012,7 @@ impl CoherentHandle { /// /// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails. pub fn alloc_with_attrs( - dev: &device::Device, + dev: &'a device::Device, size: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -1038,9 +1038,9 @@ impl CoherentHandle { // INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call // with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address, - // and we hold a refcounted reference to the device. + // and `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_handle, size, @@ -1051,7 +1051,7 @@ impl CoherentHandle { /// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping. #[inline] pub fn alloc( - dev: &device::Device, + dev: &'a device::Device, size: usize, gfp_flags: kernel::alloc::Flags, ) -> Result { @@ -1073,7 +1073,7 @@ impl CoherentHandle { } } -impl Drop for CoherentHandle { +impl Drop for CoherentHandle<'_> { fn drop(&mut self) { // SAFETY: All values are valid by the type invariants on `CoherentHandle`. // `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged. @@ -1091,12 +1091,12 @@ impl Drop for CoherentHandle { // SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle, // and a size. None of these are tied to a specific thread. -unsafe impl Send for CoherentHandle {} +unsafe impl Send for CoherentHandle<'_> {} // SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only // operations on `&CoherentHandle` are reading the DMA address and size, both of which are // plain `Copy` values. -unsafe impl Sync for CoherentHandle {} +unsafe impl Sync for CoherentHandle<'_> {} /// View type for `Coherent`. /// -- cgit v1.2.3 From c3d4fd0e9d27651d2daf223fa72128633a082b05 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sat, 29 Aug 2026 15:15:36 +0200 Subject: samples: rust_dma: separate driver type from driver data Split DmaSampleDriver into a driver type and a separate DmaSampleData struct for the driver's bus device private data, using DmaSampleData<'bound> as the Driver::Data<'bound> associated type. Store a &'bound pci::Device reference instead of an ARef, tying the data to the device's bound scope. This prepares for adding a lifetime parameter to dma::Coherent, which requires the data type to carry a lifetime. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-4-dakr@kernel.org Signed-off-by: Danilo Krummrich --- samples/rust/rust_dma.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index bd60034ded23..0fac9d4ae566 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -5,7 +5,10 @@ //! To make this driver probe, QEMU must be run with `-device pci-testdev`. use kernel::{ - device::Core, + device::{ + Bound, + Core, // + }, dma::{ Coherent, DataDirection, @@ -23,13 +26,14 @@ use kernel::{ scatterlist::{ Owned, SGTable, // - }, - sync::aref::ARef, // + }, // }; +struct DmaSampleDriver; + #[pin_data(PinnedDrop)] -struct DmaSampleDriver { - pdev: ARef, +struct DmaSampleData<'bound> { + pdev: &'bound pci::Device, ca: Coherent<[MyStruct]>, #[pin] sgt: SGTable>>, @@ -67,13 +71,13 @@ kernel::pci_device_table!( impl pci::Driver for DmaSampleDriver { type IdInfo = (); - type Data<'bound> = Self; + type Data<'bound> = DmaSampleData<'bound>; const ID_TABLE: pci::IdTable = &PCI_TABLE; fn probe<'bound>( pdev: &'bound pci::Device>, _info: Option<&'bound Self::IdInfo>, - ) -> impl PinInit + 'bound { + ) -> impl PinInit, Error> + 'bound { pin_init::pin_init_scope(move || { dev_info!(pdev, "Probe DMA test driver.\n"); @@ -94,8 +98,8 @@ impl pci::Driver for DmaSampleDriver { let sgt = SGTable::new(pdev.as_ref(), pages, DataDirection::ToDevice, GFP_KERNEL); - Ok(try_pin_init!(Self { - pdev: pdev.into(), + Ok(try_pin_init!(Self::Data { + pdev, ca, sgt <- sgt, })) @@ -103,7 +107,7 @@ impl pci::Driver for DmaSampleDriver { } } -impl DmaSampleDriver { +impl DmaSampleData<'_> { fn check_dma(&self) { for (i, value) in TEST_VALUES.into_iter().enumerate() { let val0 = io_read!(self.ca, [panic: i].h); @@ -116,7 +120,7 @@ impl DmaSampleDriver { } #[pinned_drop] -impl PinnedDrop for DmaSampleDriver { +impl PinnedDrop for DmaSampleData<'_> { fn drop(self: Pin<&mut Self>) { dev_info!(self.pdev, "Unload DMA test driver.\n"); -- cgit v1.2.3 From bed87375f90f44cd2f9fe804a469274127d38593 Mon Sep 17 00:00:00 2001 From: Danilo Krummrich Date: Sun, 30 Aug 2026 17:40:34 +0200 Subject: rust: dma: tie Coherent and CoherentBox to the device's bound lifetime Add a lifetime parameter to Coherent and CoherentBox that ties the DMA allocation to the device's bound scope, ensuring it is freed before the device is unbound. DMA allocations carry device resources (e.g. IOMMU mappings) that must not outlive the device's bound lifetime. Without a lifetime parameter, there was no compile-time enforcement that a Coherent or CoherentBox is dropped before the device is unbound. Propagate the new lifetime parameter through all users. Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260830193824.471089-5-dakr@kernel.org Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 2 +- drivers/gpu/nova-core/fb.rs | 2 +- drivers/gpu/nova-core/firmware/booter.rs | 2 +- drivers/gpu/nova-core/firmware/fsp.rs | 8 +- drivers/gpu/nova-core/firmware/fwsec/bootloader.rs | 12 +- drivers/gpu/nova-core/firmware/gsp.rs | 14 +-- drivers/gpu/nova-core/firmware/riscv.rs | 8 +- drivers/gpu/nova-core/fsp.rs | 20 ++-- drivers/gpu/nova-core/gpu.rs | 4 +- drivers/gpu/nova-core/gsp.rs | 30 ++--- drivers/gpu/nova-core/gsp/boot.rs | 10 +- drivers/gpu/nova-core/gsp/cmdq.rs | 35 +++--- drivers/gpu/nova-core/gsp/commands.rs | 2 +- drivers/gpu/nova-core/gsp/fw.rs | 12 +- drivers/gpu/nova-core/gsp/hal.rs | 14 +-- drivers/gpu/nova-core/gsp/hal/gh100.rs | 16 +-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 34 +++--- drivers/gpu/nova-core/gsp/sequencer.rs | 6 +- rust/kernel/dma.rs | 121 ++++++++++----------- rust/kernel/uaccess.rs | 4 +- samples/rust/rust_dma.rs | 4 +- 21 files changed, 176 insertions(+), 184 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..15eba039cb69 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -506,7 +506,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - dma_obj: &Coherent<[u8]>, + dma_obj: &Coherent<'_, [u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, ) -> Result { diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 9ef232a73dee..b3a6ab8b57a6 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -177,7 +177,7 @@ impl FbRanges { pub(crate) fn new( chipset: Chipset, bar: Bar0<'_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, vgpu_state: VgpuState, ) -> Result { let hal = hal::fb_hal(chipset); diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index dc071edba331..aa4458bb3312 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -186,7 +186,7 @@ impl BooterFirmware { &self, dev: &device::Device, sec2_falcon: &Falcon<'_, Sec2>, - wpr_meta: &Coherent, + wpr_meta: &Coherent<'_, T>, ) -> Result { sec2_falcon.reset()?; sec2_falcon.load(self)?; diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 5462e318410a..d47b1d2a1030 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -39,15 +39,15 @@ pub(crate) struct FmcSignatures { pub(crate) signature: [u8; FSP_SIG_SIZE], } -pub(crate) struct FspFirmware { +pub(crate) struct FspFirmware<'a> { /// FMC firmware image data - pub(crate) fmc_image: Coherent<[u8]>, + pub(crate) fmc_image: Coherent<'a, [u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox, } -impl FspFirmware { - pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result { +impl<'a> FspFirmware<'a> { + pub(crate) fn new(dev: &'a device::Device, chipset: Chipset) -> Result { let fw = request_tlv(dev, chipset, "fmc")?; let tlv = Tlv::new(fw.data())?; dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ec4d92317a93..06a7936a8c5e 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -98,9 +98,9 @@ unsafe impl AsBytes for BootloaderDmemDescV2 {} /// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load /// operation. -pub(crate) struct FwsecFirmwareWithBl { +pub(crate) struct FwsecFirmwareWithBl<'a> { /// DMA object the bootloader will copy the firmware from. - _firmware_dma: Coherent<[u8]>, + _firmware_dma: Coherent<'a, [u8]>, /// Code of the bootloader to be loaded into non-secure IMEM. ucode: KVec, /// Descriptor to be loaded into DMEM for the bootloader to read. @@ -113,12 +113,12 @@ pub(crate) struct FwsecFirmwareWithBl { start_tag: u16, } -impl FwsecFirmwareWithBl { +impl<'a> FwsecFirmwareWithBl<'a> { /// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be /// loaded using it. pub(crate) fn new( firmware: FwsecFirmware, - dev: &Device, + dev: &'a Device, chipset: Chipset, ) -> Result { let fw = request_tlv(dev, chipset, "gen_bootloader")?; @@ -272,7 +272,7 @@ impl FwsecFirmwareWithBl { } } -impl FalconFirmware for FwsecFirmwareWithBl { +impl FalconFirmware for FwsecFirmwareWithBl<'_> { type Target = Gsp; fn brom_params(&self) -> FalconBromParams { @@ -286,7 +286,7 @@ impl FalconFirmware for FwsecFirmwareWithBl { } } -impl FalconPioLoadable for FwsecFirmwareWithBl { +impl FalconPioLoadable for FwsecFirmwareWithBl<'_> { fn imem_sec_load_params(&self) -> Option> { None } diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index e8f9491e84cc..22d1f9329c9f 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -44,7 +44,7 @@ use crate::{ /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). /// Also known as "Radix3" firmware. #[pin_data] -pub(crate) struct GspFirmware { +pub(crate) struct GspFirmware<'a> { /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. #[pin] fw: SGTable>>, @@ -55,19 +55,19 @@ pub(crate) struct GspFirmware { #[pin] level1: SGTable>>, /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. - level0: Coherent<[u64]>, + level0: Coherent<'a, [u64]>, /// Size in bytes of the firmware contained in [`Self::fw`]. pub(crate) size: usize, /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. - pub(crate) signatures: Coherent<[u8]>, + pub(crate) signatures: Coherent<'a, [u8]>, /// GSP bootloader, verifies the GSP firmware before loading and running it. - pub(crate) bootloader: RiscvFirmware, + pub(crate) bootloader: RiscvFirmware<'a>, } -impl GspFirmware { +impl<'a> GspFirmware<'a> { /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. - pub(crate) fn new<'a>( + pub(crate) fn new( dev: &'a device::Device, chipset: Chipset, ) -> impl PinInit + 'a { @@ -120,7 +120,7 @@ impl GspFirmware { // Create level 0 page table data and fill its first entry with the level 1 // table. - let mut level0 = CoherentBox::<[u64]>::zeroed_slice( + let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( dev, GSP_PAGE_SIZE / size_of::(), GFP_KERNEL diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs index 1403f05a7305..f05cfb1c65da 100644 --- a/drivers/gpu/nova-core/firmware/riscv.rs +++ b/drivers/gpu/nova-core/firmware/riscv.rs @@ -13,7 +13,7 @@ use kernel::{ use crate::firmware::tlv::Tlv; /// A parsed firmware for a RISC-V core, ready to be loaded and run. -pub(crate) struct RiscvFirmware { +pub(crate) struct RiscvFirmware<'a> { /// Offset at which the code starts in the firmware image. pub(crate) code_offset: u32, /// Offset at which the data starts in the firmware image. @@ -23,12 +23,12 @@ pub(crate) struct RiscvFirmware { /// Application version. pub(crate) app_version: u32, /// Device-mapped firmware image. - pub(crate) ucode: Coherent<[u8]>, + pub(crate) ucode: Coherent<'a, [u8]>, } -impl RiscvFirmware { +impl<'a> RiscvFirmware<'a> { /// Parses the RISC-V firmware image contained in `fw`. - pub(crate) fn new(dev: &device::Device, fw: &Firmware) -> Result { + pub(crate) fn new(dev: &'a device::Device, fw: &Firmware) -> Result { let tlv = Tlv::new(fw.data())?; dev_dbg!( dev, diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ab685fb4168f..961e96fe9484 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -267,7 +267,7 @@ impl FspCotMessage { /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_info: &FbSizes, - fsp_fw: &'a FspFirmware, + fsp_fw: &'a FspFirmware<'_>, args: &'a FmcBootArgs<'_>, ) -> Result + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; @@ -345,28 +345,28 @@ impl MessageToFsp for FspPrcMessage { /// Bundled arguments for FMC boot via FSP Chain of Trust. pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, - fmc_boot_params: Coherent, + fmc_boot_params: Coherent<'a, GspFmcBootParams>, resume: bool, // Additional dependencies required to be kept alive for FMC boot. - _wpr_meta: Coherent, - _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + _wpr_meta: Coherent<'a, GspFwWprMeta>, + _libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, } impl<'a> FmcBootArgs<'a> { /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter /// structure that FSP will read. pub(crate) fn new( - dev: &device::Device, + dev: &'a device::Device, chipset: Chipset, - wpr_meta: Coherent, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + wpr_meta: Coherent<'a, GspFwWprMeta>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result { let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address()); Ok(Self { chipset, - fmc_boot_params: Coherent::::init(dev, GFP_KERNEL, init)?, + fmc_boot_params: Coherent::init(dev, GFP_KERNEL, init)?, resume, _wpr_meta: wpr_meta, _libos: libos, @@ -374,7 +374,7 @@ impl<'a> FmcBootArgs<'a> { } /// Returns the FMC boot parameters allocation. - pub(crate) fn boot_params(&self) -> &Coherent { + pub(crate) fn boot_params(&self) -> &Coherent<'_, GspFmcBootParams> { &self.fmc_boot_params } } @@ -386,7 +386,7 @@ impl<'a> FmcBootArgs<'a> { /// Chain of Trust boot. pub(crate) struct Fsp<'a> { falcon: Falcon<'a, FspEngine>, - fsp_fw: FspFirmware, + fsp_fw: FspFirmware<'a>, } impl<'a> Fsp<'a> { diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index fd1414004dd0..6de75d16488d 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -272,9 +272,9 @@ struct GspResources<'gpu> { vgpu: VgpuManager, /// GSP runtime data. #[pin] - gsp: Gsp, + gsp: Gsp<'gpu>, /// GSP unload firmware bundle, if any. - unload_bundle: Option, + unload_bundle: Option>, } /// Structure holding the resources required to operate the GPU. diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 13f361406a6c..25ea43f1cbe9 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -115,11 +115,11 @@ impl PteArray { /// then pp points to index into the buffer where the next logging entry will /// be written. Therefore, the logging data is valid if: /// 1 <= pp < sizeof(buffer)/sizeof(u64) -struct LogBuffer(Coherent<[u8; LOG_BUFFER_SIZE]>); +struct LogBuffer<'a>(Coherent<'a, [u8; LOG_BUFFER_SIZE]>); -impl LogBuffer { +impl<'a> LogBuffer<'a> { /// Creates a new `LogBuffer` mapped on `dev`. - fn new(dev: &device::Device) -> Result { + fn new(dev: &'a device::Device) -> Result { let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?); let start_addr = obj.0.dma_address(); @@ -135,33 +135,33 @@ impl LogBuffer { } } -struct LogBuffers { +struct LogBuffers<'a> { /// Init log buffer. - loginit: LogBuffer, + loginit: LogBuffer<'a>, /// Interrupts log buffer. - logintr: LogBuffer, + logintr: LogBuffer<'a>, /// RM log buffer. - logrm: LogBuffer, + logrm: LogBuffer<'a>, } /// GSP runtime data. #[pin_data] -pub(crate) struct Gsp { +pub(crate) struct Gsp<'gsp> { /// Libos arguments. - pub(crate) libos: Coherent<[LibosMemoryRegionInitArgument]>, + pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>, /// Log buffers, optionally exposed via debugfs. #[pin] - logs: debugfs::Scope, + logs: debugfs::Scope>, /// Command queue. #[pin] - pub(crate) cmdq: Cmdq, + pub(crate) cmdq: Cmdq<'gsp>, /// RM arguments. - rmargs: Coherent, + rmargs: Coherent<'gsp, GspArgumentsPadded>, } -impl Gsp { +impl<'gsp> Gsp<'gsp> { // Creates an in-place initializer for a `Gsp` manager for `pdev`. - pub(crate) fn new(pdev: &pci::Device) -> impl PinInit + '_ { + pub(crate) fn new(pdev: &'gsp pci::Device) -> impl PinInit + 'gsp { pin_init::pin_init_scope(move || { let dev = pdev.as_ref(); @@ -223,4 +223,4 @@ impl Gsp { } /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`]. -pub(crate) struct UnloadBundle(KBox); +pub(crate) struct UnloadBundle<'a>(KBox); diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e03700ee7bea..e32c9e1f33ab 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -22,7 +22,7 @@ use crate::{ }, }; -impl super::Gsp { +impl<'gsp> super::Gsp<'gsp> { /// Attempt to boot the GSP. /// /// This is a GPU-dependent and complex procedure that involves loading firmware files from @@ -33,8 +33,8 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - mut ctx: super::GspBootContext<'_, '_>, - ) -> Result> { + mut ctx: super::GspBootContext<'_, 'gsp>, + ) -> Result>> { let pdev = ctx.pdev; let bar = ctx.bar; let chipset = ctx.chipset; @@ -88,7 +88,7 @@ impl super::Gsp { /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, bar: Bar0<'_>, gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, @@ -113,7 +113,7 @@ impl super::Gsp { pub(crate) fn unload( &self, mut ctx: super::GspBootContext<'_, '_>, - unload_bundle: Option, + unload_bundle: Option>, ) -> Result { let dev = ctx.dev(); diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 6da728201281..fe087ce315b0 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -25,10 +25,7 @@ use kernel::{ new_mutex, prelude::*, ptr, - sync::{ - aref::ARef, - Mutex, // - }, + sync::Mutex, time::Delta, transmute::{ AsBytes, @@ -230,19 +227,19 @@ unsafe impl FromBytes for GspMem {} /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. -struct DmaGspMem(Coherent); +struct DmaGspMem<'a>(Coherent<'a, GspMem>); -impl DmaGspMem { +impl<'a> DmaGspMem<'a> { /// Allocate a new instance and map it for `dev`. - fn new(dev: &device::Device) -> Result { + fn new(dev: &'a device::Device) -> Result { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let mut gsp_mem = CoherentBox::::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?; gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let gsp_mem: Coherent<_> = gsp_mem.into(); + let gsp_mem: Coherent<'_, _> = gsp_mem.into(); PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; Ok(Self(gsp_mem)) @@ -483,15 +480,15 @@ struct GspMessage<'a> { /// Provides the ability to send commands and receive messages from the GSP using a shared memory /// area. #[pin_data] -pub(crate) struct Cmdq { +pub(crate) struct Cmdq<'cmdq> { /// Inner mutex-protected state. #[pin] - inner: Mutex, + inner: Mutex>, /// DMA address of the command queue's shared memory region. pub(super) dma_addr: DmaAddress, } -impl Cmdq { +impl<'cmdq> Cmdq<'cmdq> { /// Offset of the data after the PTEs. const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); @@ -512,14 +509,16 @@ impl Cmdq { pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); /// Creates a new command queue for `dev`. - pub(crate) fn new(dev: &device::Device) -> impl PinInit + '_ { + pub(crate) fn new( + dev: &'cmdq device::Device, + ) -> impl PinInit + 'cmdq { pin_init_scope(move || { let gsp_mem = DmaGspMem::new(dev)?; Ok(try_pin_init!(Self { dma_addr: gsp_mem.0.dma_address(), inner <- new_mutex!(CmdqInner { - dev: dev.into(), + dev, gsp_mem, seq: 0, }), @@ -610,16 +609,16 @@ impl Cmdq { } /// Inner mutex protected state of [`Cmdq`]. -struct CmdqInner { +struct CmdqInner<'a> { /// Device this command queue belongs to. - dev: ARef, + dev: &'a device::Device, /// Current command sequence number. seq: u32, /// Memory area shared with the GSP for communicating commands and messages. - gsp_mem: DmaGspMem, + gsp_mem: DmaGspMem<'a>, } -impl CmdqInner { +impl CmdqInner<'_> { /// Timeout for waiting for space on the command queue. const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ffc25fd8c47b..69d7d41c1791 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -187,7 +187,7 @@ impl MessageFromGsp for GspInitDone { } /// Waits for GSP initialization to complete. -pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { +pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result { loop { match cmdq.receive_msg::(Cmdq::RECEIVE_TIMEOUT) { Ok(_) => break Ok(()), diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 05f54fee6186..8778c4bf79c0 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -179,7 +179,7 @@ impl GspFwWprMeta { /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the /// framebuffer ranges `ranges`. pub(crate) fn from_ranges<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, ranges: &'a FbRanges, ) -> impl Init + 'a { let init_inner = init!(bindings::GspFwWprMeta { @@ -231,7 +231,7 @@ impl GspFwWprMeta { /// /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. pub(crate) fn from_sizes<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, sizes: &'a FbSizes, ) -> impl Init + 'a { /// VGA workspace size to reserve at the end of the framebuffer, in bytes. @@ -665,7 +665,7 @@ unsafe impl FromBytes for LibosMemoryRegionInitArgument {} impl LibosMemoryRegionInitArgument { pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( name: &'static str, - obj: &'a Coherent, + obj: &'a Coherent<'_, A>, ) -> impl Init + 'a { /// Generates the `ID8` identifier required for some GSP objects. fn id8(name: &str) -> u64 { @@ -897,7 +897,7 @@ pub(crate) struct GspArgumentsCached { impl GspArgumentsCached { /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. - pub(crate) fn new(cmdq: &Cmdq) -> impl Init + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), bDmemStack: 1, @@ -924,7 +924,7 @@ pub(crate) struct GspArgumentsPadded { } impl GspArgumentsPadded { - pub(crate) fn new(cmdq: &Cmdq) -> impl Init + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { init!(GspArgumentsPadded { inner <- GspArgumentsCached::new(cmdq), ..Zeroable::init_zeroed() @@ -944,7 +944,7 @@ type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. - fn new(cmdq: &Cmdq) -> impl Init + '_ { + fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init + use<'a, 'b> { init!(MessageQueueInitArguments { sharedMemPhysAddr: cmdq.dma_addr, pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 5850fa0fe0e9..d8329f6fcc65 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -35,12 +35,12 @@ pub(super) trait GspHal: Send { /// /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one /// could be created. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result>; + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>>; /// Performs HAL-specific post-GSP boot tasks. /// @@ -48,9 +48,9 @@ pub(super) trait GspHal: Send { /// after the initialization commands have been pushed onto its queue. fn post_boot( &self, - _gsp: &Gsp, + _gsp: &Gsp<'_>, _ctx: &mut GspBootContext<'_, '_>, - _gsp_fw: &GspFirmware, + _gsp_fw: &GspFirmware<'_>, ) -> Result { Ok(()) } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index e283429a95dd..91201b51030e 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -58,7 +58,7 @@ impl GspMbox { fn lockdown_released_or_error( &self, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during // boot. If the address is still there, keep polling rather than treating it as an error. @@ -75,7 +75,7 @@ impl GspMbox { fn wait_for_gsp_lockdown_release( dev: &device::Device, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -141,12 +141,12 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>> { let dev = ctx.dev(); let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; @@ -159,7 +159,7 @@ impl GspHal for Gh100 { let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; let unload_bundle = crate::gsp::UnloadBundle( - KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox + KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox ); // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a5c0ca355493..f315013cff86 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -52,12 +52,12 @@ use crate::{ // // Since there are two variants of the prepared firmware (with and without a bootloader), this type // abstracts the difference. -enum FwsecUnloadFirmware { +enum FwsecUnloadFirmware<'a> { WithoutBl(FwsecFirmware), - WithBl(FwsecFirmwareWithBl), + WithBl(FwsecFirmwareWithBl<'a>), } -impl FwsecUnloadFirmware { +impl FwsecUnloadFirmware<'_> { /// Runs the FWSEC SB firmware. fn run( &self, @@ -74,12 +74,12 @@ impl FwsecUnloadFirmware { // Contains the firmware required to fully reset GSP on chipsets where the GSP is started using // FWSEC/Booter. -struct Sec2UnloadBundle { - fwsec_sb: FwsecUnloadFirmware, +struct Sec2UnloadBundle<'a> { + fwsec_sb: FwsecUnloadFirmware<'a>, booter_unloader: BooterFirmware, } -impl UnloadBundle for Sec2UnloadBundle { +impl UnloadBundle for Sec2UnloadBundle<'_> { fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { let dev = ctx.dev(); let bar = ctx.bar; @@ -213,14 +213,14 @@ impl Tu102 { } /// Load and prepare the resources required to properly reset the GSP after it has been stopped. - fn build_unload_bundle( + fn build_unload_bundle<'gpu>( &self, - dev: &device::Device, + dev: &'gpu device::Device, chipset: Chipset, bios: &Vbios, gsp_falcon: &Falcon<'_, GspEngine>, sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result { + ) -> Result> { // Load the FWSEC SB firmware, as well as its bootloader if required. let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; let fwsec_sb = if self.needs_fwsec_bootloader { @@ -241,18 +241,18 @@ impl Tu102 { }, GFP_KERNEL, ) - .map(|b| crate::gsp::UnloadBundle(b)) + .map(|b| crate::gsp::UnloadBundle(b as KBox)) .map_err(Into::into) } } impl GspHal for Tu102 { - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result>> { let dev = ctx.dev(); let bar = ctx.bar; let chipset = ctx.chipset; @@ -317,9 +317,9 @@ impl GspHal for Tu102 { fn post_boot( &self, - gsp: &Gsp, + gsp: &Gsp<'_>, ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, ) -> Result { GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?; diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index bcad1421953a..dae34c11eb05 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -138,7 +138,7 @@ pub(crate) struct GspSequencer<'a> { /// GSP falcon for core operations. gsp_falcon: &'a Falcon<'a, Gsp>, /// LibOS memory region init arguments. - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. @@ -338,9 +338,9 @@ impl<'a> Iterator for GspSeqIter<'a> { impl<'a> GspSequencer<'a> { pub(crate) fn run( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, ctx: &'a GspBootContext<'_, '_>, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, bootloader_app_version: u32, ) -> Result { let seq_info = loop { diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs index 79f453e9ec0b..4ce914b7d1da 100644 --- a/rust/kernel/dma.rs +++ b/rust/kernel/dma.rs @@ -24,7 +24,6 @@ use crate::{ }, prelude::*, ptr::KnownSize, - sync::aref::ARef, transmute::{ AsBytes, FromBytes, // @@ -223,7 +222,7 @@ impl DmaMask { /// /// # fn test(dev: &Device) -> Result { /// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN; -/// let c: Coherent<[u64]> = +/// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?; /// # Ok::<(), Error>(()) } /// ``` @@ -390,9 +389,9 @@ impl From for bindings::dma_data_direction { /// }; /// /// # fn test(dev: &Device) -> Result { -/// let mut dmem: CoherentBox = CoherentBox::zeroed(dev, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?; /// *dmem = 42; -/// let dmem: Coherent = dmem.into(); +/// let dmem: Coherent<'_, u64> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` /// @@ -410,18 +409,18 @@ impl From for bindings::dma_data_direction { /// }; /// /// # fn test(dev: &Device) -> Result { -/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; +/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?; /// dmem.fill(42); -/// let dmem: Coherent<[u64]> = dmem.into(); +/// let dmem: Coherent<'_, [u64]> = dmem.into(); /// # Ok::<(), Error>(()) } /// ``` -pub struct CoherentBox(Coherent); +pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>); -impl CoherentBox<[T]> { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> { /// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`]. #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, count: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -432,7 +431,7 @@ impl CoherentBox<[T]> { /// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`. #[inline] pub fn zeroed_slice( - dev: &device::Device, + dev: &'a device::Device, count: usize, gfp_flags: kernel::alloc::Flags, ) -> Result { @@ -480,14 +479,14 @@ impl CoherentBox<[T]> { /// /// # fn test(dev: &Device) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; - /// let c: CoherentBox<[u8]> = + /// let c: CoherentBox<'_, [u8]> = /// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// assert_eq!(c.deref(), &data); /// # Ok::<(), Error>(()) } /// ``` pub fn from_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, @@ -512,7 +511,7 @@ impl CoherentBox<[T]> { /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, ) -> Result @@ -523,11 +522,11 @@ impl CoherentBox<[T]> { } } -impl CoherentBox { +impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> { /// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element. #[inline] pub fn zeroed_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -536,12 +535,12 @@ impl CoherentBox { /// Same as [`CoherentBox::zeroed_slice`], but for a single element. #[inline] - pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + pub fn zeroed(dev: &'a device::Device, gfp_flags: kernel::alloc::Flags) -> Result { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } } -impl Deref for CoherentBox { +impl Deref for CoherentBox<'_, T> { type Target = T; #[inline] @@ -554,7 +553,7 @@ impl Deref for CoherentBox { } } -impl DerefMut for CoherentBox { +impl DerefMut for CoherentBox<'_, T> { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: @@ -565,9 +564,9 @@ impl DerefMut for CoherentBox { } } -impl From> for Coherent { +impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From> for Coherent<'a, T> { #[inline] - fn from(value: CoherentBox) -> Self { + fn from(value: CoherentBox<'a, T>) -> Self { value.0 } } @@ -588,26 +587,20 @@ impl From> for Coher /// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the /// region. /// - The size in bytes of the allocation is equal to size information via pointer. -// TODO // -// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness -// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure -// that device resources can never survive device unbind. -// -// However, it is neither desirable nor necessary to protect the allocated memory of the DMA -// allocation from surviving device unbind; it would require RCU read side critical sections to -// access the memory, which may require subsequent unnecessary copies. -// -// Hence, find a way to revoke the device resources of a `Coherent`, but not the -// entire `Coherent` including the allocated memory itself. -pub struct Coherent { - dev: ARef, +// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed +// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g. +// via `mem::forget`), device resources such as IOMMU mappings will not be released. Making all +// constructors `unsafe` to prevent this is considered too restrictive for the common case; this +// soundness hole is accepted for now. +pub struct Coherent<'a, T: KnownSize + ?Sized> { + dev: &'a device::Device, dma_addr: DmaAddress, cpu_addr: NonNull, dma_attrs: Attrs, } -impl Coherent { +impl Coherent<'_, T> { /// Returns the size in bytes of this allocation. #[inline] pub fn size(&self) -> usize { @@ -663,10 +656,10 @@ impl Coherent { } } -impl Coherent { +impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> { /// Allocates a region of `T` of coherent memory. fn alloc_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -692,9 +685,9 @@ impl Coherent { // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for `T`, // hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Self { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -716,13 +709,13 @@ impl Coherent { /// }; /// /// # fn test(dev: &Device) -> Result { - /// let c: Coherent<[u64; 4]> = + /// let c: Coherent<'_, [u64; 4]> = /// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, ) -> Result { @@ -732,14 +725,14 @@ impl Coherent { /// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the /// `dma_attrs` is 0 by default. #[inline] - pub fn zeroed(dev: &device::Device, gfp_flags: kernel::alloc::Flags) -> Result { + pub fn zeroed(dev: &'a device::Device, gfp_flags: kernel::alloc::Flags) -> Result { Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0)) } /// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is /// initialized with `init`. pub fn init_with_attrs( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, init: impl Init, @@ -764,7 +757,7 @@ impl Coherent { /// with `init`. #[inline] pub fn init( - dev: &device::Device, + dev: &'a device::Device, gfp_flags: kernel::alloc::Flags, init: impl Init, ) -> Result @@ -776,11 +769,11 @@ impl Coherent { /// Allocates a region of `[T; len]` of coherent memory. fn alloc_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { + ) -> Result> { const { assert!( core::mem::size_of::() > 0, @@ -809,9 +802,9 @@ impl Coherent { // INVARIANT: // - We just successfully allocated a coherent region which is adequately sized for // `[T; len]`, hence the cpu address is valid. - // - We also hold a refcounted reference to the device. + // - `dev` is a valid reference to a bound device that outlives this allocation. Ok(Coherent { - dev: dev.into(), + dev, dma_addr, cpu_addr, dma_attrs, @@ -836,17 +829,17 @@ impl Coherent { /// }; /// /// # fn test(dev: &Device) -> Result { - /// let c: Coherent<[u64]> = + /// let c: Coherent<'_, [u64]> = /// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn zeroed_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> { + ) -> Result> { Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs) } @@ -854,10 +847,10 @@ impl Coherent { /// `dma_attrs` is 0 by default. #[inline] pub fn zeroed_slice( - dev: &device::Device, + dev: &'a device::Device, len: usize, gfp_flags: kernel::alloc::Flags, - ) -> Result> { + ) -> Result> { Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0)) } @@ -876,18 +869,18 @@ impl Coherent { /// # fn test(dev: &Device) -> Result { /// let data = [0u8, 1u8, 2u8, 3u8]; /// // `c` has the same content as `data`. - /// let c: Coherent<[u8]> = + /// let c: Coherent<'_, [u8]> = /// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?; /// /// # Ok::<(), Error>(()) } /// ``` #[inline] pub fn from_slice_with_attrs( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, dma_attrs: Attrs, - ) -> Result> + ) -> Result> where T: Copy, { @@ -898,10 +891,10 @@ impl Coherent { /// `dma_attrs` is 0 by default. #[inline] pub fn from_slice( - dev: &device::Device, + dev: &'a device::Device, data: &[T], gfp_flags: kernel::alloc::Flags, - ) -> Result> + ) -> Result> where T: Copy, { @@ -909,7 +902,7 @@ impl Coherent { } } -impl Coherent<[T]> { +impl Coherent<'_, [T]> { /// Returns the number of elements `T` in this allocation. /// /// Note that this is not the size of the allocation in bytes, which is provided by @@ -922,10 +915,10 @@ impl Coherent<[T]> { } /// Note that the device configured to do DMA must be halted before this object is dropped. -impl Drop for Coherent { +impl Drop for Coherent<'_, T> { fn drop(&mut self) { let size = T::size(self.cpu_addr.as_ptr()); - // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`. + // SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`. // The cpu address, and the dma address are valid due to the type invariants on // `Coherent`. unsafe { @@ -942,15 +935,15 @@ impl Drop for Coherent { // SAFETY: It is safe to send a `Coherent` to another thread if `T` // can be sent to another thread. -unsafe impl Send for Coherent {} +unsafe impl Send for Coherent<'_, T> {} // SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all // methods that access the buffer contents (`field_read`, `field_write`, `as_slice`, // `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur. // The safe methods only return metadata or raw pointers whose use requires `unsafe`. -unsafe impl Sync for Coherent {} +unsafe impl Sync for Coherent<'_, T> {} -impl debugfs::BinaryWriter for Coherent { +impl debugfs::BinaryWriter for Coherent<'_, T> { fn write_to_slice( &self, writer: &mut UserSliceWriter, @@ -1236,7 +1229,7 @@ impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> { } } -impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent { +impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> { type Backend = CoherentIoBackend; type Target = T; diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs index 5f6c4d7a1a51..f09078228c53 100644 --- a/rust/kernel/uaccess.rs +++ b/rust/kernel/uaccess.rs @@ -520,14 +520,14 @@ impl UserSliceWriter { /// /// fn copy_dma_to_user( /// mut writer: UserSliceWriter, - /// alloc: &Coherent<[u8]>, + /// alloc: &Coherent<'_, [u8]>, /// ) -> Result { /// writer.write_dma(alloc, 0, 256) /// } /// ``` pub fn write_dma( &mut self, - alloc: &Coherent, + alloc: &Coherent<'_, T>, offset: usize, count: usize, ) -> Result { diff --git a/samples/rust/rust_dma.rs b/samples/rust/rust_dma.rs index 0fac9d4ae566..ffb693544673 100644 --- a/samples/rust/rust_dma.rs +++ b/samples/rust/rust_dma.rs @@ -34,7 +34,7 @@ struct DmaSampleDriver; #[pin_data(PinnedDrop)] struct DmaSampleData<'bound> { pdev: &'bound pci::Device, - ca: Coherent<[MyStruct]>, + ca: Coherent<'bound, [MyStruct]>, #[pin] sgt: SGTable>>, } @@ -86,7 +86,7 @@ impl pci::Driver for DmaSampleDriver { // SAFETY: There are no concurrent calls to DMA allocation and mapping primitives. unsafe { pdev.dma_set_mask_and_coherent(mask)? }; - let ca: Coherent<[MyStruct]> = + let ca: Coherent<'_, [MyStruct]> = Coherent::zeroed_slice(pdev.as_ref(), TEST_VALUES.len(), GFP_KERNEL)?; for (i, value) in TEST_VALUES.into_iter().enumerate() { -- cgit v1.2.3 From e1df8a1560a97d2f5c1965d4844ab64769e92a91 Mon Sep 17 00:00:00 2001 From: John Hubbard Date: Fri, 4 Sep 2026 19:38:00 -0700 Subject: gpu: nova-core: gsp: fix a broken doc link on GspMessage The doc comment on the GSP message type names Cmdq as the type that defines wait_for_msg. That method belongs to CmdqInner, so the link has no target. Nothing catches the break, because rustdoc resolves a link only in an item it documents, and the kernel's rustdoc run skips private items. Point the link at CmdqInner. Assisted-by: LLM Signed-off-by: John Hubbard Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260905023800.1293631-1-jhubbard@nvidia.com Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/gsp/cmdq.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 14a711307654..28876c4aa42c 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -478,7 +478,7 @@ struct GspCommand<'a> { /// A message ready to be processed from the message queue. /// -/// This is the type returned by [`Cmdq::wait_for_msg`]. +/// This is the type returned by [`CmdqInner::wait_for_msg`]. struct GspMessage<'a> { // Reference to the header of the message. header: &'a GspMsgElement, -- cgit v1.2.3 From 1fc1f82756b1ff872c04950747925f7dcf5c1d8a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:26 +0100 Subject: rust: mem: add `transmute` with deferred size check Implement a `transmute/safe_transmute` that checks size at monomorphization time instead of type-checking time. This allows more cases where we know that the size matches but this is not generically checkable. The signature is equivalent to the unstable `transmute_neo` function in the standard library. A safe variant is provided to use with types implementing `FromBytes` and `IntoBytes`. Existing users of `transmute_copy` to bypass size checks are converted. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260901-typed_register-v4-2-5552b1d59525@garyguo.net [ Fix doc heading to use plural "Examples", add missing closing code block delimiter and fix a few other nits. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/device_id.rs | 3 +- rust/kernel/lib.rs | 1 + rust/kernel/mem.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++ rust/kernel/sync/atomic.rs | 4 +- 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 rust/kernel/mem.rs diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs index c81fca5b4986..f0b9cb84e58e 100644 --- a/rust/kernel/device_id.rs +++ b/rust/kernel/device_id.rs @@ -146,8 +146,7 @@ impl IdArray { /// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead. pub const fn new_without_index(ids: [T; N]) -> Self { // SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them. - let raw_ids: [MaybeUninit; N] = unsafe { core::mem::transmute_copy(&ids) }; - core::mem::forget(ids); + let raw_ids: [MaybeUninit; N] = unsafe { crate::mem::transmute(ids) }; Self { ids: raw_ids, diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 4d5c96ddc49c..7225abc64084 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -98,6 +98,7 @@ pub mod jump_label; pub mod kunit; pub mod list; pub mod maple_tree; +pub mod mem; pub mod miscdevice; pub mod mm; pub mod module; diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs new file mode 100644 index 000000000000..35426f4d514e --- /dev/null +++ b/rust/kernel/mem.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Basic utilities for dealing with memory, values, and types. + +use crate::prelude::*; + +/// Transmute between two types. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler. +/// +/// This is equivalent to Rust's `transmute_unchecked` intrinsics. +/// +/// # Safety +/// +/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst` +/// must match. +/// +/// # Examples +/// +/// This can be used when types are known to have the same size, but only at runtime. +/// +/// ```no_run +/// # use core::any::TypeId; +/// fn to_u32(v: T) -> Option { +/// if TypeId::of::() != TypeId::of::() { +/// return None; +/// } +/// +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We've checked that `T` is `u32`! +/// Some(unsafe { kernel::mem::transmute_unchecked(v) }) +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute_unchecked(val: Src) -> Dst { + // SAFETY: This is identical to `transmute` except that we bypassed the size check; which is + // true per safety requirement. + unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) } +} + +/// Version of `transmute` that performs size check at monomorphization-time. +/// +/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this +/// cannot be proven by the compiler during type checking and can be proven during monomorphization. +/// +/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of +/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844). +/// +/// # Safety +/// +/// Same as [`core::mem::transmute`]. +/// +/// # Examples +/// +/// This is typically used in generic code where it's known that type will have the same size, but +/// the compiler cannot prove it generically. +/// +/// ```no_run +/// trait IsU32 {} +/// impl IsU32 for u32 {} +/// +/// fn to_u32(v: T) -> u32 { +/// // `core::mem::transmute` won't work here. +/// // SAFETY: We know that `v` is u32! +/// unsafe { kernel::mem::transmute(v) } +/// } +/// +/// to_u32(1u32); +/// ``` +#[inline(always)] +pub const unsafe fn transmute(val: Src) -> Dst { + const_assert!(size_of::() == size_of::()); + + // SAFETY: Size is checked above. Other safety requirements follow those of the function. + unsafe { transmute_unchecked(val) } +} + +/// Safely transmutes a value of one type to a value of another type of the same size. +/// +/// The sizes are checked during monomorphization. +/// +/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size +/// check and thus can be used in more cases. +/// +/// # Examples +/// +/// ```no_run +/// fn to_u32(v: T) -> u32 { +/// // `zerocopy::transmute!` won't work here. +/// kernel::mem::safe_transmute(v) +/// } +/// +/// to_u32(1i32); +/// ``` +#[inline(always)] +pub const fn safe_transmute(val: Src) -> Dst { + // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds. + unsafe { transmute(val) } +} diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs index 9cd009d57e35..6d27898add42 100644 --- a/rust/kernel/sync/atomic.rs +++ b/rust/kernel/sync/atomic.rs @@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd: AtomicType { const fn into_repr(v: T) -> T::Repr { // SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to // `T::Repr`, therefore the transmute operation is sound. - unsafe { core::mem::transmute_copy(&v) } + unsafe { crate::mem::transmute(v) } } /// # Safety @@ -149,7 +149,7 @@ const fn into_repr(v: T) -> T::Repr { #[inline(always)] const unsafe fn from_repr(r: T::Repr) -> T { // SAFETY: Per the safety requirement of the function, the transmute operation is sound. - unsafe { core::mem::transmute_copy(&r) } + unsafe { crate::mem::transmute(r) } } impl Atomic { -- cgit v1.2.3 From ef605a433d80987e97fcf62a41bdf437abc61e2c Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:27 +0100 Subject: rust: mem: add `AsRepr` and `AsReprMut` Some API like atomics and I/O operate on primitives only; therefore other types would need to converted to these primitive first. Add two traits `AsRepr` and `AsReprMut` to indicate that the type can be turned into a primitive for these operations. `T: AsRepr` means that `&T` can be viewed as `&T::Repr` and thus it needs to support transmutability in one direction. `T: AsReprMut` means that `&mut T` can be viewed as `&mut T::Repr` and thus it needs to support bi-directional transmutability. To avoid duplicating implementations, all repr types are normalized to unsigned integers. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Acked-by: Miguel Ojeda Link: https://patch.msgid.link/20260901-typed_register-v4-3-5552b1d59525@garyguo.net [ Fix grammar in from_repr_unchecked() documentation, rephrase confusing safety comment, use rustdoc links for Self, fix backtick formatting in comments and use consistent 'must' wording in safety sections. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/mem.rs | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs index 35426f4d514e..f2d4cdf87d00 100644 --- a/rust/kernel/mem.rs +++ b/rust/kernel/mem.rs @@ -100,3 +100,135 @@ pub const fn safe_transmute(val: Src) -> Dst { // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds. unsafe { transmute(val) } } + +/// Type that is layout-compatible with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must have the same size and alignment as [`Self::Repr`]. +/// - [`Self`] must be [transmutable] to [`Self::Repr`]. +/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability. +/// +/// The above basically says that `&Self` can be transmuted to `&Self::Repr`. +/// +/// [transmutable]: core::mem::transmute +pub unsafe trait AsRepr: Sized { + /// Primitive representation of this type. + type Repr; + + /// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr(this: &Self) -> &Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self`] to [`Self::Repr`]. + #[inline(always)] + fn into_repr(this: Self) -> Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(this) } + } + + /// Convert from [`Self::Repr`] to [`Self`]. + /// + /// # Safety + /// + /// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it. + /// + /// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each + /// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to + /// call as well (this means that we're undoing a `into_repr` call getting the exact bytes + /// back). + /// + /// No guarantee is made if the result of a `into_repr` is passed to multiple + /// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer + /// and the user of the API wants ownership transfer. Users that want the ability to call + /// `from_repr_unchecked` after copying can require `Copy` bound explicitly. + #[inline(always)] + unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from + // `into_repr`, in which case we're undoing the transmute so it is also safe. + unsafe { transmute(repr) } + } +} + +/// Type that is bi-directionally transmutable with a primitive representation. +/// +/// # Safety +/// +/// - [`Self`] must be [transmutable] from [`Self::Repr`]. +/// +/// [transmutable]: core::mem::transmute +/// [`Self::Repr`]: AsRepr::Repr +pub unsafe trait AsReprMut: AsRepr { + /// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr). + #[inline(always)] + fn as_repr_mut(this: &mut Self) -> &mut Self::Repr { + // SAFETY: Per safety requirement of the trait. + unsafe { core::mem::transmute(this) } + } + + /// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`. + #[inline(always)] + fn from_repr(repr: Self::Repr) -> Self { + // SAFETY: Per safety requirement of the trait. + unsafe { transmute(repr) } + } +} + +// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`. +// Neither types contain interior mutability. +unsafe impl AsRepr for bool { + type Repr = u8; +} + +// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is +// transmutable to `*const c_void`. Neither types contain interior mutability. +unsafe impl AsRepr for *mut T { + type Repr = *const c_void; +} + +// SAFETY: `*mut T` is transmutable from `*const c_void`. +unsafe impl AsReprMut for *mut T {} + +// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to +// `*const c_void`. Neither types contain interior mutability. +unsafe impl AsRepr for *const T { + type Repr = *const c_void; +} + +// SAFETY: `*const T` is transmutable from `*const c_void`. +unsafe impl AsReprMut for *const T {} + +macro_rules! int_impl { + ($($unsigned:ident $signed:ident ,)*) => {$( + // SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to + // itself. It does not contain interior mutability. + unsafe impl AsRepr for $unsigned { + type Repr = $unsigned; + } + + // SAFETY: `$unsigned` is transmutable from itself. + unsafe impl AsReprMut for $unsigned {} + + // SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable + // to it Neither types contain interior mutability. + unsafe impl AsRepr for $signed { + type Repr = $unsigned; + } + + // SAFETY: `$signed` is transmutable from `$unsigned`. + unsafe impl AsReprMut for $signed {} + )*}; +} + +int_impl! { + u8 i8, + u16 i16, + u32 i32, + u64 i64, + // `usize` is not normalized to particular integer for portability. + usize isize, +} -- cgit v1.2.3 From ff067ace5fbeada7e629d4aaccfa1d04da3eeebc Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:28 +0100 Subject: rust: io: perform conversions using `AsRepr` For types that are layout-compatible with an I/O capable type, we would want the ability to use them directly for I/O operations. E.g. bitfield! { pub struct Foo(u32) { ... } } #[repr(C)] struct Bar { foo: Foo, } let mmio: Mmio<'_, Bar> = ...; io_read!(mmio, .foo) Currently this feature is available from `register!()` macro but not otherwise available with `io_read!`, `io_write!`. Support this by performing conversions to I/O primitives via the `AsRepr`/`AsReprMut` trait. This makes the `IoLoc::IoType` and `Register::Storage` redundant; thus remove them; also convert register methods to use the `read_val` and `write_val` instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-4-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/bitfield.rs | 9 ++++ rust/kernel/io.rs | 128 ++++++++++++++++++++++++--------------------- rust/kernel/io/register.rs | 15 ------ rust/macros/io/register.rs | 2 - 4 files changed, 78 insertions(+), 76 deletions(-) diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs index a0d089423f21..15c78790e151 100644 --- a/rust/kernel/bitfield.rs +++ b/rust/kernel/bitfield.rs @@ -346,6 +346,15 @@ macro_rules! bitfield { Self::from_raw(val) } } + + // SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability. + unsafe impl $crate::mem::AsRepr for $name { + // Normalize `$storage` to the canonical repr type in case it is signed. + type Repr = <$storage as $crate::mem::AsRepr>::Repr; + } + + // SAFETY: `$name` is transparent over `$storage`. + unsafe impl $crate::mem::AsReprMut for $name {} }; // Definitions requiring knowledge of individual fields: private and public field accessors, diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 5ce9fd129068..18c71bd765b2 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -11,6 +11,10 @@ use core::{ use crate::{ bindings, + mem::{ + AsRepr, + AsReprMut, // + }, prelude::*, ptr::{ Alignment, @@ -226,6 +230,17 @@ fn io_view<'a, IO: Io<'a>, U>( Ok(unsafe { IO::Backend::project_view(view, projected_ptr) }) } +/// Returns the primitive view of a I/O view. +#[inline] +fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>( + this: IO, +) -> ::View<'a, T::Repr> { + let view = this.as_view(); + + // SAFETY: `AsRepr` guarantees layout compatibility. + unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::()) } +} + /// I/O backends. /// /// This is an abstract representation to be implemented by arbitrary I/O @@ -353,15 +368,12 @@ pub trait IoCopyable: IoBackend { /// /// - The valid `Base` to operate on. For most registers, this should be [`Region`]. /// - The offset to access (returned by [`IoLoc::offset`]), -/// - The width of the access (determined by [`IoLoc::IoType`]), -/// - The type `T` in which the raw data is returned or provided. +/// - The type `T` in which the data is returned or provided. /// -/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type -/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`). +/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T: +/// AsRepr` bound and the `::Repr` type would be used to perform I/O and converted to +/// `T` instead. pub trait IoLoc { - /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset). - type IoType: Into + From; - /// Consumes `self` and returns the offset of this location. fn offset(self) -> usize; } @@ -372,8 +384,6 @@ macro_rules! impl_usize_ioloc { ($($ty:ty),*) => { $( impl IoLoc, $ty> for usize { - type IoType = $ty; - #[inline(always)] fn offset(self) -> usize { self @@ -498,10 +508,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline] fn read_val(self) -> Self::Target where - Self::Backend: IoCapable, - Self::Target: Sized, + Self::Target: AsReprMut, + Self::Backend: IoCapable<::Repr>, { - Self::Backend::io_read(self.as_view()) + Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self))) } /// Write a value to I/O. @@ -520,10 +530,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline] fn write_val(self, value: Self::Target) where - Self::Backend: IoCapable, - Self::Target: Sized, + Self::Target: AsRepr, + Self::Backend: IoCapable<::Repr>, { - Self::Backend::io_write(self.as_view(), value) + Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value)) } /// Copy-read from I/O memory. @@ -645,7 +655,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read8(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -655,7 +665,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read16(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -665,7 +675,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read32(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -675,7 +685,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read64(self, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_read(offset) @@ -685,7 +695,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write8(self, value: u8, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -695,7 +705,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write16(self, value: u16, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -705,7 +715,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write32(self, value: u32, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -715,7 +725,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write64(self, value: u64, offset: usize) -> Result where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.try_write(offset, value) @@ -727,7 +737,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read8(self, offset: usize) -> u8 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -739,7 +749,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read16(self, offset: usize) -> u16 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -751,7 +761,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read32(self, offset: usize) -> u32 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -763,7 +773,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read64(self, offset: usize) -> u64 where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.read(offset) @@ -775,7 +785,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write8(self, value: u8, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -787,7 +797,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write16(self, value: u16, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -799,7 +809,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write32(self, value: u32, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -811,7 +821,7 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write64(self, value: u64, offset: usize) where - usize: IoLoc, + usize: IoLoc, Self::Backend: IoCapable, { self.write(offset, value) @@ -843,11 +853,12 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_read(self, location: L) -> Result where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - Ok(Self::Backend::io_read(view).into()) + let view = io_view::(self, location.offset())?; + Ok(view.read_val()) } /// Generic fallible write with runtime bounds check. @@ -876,12 +887,12 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write(self, location: L, value: T) -> Result where + T: AsRepr, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + let view = io_view::(self, location.offset())?; + view.write_val(value); Ok(()) } @@ -920,9 +931,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_write_reg(self, value: V) -> Result where + T: AsRepr, L: IoLoc, V: LocatedRegister, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { let (location, value) = value.into_io_op(); @@ -954,16 +966,13 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn try_update(self, location: L, f: F) -> Result where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view::(self, location.offset())?; - - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); - + let view = io_view::(self, location.offset())?; + view.write_val(f(view.read_val())); Ok(()) } @@ -991,11 +1000,12 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn read(self, location: L) -> T where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - Self::Backend::io_read(view).into() + let view = io_view_assert::(self, location.offset()); + view.read_val() } /// Generic infallible write with compile-time bounds check. @@ -1022,12 +1032,12 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write(self, location: L, value: T) where + T: AsRepr, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - let io_value = value.into(); - Self::Backend::io_write(view, io_value); + let view = io_view_assert::(self, location.offset()); + view.write_val(value) } /// Generic infallible write of a fully-located register value. @@ -1064,9 +1074,10 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn write_reg(self, value: V) where + T: AsRepr, L: IoLoc, V: LocatedRegister, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, { let (location, value) = value.into_io_op(); @@ -1098,14 +1109,13 @@ pub trait Io<'a>: IoBase<'a> { #[inline(always)] fn update(self, location: L, f: F) where + T: AsReprMut, L: IoLoc, - Self::Backend: IoCapable, + Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view_assert::(self, location.offset()); - let value: T = Self::Backend::io_read(view).into(); - let io_value = f(value).into(); - Self::Backend::io_write(view, io_value); + let view = io_view_assert::(self, location.offset()); + view.write_val(f(view.read_val())); } } diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 6a19552ffb95..45d2e73a92e7 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -117,9 +117,6 @@ use super::Region; /// Trait implemented by all registers. pub trait Register: Sized { - /// Backing primitive type of the register. - type Storage: Into + From; - /// Start offset of the register. /// /// The interpretation of this offset depends on the type of the register. @@ -135,8 +132,6 @@ impl IoLoc, T> for () where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -149,8 +144,6 @@ impl IoLoc, T> for T where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -174,8 +167,6 @@ impl IoLoc, T> for FixedRegisterLoc where T: FixedRegister, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET @@ -246,8 +237,6 @@ where T: RelativeRegister, B: RegisterBase + ?Sized, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { RelativeRegisterLoc::offset(self) @@ -289,8 +278,6 @@ impl IoLoc, T> for RegisterArrayLoc where T: RegisterArray, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { T::OFFSET + self.0 * T::STRIDE @@ -377,8 +364,6 @@ where T: RelativeRegisterArray, B: RegisterBase + ?Sized, { - type IoType = T::Storage; - #[inline(always)] fn offset(self) -> usize { self.0.offset() + self.1 * T::STRIDE diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index cc5bcad31ae8..2e4cdabb2795 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -213,8 +213,6 @@ pub(crate) fn register(def: RegDef) -> Result { ); impl ::kernel::io::register::Register for #name { - type Storage = #storage; - const OFFSET: usize = #offset; } )); -- cgit v1.2.3 From 2a750c56179b416f3501c2644f787265ad7a6785 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:29 +0100 Subject: rust: io: support register projections `IoLoc`s themselves just describe a projection from a region to a concrete register. Thus, support it in `io_project` macro too. Also, update methods that operate on `IoLoc` to use I/O projection. Documentation of `io_project!` is not expanded yet as the example works better when `register!` type can specify base type. `io_read!` and `io_write!` gains the ability to operate on registers as corollary of the capability of `io_project!`. Examples are not added because `read` and `write` is still preferably used instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-5-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 18c71bd765b2..8df268969566 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -857,8 +857,7 @@ pub trait Io<'a>: IoBase<'a> { L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - Ok(view.read_val()) + Ok(io_read!(self, try: location)) } /// Generic fallible write with runtime bounds check. @@ -891,8 +890,7 @@ pub trait Io<'a>: IoBase<'a> { L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view::(self, location.offset())?; - view.write_val(value); + io_write!(self, try: location, value); Ok(()) } @@ -971,7 +969,7 @@ pub trait Io<'a>: IoBase<'a> { Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view::(self, location.offset())?; + let view = io_project!(self, try: location); view.write_val(f(view.read_val())); Ok(()) } @@ -1004,8 +1002,7 @@ pub trait Io<'a>: IoBase<'a> { L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - view.read_val() + io_read!(self, build: location) } /// Generic infallible write with compile-time bounds check. @@ -1036,8 +1033,7 @@ pub trait Io<'a>: IoBase<'a> { L: IoLoc, Self::Backend: IoCapable<::Repr>, { - let view = io_view_assert::(self, location.offset()); - view.write_val(value) + io_write!(self, build: location, value); } /// Generic infallible write of a fully-located register value. @@ -1114,7 +1110,7 @@ pub trait Io<'a>: IoBase<'a> { Self::Backend: IoCapable<::Repr>, F: FnOnce(T) -> T, { - let view = io_view_assert::(self, location.offset()); + let view = io_project!(self, build: location); view.write_val(f(view.read_val())); } } @@ -1659,6 +1655,25 @@ where // SAFETY: Per safety requirement. unsafe { T::Backend::project_view::(self.0, ptr) } } + + #[inline(always)] + pub fn try_project_loc( + self, + location: L, + ) -> Result<::View<'a, U>> + where + L: IoLoc, + { + io_view::<_, U>(self.0, location.offset()) + } + + #[inline(always)] + pub fn project_loc(self, location: L) -> ::View<'a, U> + where + L: IoLoc, + { + io_view_assert::<_, U>(self.0, location.offset()) + } } /// Project an I/O type to a subview of it. @@ -1686,6 +1701,21 @@ where #[macro_export] #[doc(hidden)] macro_rules! io_project { + // Register projection + ($io:expr, try: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.try_project_loc($ioloc)? + }}; + ($io:expr, build: $ioloc:expr) => {{ + #[allow(unused)] + use $crate::io::IoBase as _; + let view = $crate::io::ProjectHelper($io.as_view()); + view.project_loc($ioloc) + }}; + + // Field or index projection ($io:expr, $($proj:tt)*) => {{ #[allow(unused)] use $crate::io::IoBase as _; @@ -1756,6 +1786,12 @@ macro_rules! io_write { (@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => { $crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*]) }; + (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*]) + }; + (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => { + $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*]) + }; ($io:expr, $($rest:tt)*) => { $crate::io_write!(@parse [$io] [] [$($rest)*]) }; -- cgit v1.2.3 From 12c78173d8b21fd53cf98e456c14b005e6a3675e Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:30 +0100 Subject: rust: io: register: allow explicit base type specification Currently registers work for all untyped I/O regions, which is not ideal. It allows registers defined for device A to work for another device B and there is no safeguarding at all. All users of the `register!` macro know what type it will be operating on, and that type is consistent across the driver. Therefore, add a `base` parameter to `register!`. Currently this parameter is unused in the generated code; it will be used when all users of `register!` is converted to gain the parameter. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-6-5552b1d59525@garyguo.net [ Remove unnecessary #[allow(unused)] from the 'base' field. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 4 ++++ rust/kernel/io/register.rs | 51 +++++++++++++++++++++++++++++++++++++++------- rust/macros/io/register.rs | 22 +++++++++++++++++++- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 8df268969566..05c7fa12fdbe 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -909,6 +909,8 @@ pub trait Io<'a>: IoBase<'a> { /// }; /// /// register! { + /// base: Region; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; @@ -1051,6 +1053,8 @@ pub trait Io<'a>: IoBase<'a> { /// }; /// /// register! { + /// base: Region<0x1000>; + /// /// VERSION(u32) @ 0x100 { /// 15:8 major; /// 7:0 minor; diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 45d2e73a92e7..49a5fe069e47 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -13,9 +13,14 @@ //! # Simple example //! //! ```no_run -//! use kernel::io::register; +//! use kernel::io::{ +//! register, +//! Region, +//! }; //! //! register! { +//! base: Region<0x1000>; +//! //! /// Basic information about the chip. //! pub BOOT_0(u32) @ 0x00000100 { //! /// Vendor ID. @@ -55,11 +60,14 @@ //! register, //! Io, //! IoLoc, +//! Region, //! }, //! num::Bounded, //! }; -//! # use kernel::io::{Mmio, Region}; +//! # use kernel::io::Mmio; //! # register! { +//! # base: Region<0x1000>; +//! # //! # pub BOOT_0(u32) @ 0x00000100 { //! # 15:8 vendor_id; //! # 7:4 major_revision; @@ -429,11 +437,14 @@ where /// io::{ /// register, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// /// register! { +/// base: Region<0x1000>; +/// /// FIXED_REG(u32) @ 0x100 { /// 15:8 high_byte; /// 7:0 low_byte; @@ -464,9 +475,14 @@ where /// the context: /// /// ```no_run -/// use kernel::io::register; +/// use kernel::io::{ +/// register, +/// Region, +/// }; /// /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch register. /// pub SCRATCH(u32) @ 0x00000200 { /// 31:0 value; @@ -516,6 +532,7 @@ where /// /// ```ignore /// register! { +/// ... /// pub RELATIVE_REG(u32) @ Base + 0x80 { /// ... /// } @@ -542,9 +559,10 @@ where /// WithBase, /// }, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// /// // Type used to identify the base. /// pub struct CpuCtlBase; @@ -563,6 +581,8 @@ where /// /// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. /// register! { +/// base: Region<0x1000>; +/// /// /// CPU core control. /// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { /// 0:0 start; @@ -579,6 +599,8 @@ where /// /// // Aliases can also be defined for relative register. /// register! { +/// base: Region<0x1000>; +/// /// /// Alias to CPU core control. /// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { /// /// Start the aliased CPU core. @@ -621,15 +643,18 @@ where /// register, /// register::Array, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } /// /// // Array of 64 consecutive registers with the same layout starting at offset `0x80`. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers. /// pub SCRATCH(u32)[64] @ 0x00000080 { /// 31:0 value; @@ -655,6 +680,8 @@ where /// // Alias to a specific register in an array. /// // Here `SCRATCH[8]` is used to convey the firmware exit code. /// register! { +/// base: Region<0x1000>; +/// /// /// Firmware exit status code. /// pub FIRMWARE_STATUS(u32) => SCRATCH[8] { /// 7:0 status; @@ -667,6 +694,8 @@ where /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the /// // registers of the two declarations below are interleaved. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers bank 0. /// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 { /// 31:0 value; @@ -688,6 +717,7 @@ where /// /// ```ignore /// register! { +/// ... /// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 { /// ... /// } @@ -707,9 +737,10 @@ where /// WithBase, /// }, /// Io, +/// Region, /// }, /// }; -/// # use kernel::io::{Mmio, Region}; +/// # use kernel::io::Mmio; /// # fn get_scratch_idx() -> usize { /// # 0x15 /// # } @@ -731,6 +762,8 @@ where /// /// // 64 per-cpu scratch registers, arranged as a contiguous array. /// register! { +/// base: Region<0x1000>; +/// /// /// Per-CPU scratch registers. /// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { /// 31:0 value; @@ -758,6 +791,8 @@ where /// /// // Alias to `SCRATCH[8]` used to convey the firmware exit code. /// register! { +/// base: Region<0x1000>; +/// /// /// Per-CPU firmware exit status code. /// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { /// 7:0 status; @@ -768,6 +803,8 @@ where /// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the /// // registers of the two declarations below are interleaved. /// register! { +/// base: Region<0x1000>; +/// /// /// Scratch registers bank 0. /// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { /// 31:0 value; diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 2e4cdabb2795..403f56692f6b 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -29,6 +29,7 @@ use syn::{ }; mod kw { + syn::custom_keyword!(base); syn::custom_keyword!(stride); } @@ -152,22 +153,41 @@ impl Parse for Reg { } pub(crate) struct RegDef { + base: Option, regs: Vec, } impl Parse for RegDef { fn parse(input: syn::parse::ParseStream<'_>) -> Result { + let base = if input.peek(kw::base) { + let _: kw::base = input.parse()?; + let _: Token![:] = input.parse()?; + let base = input.parse()?; + let _: Token![;] = input.parse()?; + Some(base) + } else { + None + }; let mut regs = Vec::new(); while !input.is_empty() { regs.push(input.parse()?); } - Ok(RegDef { regs }) + Ok(RegDef { base, regs }) } } pub(crate) fn register(def: RegDef) -> Result { let mut outputs = TokenStream::new(); + if let Some(base) = &def.base { + outputs.extend(quote_spanned!(base.span() => + const _: () = { + #[allow(unused)] + type Base = #base; + }; + )); + } + for reg in def.regs { let Reg { attrs, -- cgit v1.2.3 From f7477c8324041757d2969d230be67e77a9334437 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:31 +0100 Subject: gpu: nova-core: specify base type for registers All registers use the same base type, which is `::Target`. Thus add the base parameter to `register!` invocation. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-7-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/driver.rs | 1 + drivers/gpu/nova-core/fb/regs.rs | 12 ++++++++++++ drivers/gpu/nova-core/gsp/regs.rs | 9 ++++++++- drivers/gpu/nova-core/regs.rs | 37 ++++++++++++++++++++++++++++++++++++- drivers/gpu/nova-core/vbios.rs | 11 ++++++++++- 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index bbd93959e0b2..cf3534dd47d4 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -37,6 +37,7 @@ pub(crate) struct NovaCoreDriver; const BAR0_SIZE: usize = SZ_16M; pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; +pub(crate) type NovaRegisters = kernel::io::Region; kernel::pci_device_table!( PCI_TABLE, diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index 95adbe124a30..c27582e376e2 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -5,9 +5,13 @@ use kernel::{ sizes::SizeConstants, // }; +use crate::driver::NovaRegisters; + // PDISP register! { + base: NovaRegisters; + pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { /// VGA workspace base address divided by 0x10000. 31:8 addr; @@ -30,6 +34,8 @@ impl NV_PDISP_VGA_WORKSPACE_BASE { // PFB register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform sysmembar /// operations (see [`crate::fb::SysmemFlush`]). pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { @@ -65,6 +71,8 @@ register! { pub(super) struct Hshub0Base(()); register! { + base: NovaRegisters; + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed @@ -87,6 +95,8 @@ register! { } register! { + base: NovaRegisters; + // GB20x FBHUB0 sysmem flush registers. Unlike the older // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an // 8-bit right-shift, these take the raw address split into lower and upper @@ -101,6 +111,8 @@ register! { } register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform /// sysmembar operations on Hopper. /// diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs index 9a48aa87e7fb..3c410d65e8e4 100644 --- a/drivers/gpu/nova-core/gsp/regs.rs +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -2,11 +2,16 @@ use kernel::io::register; -use crate::regs::NV_PBUS_SW_SCRATCH; +use crate::{ + driver::NovaRegisters, + regs::NV_PBUS_SW_SCRATCH, // +}; // PGSP register! { + base: NovaRegisters; + pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { 31:0 address; } @@ -15,6 +20,8 @@ register! { // PBUS register! { + base: NovaRegisters; + /// Scratch register 0xe used as FRTS firmware error code. pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { 31:16 frts_err_code; diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index caeef4d85874..1af073f3861f 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -13,7 +13,10 @@ use kernel::{ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ DmaTrfCmdSize, FalconCoreRev, @@ -37,6 +40,8 @@ use crate::{ // PMC register! { + base: NovaRegisters; + /// Basic revision information about the GPU. pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 { /// Lower bits of the architecture. @@ -108,6 +113,8 @@ impl kernel::fmt::Display for NV_PMC_BOOT_42 { // PBUS register! { + base: NovaRegisters; + pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} } @@ -121,6 +128,8 @@ register! { // number. register! { + base: NovaRegisters; + /// Boot Sequence Interface (BSI) register used to determine /// if GSP reload/resume has completed during the boot process. pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 { @@ -175,6 +184,8 @@ impl NV_USABLE_FB_SIZE_IN_MB { pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; register! { + base: NovaRegisters; + pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 { 15:0 data => u16; } @@ -191,6 +202,8 @@ register! { // PFALCON register! { + base: NovaRegisters; + pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; @@ -392,6 +405,8 @@ impl NV_PFALCON_FALCON_HWCFG2 { /* PFALCON2 */ register! { + base: NovaRegisters; + pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 { 7:0 algo ?=> FalconModSelAlgo; } @@ -414,6 +429,8 @@ register! { // PRISCV register! { + base: NovaRegisters; + /// RISC-V status register for debug (Turing and GA100 only). /// Reflects current RISC-V core status. pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 { @@ -439,6 +456,8 @@ register! { // These registers manage falcon EMEM communication queues. register! { + base: NovaRegisters; + pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { 31:0 address => u32; } @@ -462,9 +481,13 @@ register! { pub(crate) mod gm107 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 { 0:0 display_disabled => bool; } @@ -474,9 +497,13 @@ pub(crate) mod gm107 { pub(crate) mod ga100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 { 0:0 display_disabled => bool; } @@ -488,9 +515,13 @@ pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0 pub(crate) mod gh100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { 31:0 data; } @@ -505,9 +536,13 @@ pub(crate) mod gh100 { pub(crate) mod gb202 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { 31:0 data; } diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c03650ee5226..9c214b9f4dd9 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -16,7 +16,10 @@ use kernel::{ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, firmware::{ fwsec::Bcrt30Rsa3kSignature, FalconUCodeDesc, @@ -92,12 +95,16 @@ impl<'a> VbiosIterator<'a> { fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result { // IFR Header in VBIOS. register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { 31:0 signature; } } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { 30:16 fixed_data_size; 15:8 version => u8; @@ -105,6 +112,8 @@ impl<'a> VbiosIterator<'a> { } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { 19:0 total_data_size; } -- cgit v1.2.3 From be5ffd2696e4e0359ba78fde75eef739befa4fab Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:32 +0100 Subject: drm/tyr: specify base type for registers All registers use the same base type, which is `::Target`. Thus add the base parameter to `register!` invocation. Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-8-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/drm/tyr/driver.rs | 1 + drivers/gpu/drm/tyr/fw.rs | 5 ++++- drivers/gpu/drm/tyr/regs.rs | 45 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index bfb0ba19caff..730b84e37a54 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -46,6 +46,7 @@ use crate::{ }; pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>; +pub(crate) type TyrRegisters = kernel::io::Region; pub(crate) struct TyrDrmDriver; diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs index 47d25c901bd0..7edb5eff1707 100644 --- a/drivers/gpu/drm/tyr/fw.rs +++ b/drivers/gpu/drm/tyr/fw.rs @@ -39,7 +39,8 @@ use kernel::{ use crate::{ driver::{ IoMem, - TyrDrmDevice, // + TyrDrmDevice, + TyrRegisters, // }, fw::parser::{ FwParser, @@ -101,6 +102,8 @@ impl From for Bounded { } register! { + base: TyrRegisters; + #[allow(non_upper_case_globals)] pub(super) SectionFlags(u32) @ 0x0 { 0:0 read => bool; diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index a62724378ced..0c419c4e1186 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -57,7 +57,11 @@ pub(crate) mod gpu_control { uapi, // }; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// GPU identification register. pub(crate) GPU_ID(u32) @ 0x0 { /// Status of the GPU release. @@ -315,6 +319,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU command register. /// /// Use the constructor methods to create commands: @@ -380,6 +386,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU status register. Read only. pub(crate) GPU_STATUS(u32) @ 0x34 { /// GPU active, a 1-bit boolean flag. @@ -463,6 +471,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU fault status register. Read only. pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { /// Exception type. @@ -768,6 +778,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// Coherency enable. An index of which coherency protocols should be used. /// This register only selects the protocol for coherency messages on the /// interconnect. This is not to enable or disable coherency controlled by MMU. @@ -808,6 +820,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// MCU control. pub(crate) MCU_CONTROL(u32) @ 0x700 { /// Request MCU state change. @@ -849,6 +863,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// MCU status. Read only. pub(crate) MCU_STATUS(u32) @ 0x704 { /// Read current state of MCU. @@ -862,7 +878,11 @@ pub(crate) mod gpu_control { pub(crate) mod job_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// Raw status of job interrupts. /// /// Write to this register to trigger these interrupts. @@ -912,7 +932,11 @@ pub(crate) mod job_control { pub(crate) mod mmu_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// IRQ sources raw status. /// /// This register contains the raw unmasked interrupt sources for MMU status and exception @@ -966,9 +990,10 @@ pub(crate) mod mmu_control { prelude::*, register, // }; - use pin_init::Zeroable; + use crate::driver::TyrRegisters; + /// Maximum number of hardware address space slots. /// The actual number of slots available is usually lower. pub(crate) const MAX_AS: usize = 16; @@ -977,6 +1002,8 @@ pub(crate) mod mmu_control { const STRIDE: usize = 0x40; register! { + base: TyrRegisters; + /// Translation table base address. A 64-bit pointer. /// /// This field contains the address of the top level of a translation table structure. @@ -1104,6 +1131,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Stage 1 memory attributes (8-bit bitfield). /// /// This is not an actual register, but a bitfield definition used by the MEMATTR @@ -1137,6 +1166,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Memory attributes. /// /// Each address space can configure up to 8 different memory attribute profiles. @@ -1292,6 +1323,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Lock region address for each address space. pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { /// Lock region size. @@ -1353,6 +1386,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// MMU command register for each address space. Write only. pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 7:0 command ?=> MmuCommand; @@ -1480,6 +1515,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Fault status register for each address space. Read only. pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { /// Exception type. @@ -1705,6 +1742,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Translation configuration and control. pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { /// Address space mode. @@ -1760,6 +1799,8 @@ pub(crate) mod mmu_control { pub(crate) mod doorbell_block { use kernel::register; + use crate::driver::TyrRegisters; + /// Number of doorbells available. pub(crate) const NUM_DOORBELLS: usize = 64; @@ -1770,6 +1811,8 @@ pub(crate) mod doorbell_block { const STRIDE: usize = 0x10000; register! { + base: TyrRegisters; + /// Doorbell request register. Write-only. pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { /// Doorbell set. Writing 1 triggers the doorbell. -- cgit v1.2.3 From 9b27311fa940839b0839e5a96c27ffa24d8374cf Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:33 +0100 Subject: samples: rust: pci: specify base type for registers The `register!` macro is going to require explicit base type, specify it for both `register!` usages in PCI sample driver. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-9-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- samples/rust/rust_driver_pci.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/rust/rust_driver_pci.rs b/samples/rust/rust_driver_pci.rs index 2282191e6292..13b035a95756 100644 --- a/samples/rust/rust_driver_pci.rs +++ b/samples/rust/rust_driver_pci.rs @@ -23,6 +23,8 @@ mod regs { use super::*; register! { + base: kernel::io::Region; + pub(super) TEST(u8) @ 0x0 { 7:0 index => TestIndex; } @@ -102,6 +104,8 @@ impl SampleDriverData<'_> { // Some PCI configuration space registers. register! { + base: pci::Normal; + VENDOR_ID(u16) @ 0x0 { 15:0 vendor_id; } -- cgit v1.2.3 From 4f620881808e9d3d951343e2edc9c8c41286391a Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:34 +0100 Subject: rust: io: register: make register have a typed base Previously `register!` defined registers can be used on any untyped I/O regions. With all users specifying their desired register type now, propagate the specified type and restrict I/O access only when type matches. Also, add an `io_project!` example which is enabled by this change. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-10-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io.rs | 13 +++++++++++ rust/kernel/io/register.rs | 54 ++++++++++++++++++++++++++++++++-------------- rust/macros/io/register.rs | 44 +++++++++++++++---------------------- 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 05c7fa12fdbe..7a0e0ebc6c85 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -1685,21 +1685,34 @@ where /// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that /// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!). /// +/// `io_project!` can also project to a subview of registers defined with [`register!`] macro. +/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and +/// `io_project!(io, build: REGISTER)` for infallible projection. +/// /// # Examples /// /// ``` /// use kernel::io::{ /// io_project, +/// register, /// Mmio, /// }; /// #[repr(C)] /// struct MyStruct { field: u32, } /// +/// register! { +/// base: MyStruct; +/// FIELD(u32) @ 0 { +/// 31:0 val; +/// } +/// } +/// /// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result { /// // let mmio: Mmio<[MyStruct]>; /// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field); /// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]); /// let nested: Mmio<'_, u32> = io_project!(whole, .field); +/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD); /// # Ok::<(), Error>(()) } /// ``` #[macro_export] diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 49a5fe069e47..58dcf651dfd0 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,10 +121,11 @@ use crate::{ io::IoLoc, // }; -use super::Region; - /// Trait implemented by all registers. pub trait Register: Sized { + /// Base type for this register. + type Base: ?Sized; + /// Start offset of the register. /// /// The interpretation of this offset depends on the type of the register. @@ -136,9 +137,9 @@ pub trait FixedRegister: Register {} /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. -impl IoLoc, T> for () +impl IoLoc for () where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -148,9 +149,9 @@ where /// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used /// as an [`IoLoc`]. -impl IoLoc, T> for T +impl IoLoc for T where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -171,9 +172,9 @@ impl FixedRegisterLoc { } } -impl IoLoc, T> for FixedRegisterLoc +impl IoLoc for FixedRegisterLoc where - T: FixedRegister, + T: FixedRegister, { #[inline(always)] fn offset(self) -> usize { @@ -240,9 +241,9 @@ where } } -impl IoLoc, T> for RelativeRegisterLoc +impl IoLoc for RelativeRegisterLoc where - T: RelativeRegister, + T: RelativeRegister, B: RegisterBase + ?Sized, { #[inline(always)] @@ -282,9 +283,9 @@ impl RegisterArrayLoc { } } -impl IoLoc, T> for RegisterArrayLoc +impl IoLoc for RegisterArrayLoc where - T: RegisterArray, + T: RegisterArray, { #[inline(always)] fn offset(self) -> usize { @@ -367,9 +368,9 @@ where } } -impl IoLoc, T> for RelativeRegisterArrayLoc +impl IoLoc for RelativeRegisterArrayLoc where - T: RelativeRegisterArray, + T: RelativeRegisterArray, B: RegisterBase + ?Sized, { #[inline(always)] @@ -393,9 +394,9 @@ pub trait LocatedRegister { fn into_io_op(self) -> (Self::Location, Self::Value); } -impl LocatedRegister> for T +impl LocatedRegister for T where - T: FixedRegister, + T: FixedRegister, { type Location = FixedRegisterLoc; type Value = T; @@ -406,6 +407,27 @@ where } } +/// Helper function for register alias implementation. +/// +/// This is used to enforce base matching. +#[doc(hidden)] +#[inline(always)] // for const eval only +pub const fn alias_offset>() -> usize { + Alias::OFFSET +} + +/// Helper function for register element alias implementation. +/// +/// This is used to enforce base matching and provide bounds checking. +#[doc(hidden)] +#[inline(always)] // for const eval only +pub const fn element_alias_offset>( + idx: usize, +) -> usize { + assert!(idx < Alias::SIZE); + Alias::OFFSET + idx * Alias::STRIDE +} + /// Defines a dedicated type for a register, including getter and setter methods for its fields and /// methods to read and write it from an [`Io`](kernel::io::Io) region. /// diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 403f56692f6b..a05671cba2dc 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -19,6 +19,7 @@ use syn::{ spanned::Spanned, token, Attribute, + Error, Expr, Ident, Path, @@ -153,21 +154,23 @@ impl Parse for Reg { } pub(crate) struct RegDef { - base: Option, + base: Type, regs: Vec, } impl Parse for RegDef { fn parse(input: syn::parse::ParseStream<'_>) -> Result { - let base = if input.peek(kw::base) { - let _: kw::base = input.parse()?; - let _: Token![:] = input.parse()?; - let base = input.parse()?; - let _: Token![;] = input.parse()?; - Some(base) - } else { - None - }; + let _: kw::base = input.parse().map_err(|e| { + Error::new( + e.span(), + "a base type needs to be specified for `register!` invocation with `base: ty;`", + ) + })?; + + let _: Token![:] = input.parse()?; + let base = input.parse()?; + let _: Token![;] = input.parse()?; + let mut regs = Vec::new(); while !input.is_empty() { regs.push(input.parse()?); @@ -179,15 +182,7 @@ impl Parse for RegDef { pub(crate) fn register(def: RegDef) -> Result { let mut outputs = TokenStream::new(); - if let Some(base) = &def.base { - outputs.extend(quote_spanned!(base.span() => - const _: () = { - #[allow(unused)] - type Base = #base; - }; - )); - } - + let base = &def.base; for reg in def.regs { let Reg { attrs, @@ -208,18 +203,12 @@ pub(crate) fn register(def: RegDef) -> Result { RegOffset::Fixed { offset } => quote!(#offset), RegOffset::Alias { alias } => { quote_spanned!(alias.span().resolved_at(span) => - <#alias as ::kernel::io::register::Register>::OFFSET + ::kernel::io::register::alias_offset::<#base, #alias>() ) } RegOffset::ElementAlias { alias, idx } => { - outputs.extend(quote_spanned!(idx.span().resolved_at(span) => - ::kernel::build_assert::static_assert!( - (#idx) < <#alias as ::kernel::io::register::RegisterArray>::SIZE - ); - )); quote_spanned!(alias.span().resolved_at(span) => - <#alias as ::kernel::io::register::Register>::OFFSET - + (#idx) * <#alias as ::kernel::io::register::RegisterArray>::STRIDE + ::kernel::io::register::element_alias_offset::<#base, #alias>(#idx) ) } }; @@ -233,6 +222,7 @@ pub(crate) fn register(def: RegDef) -> Result { ); impl ::kernel::io::register::Register for #name { + type Base = #base; const OFFSET: usize = #offset; } )); -- cgit v1.2.3 From c09d3a26e1b810d43ee8a15c4f09ee752fee55a6 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:35 +0100 Subject: rust: io: register: support fixed offset register without bitfield Add a rule to allow creating `IoLoc` in `register!()` using an existing type and not create a bitfield. Add an example to demonstrate this for FIFO registers. This rule is also going to be used to create subregions for registers; the example of doing so will be added later when relative registers are removed. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-11-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 31 +++++++++++++++++ rust/macros/io/register.rs | 84 +++++++++++++++++++++++++++++++--------------- 2 files changed, 88 insertions(+), 27 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 58dcf651dfd0..b9cb4fce8628 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -182,6 +182,24 @@ where } } +// Provides a `IoLoc` impl that for a fixed offset. +#[doc(hidden)] +pub struct OffsetLoc(usize, PhantomData<(T, Base)>); + +impl OffsetLoc { + #[inline] + pub const fn new(offset: usize) -> Self { + Self(offset, PhantomData) + } +} + +impl IoLoc for OffsetLoc { + #[inline(always)] + fn offset(self) -> usize { + self.0 + } +} + /// Trait providing a base address to be added to the offset of a relative register to obtain /// its actual offset. /// @@ -520,6 +538,19 @@ pub const fn element_alias_offset; +/// +/// /// UART RX register. +/// pub UART_RX: u8 @ 0x100; +/// } +/// ``` +/// /// ## Relative registers /// /// Relative registers can be instantiated several times at a relative offset of a group of bases. diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index a05671cba2dc..e948b0959154 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -16,6 +16,7 @@ use syn::{ bracketed, parenthesized, parse::Parse, + parse_quote, spanned::Spanned, token, Attribute, @@ -62,11 +63,11 @@ struct Reg { attrs: Vec, vis: Visibility, name: Ident, - storage: Type, + ty: Type, array: Option, relative_base: Option, offset: RegOffset, - bitfield_args: Group, + bitfield: Option<(Type, Group)>, } impl Parse for Reg { @@ -74,11 +75,22 @@ impl Parse for Reg { let attrs = input.call(Attribute::parse_outer)?; let vis = input.parse()?; let name = input.parse()?; - let storage = { + + let lh = input.lookahead1(); + let (ty, bitfield_storage) = if lh.peek(Token![:]) { + let _: Token![:] = input.parse()?; + (input.parse()?, None) + } else if lh.peek(token::Paren) { let content; parenthesized!(content in input); - content.parse()? + let bitfield_storage = Some(content.parse()?); + + // For bitfields, bitfield macro will generate a type with the same name as `name`. + (parse_quote!(#name), bitfield_storage) + } else { + Err(lh.error())? }; + let array = if input.peek(token::Bracket) { let content; bracketed!(content in input); @@ -133,22 +145,28 @@ impl Parse for Reg { Err(lh.error())? }; - let lh = input.lookahead1(); - let bitfield_args = if lh.peek(token::Brace) { - input.parse()? + let bitfield = if let Some(storage) = bitfield_storage { + let lh = input.lookahead1(); + let args = if lh.peek(token::Brace) { + input.parse()? + } else { + Err(lh.error())? + }; + Some((storage, args)) } else { - Err(lh.error())? + let _: Token![;] = input.parse()?; + None }; Ok(Self { attrs, vis, name, - storage, + ty, array, relative_base, offset, - bitfield_args, + bitfield, }) } } @@ -188,11 +206,11 @@ pub(crate) fn register(def: RegDef) -> Result { attrs, vis, name, - storage, + ty, array, relative_base, offset, - bitfield_args, + bitfield, } = reg; // Use register name's span for generated code, so error messages (if any) can point to it @@ -213,21 +231,33 @@ pub(crate) fn register(def: RegDef) -> Result { } }; - outputs.extend(quote_spanned!(span => - ::kernel::bitfield!( - // `#[allow(non_camel_case_types)]` is added since register names typically use - // `SCREAMING_CASE`. - #[allow(non_camel_case_types)] - #(#attrs)* #vis struct #name(#storage) #bitfield_args - ); - - impl ::kernel::io::register::Register for #name { - type Base = #base; - const OFFSET: usize = #offset; - } - )); + if let Some((storage, args)) = &bitfield { + outputs.extend(quote_spanned!(span => + ::kernel::bitfield!( + // `#[allow(non_camel_case_types)]` is added since register names typically use + // `SCREAMING_CASE`. + #[allow(non_camel_case_types)] + #(#attrs)* #vis struct #name(#storage) #args + ); + + impl ::kernel::io::register::Register for #name { + type Base = #base; + const OFFSET: usize = #offset; + } + )); + } match array { + None if bitfield.is_none() && relative_base.is_none() => outputs.extend(quote!( + #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = + ::kernel::io::register::OffsetLoc::new(#offset); + )), + + _ if bitfield.is_none() => Err(Error::new_spanned( + ty, + "defining without bitfield is not yet supported for this type of register", + ))?, + None => match relative_base { None => outputs.extend(quote_spanned!(span => impl ::kernel::io::register::FixedRegister for #name {} @@ -249,12 +279,12 @@ pub(crate) fn register(def: RegDef) -> Result { let stride = if let Some(stride) = &def.stride { outputs.extend(quote_spanned!(stride.span().resolved_at(span) => ::kernel::build_assert::static_assert!( - ::core::mem::size_of::<#storage>() <= #stride + ::core::mem::size_of::<#ty>() <= #stride ); )); quote!(#stride) } else { - quote_spanned!(span => ::core::mem::size_of::<#storage>()) + quote_spanned!(span => ::core::mem::size_of::<#ty>()) }; outputs.extend(quote_spanned!(span => -- cgit v1.2.3 From ea79a50124315589da68cbcaf518455a142d6d23 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:36 +0100 Subject: gpu: nova-core: use projection for PFALCON and PFALCON2 registers Add fixed size region types `PFalconRegisters` and `PFalcon2Registers` and update PFALCON and PFALCON2 registers to be fixed register on them and not relative registers on `NovaRegisters`. Update `Falcon` struct to store projected views when constructing and access with `self.pfalcon` and `self.pfalcon2`. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-12-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/falcon.rs | 161 +++++++++------------ drivers/gpu/nova-core/falcon/fsp.rs | 63 ++++---- drivers/gpu/nova-core/falcon/gsp.rs | 51 ++++--- drivers/gpu/nova-core/falcon/hal/ga102.rs | 59 +++----- drivers/gpu/nova-core/falcon/hal/tu102.rs | 9 +- drivers/gpu/nova-core/falcon/sec2.rs | 37 +++-- drivers/gpu/nova-core/firmware/fwsec/bootloader.rs | 18 +-- drivers/gpu/nova-core/gsp/hal/tu102.rs | 7 +- drivers/gpu/nova-core/regs.rs | 92 ++++++------ 9 files changed, 239 insertions(+), 258 deletions(-) diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..e3af59fd57e3 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -14,13 +14,12 @@ use kernel::{ io::{ io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register::Array, Io, + Mmio, // }, prelude::*, + sizes::SZ_4K, time::Delta, }; @@ -165,18 +164,25 @@ bounded_enum! { } } -/// Type used to represent the `PFALCON` registers address base for a given falcon engine. -pub(crate) struct PFalconBase(()); +const PFALCON_REGION_SIZE: usize = SZ_4K; +const PFALCON2_REGION_SIZE: usize = SZ_4K; -/// Type used to represent the `PFALCON2` registers address base for a given falcon engine. -pub(crate) struct PFalcon2Base(()); +/// Type used to represent the `PFALCON` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalconRegisters([u8; PFALCON_REGION_SIZE]); + +/// Type used to represent the `PFALCON2` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalcon2Registers([u8; PFALCON2_REGION_SIZE]); /// Trait defining the parameters of a given Falcon engine. /// /// Each engine provides one base for `PFALCON` and `PFALCON2` registers. -pub(crate) trait FalconEngine: - Send + Sync + RegisterBase + RegisterBase + Sized -{ +pub(crate) trait FalconEngine: Send + Sync + Sized { + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, PFalconRegisters>; + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, PFalcon2Registers>; } /// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM) @@ -358,6 +364,9 @@ pub(crate) struct Falcon<'a, E: FalconEngine> { hal: KBox>, dev: &'a device::Device, bar: Bar0<'a>, + // TODO: make private + pub(crate) pfalcon: Mmio<'a, PFalconRegisters>, + pfalcon2: Mmio<'a, PFalcon2Registers>, } impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { @@ -371,19 +380,19 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { hal: hal::falcon_hal(chipset)?, dev, bar, + pfalcon: E::pfalcon(bar), + pfalcon2: E::pfalcon2(bar), }) } /// Resets DMA-related registers. pub(crate) fn dma_reset(&self) { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. @@ -392,10 +401,9 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.select_core(self)?; self.hal.reset_wait_mem_scrubbing(self)?; - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), - ); + self.pfalcon.write_reg(regs::NV_PFALCON_FALCON_RM::from( + self.bar.read(regs::NV_PMC_BOOT_0).into_raw(), + )); Ok(()) } @@ -413,8 +421,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { return Err(EINVAL); } - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMC::zeroed() .with_secure(load_offsets.secure) .with_aincw(true) @@ -424,14 +432,14 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() { let n = u16::try_from(n)?; let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag), ); for word in block.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -450,8 +458,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { return Err(EINVAL); } - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMC::zeroed() .with_aincw(true) .with_offs(load_offsets.dst_start), @@ -459,8 +467,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for word in load_offsets.data.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -473,14 +481,12 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { &self, fw: &F, ) -> Result { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); if let Some(imem_ns) = fw.imem_ns_load_params() { self.pio_wr_imem_slice(imem_ns)?; @@ -492,10 +498,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.program_brom(self, &fw.brom_params()); - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -547,16 +551,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { // Set up the base source DMA address. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`. (dma_address >> 8) as u32, - ), - ); - self.bar.write( - WithBase::of::(), + )); + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?, ); @@ -566,23 +567,21 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for pos in (0..num_transfers).map(|i| i * DMA_LEN) { // Perform a transfer of size `DMA_LEN`. - self.bar.write( - WithBase::of::(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed() .try_with_offs(load_offsets.dst_start + pos)?, ); - self.bar.write( - WithBase::of::(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos), ); - self.bar.write(WithBase::of::(), cmd); + self.pfalcon.write_reg(cmd); // Wait for the transfer to complete. // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories // should ever take that long. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)), |r| r.idle(), Delta::ZERO, Delta::from_secs(2), @@ -614,8 +613,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { }; self.dma_reset(); - self.bar - .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::().at(0), |v| { + self.pfalcon + .update(regs::NV_PFALCON_FBIF_TRANSCFG::at(0), |v| { v.with_target(FalconFbifTarget::CoherentSysmem) .with_mem_type(FalconFbifMemType::Physical) }); @@ -626,10 +625,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.program_brom(self, &fw.brom_params()); // Set `BootVec` to start of non-secure code. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -638,7 +635,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { pub(crate) fn wait_till_halted(&self) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL)), |r| r.halted(), Delta::ZERO, Delta::from_secs(2), @@ -649,19 +646,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Start the falcon CPU. pub(crate) fn start(&self) -> Result<()> { - match self - .bar - .read(regs::NV_PFALCON_FALCON_CPUCTL::of::()) - .alias_en() - { - true => self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true), - ), - false => self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true), - ), + match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() { + true => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true)), + false => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true)), } Ok(()) @@ -670,32 +661,24 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Writes values to the mailbox registers if provided. pub(crate) fn write_mailboxes(&self, mbox0: Option, mbox1: Option) { if let Some(mbox0) = mbox0 { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0)); } if let Some(mbox1) = mbox1 { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1)); } } /// Reads the value from `mbox0` register. pub(crate) fn read_mailbox0(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX0).value() } /// Reads the value from `mbox1` register. pub(crate) fn read_mailbox1(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX1).value() } /// Reads values from both mailbox registers. @@ -760,9 +743,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Write the application version to the OS register. pub(crate) fn write_os_version(&self, app_version: u32) { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version)); } } diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 0437180b8829..85f9c8c5d60e 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -8,13 +8,12 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - Array, - RegisterBase, - WithBase, // - }, - Io, // + register, + register::Array, + Io, + Mmio, // }, prelude::*, sizes::SZ_1K, @@ -22,11 +21,13 @@ use kernel::{ }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, num, regs, // @@ -41,15 +42,24 @@ const FSP_EMEM_CHANNEL_0_SIZE: usize = SZ_1K; /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); -impl RegisterBase for Fsp { - const BASE: usize = 0x8f2000; -} +register! { + base: NovaRegisters; -impl RegisterBase for Fsp { - const BASE: usize = 0x8f3000; + PFALCON: super::PFalconRegisters @ 0x8f2000; + PFALCON2: super::PFalcon2Registers @ 0x8f3000; } -impl FalconEngine for Fsp {} +impl FalconEngine for Fsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Fsp> { /// Writes `data` to FSP external memory at offset `0`. @@ -62,19 +72,15 @@ impl<'a> Falcon<'a, Fsp> { } // Begin a write burst at offset `0`, auto-incrementing on each write. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true)); for chunk in data.chunks_exact(4) { let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); // Write the next 32-bit `value`; hardware advances the offset. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value)); } Ok(()) @@ -90,17 +96,12 @@ impl<'a> Falcon<'a, Fsp> { } // Begin a read burst at offset `0`, auto-incrementing on each read. - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true)); for chunk in data.chunks_exact_mut(4) { // Read the next 32-bit word; hardware advances the offset. - let value = self - .bar - .read(regs::NV_PFALCON_FALCON_EMEMD::of::()) - .data(); + let value = self.pfalcon.read(regs::NV_PFALCON_FALCON_EMEMD).data(); chunk.copy_from_slice(&value.to_le_bytes()); } diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index ae32f401aeb0..4c96ae325fda 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -2,23 +2,24 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register, Io, + Mmio, // }, prelude::*, time::Delta, // }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, regs, }; @@ -26,24 +27,31 @@ use crate::{ /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); -impl RegisterBase for Gsp { - const BASE: usize = 0x00110000; -} +register! { + base: NovaRegisters; -impl RegisterBase for Gsp { - const BASE: usize = 0x00111000; + PFALCON: super::PFalconRegisters @ 0x00110000; + PFALCON2: super::PFalcon2Registers @ 0x00111000; } -impl FalconEngine for Gsp {} +impl FalconEngine for Gsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. pub(crate) fn clear_swgen0_intr(&self) { - self.bar.write( - WithBase::of::(), - regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true)); } /// Checks if GSP reload/resume has completed during the boot process. @@ -59,8 +67,8 @@ impl<'a> Falcon<'a, Gsp> { /// Returns whether the RISC-V branch privilege lockdown bit is set. pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool { - self.bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) + self.pfalcon + .read(regs::NV_PFALCON_FALCON_HWCFG2) .riscv_br_priv_lockdown() } @@ -71,10 +79,7 @@ impl<'a> Falcon<'a, Gsp> { const LOCKED_PATTERN: u32 = 0xbadf_4100; const LOCKED_MASK: u32 = 0xffff_ff00; - let hwcfg2 = self - .bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::()) - .into_raw(); + let hwcfg2 = self.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2).into_raw(); hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN } diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 7600ee07ca2e..f9a8444cf840 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -6,11 +6,9 @@ use kernel::{ device, io::{ poll::read_poll_timeout, - register::{ - Array, - WithBase, // - }, - Io, // + register::Array, + Io, + Mmio, // }, prelude::*, time::Delta, // @@ -24,6 +22,7 @@ use crate::{ FalconBromParams, FalconEngine, FalconModSelAlgo, + PFalcon2Registers, PeregrineCoreSelect, // }, regs, @@ -31,17 +30,16 @@ use crate::{ use super::FalconHal; -fn select_core_ga102(bar: Bar0<'_>) -> Result { - let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::()); +fn select_core_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>) -> Result { + let bcr_ctrl = pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL); if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon { - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PRISCV_RISCV_BCR_CTRL::zeroed().with_core_select(PeregrineCoreSelect::Falcon), ); // TIMEOUT: falcon core should take less than 10ms to report being enabled. read_poll_timeout( - || Ok(bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::())), + || Ok(pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL)), |r| r.valid(), Delta::ZERO, Delta::from_millis(10), @@ -86,24 +84,20 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102(bar: Bar0<'_>, params: &FalconBromParams) { - bar.write( - WithBase::of::().at(0), +fn program_brom_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>, params: &FalconBromParams) { + pfalcon2.write( + Array::at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), ); - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed() .with_value(u32::from(params.engine_id_mask)), ); - bar.write( - WithBase::of::(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed().with_ucode_id(params.ucode_id), ); - bar.write( - WithBase::of::(), - regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k), - ); + pfalcon2 + .write_reg(regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k)); } pub(super) struct Ga102(PhantomData); @@ -116,7 +110,7 @@ impl Ga102 { impl FalconHal for Ga102 { fn select_core(&self, falcon: &Falcon<'_, E>) -> Result { - select_core_ga102::(falcon.bar) + select_core_ga102(falcon.pfalcon2) } fn signature_reg_fuse_version( @@ -129,27 +123,24 @@ impl FalconHal for Ga102 { } fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) { - program_brom_ga102::(falcon.bar, params); + program_brom_ga102(falcon.pfalcon2, params); } fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CPUCTL) .active_stat() } fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result { - Ok(falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::()) - .halted()) + Ok(falcon.pfalcon2.read(regs::NV_PRISCV_RISCV_CPUCTL).halted()) } fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(20), @@ -158,20 +149,18 @@ impl FalconHal for Ga102 { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - let bar = falcon.bar; - - let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::()); + let _ = falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set // RESET_READY so a non-failing timeout is used. let _ = read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.reset_ready(), Delta::ZERO, Delta::from_micros(150), ); - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 5291598fedf7..7fc6e83c2566 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -5,7 +5,6 @@ use core::marker::PhantomData; use kernel::{ io::{ poll::read_poll_timeout, - register::WithBase, Io, // }, prelude::*, @@ -50,8 +49,8 @@ impl FalconHal for Tu102 { fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS) .active_stat() } @@ -62,7 +61,7 @@ impl FalconHal for Tu102 { fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_DMACTL)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(10), @@ -71,7 +70,7 @@ impl FalconHal for Tu102 { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::(falcon.bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/sec2.rs b/drivers/gpu/nova-core/falcon/sec2.rs index 91ec7d49c1f5..6648a397d38a 100644 --- a/drivers/gpu/nova-core/falcon/sec2.rs +++ b/drivers/gpu/nova-core/falcon/sec2.rs @@ -1,22 +1,37 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::io::register::RegisterBase; +use kernel::io::{ + io_project, + register, + Mmio, // +}; -use crate::falcon::{ - FalconEngine, - PFalcon2Base, - PFalconBase, // +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + falcon::FalconEngine, // }; /// Type specifying the `Sec2` falcon engine. Cannot be instantiated. pub(crate) struct Sec2(()); -impl RegisterBase for Sec2 { - const BASE: usize = 0x00840000; -} +register! { + base: NovaRegisters; -impl RegisterBase for Sec2 { - const BASE: usize = 0x00841000; + PFALCON: super::PFalconRegisters @ 0x00840000; + PFALCON2: super::PFalcon2Registers @ 0x00841000; } -impl FalconEngine for Sec2 {} +impl FalconEngine for Sec2 { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ec4d92317a93..02a36fd59902 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -12,7 +12,10 @@ use kernel::{ Device, // }, dma::Coherent, - io::{register::WithBase, Io}, + io::{ + register::Array, + Io, // + }, prelude::*, ptr::{ Alignable, @@ -23,7 +26,6 @@ use kernel::{ }; use crate::{ - driver::Bar0, falcon::{ self, gsp::Gsp, @@ -235,12 +237,7 @@ impl FwsecFirmwareWithBl { /// /// The bootloader will load the FWSEC firmware and then execute it. This function returns /// after FWSEC has reached completion. - pub(crate) fn run( - &self, - dev: &Device, - falcon: &Falcon<'_, Gsp>, - bar: Bar0<'_>, - ) -> Result<()> { + pub(crate) fn run(&self, dev: &Device, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon .reset() @@ -250,9 +247,8 @@ impl FwsecFirmwareWithBl { .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory. - bar.update( - regs::NV_PFALCON_FBIF_TRANSCFG::of::() - .try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) + falcon.pfalcon.update( + regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) .ok_or(EINVAL)?, |v| { v.with_target(FalconFbifTarget::CoherentSysmem) diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a5c0ca355493..d2ef35080689 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -62,12 +62,11 @@ impl FwsecUnloadFirmware { fn run( &self, dev: &device::Device, - bar: Bar0<'_>, gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { match self { Self::WithoutBl(fw) => fw.run(dev, gsp_falcon), - Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithBl(fw) => fw.run(dev, gsp_falcon), } } } @@ -88,7 +87,7 @@ impl UnloadBundle for Sec2UnloadBundle { // Log errors but keep going if it fails. let fwsec_sb_res = self .fwsec_sb - .run(dev, bar, ctx.gsp_falcon) + .run(dev, ctx.gsp_falcon) .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. @@ -168,7 +167,7 @@ impl Tu102 { if self.needs_fwsec_bootloader { let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; + fwsec_frts_bl.run(dev, falcon)?; } else { // Load and run FWSEC-FRTS directly. fwsec_frts.run(dev, falcon)?; diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 1af073f3861f..90ac8dac5349 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -4,8 +4,8 @@ use kernel::{ io::{ register, - register::WithBase, - Io, // + Io, + Mmio, // }, prelude::*, sizes::SizeConstants, @@ -13,22 +13,18 @@ use kernel::{ }; use crate::{ - driver::{ - Bar0, - NovaRegisters, // - }, + driver::NovaRegisters, falcon::{ DmaTrfCmdSize, FalconCoreRev, FalconCoreRevSubversion, - FalconEngine, FalconFbifMemType, FalconFbifTarget, FalconMem, FalconModSelAlgo, FalconSecurityModel, - PFalcon2Base, - PFalconBase, + PFalcon2Registers, + PFalconRegisters, PeregrineCoreSelect, // }, gpu::{ @@ -202,32 +198,32 @@ register! { // PFALCON register! { - base: NovaRegisters; + base: PFalconRegisters; - pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 { + pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; } - pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ PFalconBase + 0x00000040 { + pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ PFalconBase + 0x00000044 { + pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 { 31:0 value => u32; } /// Used to store version information about the firmware running /// on the Falcon processor. - pub(crate) NV_PFALCON_FALCON_OS(u32) @ PFalconBase + 0x00000080 { + pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_RM(u32) @ PFalconBase + 0x00000084 { + pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 { + pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 { /// Signal indicating that reset is completed (GA102+). 31:31 reset_ready => bool; /// RISC-V branch privilege lockdown bit. @@ -237,17 +233,17 @@ register! { 10:10 riscv => bool; } - pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ PFalconBase + 0x00000100 { + pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 { 6:6 alias_en => bool; 4:4 halted => bool; 1:1 startcpu => bool; } - pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ PFalconBase + 0x00000104 { + pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ PFalconBase + 0x0000010c { + pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c { 7:7 secure_stat => bool; 6:3 dmaq_num; 2:2 imem_scrubbing => bool; @@ -255,15 +251,15 @@ register! { 0:0 require_ctx => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ PFalconBase + 0x00000110 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 { 31:0 base => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ PFalconBase + 0x00000114 { + pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 { 23:0 offs; } - pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ PFalconBase + 0x00000118 { + pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 { 16:16 set_dmtag; 14:12 ctxdma; 10:8 size ?=> DmaTrfCmdSize; @@ -274,15 +270,15 @@ register! { 0:0 full => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ PFalconBase + 0x0000011c { + pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c { 31:0 offs => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ PFalconBase + 0x00000128 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 { 8:0 base; } - pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ PFalconBase + 0x0000012c { + pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c { /// Core revision subversion. 7:6 core_rev_subversion => FalconCoreRevSubversion; /// Security model. @@ -291,12 +287,12 @@ register! { 3:0 core_rev ?=> FalconCoreRev; } - pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ PFalconBase + 0x00000130 { + pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 { 1:1 startcpu => bool; } /// IMEM access control register. Up to 4 ports are available for IMEM access. - pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ PFalconBase + 0x00000180 { + pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 { /// Access secure IMEM. 28:28 secure => bool; /// Auto-increment on write. @@ -307,17 +303,17 @@ register! { /// IMEM data register. Reading/writing this register accesses IMEM at the address /// specified by the corresponding IMEMC register. - pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ PFalconBase + 0x00000184 { + pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 { 31:0 data; } /// IMEM tag register. Used to set the tag for the current IMEM block. - pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ PFalconBase + 0x00000188 { + pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 { 15:0 tag; } /// DMEM access control register. Up to 8 ports are available for DMEM access. - pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ PFalconBase + 0x000001c0 { + pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 { /// Auto-increment on write. 24:24 aincw => bool; /// DMEM block and word offset. @@ -326,29 +322,29 @@ register! { /// DMEM data register. Reading/writing this register accesses DMEM at the address /// specified by the corresponding DMEMC register. - pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ PFalconBase + 0x000001c4 { + pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 { 31:0 data; } /// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the /// falcon instance. - pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ PFalconBase + 0x000003c0 { + pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 { 0:0 reset => bool; } - pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ PFalconBase + 0x00000600 { + pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 { 2:2 mem_type => FalconFbifMemType; 1:0 target ?=> FalconFbifTarget; } - pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 { + pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 { 7:7 allow_phys_no_ctx => bool; } // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). // These provide the falcon external memory communication interface. - pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 { + pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 { /// EMEM byte offset (4-byte aligned) within the block. 7:2 offs; /// EMEM block to access. @@ -359,7 +355,7 @@ register! { 25:25 aincr => bool; } - pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 { + pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 { 31:0 data => u32; } } @@ -385,13 +381,13 @@ impl NV_PFALCON_FALCON_DMATRFCMD { impl NV_PFALCON_FALCON_ENGINE { /// Resets the falcon - pub(crate) fn reset_engine(bar: Bar0<'_>) { - bar.update(Self::of::(), |r| r.with_reset(true)); + pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) { + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true)); // TIMEOUT: falcon engine should not take more than 10us to reset. time::delay::fsleep(time::Delta::from_micros(10)); - bar.update(Self::of::(), |r| r.with_reset(false)); + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false)); } } @@ -405,23 +401,23 @@ impl NV_PFALCON_FALCON_HWCFG2 { /* PFALCON2 */ register! { - base: NovaRegisters; + base: PFalcon2Registers; - pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 { + pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 { 7:0 algo ?=> FalconModSelAlgo; } - pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ PFalcon2Base + 0x00000198 { + pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 { 7:0 ucode_id => u8; } - pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ PFalcon2Base + 0x0000019c { + pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c { 31:0 value => u32; } /// OpenRM defines this as a register array, but doesn't specify its size and only uses its /// first element. Be conservative until we know the actual size or need to use more registers. - pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ PFalcon2Base + 0x00000210 { + pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 { 31:0 value => u32; } } @@ -429,23 +425,23 @@ register! { // PRISCV register! { - base: NovaRegisters; + base: PFalcon2Registers; /// RISC-V status register for debug (Turing and GA100 only). /// Reflects current RISC-V core status. - pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 { + pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 { /// RISC-V core active/inactive status. 0:0 active_stat => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { + pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 { 7:7 active_stat => bool; 4:4 halted => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ PFalcon2Base + 0x00000668 { + pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 { 8:8 br_fetch => bool; 4:4 core_select => PeregrineCoreSelect; 0:0 valid => bool; -- cgit v1.2.3 From a6355a2ef819ae569f9b14ac808ed5c1f4b62ddd Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:37 +0100 Subject: gpu: nova-core: convert hshub0 from relative register to projection Similar to the PFALCON and PFALCON2 conversion, the hshub0 relative access can also be achieved cleanly with projection and a new base. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-13-5552b1d59525@garyguo.net [ Split nested import into kernel vertical style. - Danilo ] Signed-off-by: Danilo Krummrich --- drivers/gpu/nova-core/fb/hal/gb100.rs | 59 +++++++++++++++++------------------ drivers/gpu/nova-core/fb/regs.rs | 24 +++++++++----- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index d9e4d62ae632..9fa094939600 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -5,11 +5,10 @@ use kernel::{ io::{ - register::{ - RegisterBase, - WithBase, // - }, - Io, // + io_project, + register, + Io, + Mmio, // }, num::Bounded, prelude::*, @@ -21,7 +20,10 @@ use kernel::{ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, fb::{ hal::FbHal, regs, // @@ -31,17 +33,26 @@ use crate::{ struct Gb100; -impl RegisterBase for Gb100 { - const BASE: usize = 0x0087_0000; +register! { + base: NovaRegisters; + + HSHUB0: regs::Hshub0Registers @ 0x0087_0000; +} + +#[inline] +fn hshub0(bar: Bar0<'_>) -> Mmio<'_, regs::Hshub0Registers> { + io_project!(bar, build: HSHUB0) } -fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { +fn read_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>) -> u64 { let lo = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO) .adr(), ); let hi = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI) .adr(), ); @@ -52,7 +63,7 @@ fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { /// /// Both the primary and EG (egress) register pairs must be programmed to the same address, /// as required by hardware. -fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { +fn write_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>, addr: Bounded) { // CAST: lower 32 bits. Hardware ignores bits 7:0. let addr_lo = *addr as u32; let addr_hi = addr.shr::<32, 20>().cast::(); @@ -60,24 +71,12 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded) { // Write HI first. The hardware will trigger the flush on the LO write. // Primary HSHUB pair. - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); // EG (egress) pair -- must match the primary pair. - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); } // This PMU reservation size is r570-specific. @@ -88,13 +87,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 { impl FbHal for Gb100 { fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { - read_sysmem_flush_page_gb100(bar) + read_sysmem_flush_page_gb100(hshub0(bar)) } fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { let addr = Bounded::::try_new(addr).ok_or(EINVAL)?; - write_sysmem_flush_page_gb100(bar, addr); + write_sysmem_flush_page_gb100(hshub0(bar), addr); Ok(()) } diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index c27582e376e2..131787996a24 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -2,7 +2,11 @@ use kernel::{ io::register, - sizes::SizeConstants, // + prelude::*, + sizes::{ + SizeConstants, + SZ_4K, // + }, // }; use crate::driver::NovaRegisters; @@ -65,31 +69,35 @@ register! { } } -/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +const HSHUB0_REGION_SIZE: usize = SZ_4K; + +/// The GB10x HSHUB0 register window (Base defined as `NV_HSHUB0_PRIV_BASE` in Open RM). /// /// The base is provided by the GB10x framebuffer HAL. -pub(super) struct Hshub0Base(()); +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(super) struct Hshub0Registers([u8; HSHUB0_REGION_SIZE]); register! { - base: NovaRegisters; + base: Hshub0Registers; // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00000e50 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00000e54 { 19:0 adr; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x000006c0 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x000006c4 { 19:0 adr; } } -- cgit v1.2.3 From 631adefcac893728c278f4bd2c9634cfe3f35482 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:38 +0100 Subject: rust: io: register: remove relative registers Relative registers can be better served by projection to subregion instead of ad-hoc handling in register macro. Projection composes better (e.g. it natively allows relative registers of relative registers without needing additional support). Remove relative register support, and update the documentation to demonstrate how projection and subregions can be used to achieve this instead. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-14-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 388 ++++++--------------------------------------- rust/macros/io/register.rs | 52 ++---- 2 files changed, 61 insertions(+), 379 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index b9cb4fce8628..187f28f16614 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -8,7 +8,7 @@ //! //! Note: most of the items in this module are public so they can be referenced by the macro, but //! most are not to be used directly by users. Outside of the `register!` macro itself, the only -//! items you might want to import from this module are [`WithBase`] and [`Array`]. +//! item you might want to import from this module is [`Array`]. //! //! # Simple example //! @@ -200,76 +200,6 @@ impl IoLoc for OffsetLoc { } } -/// Trait providing a base address to be added to the offset of a relative register to obtain -/// its actual offset. -/// -/// The `T` generic argument is used to distinguish which base to use, in case a type provides -/// several bases. It is given to the `register!` macro to restrict the use of the register to -/// implementors of this particular variant. -pub trait RegisterBase { - /// Base address to which register offsets are added. - const BASE: usize; -} - -/// Trait implemented by all registers that are relative to a base. -pub trait WithBase { - /// Family of bases applicable to this register. - type BaseFamily; - - /// Returns the absolute location of this type when using `B` as its base. - #[inline(always)] - fn of>() -> RelativeRegisterLoc - where - Self: Register, - { - RelativeRegisterLoc::new() - } -} - -/// Trait implemented by relative registers. -pub trait RelativeRegister: Register + WithBase {} - -/// Location of a relative register. -/// -/// This can either be an immediately accessible regular [`RelativeRegister`], or a -/// [`RelativeRegisterArray`] that needs one additional resolution through -/// [`RelativeRegisterLoc::at`]. -pub struct RelativeRegisterLoc(PhantomData, PhantomData); - -impl RelativeRegisterLoc -where - T: Register + WithBase, - B: RegisterBase + ?Sized, -{ - /// Returns the location of a relative register or register array. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData, PhantomData) - } - - // Returns the absolute offset of the relative register using base `B`. - // - // This is implemented as a private const method so it can be reused by the [`IoLoc`] - // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`]. - #[inline] - const fn offset(self) -> usize { - B::BASE + T::OFFSET - } -} - -impl IoLoc for RelativeRegisterLoc -where - T: RelativeRegister, - B: RegisterBase + ?Sized, -{ - #[inline(always)] - fn offset(self) -> usize { - RelativeRegisterLoc::offset(self) - } -} - /// Trait implemented by arrays of registers. pub trait RegisterArray: Register { /// Number of elements in the registers array. @@ -332,71 +262,6 @@ pub trait Array { } } -/// Trait implemented by arrays of relative registers. -pub trait RelativeRegisterArray: RegisterArray + WithBase {} - -/// Location of a relative array register. -pub struct RelativeRegisterArrayLoc< - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, ->(RelativeRegisterLoc, usize); - -impl RelativeRegisterArrayLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - /// Returns the location of register `T` from the base `B` at index `idx`, with build-time - /// validation. - #[inline(always)] - pub fn new(idx: usize) -> Self { - build_assert!(idx < T::SIZE); - - Self(RelativeRegisterLoc::new(), idx) - } - - /// Attempts to return the location of register `T` from the base `B` at index `idx`, with - /// runtime validation. - #[inline(always)] - pub fn try_new(idx: usize) -> Option { - if idx < T::SIZE { - Some(Self(RelativeRegisterLoc::new(), idx)) - } else { - None - } - } -} - -/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`]. -impl RelativeRegisterLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - /// Returns the location of the register at position `idx`, with build-time validation. - #[inline(always)] - pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc { - RelativeRegisterArrayLoc::new(idx) - } - - /// Returns the location of the register at position `idx`, with runtime validation. - #[inline(always)] - pub fn try_at(self, idx: usize) -> Option> { - RelativeRegisterArrayLoc::try_new(idx) - } -} - -impl IoLoc for RelativeRegisterArrayLoc -where - T: RelativeRegisterArray, - B: RegisterBase + ?Sized, -{ - #[inline(always)] - fn offset(self) -> usize { - self.0.offset() + self.1 * T::STRIDE - } -} - /// Trait implemented by items that contain both a register value and the absolute I/O location at /// which to write it. /// @@ -452,8 +317,7 @@ pub const fn element_alias_offset+------------CPU0-------------+ -/// | | -/// 0x110--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// | | -/// | | -/// 0x200--->+------------CPU1-------------+ -/// | | -/// 0x210--->+-----------------------------+ -/// | CPU_CTL | -/// +-----------------------------+ -/// | ... | -/// +-----------------------------+ -/// ``` -/// -/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O -/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define -/// them twice and would prefer a way to select which one to use from a single definition. -/// -/// This can be done using the `Base + Offset` syntax when specifying the register's address: -/// -/// ```ignore -/// register! { -/// ... -/// pub RELATIVE_REG(u32) @ Base + 0x80 { -/// ... -/// } -/// } -/// ``` -/// -/// This creates a register with an offset of `0x80` from a given base. -/// -/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the -/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for -/// this register needs to implement `RegisterBase`. -/// -/// The location of relative registers can be built using the [`WithBase::of`] method to specify -/// its base. All relative registers implement [`WithBase`]. -/// -/// Here is the above layout translated into code: -/// -/// ```no_run -/// use kernel::{ -/// io::{ -/// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, -/// Io, -/// Region, -/// }, -/// }; -/// # use kernel::io::Mmio; -/// -/// // Type used to identify the base. -/// pub struct CpuCtlBase; -/// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase for Cpu0 { -/// const BASE: usize = 0x100; -/// } -/// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase for Cpu1 { -/// const BASE: usize = 0x200; -/// } -/// -/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase`. -/// register! { -/// base: Region<0x1000>; -/// -/// /// CPU core control. -/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 { -/// 0:0 start; -/// } -/// } -/// -/// # fn test(io: Mmio<'_, Region<0x1000>>) { -/// // Read the status of `Cpu0`. -/// let cpu0_started = io.read(CPU_CTL::of::()); -/// -/// // Stop `Cpu0`. -/// io.write(WithBase::of::(), CPU_CTL::zeroed()); -/// # } -/// -/// // Aliases can also be defined for relative register. -/// register! { -/// base: Region<0x1000>; -/// -/// /// Alias to CPU core control. -/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL { -/// /// Start the aliased CPU core. -/// 1:1 alias_start; -/// } -/// } -/// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) { -/// // Start the aliased `CPU0`, leaving its other fields untouched. -/// io.update(CPU_CTL_ALIAS::of::(), |r| r.with_alias_start(true)); -/// # } -/// ``` -/// /// ## Arrays of registers /// /// Some I/O areas contain consecutive registers that share the same field layout. These areas can @@ -763,115 +511,83 @@ pub const fn element_alias_offset+------------CPU0-------------+ +/// | | +/// 0x110--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// | | +/// | | +/// 0x200--->+------------CPU1-------------+ +/// | | +/// 0x210--->+-----------------------------+ +/// | CPU_CTL | +/// +-----------------------------+ +/// | ... | +/// +-----------------------------+ /// ``` /// -/// Like relative registers, they implement the [`WithBase`] trait. However the return value of -/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the -/// [`at`](RelativeRegisterLoc::at) method. +/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O +/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define +/// them twice and would prefer a way to select which one to use from a single definition. +/// +/// This can be done by defining a new type for the subregion, and then defining registers that use +/// the new type as the base: /// /// ```no_run /// use kernel::{ /// io::{ +/// io_project, /// register, -/// register::{ -/// RegisterBase, -/// WithBase, -/// }, /// Io, /// Region, /// }, /// }; /// # use kernel::io::Mmio; -/// # fn get_scratch_idx() -> usize { -/// # 0x15 -/// # } -/// -/// // Type used as parameter of `RegisterBase` to specify the base. -/// pub struct CpuCtlBase; /// -/// // ZST describing `CPU0`. -/// struct Cpu0; -/// impl RegisterBase for Cpu0 { -/// const BASE: usize = 0x100; -/// } +/// // Subregion type. Make sure it has adequate size and alignment. +/// #[repr(align(4))] +/// #[derive(FromBytes, IntoBytes)] +/// pub struct CpuCtl([u8; 0x100]); /// -/// // ZST describing `CPU1`. -/// struct Cpu1; -/// impl RegisterBase for Cpu1 { -/// const BASE: usize = 0x200; -/// } -/// -/// // 64 per-cpu scratch registers, arranged as a contiguous array. /// register! { /// base: Region<0x1000>; /// -/// /// Per-CPU scratch registers. -/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 { -/// 31:0 value; -/// } +/// // Subregions can just be defined like normal registers. +/// CPU0: CpuCtl @ 0x100; +/// CPU1: CpuCtl @ 0x200; /// } /// -/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// // Read scratch register 0 of CPU0. -/// let scratch = io.read(CPU_SCRATCH::of::().at(0)); -/// -/// // Write the retrieved value into scratch register 15 of CPU1. -/// io.write(WithBase::of::().at(15), scratch); -/// -/// // This won't build. -/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::().at(128)).value(); -/// -/// // Runtime-obtained array index. -/// let scratch_idx = get_scratch_idx(); -/// // Access on a runtime index returns an error if it is out-of-bounds. -/// let cpu0_scratch = io.read( -/// CPU_SCRATCH::of::().try_at(scratch_idx).ok_or(EINVAL)? -/// ).value(); -/// # Ok(()) -/// # } -/// -/// // Alias to `SCRATCH[8]` used to convey the firmware exit code. +/// // Then you can define new registers on the subregion. /// register! { -/// base: Region<0x1000>; +/// base: CpuCtl; /// -/// /// Per-CPU firmware exit status code. -/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] { -/// 7:0 status; +/// /// CPU core control. +/// pub CPU_CTL(u32) @ 0x10 { +/// 0:0 start; /// } /// } /// -/// // Non-contiguous relative register arrays can be defined by adding a stride parameter. -/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the -/// // registers of the two declarations below are interleaved. -/// register! { -/// base: Region<0x1000>; -/// -/// /// Scratch registers bank 0. -/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 { -/// 31:0 value; -/// } -/// -/// /// Scratch registers bank 1. -/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 { -/// 31:0 value; -/// } -/// } +/// # fn test(io: Mmio<'_, Region<0x1000>>) { +/// // Read the status of `Cpu0`. +/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL); /// -/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> { -/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::()).status(); -/// # Ok(()) +/// // Stop `Cpu0`. +/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed()); /// # } /// ``` #[macro_export] diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index e948b0959154..500805aa6b23 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -65,7 +65,6 @@ struct Reg { name: Ident, ty: Type, array: Option, - relative_base: Option, offset: RegOffset, bitfield: Option<(Type, Group)>, } @@ -110,26 +109,15 @@ impl Parse for Reg { // Parse offset and the base it's relative to. let lh = input.lookahead1(); - let mut relative_base = None; let offset = if lh.peek(Token![@]) { let _: Token![@] = input.parse()?; - if input.peek(Ident) { - relative_base = Some(input.parse()?); - let _: Token![+] = input.parse()?; - } - RegOffset::Fixed { offset: input.parse()?, } } else if lh.peek(Token![=>]) { let _: Token![=>] = input.parse()?; - let mut alias: Path = input.parse()?; - if input.peek(Token![+]) { - let _: Token![+] = input.parse()?; - relative_base = Some(alias); - alias = input.parse()?; - } + let alias: Path = input.parse()?; if input.peek(token::Bracket) { let content; @@ -164,7 +152,6 @@ impl Parse for Reg { name, ty, array, - relative_base, offset, bitfield, }) @@ -208,7 +195,6 @@ pub(crate) fn register(def: RegDef) -> Result { name, ty, array, - relative_base, offset, bitfield, } = reg; @@ -248,7 +234,7 @@ pub(crate) fn register(def: RegDef) -> Result { } match array { - None if bitfield.is_none() && relative_base.is_none() => outputs.extend(quote!( + None if bitfield.is_none() => outputs.extend(quote!( #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = ::kernel::io::register::OffsetLoc::new(#offset); )), @@ -258,21 +244,12 @@ pub(crate) fn register(def: RegDef) -> Result { "defining without bitfield is not yet supported for this type of register", ))?, - None => match relative_base { - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name {} - - #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = - ::kernel::io::register::FixedRegisterLoc::<#name>::new(); - )), - Some(relative_base) => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::WithBase for #name { - type BaseFamily = #relative_base; - } + None => outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::FixedRegister for #name {} - impl ::kernel::io::register::RelativeRegister for #name {} - )), - }, + #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = + ::kernel::io::register::FixedRegisterLoc::<#name>::new(); + )), Some(def) => { let size = &def.size; @@ -288,24 +265,13 @@ pub(crate) fn register(def: RegDef) -> Result { }; outputs.extend(quote_spanned!(span => + impl ::kernel::io::register::Array for #name {} + impl ::kernel::io::register::RegisterArray for #name { const SIZE: usize = #size; const STRIDE: usize = #stride; } )); - - match relative_base { - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::Array for #name {} - )), - Some(relative_base) => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::WithBase for #name { - type BaseFamily = #relative_base; - } - - impl ::kernel::io::register::RelativeRegisterArray for #name {} - )), - } } }; } -- cgit v1.2.3 From a8ab43a1089e289b58d303a1c10d0f81284087e7 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:39 +0100 Subject: rust: io: register: remove `Register` trait and cleanup macro With the removal of relative registers, there are only two type of registers left, fixed register and register arrays. There is not much benefit in having a common super trait for them anymore, thus remove it, and cleanup the macro rules associated with it. Signed-off-by: Gary Guo Reviewed-by: Alexandre Courbot Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-15-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 22 +++++++++++++--------- rust/macros/io/register.rs | 12 ++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 187f28f16614..9df7572aee9d 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,8 +121,8 @@ use crate::{ io::IoLoc, // }; -/// Trait implemented by all registers. -pub trait Register: Sized { +/// Trait implemented by registers with a fixed offset. +pub trait FixedRegister: Sized { /// Base type for this register. type Base: ?Sized; @@ -132,9 +132,6 @@ pub trait Register: Sized { const OFFSET: usize; } -/// Trait implemented by registers with a fixed offset. -pub trait FixedRegister: Register {} - /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when /// passing a [`FixedRegister`] value. impl IoLoc for () @@ -201,7 +198,14 @@ impl IoLoc for OffsetLoc { } /// Trait implemented by arrays of registers. -pub trait RegisterArray: Register { +pub trait RegisterArray: Sized { + /// Base type for this register. + type Base: ?Sized; + + /// Start offset of the register. + /// + /// The interpretation of this offset depends on the type of the register. + const OFFSET: usize; /// Number of elements in the registers array. const SIZE: usize; /// Number of bytes between the start of elements in the registers array. @@ -267,8 +271,8 @@ pub trait Array { /// /// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). pub trait LocatedRegister { - /// Register value to write. - type Value: Register; + /// Value to write. + type Value; /// Full location information at which to write the value. type Location: IoLoc; @@ -295,7 +299,7 @@ where /// This is used to enforce base matching. #[doc(hidden)] #[inline(always)] // for const eval only -pub const fn alias_offset>() -> usize { +pub const fn alias_offset>() -> usize { Alias::OFFSET } diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 500805aa6b23..5511b8e5b875 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -225,11 +225,6 @@ pub(crate) fn register(def: RegDef) -> Result { #[allow(non_camel_case_types)] #(#attrs)* #vis struct #name(#storage) #args ); - - impl ::kernel::io::register::Register for #name { - type Base = #base; - const OFFSET: usize = #offset; - } )); } @@ -245,7 +240,10 @@ pub(crate) fn register(def: RegDef) -> Result { ))?, None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name {} + impl ::kernel::io::register::FixedRegister for #name { + type Base = #base; + const OFFSET: usize = #offset; + } #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = ::kernel::io::register::FixedRegisterLoc::<#name>::new(); @@ -268,6 +266,8 @@ pub(crate) fn register(def: RegDef) -> Result { impl ::kernel::io::register::Array for #name {} impl ::kernel::io::register::RegisterArray for #name { + type Base = #base; + const OFFSET: usize = #offset; const SIZE: usize = #size; const STRIDE: usize = #stride; } -- cgit v1.2.3 From a12f711dc53c1ca1d7ff4184e1609182174de87b Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Tue, 1 Sep 2026 17:50:40 +0100 Subject: rust: io: register: unify handling of register with/without bitfields Move the `FixedRegister` from a property of register to become a property of type. Name the new trait `FixedIoLoc` indicating if I/O location of a type is unique for a specific base. Thus, bitfields become just a special case of this (where type is unique because we're generating it in the register macro), and expose feature to registers without inline bitfield definition with the `#[unique]` attribute. Reviewed-by: Alexandre Courbot Signed-off-by: Gary Guo Tested-by: Alexandre Courbot Link: https://patch.msgid.link/20260901-typed_register-v4-16-5552b1d59525@garyguo.net Signed-off-by: Danilo Krummrich --- rust/kernel/io/register.rs | 114 +++++++++++++++++++++------------------------ rust/macros/io/register.rs | 58 ++++++++++++++--------- 2 files changed, 89 insertions(+), 83 deletions(-) diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs index 9df7572aee9d..b6513fa0f412 100644 --- a/rust/kernel/io/register.rs +++ b/rust/kernel/io/register.rs @@ -121,61 +121,15 @@ use crate::{ io::IoLoc, // }; -/// Trait implemented by registers with a fixed offset. -pub trait FixedRegister: Sized { - /// Base type for this register. - type Base: ?Sized; - - /// Start offset of the register. - /// - /// The interpretation of this offset depends on the type of the register. - const OFFSET: usize; -} - /// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when -/// passing a [`FixedRegister`] value. +/// passing a [`FixedIoLoc`] value. impl IoLoc for () where - T: FixedRegister, + T: FixedIoLoc, { #[inline(always)] fn offset(self) -> usize { - T::OFFSET - } -} - -/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used -/// as an [`IoLoc`]. -impl IoLoc for T -where - T: FixedRegister, -{ - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET - } -} - -/// Location of a fixed register. -pub struct FixedRegisterLoc(PhantomData); - -impl FixedRegisterLoc { - /// Returns the location of `T`. - #[inline(always)] - // We do not implement `Default` so we can be const. - #[expect(clippy::new_without_default)] - pub const fn new() -> Self { - Self(PhantomData) - } -} - -impl IoLoc for FixedRegisterLoc -where - T: FixedRegister, -{ - #[inline(always)] - fn offset(self) -> usize { - T::OFFSET + T::LOCATION.offset() } } @@ -188,6 +142,11 @@ impl OffsetLoc { pub const fn new(offset: usize) -> Self { Self(offset, PhantomData) } + + #[inline] + pub const fn const_offset(self) -> usize { + self.0 + } } impl IoLoc for OffsetLoc { @@ -266,6 +225,17 @@ pub trait Array { } } +/// Trait implemented by types that indicate there is a fixed I/O location for this given type. +/// +/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg). +pub trait FixedIoLoc: Sized { + /// Type of [`FixedIoLoc::LOCATION`]. + type Location: IoLoc; + + /// Location of this type within given base. + const LOCATION: Self::Location; +} + /// Trait implemented by items that contain both a register value and the absolute I/O location at /// which to write it. /// @@ -283,26 +253,17 @@ pub trait LocatedRegister { impl LocatedRegister for T where - T: FixedRegister, + T: FixedIoLoc, { - type Location = FixedRegisterLoc; + type Location = T::Location; type Value = T; #[inline(always)] - fn into_io_op(self) -> (FixedRegisterLoc, T) { - (FixedRegisterLoc::new(), self) + fn into_io_op(self) -> (T::Location, T) { + (T::LOCATION, self) } } -/// Helper function for register alias implementation. -/// -/// This is used to enforce base matching. -#[doc(hidden)] -#[inline(always)] // for const eval only -pub const fn alias_offset>() -> usize { - Alias::OFFSET -} - /// Helper function for register element alias implementation. /// /// This is used to enforce base matching and provide bounds checking. @@ -419,6 +380,35 @@ pub const fn element_alias_offset; +/// +/// pub RESET: #[unique] Reset @ 0x100; +/// } +/// +/// # fn test(mmio: Mmio<'_, Region<0x1000>>) { +/// // let mmio: Mmio<'_, Region<0x1000>>; +/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>()); +/// # } +/// ``` +/// /// ## Arrays of registers /// /// Some I/O areas contain consecutive registers that share the same field layout. These areas can diff --git a/rust/macros/io/register.rs b/rust/macros/io/register.rs index 5511b8e5b875..420c3ad052b8 100644 --- a/rust/macros/io/register.rs +++ b/rust/macros/io/register.rs @@ -63,6 +63,7 @@ struct Reg { attrs: Vec, vis: Visibility, name: Ident, + unique: bool, ty: Type, array: Option, offset: RegOffset, @@ -76,16 +77,26 @@ impl Parse for Reg { let name = input.parse()?; let lh = input.lookahead1(); - let (ty, bitfield_storage) = if lh.peek(Token![:]) { + let (unique, ty, bitfield_storage) = if lh.peek(Token![:]) { let _: Token![:] = input.parse()?; - (input.parse()?, None) + + let mut attrs = input.call(Attribute::parse_outer)?; + let unique = attrs + .extract_if(.., |attr| attr.path().is_ident("unique")) + .count() + != 0; + if !attrs.is_empty() { + Err(Error::new_spanned(&attrs[0], "unexpected attributes"))? + } + + (unique, input.parse()?, None) } else if lh.peek(token::Paren) { let content; parenthesized!(content in input); let bitfield_storage = Some(content.parse()?); // For bitfields, bitfield macro will generate a type with the same name as `name`. - (parse_quote!(#name), bitfield_storage) + (true, parse_quote!(#name), bitfield_storage) } else { Err(lh.error())? }; @@ -150,6 +161,7 @@ impl Parse for Reg { attrs, vis, name, + unique, ty, array, offset, @@ -193,6 +205,7 @@ pub(crate) fn register(def: RegDef) -> Result { attrs, vis, name, + unique, ty, array, offset, @@ -207,7 +220,7 @@ pub(crate) fn register(def: RegDef) -> Result { RegOffset::Fixed { offset } => quote!(#offset), RegOffset::Alias { alias } => { quote_spanned!(alias.span().resolved_at(span) => - ::kernel::io::register::alias_offset::<#base, #alias>() + ::kernel::io::register::OffsetLoc::<#base, _>::const_offset(#alias) ) } RegOffset::ElementAlias { alias, idx } => { @@ -229,27 +242,30 @@ pub(crate) fn register(def: RegDef) -> Result { } match array { - None if bitfield.is_none() => outputs.extend(quote!( - #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = - ::kernel::io::register::OffsetLoc::new(#offset); - )), - - _ if bitfield.is_none() => Err(Error::new_spanned( - ty, - "defining without bitfield is not yet supported for this type of register", - ))?, - - None => outputs.extend(quote_spanned!(span => - impl ::kernel::io::register::FixedRegister for #name { - type Base = #base; - const OFFSET: usize = #offset; + None => { + if unique { + outputs.extend(quote!( + impl ::kernel::io::register::FixedIoLoc<#base> for #ty { + type Location = ::kernel::io::register::OffsetLoc<#base, #ty>; + const LOCATION: Self::Location = #name; + } + )) } - #(#attrs)* #vis const #name: ::kernel::io::register::FixedRegisterLoc<#name> = - ::kernel::io::register::FixedRegisterLoc::<#name>::new(); - )), + outputs.extend(quote_spanned!(span => + #(#attrs)* #vis const #name: ::kernel::io::register::OffsetLoc<#base, #ty> = + ::kernel::io::register::OffsetLoc::new(#offset); + )); + } Some(def) => { + if bitfield.is_none() { + Err(Error::new_spanned( + &ty, + "defining without bitfield is not yet supported for this type of register", + ))? + } + let size = &def.size; let stride = if let Some(stride) = &def.stride { outputs.extend(quote_spanned!(stride.span().resolved_at(span) => -- cgit v1.2.3 From 1473ea0e38b3e3bc76d5e9f89ea4378550ed6328 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:42 +0900 Subject: gpu: nova-core: mm: Add VramAddress type Add the `VramAddress` type representing a physical address in VRAM. Also add an arithmetic helper, comparison, and operator overloads which are required in later patches for address arithmetic. Signed-off-by: Joel Fernandes [ecourtney: create mm.rs here, squashing in the arithmetic patch] [ecourtney: splice the two commit bodies] [ecourtney: drop the Pfn fields, open-coding what bitfield! generated] [ecourtney: drop align_down and the IntoVramOffset/IntoVramRange traits] [ecourtney: make checked_add() const over a plain u64, derive the ordering] [ecourtney: doc wording, header, import, and signature cleanups] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-1-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 60 ++++++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/nova_core.rs | 1 + 2 files changed, 61 insertions(+) create mode 100644 drivers/gpu/nova-core/mm.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs new file mode 100644 index 000000000000..dcd5e5e919bf --- /dev/null +++ b/drivers/gpu/nova-core/mm.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management subsystems. + +#![expect(dead_code)] + +use core::{ + fmt::LowerHex, + ops, // +}; + +use kernel::fmt; + +/// Physical VRAM address in GPU video memory. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub(crate) struct VramAddress(u64); + +impl VramAddress { + /// Creates an address from a raw value. + pub(crate) const fn from_raw(addr: u64) -> Self { + Self(addr) + } + + /// Returns the address as a raw value. + pub(crate) const fn into_raw(self) -> u64 { + self.0 + } + + /// Adds `rhs` to this address, returning [`None`] on overflow. + pub(crate) const fn checked_add(self, rhs: u64) -> Option { + match self.into_raw().checked_add(rhs) { + Some(addr) => Some(Self::from_raw(addr)), + None => None, + } + } +} + +impl LowerHex for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + LowerHex::fmt(&self.into_raw(), f) + } +} + +impl ops::Add for VramAddress { + type Output = Self; + + fn add(self, rhs: u64) -> Self::Output { + Self::from_raw(self.into_raw() + rhs) + } +} + +impl ops::Sub for VramAddress { + type Output = u64; + + fn sub(self, rhs: Self) -> Self::Output { + self.into_raw() - rhs.into_raw() + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 35a8b1214b0e..8f59cfa97017 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,6 +18,7 @@ mod fsp; mod gpu; mod gsp; mod mctp; +mod mm; #[macro_use] mod num; mod regs; -- cgit v1.2.3 From 8e43629439b7bbdbe4336300aa33727bae9792ca Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:43 +0900 Subject: gpu: nova-core: mm: Implement Alignable and Debug for VramAddress Later patches align VRAM addresses down to the PRAMIN window. Implement `Alignable` trait for `VramAddress` and plus add a `ZERO` constant. Also print the address in hex under `{:?}`, so it reads well in debug output. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-2-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index dcd5e5e919bf..f9d80ffbe01d 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -10,7 +10,14 @@ use core::{ ops, // }; -use kernel::fmt; +use kernel::{ + fmt, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, +}; /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -18,6 +25,9 @@ use kernel::fmt; pub(crate) struct VramAddress(u64); impl VramAddress { + /// The zero address. + pub(crate) const ZERO: Self = Self::from_raw(0); + /// Creates an address from a raw value. pub(crate) const fn from_raw(addr: u64) -> Self { Self(addr) @@ -37,12 +47,28 @@ impl VramAddress { } } +impl Alignable for VramAddress { + fn align_down(self, alignment: Alignment) -> Self { + Self::from_raw(self.into_raw().align_down(alignment)) + } + + fn align_up(self, alignment: Alignment) -> Option { + self.into_raw().align_up(alignment).map(Self::from_raw) + } +} + impl LowerHex for VramAddress { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { LowerHex::fmt(&self.into_raw(), f) } } +impl fmt::Debug for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(fmt!("{:#x}", self)) + } +} + impl ops::Add for VramAddress { type Output = Self; -- cgit v1.2.3 From d3bd73dcf51d88ae116a574ce2f49209c7b48944 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:44 +0900 Subject: gpu: nova-core: mm: Add PRAMIN window registers PRAMIN apertures are a crucial mechanism for direct CPU read/write to VRAM. Add the BAR0 window registers that position the PRAMIN aperture on all supported GPU architectures: Turing, Ampere, Ada (via `NV_PBUS_BAR0_WINDOW`), Hopper (via `gh100::NV_XAL_EP_BAR0_WINDOW`), and Blackwell (via `gb100::NV_XAL_EP_BAR0_WINDOW`). Hopper/Blackwell window-base registers are based on Eliot Courtney's offlist reference patch. Signed-off-by: Joel Fernandes [ecourtney: split the registers out of the PRAMIN patch into mm/regs.rs] [ecourtney: drop the register read path] [ecourtney: reword the message for the split, narrow visibility to mm] [ecourtney: plain base fields, as the bitfield cast+shift patch is dropped] [ecourtney: rename the target to VidMem, fix derives, redo the target docs] [ecourtney: annotate the register base types] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-3-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 2 ++ drivers/gpu/nova-core/mm/regs.rs | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/regs.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index f9d80ffbe01d..7d24ad790310 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,6 +19,8 @@ use kernel::{ }, }; +mod regs; + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] diff --git a/drivers/gpu/nova-core/mm/regs.rs b/drivers/gpu/nova-core/mm/regs.rs new file mode 100644 index 000000000000..82de6dfa4e8b --- /dev/null +++ b/drivers/gpu/nova-core/mm/regs.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Registers used by the memory management subsystems: the BAR0 PRAMIN window. + +use kernel::io::register; + +use crate::{ + bounded_enum, + driver::NovaRegisters, // +}; + +// PRAMIN window + +bounded_enum! { + /// Target memory type for the BAR0 window register. + /// + /// Only VRAM is needed by the driver. Pre-Hopper window registers also define + /// system-memory targets that are unused here; Hopper+ uses a separate register + /// without a target field. + #[derive(Debug, Copy, Clone)] + pub(super) enum Bar0WindowTarget with TryFrom> { + /// Video memory (GPU framebuffer memory). + VidMem = 0, + } +} + +register! { + base: NovaRegisters; + + /// BAR0 window control for PRAMIN access. + pub(super) NV_PBUS_BAR0_WINDOW(u32) @ 0x00001700 { + /// Target memory aperture for the window. + 25:24 target ?=> Bar0WindowTarget; + /// PRAMIN window base bits 39:16. + 23:0 base; + } +} + +pub(super) mod gh100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Hopper register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 37:16. + 21:0 base; + } + } +} + +pub(super) mod gb100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Blackwell GB10x/GB20x register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 38:16. + 22:0 base; + } + } +} -- cgit v1.2.3 From b05ff82081b7b90fdd70cc7d07b3da9d85184f88 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:45 +0900 Subject: gpu: nova-core: mm: Add the memory management HAL Positioning the PRAMIN window requires writing an architecture-specific register: `NV_PBUS_BAR0_WINDOW` on Turing, Ampere and Ada, and `NV_XAL_EP_BAR0_WINDOW` with a different field width on Hopper and on Blackwell. A `MmHal` trait with one implementation per hardware family hides the register choice from the rest of the mm code, matching the layout of the driver's other HALs. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-4-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/hal.rs | 56 +++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/mm/hal/gb100.rs | 35 ++++++++++++++++++++++ drivers/gpu/nova-core/mm/hal/gh100.rs | 35 ++++++++++++++++++++++ drivers/gpu/nova-core/mm/hal/tu102.rs | 37 +++++++++++++++++++++++ 5 files changed, 164 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/hal.rs create mode 100644 drivers/gpu/nova-core/mm/hal/gb100.rs create mode 100644 drivers/gpu/nova-core/mm/hal/gh100.rs create mode 100644 drivers/gpu/nova-core/mm/hal/tu102.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 7d24ad790310..07dce4ce2473 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,6 +19,7 @@ use kernel::{ }, }; +mod hal; mod regs; /// Physical VRAM address in GPU video memory. diff --git a/drivers/gpu/nova-core/mm/hal.rs b/drivers/gpu/nova-core/mm/hal.rs new file mode 100644 index 000000000000..e7fd1e38bd38 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management HAL. + +use kernel::{ + num::Bounded, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, + mm::VramAddress, // +}; + +mod gb100; +mod gh100; +mod tu102; + +/// Trait implemented by per-architecture MM HALs. +/// +/// `Sync` is required so that the `&'static dyn MmHal` references can be stored in `Send` +/// structures. +pub(super) trait MmHal: Sync { + /// Positions the PRAMIN window at `base`. + /// + /// This fails if `base` is not aligned to the 64 KiB window alignment or is too large for + /// the receiving register. + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result; +} + +/// Returns the HAL corresponding to `chipset`. +pub(super) fn mm_hal(chipset: Chipset) -> &'static dyn MmHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper => gh100::GH100_HAL, + Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL, + } +} + +/// Converts `base` into the value of the window-base register field. +/// +/// Fails with [`EINVAL`] if `base` is not aligned to the window alignment required by the register +/// field's shift, or if the shifted value does not fit within `RES` bits. +fn window_base(base: VramAddress) -> Result> { + const WINDOW_BASE_SHIFT: u32 = 16; + + Bounded::::from(base.into_raw()) + .shr_exact::() + .and_then(Bounded::try_shrink) + .ok_or(EINVAL) +} diff --git a/drivers/gpu/nova-core/mm/hal/gb100.rs b/drivers/gpu/nova-core/mm/hal/gb100.rs new file mode 100644 index 000000000000..3781e143dea7 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gb100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB10x/GB20x memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gb100; + +impl MmHal for Gb100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gb100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn MmHal = &GB100; diff --git a/drivers/gpu/nova-core/mm/hal/gh100.rs b/drivers/gpu/nova-core/mm/hal/gh100.rs new file mode 100644 index 000000000000..8af384db2921 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gh100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Hopper memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gh100; + +impl MmHal for Gh100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gh100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn MmHal = &GH100; diff --git a/drivers/gpu/nova-core/mm/hal/tu102.rs b/drivers/gpu/nova-core/mm/hal/tu102.rs new file mode 100644 index 000000000000..e4fe7561223c --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/tu102.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Turing, Ampere and Ada memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Tu102; + +impl MmHal for Tu102 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::NV_PBUS_BAR0_WINDOW::zeroed() + .with_target(regs::Bar0WindowTarget::VidMem) + .with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn MmHal = &TU102; -- cgit v1.2.3 From 9aa6069d624ea877a17b7d0952c993d2eb07dc22 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 27 Aug 2026 17:51:46 +0900 Subject: rust: io: add static `cast()` method for views Add a compile-time checked variant of `try_cast()` using the minimum size and alignment information. Signed-off-by: Gary Guo Link: https://patch.msgid.link/20260805-typed_register-v2-1-c3ca142220a0@garyguo.net [ecourtney: fix the doc example type and doc grammar per v2 review] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-5-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- rust/kernel/io.rs | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs index 7a0e0ebc6c85..de8ef8e2aec4 100644 --- a/rust/kernel/io.rs +++ b/rust/kernel/io.rs @@ -447,6 +447,45 @@ pub trait Io<'a>: IoBase<'a> { self.len() == 0 } + /// Convert into a different typed I/O view. + /// + /// The target type must be known (statically) to be of the same or smaller size to current + /// type, and the current view must be properly aligned for the target type. + /// + /// # Examples + /// + /// ```no_run + /// use kernel::io::{ + /// io_project, + /// Mmio, + /// Io, + /// Region, + /// }; + /// #[derive(FromBytes, IntoBytes)] + /// #[repr(C)] + /// struct MyStruct { field: u32, } + /// + /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) { + /// // let mmio: Mmio<'_, Region<0x1000>>; + /// let whole: Mmio<'_, MyStruct> = mmio.cast(); + /// # } + /// ``` + #[inline] + fn cast(self) -> ::View<'a, U> + where + Self::Target: FromBytes + IntoBytes, + U: FromBytes + IntoBytes, + { + let view = self.as_view(); + let ptr = Self::Backend::as_ptr(view); + + const_assert!(size_of::() <= Self::Target::MIN_SIZE); + const_assert!(align_of::() <= Self::Target::MIN_ALIGN.as_usize()); + + // SAFETY: We have checked bounds and alignment, so this is a valid projection. + unsafe { Self::Backend::project_view(view, ptr.cast()) } + } + /// Try to convert into a different typed I/O view. /// /// A runtime check is performed to ensure that the target type is of same or smaller size to -- cgit v1.2.3 From a14a957c0e619437fd3a67bc435120b0b6270914 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:47 +0900 Subject: gpu: nova-core: mm: Add support to use PRAMIN windows to write to VRAM PRAMIN apertures are a crucial mechanism for direct CPU read/write to VRAM. Add a `Pramin` manager whose `window_at()` returns a typed MMIO view of VRAM through the 1 MiB PRAMIN aperture in BAR0, validating the view against the VRAM region and repositioning the window as needed for the accessed address. A view borrows `Pramin` mutably, so the window cannot move while the view is in use, and it inserts an ordering point on Drop. Signed-off-by: Joel Fernandes [ecourtney: split the registers and HAL into the two preceding patches] [ecourtney: rebase w.r.t. Bar0 lifetime changes and register projections] [ecourtney: drop the window guard and mutex, use &mut self] [ecourtney: position at init to avoid reads, reposition in window_offset] [ecourtney: return typed MMIO views instead of read/write accessors] [ecourtney: insert an ordering read when a view drops] [ecourtney: declare the window location, drop the doc examples] [ecourtney: add the copyright header, doc and naming cleanups] [ecourtney: the pramin module is mm-internal] Co-developed-by: Eliot Courtney Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-6-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/pramin.rs | 178 +++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pramin.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 07dce4ce2473..ef5b1cad56c3 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -20,6 +20,7 @@ use kernel::{ }; mod hal; +mod pramin; mod regs; /// Physical VRAM address in GPU video memory. diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs new file mode 100644 index 000000000000..20be3fc471ba --- /dev/null +++ b/drivers/gpu/nova-core/mm/pramin.rs @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Utilities for accessing VRAM through the PRAMIN window. + +use core::ops::Range; + +use kernel::{ + io::{ + io_project, + register, + register::OffsetLoc, + Io, + Mmio, // + }, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::{ + SZ_1M, + SZ_64K, // + }, +}; + +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + gpu::Chipset, + mm::{ + hal::{ + self, + MmHal, // + }, + VramAddress, // + }, + num::IntoSafeCast, // +}; + +/// Size of the PRAMIN window (1 MiB). +const WINDOW_SIZE: usize = SZ_1M; + +/// The PRAMIN window, which is a 1 MiB window into VRAM at a fixed BAR0 offset. +#[derive(FromBytes, IntoBytes)] +struct PraminWindow([u8; WINDOW_SIZE]); + +register! { + base: NovaRegisters; + + /// Location of the window inside BAR0. + PRAMIN: PraminWindow @ 0x700000; +} + +/// Owner of the PRAMIN window state. +/// +/// [`Pramin::window_at()`] repositions the window as needed and returns a typed MMIO view into +/// it, holding the manager borrowed for the lifetime of the view. +pub(super) struct Pramin<'gpu> { + bar: Bar0<'gpu>, + hal: &'static dyn MmHal, + /// MMIO view of the PRAMIN window in BAR0. + window: Mmio<'gpu, PraminWindow>, + /// VRAM range to keep the PRAMIN window inside. + vram_range: Range, + /// Cached window position. + window_range: Range, +} + +/// Typed view of VRAM through the PRAMIN window. +/// +/// Inserts an ordering point after previous writes through the window on drop. Views returned +/// by [`PraminAccess::view()`] cannot outlive this access, so the ordering point covers every +/// write made through them. +pub(super) struct PraminAccess<'a, T> +where + T: FromBytes + IntoBytes, +{ + view: Mmio<'a, T>, +} + +impl PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + /// Returns the MMIO view of the accessed location. + pub(super) fn view(&self) -> Mmio<'_, T> { + self.view + } +} + +impl Drop for PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + fn drop(&mut self) { + // Insert an ordering point after previous writes through this window. + self.view.cast::().read_val(); + } +} + +impl<'gpu> Pramin<'gpu> { + /// Alignment required by the PRAMIN window. + const BASE_ALIGN: Alignment = Alignment::new::(); + + /// Creates the window manager for the given VRAM region. + pub(super) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + vram_range: Range, + ) -> Result { + let hal = hal::mm_hal(chipset); + let window = io_project!(bar, build: PRAMIN); + let base = vram_range.start.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + hal.write_pramin_window_base(bar, base)?; + + Ok(Self { + bar, + hal, + window, + vram_range, + window_range, + }) + } + + /// Returns the VRAM range a window based at `base` exposes. + fn window_range(base: VramAddress) -> Result> { + let end = base + .checked_add(WINDOW_SIZE.into_safe_cast()) + .ok_or(EINVAL)?; + Ok(base..end) + } + + /// Check the window covers `len` bytes at `addr`, moving it if needed. + /// + /// Returns the window offset at which to perform the access. + fn window_offset(&mut self, addr: VramAddress, len: usize) -> Result { + let end = addr.checked_add(len.into_safe_cast()).ok_or(EINVAL)?; + + let inside = |r: &Range| r.contains(&addr) && end <= r.end; + if !inside(&self.vram_range) { + return Err(EINVAL); + } + + // Reposition the window if the access falls outside it. + if !inside(&self.window_range) { + let base = addr.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + if !inside(&window_range) { + return Err(EINVAL); + } + self.hal.write_pramin_window_base(self.bar, base)?; + self.window_range = window_range; + } + + Ok((addr - self.window_range.start).into_safe_cast()) + } + + /// Return a typed MMIO view of a `T` at `vram_addr`. + /// + /// Returns an error if `vram_addr` is not aligned to `T`'s alignment, or if + /// a `T` at `vram_addr` does not fit within the VRAM region. + pub(super) fn window_at<'a, T>( + &'a mut self, + vram_addr: VramAddress, + ) -> Result> + where + T: FromBytes + IntoBytes, + { + let offset = self.window_offset(vram_addr, size_of::())?; + let view = io_project!(self.window, try: OffsetLoc::new(offset)); + + Ok(PraminAccess { view }) + } +} -- cgit v1.2.3 From b4481b9e3b5a3e1b45b5a70880cc2b730bc3700a Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:48 +0900 Subject: docs: gpu: nova-core: Document the PRAMIN aperture mechanism Add documentation for the PRAMIN aperture mechanism used by nova-core for direct VRAM access. Nova only uses TARGET=VRAM for VRAM access. The SYS_MEM target values are documented for completeness but not used by the driver. Signed-off-by: Joel Fernandes [ecourtney: fix the TARGET encodings, KiB/MiB units, wording nits] [ecourtney: wrap at 80 columns] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-7-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/pramin.rst | 128 +++++++++++++++++++++++++++++++++ Documentation/gpu/nova/index.rst | 1 + 2 files changed, 129 insertions(+) create mode 100644 Documentation/gpu/nova/core/pramin.rst diff --git a/Documentation/gpu/nova/core/pramin.rst b/Documentation/gpu/nova/core/pramin.rst new file mode 100644 index 000000000000..f50b052d73ba --- /dev/null +++ b/Documentation/gpu/nova/core/pramin.rst @@ -0,0 +1,128 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========================= +PRAMIN aperture mechanism +========================= + +.. note:: + The following description is approximate and current as of the Ampere + family. It may change for future generations and is intended to assist in + understanding the driver code. + +Introduction +============ + +PRAMIN is a hardware aperture mechanism that provides CPU access to GPU Video +RAM (VRAM) before the GPU's Memory Management Unit (MMU) and page tables are +initialized. This 1 MiB sliding window, located at a fixed offset within BAR0, +is essential for setting up page tables and other critical GPU data structures +without relying on the GPU's MMU. + +Architecture Overview +===================== + +The PRAMIN aperture mechanism is logically implemented by the GPU's PBUS (PCIe +Bus Controller Unit) and provides a CPU-accessible window into VRAM through the +PCIe interface:: + + +-----------------+ PCIe +------------------------------+ + | CPU |<----------->| GPU | + +-----------------+ | | + | +----------------------+ | + | | PBUS | | + | | (Bus Controller) | | + | | | | + | | +--------------+ <------------ [1] + | | | PRAMIN | | | + | | | Window | | | + | | | (1 MiB) | | | + | | +--------------+ | | + | | | | | + | +---------|------------+ | + | | | + | v | + | +----------------------+ <------- [2] + | | VRAM | | + | | (Several GiB) | | + | | | | + | | FB[0x0000000000] | | + | | ... | | + | | FB[0xFFFFFFFFFF] | | + | +----------------------+ | + +------------------------------+ + + [1] Window starts at BAR0 + 0x700000. + [2] Program PRAMIN to any 64 KiB-aligned VRAM boundary. + +PBUS is responsible for, among other things, handling MMIO +accesses to the BAR registers. + +PRAMIN Window Operation +======================= + +The PRAMIN window provides a 1 MiB sliding aperture that can be repositioned +over the entire VRAM address space using the ``NV_PBUS_BAR0_WINDOW`` register. + +Window Control Mechanism +------------------------- + +:: + + NV_PBUS_BAR0_WINDOW Register (0x1700): + +-------+--------+--------------------------------------+ + | 31:26 | 25:24 | 23:0 | + | RSVD | TARGET | BASE_ADDR | + | | | (bits 39:16 of VRAM address) | + +-------+--------+--------------------------------------+ + + The 24-bit BASE_ADDR field encodes bits [39:16] of the target VRAM address, + providing 40-bit (1 TiB) address space coverage with 64 KiB alignment. + + TARGET field (bits 25:24): + - 0x0: VRAM (Video Memory) + - 0x1: Reserved (unused) + - 0x2: SYS_MEM_COH (Coherent System Memory) + - 0x3: SYS_MEM_NONCOH (Non-coherent System Memory) + +.. note:: + Nova only uses TARGET=VRAM (0x0) for video memory access. The SYS_MEM + target values are documented here for hardware completeness but are + not used by the driver. + +64 KiB Alignment Requirement +---------------------------- + +The PRAMIN window must be aligned to 64 KiB boundaries in VRAM. This is enforced +by the ``BASE_ADDR`` field representing bits [39:16] of the target address:: + + VRAM Address Calculation: + actual_vram_addr = (BASE_ADDR << 16) + pramin_offset + Where: + - BASE_ADDR: 24-bit value from NV_PBUS_BAR0_WINDOW[23:0] + - pramin_offset: 20-bit offset within the PRAMIN window [0x00000-0xFFFFF] + + Example Window Positioning: + +---------------------------------------------------------+ + | VRAM Space | + | | + | 0x0000000000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0x00000FFFFF +-----------------+ | + | | + | | ^ | + | | | Window can slide | + | v | to any 64 KiB-aligned boundary | + | | + | 0x0123400000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0x01234FFFFF +-----------------+ | + | | + | ... | + | | + | 0xFFFFF00000 +-----------------+ <-- 64 KiB aligned | + | | PRAMIN Window | | + | | (1 MiB) | | + | 0xFFFFFFFFFF +-----------------+ | + +---------------------------------------------------------+ diff --git a/Documentation/gpu/nova/index.rst b/Documentation/gpu/nova/index.rst index 2afa58e8f08d..59b206238498 100644 --- a/Documentation/gpu/nova/index.rst +++ b/Documentation/gpu/nova/index.rst @@ -34,3 +34,4 @@ vGPU manager VFIO driver and the nova-drm driver. core/fwsec core/falcon core/tlv + core/pramin -- cgit v1.2.3 From 5958450e98e52adb61c4967d67b76734ff9052c0 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:49 +0900 Subject: gpu: nova-core: mm: Add GpuMm centralized memory manager Introduce GpuMm as the centralized GPU memory manager. At this point in the series, GpuMm only owns the PRAMIN window for direct VRAM access; the buddy allocator and TLB manager are added later when those backing types become available. This provides a clean ownership model where GpuMm provides accessor methods for its components that can be used for memory management operations, and lets follow-on patches (such as the PRAMIN aperture self-tests) reference `pramin_mut()` cleanly. Signed-off-by: Joel Fernandes [ecourtney: squash the total VRAM end patch, drop its dev_info print] [ecourtney: take the maximum FB limit before adding one, fail with EINVAL] [ecourtney: rebase for the Bar0 lifetime and ownership changes, drop Arcs] [ecourtney: source the chipset through gsp_resources] [ecourtney: build the VRAM region in new(), pramin() becomes pramin_mut()] [ecourtney: declare mm before gsp_resources, doc wording cleanups] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-8-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 18 +++++++++++++++- drivers/gpu/nova-core/gsp/commands.rs | 4 ++++ drivers/gpu/nova-core/gsp/fw/commands.rs | 5 +++++ drivers/gpu/nova-core/mm.rs | 35 ++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index e1ac8ee9ba4d..0e270fe1dd87 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -29,6 +29,10 @@ use crate::{ Gsp, GspBootContext, // }, + mm::{ + GpuMm, + VramAddress, // + }, vgpu::VgpuManager, // }; @@ -283,6 +287,11 @@ pub(crate) struct Gpu<'gpu> { spec: Spec, /// Static GPU information as provided by the GSP. gsp_static_info: GetGspStaticInfoReply, + /// GPU memory manager owning memory management resources. + /// + /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while + /// the GSP is still operational. + mm: GpuMm<'gpu>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -410,7 +419,14 @@ impl<'gpu> Gpu<'gpu> { } info - } + }, + + // Create GPU memory manager owning memory management resources. + mm: GpuMm::new( + bar, + gsp_resources.spec.chipset, + VramAddress::from_raw(gsp_static_info.total_fb_end), + )?, }) } } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ffc25fd8c47b..eea1c9ed4684 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -216,6 +216,8 @@ pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, + /// Exclusive end of the FB physical address space. + pub(crate) total_fb_end: u64, } impl MessageFromGsp for GetGspStaticInfoReply { @@ -231,10 +233,12 @@ impl MessageFromGsp for GetGspStaticInfoReply { for region in msg.usable_fb_regions() { usable_fb_regions.push(region, GFP_KERNEL)?; } + let total_fb_end = msg.total_fb_end().ok_or(EINVAL)?; Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), usable_fb_regions, + total_fb_end, }) } } diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 6dc31d1bf5ae..6e85442a4b13 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -165,6 +165,11 @@ impl GspStaticConfigInfo { } }) } + + /// Computes the exclusive end of the FB physical address space. + pub(crate) fn total_fb_end(&self) -> Option { + self.fb_regions().map(|reg| reg.limit).max()?.checked_add(1) + } } // SAFETY: Padding is explicit and will not contain uninitialized data. diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index ef5b1cad56c3..58dce211a337 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -19,10 +19,45 @@ use kernel::{ }, }; +use crate::{ + driver::Bar0, + gpu::Chipset, // +}; + mod hal; mod pramin; mod regs; +/// GPU Memory Manager - owns all core MM components. +/// +/// Provides centralized ownership of memory management resources: +/// - [`pramin::Pramin`] for direct VRAM access. +pub(crate) struct GpuMm<'gpu> { + pramin: pramin::Pramin<'gpu>, +} + +impl<'gpu> GpuMm<'gpu> { + /// Creates the GPU memory manager. + pub(crate) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + total_fb_end: VramAddress, + ) -> Result { + // PRAMIN covers all physical VRAM (including GSP-reserved areas + // above the usable region, e.g. the BAR1 page directory). + let vram_region = VramAddress::ZERO..total_fb_end; + + Ok(Self { + pramin: pramin::Pramin::new(bar, chipset, vram_region)?, + }) + } + + /// Access the [`pramin::Pramin`]. + fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { + &mut self.pramin + } +} + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] -- cgit v1.2.3 From 2c76c4ec9eb0c91f7b37795f83df8a34d4172285 Mon Sep 17 00:00:00 2001 From: Eliot Courtney Date: Thu, 27 Aug 2026 17:51:50 +0900 Subject: gpu: nova-core: Add self-test assertion macros and config option The existing assert! and assert_eq! macros cause a panic. For self tests in nova-core, it's inconvenient to cause a panic since these need to be run on actual hardware. Instead, define similar macros that log an error then return an Err. Also add the NOVA_CORE_SELFTESTS Kconfig option that gates the driver self-tests. Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-9-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/Kconfig | 9 ++++++ drivers/gpu/nova-core/nova_core.rs | 2 ++ drivers/gpu/nova-core/selftest.rs | 64 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 drivers/gpu/nova-core/selftest.rs diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index f918f69e0599..cb7f0b00f796 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -15,3 +15,12 @@ config NOVA_CORE This driver is work in progress and may not be functional. If M is selected, the module will be called nova-core. + +config NOVA_CORE_SELFTESTS + bool "Nova Core driver self-tests" + depends on NOVA_CORE + default n + help + Build the driver self-tests and run them when the GPU is probed. + + If unsure, say N. diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 8f59cfa97017..1133c6ce5c55 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -23,6 +23,8 @@ mod mm; mod num; mod regs; mod sbuffer; +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +mod selftest; mod vbios; mod vgpu; diff --git a/drivers/gpu/nova-core/selftest.rs b/drivers/gpu/nova-core/selftest.rs new file mode 100644 index 000000000000..f5b5965b7e6a --- /dev/null +++ b/drivers/gpu/nova-core/selftest.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Assertion macros for driver self-tests. +//! +//! Self-tests run against live hardware during probe, so a failed assertion should not panic. These +//! macros log the failure on the device and fail the enclosing test by returning +//! [`EIO`](kernel::error::code::EIO) instead. + +/// Like [`assert!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert!`], a custom message with format arguments can follow the condition. +#[macro_export] +macro_rules! selftest_assert { + ($dev:expr, $cond:expr $(,)?) => { + $crate::selftest_assert!($dev, $cond, "assertion failed: {}", ::core::stringify!($cond)) + }; + ($dev:expr, $cond:expr, $($arg:tt)+) => {{ + if !$cond { + ::kernel::dev_err!( + $dev, + "Selftest: {}:{}: {}\n", + ::core::file!(), + ::core::line!(), + ::kernel::prelude::fmt!($($arg)+) + ); + return Err(::kernel::error::code::EIO); + } + }}; +} + +/// Like [`assert_eq!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert_eq!`], a custom message with format arguments can follow the compared values. +#[macro_export] +macro_rules! selftest_assert_eq { + ($dev:expr, $left:expr, $right:expr $(,)?) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `{} == {}` failed: left {:?}, right {:?}", + ::core::stringify!($left), + ::core::stringify!($right), + left, + right + ), + } + }; + ($dev:expr, $left:expr, $right:expr, $($arg:tt)+) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `left == right` failed: {}: left {:?}, right {:?}", + ::kernel::prelude::fmt!($($arg)+), + left, + right + ), + } + }; +} -- cgit v1.2.3 From e6a2c988ed96a5a3af51ed97ecba980521bf2fc0 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 27 Aug 2026 17:51:51 +0900 Subject: gpu: nova-core: mm: Add PRAMIN aperture self-tests Add self-tests for the PRAMIN aperture mechanism to verify correct operation during GPU probe. The tests validate various alignment requirements and corner cases. The tests are default disabled and behind CONFIG_NOVA_CORE_SELFTESTS. When enabled, tests run after GSP boot during probe. Signed-off-by: Joel Fernandes [ecourtney: convert the tests to window_at(), macros, and the new types] [ecourtney: cfg-gate the tests and expect(dead_code), not a runtime no-op] [ecourtney: run the self-tests on all architectures, drop the chipset arg] [ecourtney: test within a usable FB region, skip when none is large enough] [ecourtney: report failures without failing probe, start banner at dev_dbg] [ecourtney: removed the mm-specific Kconfig option] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260827-pramin-split-v3-10-24b24d7afc52@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 3 + drivers/gpu/nova-core/gpu.rs | 12 ++++ drivers/gpu/nova-core/mm.rs | 40 ++++++++++- drivers/gpu/nova-core/mm/pramin.rs | 134 +++++++++++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index cf3534dd47d4..8cef4c284277 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -86,6 +86,9 @@ impl pci::Driver for NovaCoreDriver { // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned // stable address, and is dropped after `gpu` (struct field drop order). gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + // Run optional GPU selftests. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _: { gpu.run_selftests(pdev) }, _reg: auxiliary::Registration::new( pdev.as_ref(), c"nova-drm", diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 0e270fe1dd87..a52a3d4d86af 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -429,6 +429,18 @@ impl<'gpu> Gpu<'gpu> { )?, }) } + + /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device) { + let this = self.project(); + let dev = pdev.as_ref(); + let regions = &this.gsp_static_info.usable_fb_regions; + + if let Err(err) = crate::mm::selftest::run(dev, this.mm, regions) { + dev_err!(dev, "self-tests failed: {:?}\n", err); + } + } } /// Reads the boot0 register and returns its raw value. diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 58dce211a337..9e4338c7c393 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -3,7 +3,7 @@ //! Memory management subsystems. -#![expect(dead_code)] +#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] use core::{ fmt::LowerHex, @@ -123,3 +123,41 @@ impl ops::Sub for VramAddress { self.into_raw() - rhs.into_raw() } } + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) mod selftest { + use core::ops::Range; + + use kernel::{ + device, + sizes::SizeConstants, // + }; + + use super::*; + + /// Run MM subsystem self-tests during probe. + pub(crate) fn run( + dev: &device::Device, + mm: &mut GpuMm<'_>, + usable_fb_regions: &[Range], + ) -> Result { + // VRAM span the self-tests are free to overwrite, from the chosen test base. + const SELFTEST_SPAN: u64 = u64::SZ_64M; + + let base = usable_fb_regions.iter().find_map(|region| { + // Tests rely on this being 8 byte aligned for checking misalignment handling. + let base = region.start.align_up(Alignment::new::<8>())?; + (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) + }); + let Some(base) = base else { + dev_warn!( + dev, + "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", + SELFTEST_SPAN + ); + return Ok(()); + }; + + pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) + } +} diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs index 20be3fc471ba..7f89c093d591 100644 --- a/drivers/gpu/nova-core/mm/pramin.rs +++ b/drivers/gpu/nova-core/mm/pramin.rs @@ -176,3 +176,137 @@ impl<'gpu> Pramin<'gpu> { Ok(PraminAccess { view }) } } + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) mod selftest { + use kernel::{ + device, + io::io_read, + sizes::SizeConstants, // + }; + + use super::*; + use crate::{ + selftest_assert, + selftest_assert_eq, // + }; + + /// Test read/write at byte granularity, at unaligned addresses. + fn test_byte_readwrite( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + pramin.window_at::(addr)?.view().write_val(0xA0 + i); + } + + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + selftest_assert_eq!( + dev, + pramin.window_at::(addr)?.view().read_val(), + 0xA0 + i + ); + } + Ok(()) + } + + /// Test writing a `u32` and reading back as individual `u8`s. + fn test_u32_as_bytes( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr = base + 0x10; + let val: u32 = 0xDEADBEEF; + pramin.window_at::(addr)?.view().write_val(val); + + let window = pramin.window_at::<[u8; 4]>(addr)?; + for (i, &expected) in val.to_le_bytes().iter().enumerate() { + selftest_assert_eq!(dev, io_read!(window.view(), [build: i]), expected); + } + Ok(()) + } + + /// Test window repositioning across 1 MiB boundaries. + fn test_window_reposition( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr_a = base; + let addr_b = base + u64::SZ_2M; // base + 2 MiB (different 1 MiB region). + let val_a: u32 = 0x11111111; + let val_b: u32 = 0x22222222; + + pramin.window_at::(addr_a)?.view().write_val(val_a); + pramin.window_at::(addr_b)?.view().write_val(val_b); + + selftest_assert_eq!( + dev, + pramin.window_at::(addr_a)?.view().read_val(), + val_a + ); + selftest_assert_eq!( + dev, + pramin.window_at::(addr_b)?.view().read_val(), + val_b + ); + Ok(()) + } + + /// Test that offsets outside the VRAM region are rejected. + fn test_invalid_offset( + dev: &device::Device, + pramin: &mut Pramin<'_>, + vram_end: VramAddress, + ) -> Result { + selftest_assert!(dev, pramin.window_at::(vram_end).is_err()); + Ok(()) + } + + /// Test that misaligned accesses are rejected. + fn test_misaligned_access( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + // `u16` at odd offset (not 2-byte aligned). + selftest_assert!(dev, pramin.window_at::(base + 0x21).is_err()); + + // `u32` at 2-byte-aligned (not 4-byte-aligned) offset. + selftest_assert!(dev, pramin.window_at::(base + 2).is_err()); + + // `u64` at a 4-byte-aligned (not 8-byte-aligned) address. + selftest_assert!(dev, pramin.window_at::(base + 0x44).is_err()); + + // A `u16` view at an even address is allowed. + pramin.window_at::(base + 0x22)?; + Ok(()) + } + + /// Run PRAMIN self-tests during probe. + /// + /// `base` is the start of a driver-usable VRAM span that the tests are free to + /// overwrite. + pub(crate) fn run( + dev: &device::Device, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + dev_dbg!(dev, "PRAMIN: starting self-tests\n"); + + let vram_end = pramin.vram_range.end; + + test_byte_readwrite(dev, pramin, base)?; + test_u32_as_bytes(dev, pramin, base)?; + test_window_reposition(dev, pramin, base)?; + test_invalid_offset(dev, pramin, vram_end)?; + test_misaligned_access(dev, pramin, base)?; + + dev_info!(dev, "PRAMIN: self-tests passed\n"); + Ok(()) + } +} -- cgit v1.2.3 From 3282d591601949e2970b73024a00e54f0f6fdb05 Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Sat, 5 Sep 2026 10:53:42 +0200 Subject: rust: Add dma_fence abstractions DMA fences are synchronisation primitives that will be needed by all Rust GPU drivers. The dma_fence framework sets a number of rules, notably: - fences must only be signaled once - all fences must be signaled at some point - fence error codes must only be set before signaling - every pointer to a fence must be backed by a reference All those rules are being addressed by these abstractions. To cleanly decouple fence issuers and consumers, two types are provided: - DriverFence: the only fence type that can be signaled and that carries driver-specific data. - Fence: the fence type to be shared with other drivers and / or userspace. The only type callbacks can be registered on. Cannot be signaled. Hereby, a Fence lives in the same chunk of memory as a DriverFence. Both share the refcount of the underlying C dma_fence. Since this implementation does not provide a custom dma_fence_backend_ops.release() function, the memory is freed by the dma_fence backend once the refcount drops to 0. To create a DriverFence, the user must first allocate a DriverFenceAllocation, so that the creation of the DriverFence later on can always succeed. Otherwise, deadlocks could occur if fences need to be created in a GPU job submission path. Synchronization is ensured by the dma_fence backend. All DriverFence's created through this abstraction must be signaled by the creator with an error code. In case a DriverFence drops without being signaled beforehand, it is signaled with -ECANCELLED as its error and a warning is printed. This allows the Rust abstraction to very cleanly decouple fence issuer and consumer by relying on the decoupling mechanisms in the C backend, which ensures through RCU and the 'signaled' fence-flag that dma_fence_backend_ops functions cannot access the potentially unloaded driver code anymore. Signalling fences on drop thus grants many advantages. Not signaling fences on drop would risk deadlock and does not grant real advantages: By definition only the drivers can ensure that a fence always represents the hardware's state correctly. This implementation models a DmaFenceContext object on which fences are to be created, thereby ensuring correct sequence numbering according to the timeline. dma_fence supports a variety of callbacks. The mandatory callbacks (get_timeline_name() and get_driver_name()) are implemented in this patch. For convenience, they store those name parameters in the fence context, saving the driver from implementing these two callbacks. Support for other callbacks (like for hardware signaling) is prepared for through the fact that both DriverFence and Fence live in the same allocation, allowing for usage of container_of from the callback to access the driver-specific data. It is expected that other callbacks, added in the future, also mostly operate on the generic data in the FenceContext. To make this safe, the implementation ensures through a lifetime that a DriverFence cannot outlive its FenceContext. Synchronization for dma_fence_ops callbacks is ensured by only running the Rust deconstructor delayed with call_rcu(), which prevents UAF-bugs should a DriverFence drop while a Fence callback is currently operating on the associated driver data. Since they can also operate on the FenceContext's data, its drop implementation also performs the necessary delay with rcu_barrier(). An additional issue discovered during the review process of this code is that there is (currently) no mechanism in Rust to prevent someone from circumventing the DriverFence's FenceContext-reference's lifetime by "forgetting" the fence, e.g. with core::mem::forget(). This would enable UAF bugs on the FenceContext. Throw a panic if this happens and document a path towards a more robust solution. Add abstractions for dma_fence in Rust. Signed-off-by: Philipp Stanner Tested-by: Daniel Almeida Reviewed-by: Daniel Almeida Link: https://patch.msgid.link/20260905085343.1827305-3-phasta@kernel.org [ In dma_fence_callback(), split combined unsafe block into separate blocks for container_of!() and pointer dereference. - Danilo ] Signed-off-by: Danilo Krummrich --- rust/bindings/bindings_helper.h | 1 + rust/helpers/dma_fence.c | 49 ++ rust/helpers/helpers.c | 1 + rust/kernel/dma_buf/dma_fence.rs | 1022 ++++++++++++++++++++++++++++++++++++++ rust/kernel/dma_buf/mod.rs | 14 + rust/kernel/lib.rs | 1 + 6 files changed, 1088 insertions(+) create mode 100644 rust/helpers/dma_fence.c create mode 100644 rust/kernel/dma_buf/dma_fence.rs create mode 100644 rust/kernel/dma_buf/mod.rs diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 4b31aa7f432f..930e63290cdd 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include diff --git a/rust/helpers/dma_fence.c b/rust/helpers/dma_fence.c new file mode 100644 index 000000000000..549f6b6a7171 --- /dev/null +++ b/rust/helpers/dma_fence.c @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper void rust_helper_dma_fence_get(struct dma_fence *f) +{ + dma_fence_get(f); +} + +__rust_helper void rust_helper_dma_fence_put(struct dma_fence *f) +{ + dma_fence_put(f); +} + +__rust_helper bool rust_helper_dma_fence_begin_signalling(void) +{ + return dma_fence_begin_signalling(); +} + +__rust_helper void rust_helper_dma_fence_end_signalling(bool cookie) +{ + dma_fence_end_signalling(cookie); +} + +__rust_helper bool rust_helper_dma_fence_is_signaled(struct dma_fence *f) +{ + return dma_fence_is_signaled(f); +} + +__rust_helper bool rust_helper_dma_fence_test_signaled_flag(struct dma_fence *f) +{ + return dma_fence_test_signaled_flag(f); +} + +__rust_helper void rust_helper_dma_fence_lock_irqsave(struct dma_fence *f, unsigned long *flags) +{ + dma_fence_lock_irqsave(f, *flags); +} + +__rust_helper void rust_helper_dma_fence_unlock_irqrestore(struct dma_fence *f, + unsigned long *flags) +{ + dma_fence_unlock_irqrestore(f, *flags); +} + +__rust_helper void rust_helper_dma_fence_set_error(struct dma_fence *f, int error) +{ + dma_fence_set_error(f, error); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 440fb7638e3c..609689fba3dd 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -57,6 +57,7 @@ #include "cred.c" #include "device.c" #include "dma.c" +#include "dma_fence.c" #ifdef CONFIG_DMA_SHARED_BUFFER #include "dma-resv.c" #endif diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs new file mode 100644 index 000000000000..18a43e1bb442 --- /dev/null +++ b/rust/kernel/dma_buf/dma_fence.rs @@ -0,0 +1,1022 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025-2026 Red Hat Inc. + * Author: Philipp Stanner + */ + +//! DMA Fence support. +//! +//! Reference: +//! +//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h) + +use crate::{ + alloc::AllocError, + bindings, + container_of, + error::to_result, + prelude::*, + types::ForeignOwnable, + types::Opaque, // +}; + +use core::{ + marker::PhantomData, + mem::ManuallyDrop, + ops::Deref, + ptr, + ptr::{ + drop_in_place, + NonNull, // + }, // +}; + +use kernel::{ + str::CString, + sync::{ + aref::{ + ARef, + AlwaysRefCounted, // + }, + atomic::{ + Atomic, + Relaxed, // + }, + rcu::rcu_barrier, // + }, // +}; + +/// VTable for dma_fence backend_ops callbacks. +// +// Mandatory dma_fence backend_ops are implemented implicitly through +// [`FenceContext`]. Additional ones shall get implemented on this trait. +pub trait FenceContextOps { + /// The generic payload data for [`DriverFence`]s created on this fctx. + type FenceDataType: Send + Sync; +} + +/// A dma-fence context. A fence context takes care of associating related fences +/// with each other, providing each with raising sequence numbers and a common +/// identifier. +#[pin_data(PinnedDrop)] +pub struct FenceContext { + /// The fence context number. + nr: u64, + /// The sequence number for the next fence created. + seqno: Atomic, + // The name parameters can be accessed by the dma_fence backend_ops. UAF + // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`. + /// The name of the driver this FenceContext's fences belong to. + driver_name: CString, + /// The name of the timeline this FenceContext's fences belong to. + timeline_name: CString, + /// The number of all unsignaled fences on this context. + // Used to prevent bugs due to forgotten fences. + // + // The lifetime on `DriverFence`s should typically prevent this from + // happening. + // + // However, we cannot fully guarantee in Rust that `DriverFence`s will not + // be forgotten, e.g., through `core::mem::forget()`. This could circumvent + // the lifetime which intends to enforce that all fences disappear before + // their context. + nr_of_unsignaled_fences: Atomic, + /// The user's data. + #[pin] + data: T, +} + +impl<'a, T: Send + Sync + FenceContextOps> FenceContext { + // This can later be extended as a vtable in case other parties need support + // for the more "exotic" callbacks. + const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops { + get_driver_name: Some(Self::get_driver_name), + get_timeline_name: Some(Self::get_timeline_name), + enable_signaling: None, + signaled: None, + // Deprecated. + wait: None, + // Must never be implemented for these abstractions. + release: None, + set_deadline: None, + }; + + /// Create a new `FenceContext`. + pub fn new( + initial_seqno: u64, + driver_name: &CStr, + timeline_name: &CStr, + data: impl PinInit, + ) -> impl PinInit + where + Error: From, + { + let driver_name = CString::try_from(driver_name); + let timeline_name = CString::try_from(timeline_name); + try_pin_init!(Self { + // SAFETY: `dma_fence_context_alloc()` merely works on a global + // atomic. Parameter `1` is the number of contexts we want to + // allocate. + nr: unsafe { bindings::dma_fence_context_alloc(1) }, + seqno: Atomic::new(initial_seqno), + driver_name: driver_name?, + timeline_name: timeline_name?, + nr_of_unsignaled_fences: Atomic::new(0), + data <- data, + }) + } + + fn next_seqno(&self) -> u64 { + self.seqno.fetch_add(1, Relaxed) + } + + /// Allocate the memory for a [`DriverFence`] and already store `data` inside. + /// + /// This is needed because many times, creation of a [`DriverFence`] must not + /// fail, and allocating might deadlock in some situations. + /// + /// The `data` you pass here must not perform any operations that are illegal + /// in atomic context in its [`Drop`] implementation. + pub fn new_fence_allocation( + &self, + data: T::FenceDataType, + ) -> Result> { + let fence_data = DriverFenceData { + rcu_head: Default::default(), + // `inner` remains uninitialized until a `DriverFence` takes over. + inner: Fence { + inner: Opaque::uninit(), + }, + fctx: self, + data, + }; + + // In order to support the C dma_fence callbacks, it is necessary for + // a `Fence` and a `DriverFence` to live in the same allocation, + // because the C backend passes a dma_fence, from which the driver most + // likely wants to be able to access its `data` in `DriverFence`. + // + // Hence, we need the manage the memory manually. It will be freed by the + // C backend automatically once the refcount within `Fence` drops to 0. + let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?; + + Ok(DriverFenceAllocation { + data, + ops: &Self::OPS, + }) + } + + extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.driver_name.as_char_ptr() + } + + extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char { + // SAFETY: The C backend only invokes this callback with `ptr` pointing + // to a valid, unsignaled `bindings::dma_fence`. All fences created in + // this module always reside within `Fence` which always resides in a + // `DriverFenceData`, thus satisfying the function's safety + // requirements. + let fctx = unsafe { Self::from_raw_fence(ptr) }; + + fctx.timeline_name.as_char_ptr() + } + + /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides + /// within a [`Fence`], which in turn resides in a [`DriverFenceData`]. + unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member. + let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast(); + + // SAFETY: Safe because of the comments directly above. + let fence_data = unsafe { &*fence_data_ptr }; + + fence_data.fctx + } +} + +#[pinned_drop] +impl PinnedDrop for FenceContext { + fn drop(self: Pin<&mut Self>) { + // Fence ops callbacks can be called on unsignaled fences. Since these + // callbacks can access the fence context and its data, it needs to be + // guaranteed that a context only drops after all associated + // `DriverFence`s have been dropped. This is unlikely to occur, but + // would result in silent UAF. Throw a panic to prevent that. + // + // TODO: + // It would be better if the fence context signals all forgotten fences + // itself. To do so, it would keep a list of unsignaled fences. That + // list's members would have to be pre-allocated (see + // `FenceCallback::new_fence_allocation()`). + if self.nr_of_unsignaled_fences.load(Relaxed) != 0 { + panic!("Forgotten fences in FenceContext."); + } + + // Ensure that the driver cannot unload while there are still dma_fence + // callbacks running. At the same time, the RCU barrier addresses the + // problem inherited by the C backend, in which backend ops callbacks + // might be accessing the fence while it is being signaled (or shortly + // after). This could cause UAF access on the fence context's + // `fctx.driver_name` and `fctx.timeline_name`. + // + // Wait for the RCU callbacks in `DriverFence::drop`. + rcu_barrier(); + } +} + +/// Error type for fence callback registration. +/// +/// Generic over `T` so that `AlreadySignaled` can return the callback to the +/// caller, allowing it to reclaim any resources owned by the callback (e.g., +/// a fence handle that needs to be signaled). +#[derive(Debug)] +pub enum CallbackError { + /// The fence was already signaled. The callback is returned so the caller + /// can extract owned resources without losing them. + AlreadySignaled(T), + /// Some other error occurred during registration. + Other(Error), +} + +impl From> for Error { + #[inline] + fn from(err: CallbackError) -> Self { + match err { + CallbackError::AlreadySignaled(_) => ENOENT, + CallbackError::Other(e) => e, + } + } +} + +impl From for CallbackError { + #[inline] + fn from(e: AllocError) -> Self { + CallbackError::Other(Error::from(e)) + } +} + +/// Trait for callbacks that can be registered on fences. +/// +/// When the fence signals, the callback will be invoked. +/// +/// # Example +/// +/// ```rust +/// use kernel::dma_buf::FenceCallback; +/// +/// struct MyCallback { +/// // Your callback state here +/// } +/// +/// impl FenceCallback for MyCallback { +/// fn on_signal(&mut self) { +/// pr_info!("Fence signaled!\n"); +/// // Handle fence completion +/// } +/// } +/// ``` +pub trait FenceCallback: Send + 'static { + /// Called when the fence is signaled. + /// + /// This is called from the fence signaling path, which may be in interrupt + /// context or with locks held, which is why `self` is only borrowed, so that + /// it cannot drop. Implementations must not sleep or perform + /// long-running operations. + /// + /// An implementation likely wants to inform itself (e.g., through a work item) + /// within this callback that the associated [`FenceCallbackRegistration`] + /// can now be dropped. + fn on_signal(&mut self); +} + +/// A callback registration on a fence. +/// +/// When this object is dropped, the callback is automatically removed if it +/// hasn't been called yet. +#[pin_data(PinnedDrop)] +pub struct FenceCallbackRegistration { + #[pin] + callback_foreign: Opaque, + callback: ManuallyDrop, + fence: ARef, +} + +impl FenceCallbackRegistration { + /// Create a [`PinInit`] closure for registering a callback on a fence. + /// + /// The actual attempt at registering the callback will take place once you + /// call an allocator's `pin_init()` function. + /// + /// On success the callback is pinned in place and will fire when the fence + /// signals. On `AlreadySignaled` the callback is returned to the caller so + /// that owned resources can be reclaimed. + pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit> + 'a + where + T: 'a, + { + try_pin_init!(Self { + // We need to fully initialize the fence because after + // `dma_fence_add_callback()` ran, the callback might immediately + // get invoked. + callback: ManuallyDrop::new(callback), + fence: ARef::from(fence), + callback_foreign <- Opaque::try_ffi_init(|ptr| { + // SAFETY: `fence.inner.get()` is a valid, initialized `struct + // dma_fence`. `ptr` points to the `struct dma_fence_cb` field + // within the pinned allocation, so it remains valid until + // `dma_fence_remove_callback()` in `PinnedDrop` or until the + // callback fires. + let ret = unsafe { + to_result(bindings::dma_fence_add_callback( + fence.inner.get(), + ptr, + Some(Self::dma_fence_callback), + )) + }; + match ret { + Ok(()) => Ok(()), + Err(e) => { + // SAFETY: We could not register the callback. Thus, + // C will not use it. So we can just take it back + // and pass it to the user again. + let cb_back = unsafe { ManuallyDrop::take(callback) }; + if e == ENOENT { + Err(CallbackError::AlreadySignaled(cb_back)) + } else { + Err(CallbackError::Other(e)) + } + }, + } + }), + }? CallbackError) + } + + /// Raw dma fence callback that is called by the C code. + /// + /// # Safety + /// + /// This is only called by the dma_fence subsystem with valid pointers. + unsafe extern "C" fn dma_fence_callback( + _fence: *mut bindings::dma_fence, + callback_foreign: *mut bindings::dma_fence_cb, + ) { + let ptr = Opaque::cast_from(callback_foreign).cast_mut(); + + // SAFETY: All callbacks we can receive here have been created in such a way that they are + // embedded into a `FenceCallbackRegistration`. + let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) }; + + // SAFETY: `reg` is a valid `Self` pointer. + // + // The backend ensures synchronisation so whoever holds the registration object cannot drop + // it while this code is running. See `FenceCallbackRegistration::drop`. + unsafe { (*reg).callback.on_signal() }; + } + + /// Returns a reference to the fence this callback is registered on. + #[inline] + pub fn fence(&self) -> &Fence { + &self.fence + } +} + +#[pinned_drop] +impl PinnedDrop for FenceCallbackRegistration { + fn drop(self: Pin<&mut Self>) { + // Always call `dma_fence_remove_callback()`, even if the callback + // already ran. This is necessary for synchronization: + // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures + // that any in-flight `dma_fence_signal()` (which calls our callback + // while holding the same lock) has completed before we free the struct. + // + // Without this, Drop can race with a concurrent signal: + // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress) + // CPU1 (drop): skips lock -> frees struct + // CPU0: accesses fence_ref -> use-after-free + // + // When the callback has already fired, the signal path detached the + // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just + // sees an empty node and returns false — the lock acquisition is the + // only thing that matters. + // + // SAFETY: The fence pointer is valid and the cb was initialized by + // `dma_fence_add_callback()` during construction. + unsafe { + bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get()) + }; + + // SAFETY: This is literally the drop implementation, so no one has + // dropped this so far; so we can do it now. + unsafe { ManuallyDrop::::drop(self.project().callback) }; + } +} + +// SAFETY: FenceCallbackRegistration can be sent between threads. +unsafe impl Send for FenceCallbackRegistration {} + +// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can. +unsafe impl Sync for FenceCallbackRegistration where T: Sync {} + +/// The receiving counterpart of a [`DriverFence`]. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +#[repr(transparent)] +pub struct Fence { + /// The actual dma_fence passed to C. + inner: Opaque, +} + +/// Guard helper for locking within this module. +/// +/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles. +/// It is never used outside of this module. +// TODO: This should be made more canonical, probably by basing it on a +// SpinLockIrqGuard once available. +struct FenceGuard<'a> { + inner: &'a Fence, + flags: usize, +} + +impl<'a> Deref for FenceGuard<'a> { + type Target = &'a Fence; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl Drop for FenceGuard<'_> { + fn drop(&mut self) { + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, which lives as long as this function. + // When a `FenceGuard` exists, the lock has been taken by definition. + unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) }; + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Send for Fence {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl Sync for Fence {} + +impl Fence { + /// Check whether the fence was signaled at the moment of the function call. + /// + /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`] + /// has not yet been dropped. The reason is that the fence ops callbacks can + /// cause the fence to get signaled by the C backend. + #[inline] + pub fn is_signaled(&self) -> bool { + // We should not use `dma_fence_is_signaled_locked()` here, because + // according to the C backend's recommendations, that function is + // problematic and we should avoid calling that function with a lock + // held. + + // SAFETY: Inner `fence` is valid because `self` is valid. + let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) }; + + // To be as robust as possible for the future we guarantee that an API + // caller can 100% rely on the signaling being completed (i.e., all + // fence callbacks ran), so we have to take the lock. + // + // The reason is that the C dma_fence backend currently does not + // carefully synchronize the `dma_fence_is_signaled()` function with the + // proper spinlock. This can lead to the function returning `true` while + // fence callbacks are still being executed. This can be mitigated by + // guarding the entire function with the spinlock. + // + // The fundamental reason is that the C backend currently does guard + // setting of the fence's signaled-bit with the fence's spinlock, but + // reading is done locklessly. + // + // See commit c8a5d5ea3ba6a. + let _ = self.lock(); + + ret + } + + /// Lock the fence. A helper only to be used internally in this module. + fn lock(&self) -> FenceGuard<'_> { + let mut guard = FenceGuard { + inner: self, + flags: 0, + }; + + // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is + // merely a pointer to an integer, whose lifetime is tied to the guard + // object. + unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) }; + + guard + } + + /// Get the fence's sequence number. + #[inline] + pub fn seqno(&self) -> u64 { + // SAFETY: Valid because `self` is valid. + unsafe { (*self.as_raw()).seqno } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.inner.get() + } + + /// Create a [`Fence`] from a raw C [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must point to an initialized fence that is embedded into a [`Fence`]. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self { + // SAFETY: Safe as per the function's overall safety requirements. + unsafe { &*ptr.cast() } + } +} + +// SAFETY: These implement the C backends refcounting methods which are proven +// to work correctly. +unsafe impl AlwaysRefCounted for Fence { + fn inc_ref(&self) { + // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`. + unsafe { bindings::dma_fence_get(self.as_raw()) } + } + + unsafe fn dec_ref(ptr: NonNull) { + // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called + // the fence is by definition still valid. + let fence = unsafe { (*ptr.as_ptr()).inner.get() }; + + // SAFETY: `fence` was created validly above. When `dec_ref()` is called, + // there is by definition still a reference alive that can be put. + unsafe { bindings::dma_fence_put(fence) } + } +} + +// Necessary to guarantee that `inner` always comes first and can be freed by C. +// Also useful for using casts instead of container_of(). +#[repr(C)] +#[pin_data] +struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> { + #[pin] + /// The inner fence. + // Must always be the first member so that unsafe casting works; but also + // necessary so that the C backend can free the allocation (coming from our + // Rust code) with kfree_rcu(). + inner: Fence, + /// Callback head for dropping this in a deferred manner through RCU. + rcu_head: bindings::callback_head, + /// Reference to access the FenceContext. + fctx: &'a FenceContext, + /// The API user's data. It is essential that the data only performs + /// operations legal in atomic context in its [`Drop`] implementation. + #[pin] + data: T::FenceDataType, +} + +/// A synchronization primitive mainly for GPU drivers. +/// +/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s +/// are the producer-side, intended to be always owned by only one party. That +/// party has the monopoly on signaling the fence. +/// +/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always +/// refcounted and can be shared with an arbitrary number of parties, including +/// userspace. A [`Fence`] can only be used for actions such as checking the +/// fence's status or for registering callbacks on it. +/// +/// Once the associated [`DriverFence`] signals, all +/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed. +/// +/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the +/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its +/// [`Fence`]s. +/// +/// It is crucial that a [`DriverFence`] always correctly represents the state +/// of the associated job on the hardware. Especially, it is strictly necessary +/// that the owner ensures that all [`DriverFence`]s eventually get signaled. +/// As a last resort, a [`DriverFence`] will signal itself if it drops +/// unsignaled and print a warning. +/// +/// This design intends to implement the [`bindings::dma_fence_ops`] in such a +/// way that the driver-data necessary to implement the callback's functionality +/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a +/// reference to the context, which can be accessed in the callbacks. The +/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its +/// [`FenceContext`]. Unfortunately, this can be circumvented under certain +/// circumstances in Rust (e.g., usage of [`core::mem::forget`]). +/// +/// In the unlikely case of such violations, a panic is thrown. +/// +/// # Examples +/// +/// ``` +/// use kernel::{ +/// dma_buf::{ +/// DriverFence, +/// FenceContext, +/// FenceContextOps, +/// FenceCallback, +/// FenceCallbackRegistration, +/// }, +/// str::CString, +/// sync::aref::ARef, // +/// }; +/// use core::fmt::Display; +/// +/// struct CallbackData { } +/// +/// impl FenceCallback for CallbackData { +/// fn on_signal(&mut self) { +/// pr_info!("DmaFence callback executed.\n"); +/// } +/// } +/// +/// #[pin_data] +/// struct FenceContextData {} +/// +/// impl FenceContextData { +/// fn new() -> impl PinInit { +/// pin_init!(Self {}) +/// } +/// } +/// +/// impl FenceContextOps for FenceContextData { +/// type FenceDataType = FenceData; +/// } +/// +/// let fctx_data = FenceContextData::new(); +/// +/// +/// let mut fctx = KBox::pin_init( +/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data), +/// GFP_KERNEL +/// )?; +/// +/// struct FenceData { +/// data: CString, +/// } +/// +/// let fence_data = FenceData { data: c"dummy_data".try_into()? }; +/// +/// let fence_alloc = fctx.new_fence_allocation(fence_data)?; +/// let mut fence = fence_alloc.new_fence(); +/// +/// let cb_data = CallbackData { }; +/// let waiting_fence = ARef::from(fence.as_fence()); +/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data); +/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?; +/// +/// // TODO signalling guards +/// assert_eq!(waiting_fence.is_signaled(), false); +/// fence.signal(Ok(())); +/// assert_eq!(waiting_fence.is_signaled(), true); +/// +/// Ok::<(), Error>(()) +/// ``` +pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> { + /// The actual content of the fence. Lives in a [`NonNull`] so that its + /// memory can be managed independently. Valid until both the [`DriverFence`] + /// and all associated [`Fence`]s have disappeared. + data: NonNull>, +} + +/// A pre-prepared DMA fence, carrying the user's data and the memory it and the +/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting +/// allocation and full initialization is necessary because fences cannot be +/// allocated dynamically in some circumstances (deadlock). +pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> { + /// The memory for the actual content of the fence. + /// Handed over to a [`DriverFence`], or deallocated once the + /// [`DriverFenceAllocation`] drops. + data: KBox>, + /// Reference for the ops for the associated [`FenceContext`] + ops: &'static bindings::dma_fence_ops, +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> { + /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`]. + /// + /// This increments the sequence number in the associated [`FenceContext`]. + pub fn new_fence(self) -> DriverFence<'a, T> { + // We feed the C dma_fence backend a NULL for the spinlock so that it + // uses per-fence locks automatically. + let null_ptr: *mut bindings::spinlock = ptr::null_mut(); + let seqno = self.data.fctx.next_seqno(); + let fence_ptr = self.as_raw(); + // SAFETY: `fence_ptr` has been created directly above. It will live + // at least as long as `Self`. The same applies to `&Self::OPS`. + unsafe { + bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno) + }; + + self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed); + + // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so + // that `DriverFence`s can always be created without allocating. In this + // method, ownership over that memory is transferred to the new + // `DriverFence` and managed through refcounting. The C dma_fence + // backend will ultimately free the memory once the refcount reaches 0. + let ptr = KBox::into_raw(self.data); + // SAFETY: `ptr` was just created validly directly above. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + + DriverFence { data: ptr } + } + + fn as_raw(&self) -> *mut bindings::dma_fence { + self.data.inner.inner.get() + } +} + +impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> { + fn as_raw(&self) -> *mut bindings::dma_fence { + // SAFETY: Valid because `self` is valid. + let fence_data = unsafe { &*self.data.as_ptr() }; + + fence_data.inner.inner.get() + } + + /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`]. + /// + /// # Safety + /// + /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through + /// a [`DriverFence`] with matching generic data for both fence and associated + /// [`FenceContext`]. + unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self { + let opaque_fence = Opaque::cast_from(ptr); + + // SAFETY: Safe due to the function's overall safety requirements. + let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) }; + + // DriverFenceData is `repr(C)` and a Fence is its first member. + let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>; + + // SAFETY: `fence_data_ptr` was created validly above. + let data = unsafe { NonNull::new_unchecked(fence_data_ptr) }; + + Self { data } + } + + /// Return the underlying [`Fence`]. + #[inline] + pub fn as_fence(&self) -> &Fence { + // SAFETY: `self` is by definition still valid, and it cannot drop until + // this new reference is gone. + unsafe { Fence::from_raw(self.as_raw()) } + } + + /// Signal the fence. This will invoke all registered callbacks. + pub fn signal(self, res: Result) { + let fence = self.as_fence().lock(); + + // SAFETY: `fence` is valid because `self` is valid. The lock must be + // held, which we acquired directly above. + if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } { + if let Err(err) = res { + // SAFETY: `fence` is valid because `self` is valid. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) }; + } + // SAFETY: `fence` is valid because `self` is valid. The lock must + // be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) }; + } + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } +} + +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {} +// SAFETY: Fences are literally designed to be shared between threads. +unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> { + type Target = T::FenceDataType; + + fn deref(&self) -> &Self::Target { + // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is. + let data = unsafe { &*self.data.as_ptr() }; + + &data.data + } +} + +/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`]. +pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> { + driver_fence: ManuallyDrop>, + _lifetime: PhantomData<&'a T>, +} + +impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> { + type Target = DriverFence<'a, T>; + + fn deref(&self) -> &Self::Target { + self.driver_fence.deref() + } +} + +// SAFETY: The Rust dma_fence abstractions are already designed around the inner +// C `dma_fence`, which can serve safely as the identification point when being +// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by +// only allowing operations without side effects on the Borrowed type. +unsafe impl ForeignOwnable for DriverFence<'_, T> { + type Borrowed<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + type BorrowedMut<'a> + = DriverFenceBorrow<'a, T> + where + Self: 'a; + + const FOREIGN_ALIGN: usize = core::mem::align_of::(); + + fn into_foreign(self) -> *mut c_void { + let fence = self; + + let ptr = fence.as_raw(); + + // DriverFence must not drop. + let _ = ManuallyDrop::new(fence); + + ptr.cast() + } + + unsafe fn from_foreign(ptr: *mut c_void) -> Self { + // SAFETY: Safe because the trait implementation only invokes this with + // a valid `ptr`, associated to a `DriverFence` with matching generic data. + unsafe { Self::from_raw(ptr.cast()) } + } + + unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a> + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } + + unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a> + // FIXME: The bound below and the one above in `borrow` should actually be + // unnecessary since the compiler should be able to completely derive all + // necessary information automatically. There is currently a compiler bug + // preventing that, though: + // + // https://github.com/rust-lang/rust/issues/155430. + // + // (Help to) fix the compiler bug and remove the bounds afterwards. + where + Self: 'a, + { + // SAFETY: The trait implementation ensures that `ptr` always resides + // within a [`Fence`] within a [`DriverFenceData`]. + let driver_fence = unsafe { Self::from_raw(ptr.cast()) }; + + let driver_fence = ManuallyDrop::new(driver_fence); + + DriverFenceBorrow { + driver_fence, + _lifetime: PhantomData, + } + } +} + +impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> { + fn drop(&mut self) { + let guard = self.as_fence().lock(); + + // Use dma_fence_test_signaled_flag() instead of + // dma_fence_is_signaled_locked() because the C backend wants to get rid + // of the latter. + + // SAFETY: `guard` is valid until the `call_rcu()` below. + let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) }; + if !signaled { + pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n"); + // SAFETY: `guard` is valid until the `call_rcu()` below. The fence + // must not have been signaled yet, which we check directly above. + unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) }; + // SAFETY: `guard` is valid until the `call_rcu()` below. The lock + // must be held, which we acquired above. + unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) }; + + // SAFETY: `self.data` is valid because `self` is valid. + let fctx = unsafe { self.data.as_ref().fctx }; + let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed); + } + drop(guard); + + // `DriverFenceData` could be accessed through some dma_fence + // callbacks right now. Access is being revoked in principle above by + // signaling the fence, but since the C backend does not guarantee + // perfect full synchronization, we have to wait for one grace period to + // ensure that all accessors of `DriverFenceData` (through the + // dma_fence_ops accessible through a `Fence`) are gone. + + if !core::mem::needs_drop::() { + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is + // valid and initialized. It is valid until the refcount drops to 0, + // which can earliest happen once we drop the `DriverFence`'s + // reference here. + unsafe { bindings::dma_fence_put(self.as_raw()) }; + return; + } + + // SAFETY: Valid because `self` is valid. + let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head }; + + // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was + // created validly above. The module must perform a `synchronize_rcu()` + // or `rcu_barrier()` call to guard against module unload. + unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::)) }; + } +} + +// TODO: +// The entire call_rcu() mechanism in the drop above and the code below would be +// unnecessary if C's dma_fence_signal() could be reworked in a way that after it +// ran, the caller knows that no fence_ops callbacks can be running anymore. +// In other words, if the dma_fence backend would use its spinlock for full +// synchronization. +// +// Then we could move the drop_in_place() and dma_fence_put() upwards into the +// drop() implementation and call it a day. + +/// Finally really drop this `DriverFence` +/// +/// # Safety +/// +/// `head` references the `rcu_head` field of an `DriverFenceData`. All +/// accessors to that `DriverFenceData` must be gone by now. This must be +/// ensured by signalling the associated `DriverFence` and then waiting +/// for a grace period until calling this function here. +unsafe extern "C" fn drop_driver_fence_data( + head: *mut bindings::callback_head, +) { + // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData`. + let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) }; + + // SAFETY: `fence_data` was created validly above. All the fence's data will + // only drop below, but the raw pointer to the raw C `dma_fence` remains + // valid because the reference count is only decremented at the end of the + // function. + let fence = unsafe { (*fence_data).inner.inner.get() }; + + // SAFETY: `fence_data` was created validly above. The user has already + // dropped the only conventional accessor to the user data, the `DriverFence`, + // one grace period ago. All accessors are gone now. + unsafe { drop_in_place(&raw mut (*fence_data).data) }; + + // The inner `Fence` explicitly does not get dropped because there may be + // many more users / consumers, each holding their own reference. + + // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid + // and initialized. It is valid until the refcount drops to 0, which can + // earliest happen once we drop the `DriverFence`'s reference here. + unsafe { bindings::dma_fence_put(fence) }; + + // The actual memory the data associated with a `DriverFence` lives in + // gets freed by the C dma_fence backend once the fence's refcount reaches 0. +} diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs new file mode 100644 index 000000000000..4764a828642e --- /dev/null +++ b/rust/kernel/dma_buf/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +//! DMA-buf subsystem abstractions. + +pub mod dma_fence; + +pub use self::dma_fence::{ + DriverFence, + Fence, + FenceCallback, + FenceCallbackRegistration, + FenceContext, + FenceContextOps, // +}; diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 7225abc64084..72f5527ba66c 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,7 @@ pub mod device; pub mod device_id; pub mod devres; pub mod dma; +pub mod dma_buf; pub mod driver; #[cfg(CONFIG_DRM = "y")] pub mod drm; -- cgit v1.2.3 From 7aa432f5121bea8fe96fb3cfedfbd7f4e448133a Mon Sep 17 00:00:00 2001 From: Philipp Stanner Date: Sat, 5 Sep 2026 10:53:43 +0200 Subject: MAINTAINERS: Add entry for Rust dma-buf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rust does now have abstractions for dma_fence. These abstractions are quite complicated and require expertise with both the C and the Rust side. Therefore, using the existing entry also for maintenance of the Rust code appears reasonable. Philipp volunteers to help maintain the dma_fence abstractions. Add a corresponding MAINTAINERS entry. Signed-off-by: Philipp Stanner Acked-by: Christian König Acked-by: Sumit Semwal Tested-by: Daniel Almeida Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260905085343.1827305-4-phasta@kernel.org Signed-off-by: Danilo Krummrich --- MAINTAINERS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index d92c7d834693..2389df80ba50 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7681,6 +7681,7 @@ F: fs/dlm/ DMA BUFFER SHARING FRAMEWORK M: Sumit Semwal M: Christian König +M: Philipp Stanner L: linux-media@vger.kernel.org L: dri-devel@lists.freedesktop.org L: linaro-mm-sig@lists.linaro.org (moderated for non-subscribers) @@ -7694,6 +7695,8 @@ F: include/linux/dma-buf.h F: include/linux/dma-buf/ F: include/linux/dma-resv.h F: rust/helpers/dma-resv.c +F: rust/helpers/dma_fence.c +F: rust/kernel/dma_buf/ K: \bdma_(?:buf|fence|resv)\b DMA GENERIC OFFLOAD ENGINE SUBSYSTEM @@ -8698,7 +8701,9 @@ T: git https://gitlab.freedesktop.org/drm/rust/kernel.git F: drivers/gpu/drm/nova/ F: drivers/gpu/drm/tyr/ F: drivers/gpu/nova-core/ +F: rust/helpers/dma_fence.c F: rust/helpers/gpu.c +F: rust/kernel/dma_buf/ F: rust/kernel/drm/ F: rust/kernel/gpu.rs F: rust/kernel/gpu/ -- cgit v1.2.3 From 73e5616f3d197c1af5a04a481fe0f13aa3913bd1 Mon Sep 17 00:00:00 2001 From: Edwin Peer Date: Wed, 9 Sep 2026 13:23:27 -0700 Subject: rust: dma_fence: gate abstractions on CONFIG_DMA_SHARED_BUFFER The dma_fence abstractions added by commit 3282d5916019 ("rust: Add dma_fence abstractions") are compiled unconditionally, but the C `dma_fence_*` symbols they reference live in drivers/dma-buf/dma-fence.c, which is only built when CONFIG_DMA_SHARED_BUFFER=y. Kernels with CONFIG_RUST=y and CONFIG_RUST_KERNEL_DOCTESTS=y but CONFIG_DMA_SHARED_BUFFER=n fail to link: ld.lld: error: undefined symbol: dma_fence_release >>> referenced by usercopy_64.c >>> vmlinux.o:(rust_helper_dma_fence_put) ... dma_fence_signal{,_locked}, dma_fence_context_alloc, dma_fence_init, dma_fence_{add,remove}_callback ... Wrap the dma_fence.c helper include in the same `pub mod dma_buf;` in rust/kernel/lib.rs with a matching #[cfg]. Fixes: 3282d5916019 ("rust: Add dma_fence abstractions") Signed-off-by: Edwin Peer Reviewed-by: John Hubbard Reviewed-by: Philipp Stanner Link: https://patch.msgid.link/20260909202327.33285-1-epeer@nvidia.com Signed-off-by: Danilo Krummrich --- rust/helpers/helpers.c | 2 +- rust/kernel/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 609689fba3dd..2c5eecc091c4 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -57,8 +57,8 @@ #include "cred.c" #include "device.c" #include "dma.c" -#include "dma_fence.c" #ifdef CONFIG_DMA_SHARED_BUFFER +#include "dma_fence.c" #include "dma-resv.c" #endif #include "drm.c" diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs index 72f5527ba66c..f9ef36217bb5 100644 --- a/rust/kernel/lib.rs +++ b/rust/kernel/lib.rs @@ -67,6 +67,7 @@ pub mod device; pub mod device_id; pub mod devres; pub mod dma; +#[cfg(CONFIG_DMA_SHARED_BUFFER)] pub mod dma_buf; pub mod driver; #[cfg(CONFIG_DRM = "y")] -- cgit v1.2.3 From 9cf1bc8bbc71a6b727ed6def13f125b07cf35210 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 8 Sep 2026 18:54:12 +0900 Subject: rust: maple_tree: implement Send and Sync for MapleTree The C maple_tree struct contains a *mut c_void, which prevents Rust from auto-deriving Send/Sync. Following is an example error message when using MapleTree in nova-core's Vmm. This propagates up through MapleTreeAlloc to Vmm, BarUser, Gpu, and NovaCore, causing NovaCore to fail the Send bound required by pci::Driver: error[E0277]: `*mut c_void` cannot be sent between threads safely --> drivers/gpu/nova-core/driver.rs:77:22 | 77 | impl pci::Driver for NovaCore { | ^^^^^^^^ `*mut c_void` cannot be sent between threads safely | = help: within `MapleTreeAlloc<()>`, the trait `Send` is not implemented for `*mut c_void` note: required because it appears within the type `kernel::bindings::maple_tree` note: required because it appears within the type `Opaque` note: required because it appears within the type `MapleTree<()>` note: required because it appears within the type `MapleTreeAlloc<()>` = note: required for `Box, Kmalloc>` to implement `Send` note: required because it appears within the type `core::pin::Pin, Kmalloc>>` note: required because it appears within the type `Vmm` note: required because it appears within the type `BarUser` note: required because it appears within the type `Gpu` note: required because it appears within the type `NovaCore` note: required by a bound in `kernel::pci::Driver` --> rust/kernel/pci.rs:294:19 Implement Send and Sync for MapleTree. The tree contains no thread-local state, and all shared access goes through the internal ma_lock spinlock. Signed-off-by: Joel Fernandes Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng [ecourtney: also require T: Sync for the Sync impl] Acked-by: Danilo Krummrich Acked-by: Liam R. Howlett (Oracle) Link: https://patch.msgid.link/20260908-maple-tree-send-sync-v2-1-24aae5b3a990@nvidia.com Signed-off-by: Eliot Courtney --- rust/kernel/maple_tree.rs | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs index 265d6396a78a..7abe41228cb7 100644 --- a/rust/kernel/maple_tree.rs +++ b/rust/kernel/maple_tree.rs @@ -16,7 +16,11 @@ use kernel::{ alloc::Flags, error::to_result, prelude::*, - types::{ForeignOwnable, Opaque}, + types::{ + ForeignOwnable, + NotThreadSafe, + Opaque, // + }, }; /// A maple tree optimized for storing non-overlapping ranges. @@ -240,7 +244,10 @@ impl MapleTree { unsafe { bindings::spin_lock(self.ma_lock()) }; // INVARIANT: We just took the spinlock. - MapleGuard(self) + MapleGuard { + tree: self, + _not_send: NotThreadSafe, + } } #[inline] @@ -302,19 +309,30 @@ impl PinnedDrop for MapleTree { } } +// SAFETY: `MapleTree` is `Send` if `T` is `Send` because `MapleTree` owns its elements. +unsafe impl Send for MapleTree {} + +// SAFETY: `&MapleTree` allows inserting and erasing entries from any thread, so `T: Send` is +// required, and shared borrows of entries require `T: Sync`. +unsafe impl Sync for MapleTree {} + /// A reference to a [`MapleTree`] that owns the inner lock. /// /// # Invariants /// /// This guard owns the inner spinlock. #[must_use = "if unused, the lock will be immediately unlocked"] -pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree); +pub struct MapleGuard<'tree, T: ForeignOwnable> { + tree: &'tree MapleTree, + // A held spinlock must be released on the same CPU that acquired it. + _not_send: NotThreadSafe, +} impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> { #[inline] fn drop(&mut self) { // SAFETY: By the type invariants, we hold this spinlock. - unsafe { bindings::spin_unlock(self.0.ma_lock()) }; + unsafe { bindings::spin_unlock(self.tree.ma_lock()) }; } } @@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> { // SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s // read/write permissions to the maple tree. - unsafe { MaState::new_raw(self.0, first, end) } + unsafe { MaState::new_raw(self.tree, first, end) } } /// Load the value at the given index. @@ -375,7 +393,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> { #[inline] pub fn load(&mut self, index: usize) -> Option> { // SAFETY: `self.tree` contains a valid maple tree. - let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) }; + let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) }; if ret.is_null() { return None; } -- cgit v1.2.3 From 3f1e12fce16f6e88fbaab3404df33f65c8f4df87 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 8 Sep 2026 16:51:48 +0900 Subject: rust: pci: add resource_flags accessor Add a `Device::resource_flags()` method to the PCI Rust abstraction, wrapping the C-side static inline `pci_resource_flags()`. The flags returned correspond to the `IORESOURCE` bitmask carried by a PCI BAR's `struct resource`. The immediate motivation is BAR layout discovery on NVIDIA GPUs: a 64-bit BAR consumes two consecutive Linux PCI resource slots (the lower 32 bits at index N and the upper 32 bits at index N+1, with the latter having no flags or size of its own). Signed-off-by: Joel Fernandes Reviewed-by: Alexandre Courbot Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260908-pci-resource-flags-v1-1-269f29f81de6@nvidia.com Signed-off-by: Eliot Courtney --- rust/helpers/pci.c | 6 ++++++ rust/kernel/io/resource.rs | 8 ++++++++ rust/kernel/pci.rs | 14 ++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/rust/helpers/pci.c b/rust/helpers/pci.c index a714cc2bfb7a..3686e405160d 100644 --- a/rust/helpers/pci.c +++ b/rust/helpers/pci.c @@ -19,6 +19,12 @@ __rust_helper resource_size_t rust_helper_pci_resource_len(struct pci_dev *pdev, return pci_resource_len(pdev, bar); } +__rust_helper unsigned long rust_helper_pci_resource_flags(const struct pci_dev *pdev, + int bar) +{ + return pci_resource_flags(pdev, bar); +} + __rust_helper bool rust_helper_dev_is_pci(const struct device *dev) { return dev_is_pci(dev); diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs index 17b0c174cfc5..0d3b34f83334 100644 --- a/rust/kernel/io/resource.rs +++ b/rust/kernel/io/resource.rs @@ -226,10 +226,18 @@ impl Flags { /// Resource represents a memory region that must be ioremaped using `ioremap_np`. pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED); + /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots). + pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64); + // Always inline to optimize out error path of `build_assert`. #[inline(always)] const fn new(value: u32) -> Self { build_assert!(value as u64 <= c_ulong::MAX as u64); Flags(value as c_ulong) } + + /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`]. + pub(crate) const fn from_raw(value: c_ulong) -> Self { + Flags(value) + } } diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs index 3ec897709e89..19a219847c17 100644 --- a/rust/kernel/pci.rs +++ b/rust/kernel/pci.rs @@ -17,6 +17,7 @@ use crate::{ from_result, to_result, // }, + io::resource, prelude::*, str::CStr, types::Opaque, @@ -439,6 +440,19 @@ impl Device { Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) }) } + /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR. + pub fn resource_flags(&self, bar: u32) -> Result { + if !Bar::index_is_valid(bar) { + return Err(EINVAL); + } + + // SAFETY: + // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`, + // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`. + let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) }; + Ok(resource::Flags::from_raw(raw)) + } + /// Returns the PCI class as a `Class` struct. #[inline] pub fn pci_class(&self) -> Class { -- cgit v1.2.3 From b8ca2d0b7fcf5b8ff0ec174b092f732a9d3d27a3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:39 +0900 Subject: gpu: nova-core: mm: Add common types for virtual memory management Add common virtual memory management types: `PAGE_SIZE` constant, `VirtualAddress` bitfield type, `Vfn` (Virtual Frame Number) type, `Pfn` (Physical Frame Number) type. Signed-off-by: Joel Fernandes [ecourtney: restore Pfn and its VramAddress conversions on the raw API] [ecourtney: make dead_code expect unconditional until the self-tests patch] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-1-8dd5d4225d2e@nvidia.com [acourbot: drop repeated "memory" in commit message.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 142 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 9e4338c7c393..4234ba4d596a 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -3,7 +3,34 @@ //! Memory management subsystems. -#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] +#![expect(dead_code)] + +/// Implements `From` conversions between a frame-number type and `Bounded`. +/// +/// Each MMU version module should invoke this for the specific bit widths used by that version's +/// PTE/PDE bitfield definitions. +macro_rules! impl_frame_number_bounded { + ($type:ty, $bits:literal) => { + impl From> for $type { + fn from(val: Bounded) -> Self { + Self::new(val.get()) + } + } + + impl From<$type> for Bounded { + fn from(v: $type) -> Self { + Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) + } + } + }; +} + +/// Implements `From` conversions between [`Pfn`] and `Bounded` for bitfield interop. +macro_rules! impl_pfn_bounded { + ($bits:literal) => { + impl_frame_number_bounded!(Pfn, $bits); + }; +} use core::{ fmt::LowerHex, @@ -11,12 +38,15 @@ use core::{ }; use kernel::{ + bitfield, fmt, + num::Bounded, prelude::*, ptr::{ Alignable, Alignment, // }, + sizes::SZ_4K, // }; use crate::{ @@ -58,6 +88,9 @@ impl<'gpu> GpuMm<'gpu> { } } +/// Page size in bytes (4 KiB). +pub(crate) const PAGE_SIZE: usize = SZ_4K; + /// Physical VRAM address in GPU video memory. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] #[repr(transparent)] @@ -124,6 +157,113 @@ impl ops::Sub for VramAddress { } } +impl From for VramAddress { + fn from(pfn: Pfn) -> Self { + Self::from_raw(pfn.raw() << 12) + } +} + +bitfield! { + /// Virtual address in GPU address space. + pub(crate) struct VirtualAddress(u64) { + /// Offset within 4KB page. + 11:0 offset; + /// Virtual frame number. + 63:12 frame_number => Vfn; + } +} + +impl VirtualAddress { + /// Create a new virtual address from a raw value. + pub(crate) const fn new(addr: u64) -> Self { + Self::from_raw(addr) + } +} + +impl From for VirtualAddress { + fn from(vfn: Vfn) -> Self { + Self::zeroed().with_frame_number(vfn) + } +} + +/// Physical Frame Number. +/// +/// Represents a physical page in VRAM. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Pfn(u64); + +impl Pfn { + /// Create a new PFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From for Pfn { + fn from(addr: VramAddress) -> Self { + Self::new(addr.into_raw() >> 12) + } +} + +impl From for Pfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From for u64 { + fn from(pfn: Pfn) -> Self { + pfn.0 + } +} + +impl_pfn_bounded!(52); + +/// Virtual Frame Number. +/// +/// Represents a virtual page in GPU address space. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Vfn(u64); + +impl Vfn { + /// Create a new VFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From for Vfn { + fn from(addr: VirtualAddress) -> Self { + addr.frame_number() + } +} + +impl From for Vfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From for u64 { + fn from(vfn: Vfn) -> Self { + vfn.0 + } +} + +impl_frame_number_bounded!(Vfn, 52); + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] pub(crate) mod selftest { use core::ops::Range; -- cgit v1.2.3 From 33be8f687bd8204965c137a5600822a1959033b8 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:40 +0900 Subject: gpu: nova-core: mm: Add buddy allocator and TLB to GpuMm Extend GpuMm with the remaining two memory-management components: - Buddy allocator for VRAM allocation. - TLB manager for translation buffer operations. PRAMIN was added in an earlier commit; this completes the centralized ownership model with accessor methods for each component. Signed-off-by: Joel Fernandes [ecourtney: rebase GpuMm for borrowed BAR0, pin Tlb, write regs directly] [ecourtney: size the buddy from the first usable FB region, drop dev_info] [ecourtney: update for the VramAddress raw API and typed register base] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-2-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/Kconfig | 1 + drivers/gpu/nova-core/gpu.rs | 27 +++++++-- drivers/gpu/nova-core/mm.rs | 24 ++++++++ drivers/gpu/nova-core/mm/tlb.rs | 120 ++++++++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/regs.rs | 68 +++++++++++++++++++++++ 5 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/tlb.rs diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index cb7f0b00f796..1934f17baa8b 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -5,6 +5,7 @@ config NOVA_CORE depends on RUST depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS + select GPU_BUDDY select RUST_FW_LOADER_ABSTRACTIONS default n help diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 98834c60f8e1..e9d3c10708f1 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -6,11 +6,16 @@ use kernel::{ device, dma::Device, fmt, + gpu::buddy::GpuBuddyParams, io::Io, num::Bounded, pci, prelude::*, - sizes::SizeConstants, // + ptr::Alignment, + sizes::{ + SizeConstants, + SZ_4K, // + }, }; use crate::{ @@ -422,11 +427,21 @@ impl<'gpu> Gpu<'gpu> { }, // Create GPU memory manager owning memory management resources. - mm: GpuMm::new( - bar, - gsp_resources.spec.chipset, - VramAddress::from_raw(gsp_static_info.total_fb_end), - )?, + mm: { + let usable_vram = gsp_static_info.usable_fb_regions.first().ok_or(ENODEV)?; + let buddy_params = GpuBuddyParams { + base_offset: usable_vram.start, + size: usable_vram.end - usable_vram.start, + chunk_size: Alignment::new::(), + }; + + GpuMm::new( + bar, + gsp_resources.spec.chipset, + buddy_params, + VramAddress::from_raw(gsp_static_info.total_fb_end), + )? + }, }) } diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 4234ba4d596a..202bb8f8ebed 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -40,6 +40,10 @@ use core::{ use kernel::{ bitfield, fmt, + gpu::buddy::{ + GpuBuddy, + GpuBuddyParams, // + }, num::Bounded, prelude::*, ptr::{ @@ -54,16 +58,23 @@ use crate::{ gpu::Chipset, // }; +pub(crate) use tlb::Tlb; + mod hal; mod pramin; mod regs; +pub(super) mod tlb; /// GPU Memory Manager - owns all core MM components. /// /// Provides centralized ownership of memory management resources: +/// - [`GpuBuddy`] allocator for VRAM page table allocation. /// - [`pramin::Pramin`] for direct VRAM access. +/// - [`Tlb`] manager for translation buffer flush operations. pub(crate) struct GpuMm<'gpu> { + buddy: GpuBuddy, pramin: pramin::Pramin<'gpu>, + tlb: Pin>>, } impl<'gpu> GpuMm<'gpu> { @@ -71,6 +82,7 @@ impl<'gpu> GpuMm<'gpu> { pub(crate) fn new( bar: Bar0<'gpu>, chipset: Chipset, + buddy_params: GpuBuddyParams, total_fb_end: VramAddress, ) -> Result { // PRAMIN covers all physical VRAM (including GSP-reserved areas @@ -78,14 +90,26 @@ impl<'gpu> GpuMm<'gpu> { let vram_region = VramAddress::ZERO..total_fb_end; Ok(Self { + buddy: GpuBuddy::new(buddy_params)?, pramin: pramin::Pramin::new(bar, chipset, vram_region)?, + tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, }) } + /// Access the [`GpuBuddy`] allocator. + pub(crate) fn buddy(&self) -> &GpuBuddy { + &self.buddy + } + /// Access the [`pramin::Pramin`]. fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { &mut self.pramin } + + /// Access the [`Tlb`] manager. + pub(crate) fn tlb(&self) -> &Tlb<'gpu> { + self.tlb.as_ref().get_ref() + } } /// Page size in bytes (4 KiB). diff --git a/drivers/gpu/nova-core/mm/tlb.rs b/drivers/gpu/nova-core/mm/tlb.rs new file mode 100644 index 000000000000..cc862e8159a1 --- /dev/null +++ b/drivers/gpu/nova-core/mm/tlb.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! TLB (Translation Lookaside Buffer) flush support for GPU MMU. +//! +//! After modifying page table entries, the GPU's TLB must be flushed to +//! ensure the new mappings take effect. This module provides TLB flush +//! functionality for virtual memory managers. +//! +//! # Examples +//! +//! ```ignore +//! use crate::mm::tlb::Tlb; +//! +//! fn page_table_update(tlb: &Tlb, pdb_addr: VramAddress) -> Result<()> { +//! // ... modify page tables ... +//! +//! // Flush TLB to make changes visible (polls for completion). +//! tlb.flush(pdb_addr)?; +//! +//! Ok(()) +//! } +//! ``` + +use kernel::{ + io::poll::read_poll_timeout, + io::Io, + new_mutex, + prelude::*, + sync::Mutex, + time::Delta, // +}; + +use crate::{ + bounded_enum, + driver::Bar0, + mm::VramAddress, + regs, // +}; + +bounded_enum! { + /// TLB invalidation acknowledgment scope. + /// + /// Controls how far the hardware waits for the invalidation to propagate + /// before clearing the `trigger` bit of `NV_TLB_FLUSH_CTRL`. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub(crate) enum TlbAckMode with TryFrom> { + /// Fire-and-forget: no acknowledgment required. + None = 0, + /// Wait for acknowledgment from all consumers, including remote GPUs + /// reachable over NVLink. + /// + /// Globally is strictly required only during unmap or permission + /// tightening, because the backing memory may be reassigned after the + /// flush returns and a stale TLB entry could let the GPU access freed + /// memory. For new mapping or relaxing permissions, a stale entry would + /// merely cause a redundant fault and retry, so [`TlbAckMode::None`] + /// would suffice. + Globally = 1, + /// Wait for acknowledgment from consumers within the local NVLink + /// fabric node only; skip cross-node ack. + Intranode = 2, + } +} + +/// TLB manager for GPU translation buffer operations. +#[pin_data] +pub(crate) struct Tlb<'gpu> { + bar: Bar0<'gpu>, + /// TLB flush serialization lock: This lock is designed to be acquired during + /// the DMA fence signalling critical path. It should NEVER be held across any + /// reclaimable CPU memory allocations because the memory reclaim path can + /// call `dma_fence_wait()` (when implemented), which would deadlock if lock held. + #[pin] + lock: Mutex<()>, +} + +impl<'gpu> Tlb<'gpu> { + /// Create a new TLB manager. + pub(super) fn new(bar: Bar0<'gpu>) -> impl PinInit { + pin_init!(Self { + bar, + lock <- new_mutex!((), "tlb_flush"), + }) + } + + /// Flush the GPU TLB for a specific page directory base. + /// + /// This invalidates all TLB entries associated with the given PDB address. + /// Must be called after modifying page table entries to ensure the GPU sees + /// the updated mappings. + pub(super) fn flush(&self, pdb_addr: VramAddress) -> Result { + let _guard = self.lock.lock(); + + // Write PDB address. + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_LO::from_pdb_addr( + pdb_addr.into_raw(), + )); + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_HI::from_pdb_addr( + pdb_addr.into_raw(), + )); + + // Trigger flush. + self.bar.write_reg( + regs::NV_TLB_FLUSH_CTRL::zeroed() + .with_all_va(true) + .with_ack(TlbAckMode::None) + .with_trigger(true), + ); + + // Poll for completion. + read_poll_timeout( + || Ok(self.bar.read(regs::NV_TLB_FLUSH_CTRL)), + |ctrl: ®s::NV_TLB_FLUSH_CTRL| !ctrl.trigger(), + Delta::ZERO, + Delta::from_secs(2), + )?; + + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index bf1f3a97c632..9978fb2803b0 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -10,6 +10,7 @@ use kernel::{ sizes::SizeConstants, time, // }; +use pin_init::Zeroable; use crate::{ driver::NovaRegisters, @@ -26,6 +27,7 @@ use crate::{ PFalconRegisters, PeregrineCoreSelect, // }, + mm::tlb::TlbAckMode, // }; // PBUS @@ -471,3 +473,69 @@ pub(crate) mod gb202 { } } } + +// MMU TLB + +register! { + base: NovaRegisters; + + /// TLB flush register: PDB address lower bits. + pub(crate) NV_TLB_FLUSH_PDB_LO(u32) @ 0x00b830a0 { + /// PDB address bits [39:8]. + 31:0 pdb_lo => u32; + } + + /// TLB flush register: PDB address higher bits. + pub(crate) NV_TLB_FLUSH_PDB_HI(u32) @ 0x00b830a4 { + /// PDB address bits [47:40]. + 7:0 pdb_hi => u8; + } + + /// TLB flush control register. + pub(crate) NV_TLB_FLUSH_CTRL(u32) @ 0x00b830b0 { + /// Invalidate every VA in the PDB selected by `NV_TLB_FLUSH_PDB_LO/HI`. + 0:0 all_va => bool; + /// Invalidate TLBs for all PDBs (ignores `NV_TLB_FLUSH_PDB_LO/HI`). + 1:1 all_pdb => bool; + /// Restrict the flush to the HUB MMU's TLBs; skip broadcasting to the + /// per-GPC L2 TLBs. + /// + /// The GPU MMU has a two-level TLB hierarchy: + /// 1. The *HUB MMU* sits at the top and serves memory requests from + /// "host-side" engines: the host/channel interface, copy engines, + /// display, and BAR1/BAR2 accesses. + /// 2. Each GPC (Graphics Processing Cluster — the block that houses + /// shader cores / SMs) has its own L2 TLB that serves requests from + /// the compute and graphics engines inside the cluster. + /// + /// When set, only the HUB TLBs are invalidated. This is a performance + /// optimization for flushes that only affect HUB-side mappings (e.g. + /// BAR1/BAR2 windows), where fanning the invalidation out to every + /// GPC's L2 TLB would be wasted work. Must be false when flushing + /// mappings that may be cached by compute/graphics engines. + 2:2 hubtlb_only => bool; + /// Invalidation acknowledgment scope. See [`TlbAckMode`] for details. + 8:7 ack ?=> TlbAckMode; + /// Write 1 to kick off the flush. Hardware clears this bit when the + /// flush completes; reads as 1 while the flush is in progress. + 31:31 trigger => bool; + } +} + +impl NV_TLB_FLUSH_PDB_LO { + /// Create a register value from a PDB address. + /// + /// Extracts bits [39:8] of the address and shifts it right by 8 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_lo(((addr >> 8) & 0xFFFF_FFFF) as u32) + } +} + +impl NV_TLB_FLUSH_PDB_HI { + /// Create a register value from a PDB address. + /// + /// Extracts bits [47:40] of the address and shifts it right by 40 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_hi(((addr >> 40) & 0xFF) as u8) + } +} -- cgit v1.2.3 From a5dfe5955c6beff8403ce966efde6822700a6634 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:41 +0900 Subject: gpu: nova-core: mm: Add common types for all page table formats Add common page table types shared between MMU v2 and v3. These types are hardware-agnostic and used by both MMU versions. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-3-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/pagetable.rs | 158 ++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 202bb8f8ebed..377cc9d44702 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -61,6 +61,7 @@ use crate::{ pub(crate) use tlb::Tlb; mod hal; +pub(super) mod pagetable; mod pramin; mod regs; pub(super) mod tlb; diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs new file mode 100644 index 000000000000..ed0f3d731c63 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Common page table types shared between MMU v2 and v3. +//! +//! This module provides foundational types used by both MMU versions: +//! - Page table level hierarchy +//! - Memory aperture types for PDEs and PTEs + +#![expect(dead_code)] + +use kernel::num::Bounded; + +use crate::gpu::Architecture; + +/// Extracts the page table index at a given level from a virtual address. +pub(super) trait VaLevelIndex { + /// Return the page table index at `level` for this virtual address. + fn level_index(&self, level: u64) -> u64; +} + +/// MMU version enumeration. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MmuVersion { + /// MMU v2 for Turing/Ampere/Ada. + V2, + /// MMU v3 for Hopper and later. + V3, +} + +impl From for MmuVersion { + fn from(arch: Architecture) -> Self { + match arch { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => Self::V2, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + Self::V3 + } + } + } +} + +/// Page Table Level hierarchy for MMU v2/v3. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PageTableLevel { + /// Level 0 - Page Directory Base (root). + Pdb, + /// Level 1 - Intermediate page directory. + L1, + /// Level 2 - Intermediate page directory. + L2, + /// Level 3 - Intermediate page directory or dual PDE (version-dependent). + L3, + /// Level 4 - PTE level for v2, intermediate page directory for v3. + L4, + /// Level 5 - PTE level used for MMU v3 only. + L5, +} + +impl PageTableLevel { + /// Number of entries per page table (512 for 4KB pages). + pub(super) const ENTRIES_PER_TABLE: usize = 512; + + /// Get the next level in the hierarchy. + pub(super) const fn next(&self) -> Option { + match self { + Self::Pdb => Some(Self::L1), + Self::L1 => Some(Self::L2), + Self::L2 => Some(Self::L3), + Self::L3 => Some(Self::L4), + Self::L4 => Some(Self::L5), + Self::L5 => None, + } + } + + /// Convert level to index. + pub(super) const fn as_index(&self) -> u64 { + match self { + Self::Pdb => 0, + Self::L1 => 1, + Self::L2 => 2, + Self::L3 => 3, + Self::L4 => 4, + Self::L5 => 5, + } + } +} + +/// Memory aperture for Page Table Entries (`PTE`s). +/// +/// Determines which memory region the `PTE` points to. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePte { + /// Local video memory (VRAM). + #[default] + VideoMemory = 0, + /// Peer GPU's video memory. + PeerMemory = 1, + /// System memory with cache coherence. + SystemCoherent = 2, + /// System memory without cache coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From> for AperturePte { + fn from(val: Bounded) -> Self { + match *val { + 0 => Self::VideoMemory, + 1 => Self::PeerMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::VideoMemory, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From for Bounded { + fn from(val: AperturePte) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} + +/// Memory aperture for Page Directory Entries (`PDE`s). +/// +/// Note: For `PDE`s, `Invalid` (0) means the entry is not valid. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePde { + /// Invalid/unused entry. + #[default] + Invalid = 0, + /// Page table is in video memory. + VideoMemory = 1, + /// Page table is in system memory with coherence. + SystemCoherent = 2, + /// Page table is in system memory without coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From> for AperturePde { + fn from(val: Bounded) -> Self { + match *val { + 1 => Self::VideoMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::Invalid, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From for Bounded { + fn from(val: AperturePde) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} -- cgit v1.2.3 From 239bb057a81dca1623857102cfd963e1305aadc1 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:42 +0900 Subject: gpu: nova-core: mm: pagetable: Add PteOps trait Introduce a trait for GPU Page Table Entries (PTEs). New `read()`/`write()` helpers are provided that go through a `Pramin`). The forthcoming MMU v2, v3 PTE structs will each implement `PteOps`, allowing the later page-table walker and mapper to call PTE operations. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-4-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 52 +++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index ed0f3d731c63..d5c02a0ea4fb 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,9 +8,20 @@ #![expect(dead_code)] -use kernel::num::Bounded; - -use crate::gpu::Architecture; +use kernel::{ + io::Io, + num::Bounded, + prelude::*, // +}; + +use crate::{ + gpu::Architecture, + mm::{ + pramin, + Pfn, + VramAddress, // + }, +}; /// Extracts the page table index at a given level from a virtual address. pub(super) trait VaLevelIndex { @@ -84,6 +95,41 @@ impl PageTableLevel { } } +// Trait abstractions for page table operations. + +/// Operations on Page Table Entries (`PTE`s). +pub(super) trait PteOps: Copy + core::fmt::Debug + Into { + /// Create a `PTE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create an invalid `PTE`. + fn invalid() -> Self; + + /// Create a valid `PTE` for the given memory aperture. + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self; + + /// Check if this `PTE` is valid. + fn is_valid(&self) -> bool; + + /// Get the physical frame number. + fn frame_number(&self) -> Pfn; + + /// Read a `PTE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let val = pramin.window_at::(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PTE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. -- cgit v1.2.3 From 85a7542510cdff8bbeef9fb016e98a2f8bcd1171 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:43 +0900 Subject: gpu: nova-core: mm: pagetable: Add PdeOps trait Introduce a trait for GPU Page Directory Entries (PDEs). Default `read()`/`write()` helpers via a `Pramin` are provided. The forthcoming MMU v2, v3 PDE structs will each implement `PdeOps`, allowing the later page-table walker and mapper to call PDE operations. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-5-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index d5c02a0ea4fb..03804da784cc 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -130,6 +130,47 @@ pub(super) trait PteOps: Copy + core::fmt::Debug + Into { } } +/// Operations on Page Directory Entries (`PDE`s). +pub(super) trait PdeOps: Copy + core::fmt::Debug + Into { + /// Create a `PDE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create a valid `PDE` pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self; + + /// Create an invalid `PDE`. + fn invalid() -> Self; + + /// Check if this `PDE` is valid. + fn is_valid(&self) -> bool; + + /// Get the memory aperture of this `PDE`. + fn aperture(&self) -> AperturePde; + + /// Get the VRAM address of the page table. + fn table_vram_address(&self) -> VramAddress; + + /// Read a `PDE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let val = pramin.window_at::(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PDE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } + + /// Check if this `PDE` is valid and points to video memory. + fn is_valid_vram(&self) -> bool { + self.is_valid() && self.aperture() == AperturePde::VideoMemory + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. -- cgit v1.2.3 From 71e99998a6559d71e5d6eff85afcc0f7f99ece25 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:44 +0900 Subject: gpu: nova-core: mm: pagetable: Add DualPdeOps trait Introduce a trait for 128-bit Dual Page Directory Entries. The `read()`/`write()` helpers issue two 64-bit accesses through a `Pramin` to load/store the 128-bit value. Signed-off-by: Joel Fernandes [ecourtney: replace PraminWindow accesses with typed MMIO views] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-6-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 03804da784cc..4e63f79b95e5 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -171,6 +171,47 @@ pub(super) trait PdeOps: Copy + core::fmt::Debug + Into { } } +/// Operations on Dual Page Directory Entries (128-bit `DualPde`s). +pub(super) trait DualPdeOps: Copy + core::fmt::Debug { + /// Create a `DualPde` from raw 128-bit value (two `u64`s). + fn from_raw(big: u64, small: u64) -> Self; + + /// Create a `DualPde` with only the small page table pointer set. + fn new_small(table_pfn: Pfn) -> Self; + + /// Check if the small page table pointer is valid. + fn has_small(&self) -> bool; + + /// Get the small page table VRAM address. + fn small_vram_address(&self) -> VramAddress; + + /// Get the raw `u64` value of the big PDE. + fn big_raw_u64(&self) -> u64; + + /// Get the raw `u64` value of the small PDE. + fn small_raw_u64(&self) -> u64; + + /// Read a dual PDE (128-bit) from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + let lo = pramin.window_at::(addr)?.view().read_val(); + let hi = pramin.window_at::(addr + 8)?.view().read_val(); + Ok(Self::from_raw(lo, hi)) + } + + /// Write this dual PDE (128-bit) to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::(addr)? + .view() + .write_val(self.big_raw_u64()); + pramin + .window_at::(addr + 8)? + .view() + .write_val(self.small_raw_u64()); + Ok(()) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. -- cgit v1.2.3 From 98b7273f5de149b2c6ff0d98bc2b3fa55b749b74 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:45 +0900 Subject: gpu: nova-core: mm: Add MMU v2 page table types Add page table entry and directory structures for MMU version 2 used by Hopper and later GPUs. The `Pte`, `Pde`, and `DualPde` types each implement the `PteOps`, `PdeOps`, and `DualPdeOps` traits introduced earlier in the series, providing the version-agnostic API used by the forthcoming page-table walker and mapper. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-7-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 2 + drivers/gpu/nova-core/mm/pagetable/ver2.rs | 275 +++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver2.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 4e63f79b95e5..6f3f39ebde13 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,6 +8,8 @@ #![expect(dead_code)] +pub(super) mod ver2; + use kernel::{ io::Io, num::Bounded, diff --git a/drivers/gpu/nova-core/mm/pagetable/ver2.rs b/drivers/gpu/nova-core/mm/pagetable/ver2.rs new file mode 100644 index 000000000000..d7169a0fcff9 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver2.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v2 page table types for Turing, Ampere and Ada GPUs. +//! +//! This module defines MMU version 2 specific types (Turing, Ampere and Ada GPUs). +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/turing/tu102/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 2 Pfn bitfield conversions: +// 25 bits for video memory frame numbers (bits 32:8). +impl_pfn_bounded!(25); +// 46 bits for system memory frame numbers (bits 53:8). +impl_pfn_bounded!(46); + +bitfield! { + /// MMU v2 49-bit virtual address layout. + pub(super) struct VirtualAddressV2(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [48:47]. + 48:47 pde3_idx; + } +} + +impl VirtualAddressV2 { + /// Create a [`VirtualAddressV2`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV2 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde3_idx(), + 1 => *self.pde2_idx(), + 2 => *self.pde1_idx(), + 3 => *self.pde0_idx(), + 4 => *self.pt_idx(), + _ => 0, + } + } +} + +/// `PDE` levels for MMU v2 (5-level hierarchy: `PDB` -> `L1` -> `L2` -> `L3` -> `L4`). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, +]; + +/// `PTE` level for MMU v2. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L4; + +/// Dual `PDE` level for MMU v2 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L3; + +// Page Table Entry (PTE) for MMU v2 - 64-bit entry at level 4. +bitfield! { + /// Page Table Entry for MMU v2. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Encryption enabled (Confidential Computing). + 4:4 encrypted; + /// Privileged access only. + 5:5 privilege; + /// Write protection. + 6:6 read_only; + /// Atomic operations disabled. + 7:7 atomic_disable; + /// Frame number for system memory. + 53:8 frame_number_sys => Pfn; + /// Frame number for video memory. + 32:8 frame_number_vid => Pfn; + /// Peer GPU ID for peer memory (0-7). + 35:33 peer_id; + /// Compression tag line bits. + 53:36 comptagline; + /// Surface kind/format. + 63:56 kind; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let base = Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_read_only(!writable); + match aperture { + AperturePte::VideoMemory => base.with_frame_number_vid(pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePte::SystemCoherent => base.with_frame_number_sys(pfn).with_volatile(true), + AperturePte::PeerMemory | AperturePte::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PTE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + match self.aperture() { + AperturePte::VideoMemory => self.frame_number_vid(), + _ => self.frame_number_sys(), + } + } +} + +// Page Directory Entry (PDE) for MMU v2 - 64-bit entry at levels 0-2. +bitfield! { + /// Page Directory Entry for MMU v2. + pub(in crate::mm) struct Pde(u64) { + /// Valid bit (inverted logic). + 0:0 valid_inverted; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Disable Address Translation Services. + 5:5 no_ats; + /// Table frame number for system memory. + 53:8 table_frame_sys => Pfn; + /// Table frame number for video memory. + 32:8 table_frame_vid => Pfn; + /// Peer GPU ID (0-7). + 35:33 peer_id; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + let base = Self::zeroed() + .with_valid_inverted(false) // 0 = valid + .with_aperture(aperture); + match aperture { + AperturePde::VideoMemory => base.with_table_frame_vid(table_pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePde::SystemCoherent => base.with_table_frame_sys(table_pfn).with_volatile(true), + AperturePde::Invalid | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed() + .with_valid_inverted(true) + .with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + !self.valid_inverted().into_bool() && self.aperture() != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame_vid()) + } +} + +/// Dual `PDE` at Level 3 - 128-bit entry of Large/Small Page Table pointers. +/// +/// The dual `PDE` supports both large (64KB) and small (4KB) page tables. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Large/Big Page Table pointer (lower 64 bits). + pub(super) big: Pde, + /// Small Page Table pointer (upper 64 bits). + pub(super) small: Pde, +} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + PdeOps::is_valid(&self.big) + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: PdeOps::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: PdeOps::from_raw(0), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} -- cgit v1.2.3 From 103fb3a4b2b85ebf30f11a433df17df708f85689 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:46 +0900 Subject: gpu: nova-core: mm: Add MMU v3 page table types Add page table entry and directory structures for MMU version 3 used by Hopper and later GPUs. The `Pte`, `Pde`, and `DualPde` types each implement the `PteOps`, `PdeOps`, and `DualPdeOps` traits introduced earlier in the series, providing the version-agnostic API used by the forthcoming page-table walker and mapper. Signed-off-by: Joel Fernandes [ecourtney: update for the VramAddress raw API, apply rustfmt] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-8-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/ver3.rs | 421 +++++++++++++++++++++++++++++ 2 files changed, 422 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/ver3.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 6f3f39ebde13..f8ec62decae3 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -9,6 +9,7 @@ #![expect(dead_code)] pub(super) mod ver2; +pub(super) mod ver3; use kernel::{ io::Io, diff --git a/drivers/gpu/nova-core/mm/pagetable/ver3.rs b/drivers/gpu/nova-core/mm/pagetable/ver3.rs new file mode 100644 index 000000000000..47ed3339026b --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver3.rs @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v3 page table types for Hopper and later GPUs. +//! +//! This module defines MMU version 3 specific types (Hopper and later GPUs). +//! +//! Key differences from MMU v2: +//! - Unified 40-bit address field for all apertures (v2 had separate sys/vid fields). +//! - PCF (Page Classification Field) replaces separate privilege/RO/atomic/cache bits. +//! - KIND field is 4 bits (not 8). +//! - IS_PTE bit in PDE to support large pages directly. +//! - No COMPTAGLINE field (compression handled differently in v3). +//! - No separate ENCRYPTED bit. +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/hopper/gh100/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, + prelude::*, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 3 Pfn conversion. +impl_pfn_bounded!(40); + +bitfield! { + /// MMU v3 57-bit virtual address layout. + pub(super) struct VirtualAddressV3(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [55:47]. + 55:47 pde3_idx; + /// PDE4 index [56]. + 56:56 pde4_idx; + } +} + +impl VirtualAddressV3 { + /// Create a [`VirtualAddressV3`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV3 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde4_idx(), + 1 => *self.pde3_idx(), + 2 => *self.pde2_idx(), + 3 => *self.pde1_idx(), + 4 => *self.pde0_idx(), + 5 => *self.pt_idx(), + _ => 0, + } + } +} + +/// PDE levels for MMU v3 (6-level hierarchy). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, + PageTableLevel::L4, +]; + +/// PTE level for MMU v3. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L5; + +/// Dual PDE level for MMU v3 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L4; + +bitfield! { + /// Page Classification Field for PTEs (5 bits) in MMU v3. + pub(in crate::mm) struct PtePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// Access counting disabled (0=enabled, 1=disabled). + 1:1 acd; + /// Read-only access (0=read-write, 1=read-only). + 2:2 read_only; + /// Atomics disabled (0=enabled, 1=disabled). + 3:3 no_atomic; + /// Privileged access only (0=regular, 1=privileged). + 4:4 privileged; + } +} + +impl PtePcf { + /// Create PCF for read-write mapping (cached, no atomics, regular mode). + fn rw() -> Self { + Self::zeroed().with_no_atomic(true) + } + + /// Create PCF for read-only mapping (cached, no atomics, regular mode). + fn ro() -> Self { + Self::zeroed().with_read_only(true).with_no_atomic(true) + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From> for PtePcf { + fn from(val: Bounded) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From for Bounded { + fn from(pcf: PtePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x1F) + } +} + +bitfield! { + /// Page Classification Field for PDEs (3 bits) in MMU v3. + /// + /// Controls Address Translation Services (ATS) and caching. + pub(in crate::mm) struct PdePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// ATS disabled (0=enabled, 1=disabled). + 1:1 no_ats; + } +} + +impl PdePcf { + /// Create PCF for cached mapping with ATS enabled (default). + fn cached() -> Self { + Self::zeroed() + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From> for PdePcf { + fn from(val: Bounded) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From for Bounded { + fn from(pcf: PdePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x7) + } +} + +bitfield! { + /// Page Table Entry for MMU v3. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Page Classification Field. + 7:3 pcf => PtePcf; + /// Surface kind (4 bits, 0x0=pitch, 0xF=invalid). + 11:8 kind; + /// Physical frame number (for all apertures). + 51:12 frame_number => Pfn; + /// Peer GPU ID for peer memory (0-7). + 63:61 peer_id; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let pcf = match (aperture, writable) { + (AperturePte::VideoMemory, true) => PtePcf::rw(), + (AperturePte::VideoMemory, false) => PtePcf::ro(), + // Sysmem PTEs use uncached+no_atomic PCF for cache coherency. + (AperturePte::SystemCoherent, true) => { + PtePcf::zeroed().with_uncached(true).with_no_atomic(true) + } + (AperturePte::SystemCoherent, false) => PtePcf::zeroed() + .with_uncached(true) + .with_no_atomic(true) + .with_read_only(true), + (AperturePte::PeerMemory | AperturePte::SystemNonCoherent, _) => { + kernel::pr_warn!("MMU v3 PTE aperture {:?} not supported\n", aperture); + return Self::invalid(); + } + }; + Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_pcf(pcf) + .with_frame_number(pfn) + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + Pte::frame_number(*self) + } +} + +bitfield! { + /// Page Directory Entry for MMU v3 (Hopper+). + /// + /// ## Note + /// + /// v3 uses a unified 40-bit address field (v2 had separate sys/vid address fields). + pub(in crate::mm) struct Pde(u64) { + /// Entry is a PTE (0=PDE, 1=large page PTE). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field (3 bits for PDE). + 5:3 pcf => PdePcf; + /// Table frame number (40-bit unified address). + 51:12 table_frame => Pfn; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + match aperture { + AperturePde::VideoMemory => Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_pfn), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + Pde::aperture(*self) != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame()) + } +} + +bitfield! { + /// Big Page Table pointer in Dual PDE (MMU v3). + /// + /// 64-bit lower word of the 128-bit Dual PDE. + pub(super) struct DualPdeBig(u64) { + /// Entry is a PTE (for large pages). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field. + 5:3 pcf => PdePcf; + /// Table frame (table address 256-byte aligned). + 51:8 table_frame; + } +} + +impl DualPdeBig { + /// Create an invalid big page table pointer. + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + /// Create a valid big PDE pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_addr: VramAddress) -> Result { + // Big page table addresses must be 256-byte aligned (shift 8). + if table_addr.into_raw() & 0xFF != 0 { + return Err(EINVAL); + } + let table_frame = Bounded::from_expr(table_addr.into_raw() >> 8); + match aperture { + AperturePde::VideoMemory => Ok(Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_frame)), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 DualPdeBig aperture {:?} not supported\n", aperture); + Ok(Self::invalid()) + } + } + } + + /// Check if this big PDE is valid. + fn is_valid(&self) -> bool { + self.aperture() != AperturePde::Invalid + } + + /// Get the VRAM address of the big page table. + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + self.aperture() == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM DualPdeBig (aperture: {:?})", + self.aperture() + ); + VramAddress::from_raw(*self.table_frame() << 8) + } +} + +/// Dual PDE at Level 4 for MMU v3 - 128-bit entry. +/// +/// Contains both big (64KB) and small (4KB) page table pointers: +/// - Lower 64 bits: Big Page Table pointer. +/// - Upper 64 bits: Small Page Table pointer. +/// +/// ## Note +/// +/// The big and small page table pointers have different address layouts: +/// - Big address = field value << 8 (256-byte alignment). +/// - Small address = field value << 12 (4KB alignment). +/// +/// This is why `DualPdeBig` is a separate type from `Pde`. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Big Page Table pointer. + pub(super) big: DualPdeBig, + /// Small Page Table pointer. + pub(super) small: Pde, +} + +// SAFETY: Both `DualPdeBig` and `Pde` fields are `Zeroable` (bitfield types are Zeroable). +unsafe impl Zeroable for DualPde {} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + self.big.is_valid() + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: DualPdeBig::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: DualPdeBig::invalid(), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} -- cgit v1.2.3 From a9eabe320b60e5af72ff10551be1b2d67ca614af Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:47 +0900 Subject: gpu: nova-core: mm: pagetable: Add MmuConfig trait Introduce `MmuConfig`, the trait that ties the entry-operation traits (`PteOps`, `PdeOps`, `DualPdeOps`) together with the version-specific constants and helpers. `MmuV2` and `MmuV3` are zero-sized marker structs that implement `MmuConfig` for Turing/Ampere/Ada and Hopper/Blackwell respectively. Dispatch is fully resolved at compile time through these markers, so version-specific code is selected without runtime overhead and without wrapper enums. This enables version-agnostic page-table operations while keeping version-specific implementation details encapsulated in the `ver2` and `ver3` modules. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-9-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 109 ++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index f8ec62decae3..89f828194c59 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -22,6 +22,7 @@ use crate::{ mm::{ pramin, Pfn, + VirtualAddress, VramAddress, // }, }; @@ -215,6 +216,114 @@ pub(super) trait DualPdeOps: Copy + core::fmt::Debug { } } +/// MMU configuration trait -- encodes version-specific constants and types. +pub(super) trait MmuConfig: 'static { + /// Page Table Entry type. + type Pte: PteOps; + /// Page Directory Entry type. + type Pde: PdeOps; + /// Dual Page Directory Entry type (128-bit). + type DualPde: DualPdeOps; + + /// PDE levels (excluding PTE level) for page table walking. + const PDE_LEVELS: &'static [PageTableLevel]; + /// PTE level for this MMU version. + const PTE_LEVEL: PageTableLevel; + /// Dual PDE level (128-bit entries) for this MMU version. + const DUAL_PDE_LEVEL: PageTableLevel; + + /// Get the number of entries per page table page for a given level. + fn entries_per_page(level: PageTableLevel) -> usize; + + /// Extract the page table index at `level` from `va`. + fn level_index(va: VirtualAddress, level: u64) -> u64; + + /// Get the entry size in bytes for a given level. + fn entry_size(level: PageTableLevel) -> usize { + if level == Self::DUAL_PDE_LEVEL { + 16 // 128-bit dual PDE + } else { + 8 // 64-bit PDE/PTE + } + } + + /// Compute upper bound on page table pages needed for `num_virt_pages`. + /// + /// Walks from PTE level up through PDE levels, accumulating the tree. + fn pt_pages_upper_bound(num_virt_pages: usize) -> usize { + let mut total = 0; + + // PTE pages at the leaf level. + let pte_epp = Self::entries_per_page(Self::PTE_LEVEL); + let mut pages_at_level = num_virt_pages.div_ceil(pte_epp); + total += pages_at_level; + + // Walk PDE levels bottom-up (reverse of PDE_LEVELS). + for &level in Self::PDE_LEVELS.iter().rev() { + let epp = Self::entries_per_page(level); + + // How many pages at this level do we need to point to + // the previous pages_at_level? + pages_at_level = pages_at_level.div_ceil(epp); + total += pages_at_level; + } + + total + } +} + +/// Marker struct for MMU v2 (Turing/Ampere/Ada). +pub(super) struct MmuV2; + +impl MmuConfig for MmuV2 { + type Pte = ver2::Pte; + type Pde = ver2::Pde; + type DualPde = ver2::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver2::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver2::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver2::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + // TODO: Calculate these values from the bitfield dynamically + // instead of hardcoding them. + match level { + PageTableLevel::Pdb => 4, // PD3 root: bits [48:47] = 2 bits + PageTableLevel::L3 => 256, // PD0 dual: bits [28:21] = 8 bits + _ => 512, // PD2, PD1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver2::VirtualAddressV2::new(va).level_index(level) + } +} + +/// Marker struct for MMU v3 (Hopper and later). +pub(super) struct MmuV3; + +impl MmuConfig for MmuV3 { + type Pte = ver3::Pte; + type Pde = ver3::Pde; + type DualPde = ver3::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver3::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver3::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver3::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + match level { + PageTableLevel::Pdb => 2, // PDE4 root: bit [56] = 1 bit, 2 entries + PageTableLevel::L4 => 256, // PDE0 dual: bits [28:21] = 8 bits + _ => 512, // PDE3, PDE2, PDE1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver3::VirtualAddressV3::new(va).level_index(level) + } +} + /// Memory aperture for Page Table Entries (`PTE`s). /// /// Determines which memory region the `PTE` points to. -- cgit v1.2.3 From 1123774f3610c45a7acf055b8c9841883ed4bf75 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:48 +0900 Subject: gpu: nova-core: mm: Add page table walker for MMU v2/v3 Add the page table walker implementation that traverses the page table hierarchy for both MMU v2 (5-level) and MMU v3 (6-level) to resolve virtual addresses to physical addresses or find PTE locations. Currently only v2 has been tested (nova-core currently boots pre-hopper) with some initial preparatory work done for v3. Signed-off-by: Joel Fernandes [ecourtney: take mutable GpuMm and Pramin, not a bound device and window] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-10-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/walk.rs | 244 +++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/pagetable/walk.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 89f828194c59..0e9ffad34c90 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -10,6 +10,7 @@ pub(super) mod ver2; pub(super) mod ver3; +pub(super) mod walk; use kernel::{ io::Io, diff --git a/drivers/gpu/nova-core/mm/pagetable/walk.rs b/drivers/gpu/nova-core/mm/pagetable/walk.rs new file mode 100644 index 000000000000..76c1729971f5 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/walk.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table walker implementation for NVIDIA GPUs. +//! +//! This module provides page table walking functionality for MMU v2 and v3. +//! The walker traverses the page table hierarchy to resolve virtual addresses +//! to physical addresses or to find PTE locations. +//! +//! # Page Table Hierarchy +//! +//! ## MMU v2 (Turing/Ampere/Ada) - 5 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 Dual |---->| L4 | +//! | (L0) | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE (big+small) PTE +//! ``` +//! +//! ## MMU v3 (Hopper+) - 6 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 |---->| L4 Dual |---->| L5 | +//! | (L0) | | | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE PDE (big+small) PTE +//! ``` +//! +//! # Result of a page table walk +//! +//! The walker returns a [`WalkResult`] indicating the outcome. + +use core::marker::PhantomData; + +use kernel::prelude::*; + +use super::{ + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + pramin, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// Result of walking to a PTE. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkResult { + /// Intermediate page tables are missing (only returned in lookup mode). + PageTableMissing, + /// PTE exists but is invalid (page not mapped). + Unmapped { pte_addr: VramAddress }, + /// PTE exists and is valid (page is mapped). + Mapped { pte_addr: VramAddress, pfn: Pfn }, +} + +/// Result of walking PDE levels only. +/// +/// Returned by [`PtWalkInner::walk_pde_levels()`] to indicate whether all PDE +/// levels resolved or a PDE is missing. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkPdeResult { + /// All PDE levels resolved -- returns PTE page table address. + Complete { + /// VRAM address of the PTE-level page table. + pte_table: VramAddress, + }, + /// A PDE is missing and no prepared page was provided by the closure. + Missing { + /// PDE slot address in the parent page table (where to install). + install_addr: VramAddress, + /// The page table level that is missing. + level: PageTableLevel, + }, +} + +/// Page table walker. +pub(in crate::mm) struct PtWalkInner { + pdb_addr: VramAddress, + _phantom: PhantomData, +} + +impl PtWalkInner { + /// Calculate the VRAM address of an entry within a page table. + fn entry_addr(table: VramAddress, level: PageTableLevel, index: u64) -> VramAddress { + let entry_size: u64 = M::entry_size(level).into_safe_cast(); + table + index * entry_size + } + + /// Create a new page table walker. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + pdb_addr, + _phantom: PhantomData, + } + } + + /// Walk PDE levels with closure-based resolution for missing PDEs. + /// + /// Traverses all PDE levels for the MMU version. At each level, reads the PDE. + /// If valid, extracts the child table address and continues. If missing, calls + /// `resolve_prepared(install_addr)` to resolve the missing PDE. + pub(super) fn walk_pde_levels( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + resolve_prepared: impl Fn(VramAddress) -> Option, + ) -> Result { + let va = VirtualAddress::from(vfn); + let mut cur_table = self.pdb_addr; + + for &level in M::PDE_LEVELS { + let idx = M::level_index(va, level.as_index()); + let install_addr = Self::entry_addr(cur_table, level, idx); + + if level == M::DUAL_PDE_LEVEL { + // 128-bit dual PDE with big+small page table pointers. + let dpde = M::DualPde::read(pramin, install_addr)?; + if dpde.has_small() { + cur_table = dpde.small_vram_address(); + continue; + } + } else { + // Regular 64-bit PDE. Use `is_valid_vram()` because + // `table_vram_address()` only reads the VRAM frame-number + // bitfield; system-memory PDEs store the address in a + // different (wider) field and would be silently truncated. + let pde = M::Pde::read(pramin, install_addr)?; + if pde.is_valid_vram() { + cur_table = pde.table_vram_address(); + continue; + } + } + + // PDE missing in HW. Ask caller for resolution. + if let Some(prepared_addr) = resolve_prepared(install_addr) { + cur_table = prepared_addr; + continue; + } + + return Ok(WalkPdeResult::Missing { + install_addr, + level, + }); + } + + Ok(WalkPdeResult::Complete { + pte_table: cur_table, + }) + } + + /// Walk to PTE for lookup only (no allocation). + /// + /// Returns [`WalkResult::PageTableMissing`] if intermediate tables don't exist. + pub(super) fn walk_to_pte_lookup(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result { + self.walk_to_pte_lookup_with_window(mm.pramin_mut(), vfn) + } + + /// Walk to PTE using a caller-provided PRAMIN manager (lookup only). + pub(super) fn walk_to_pte_lookup_with_window( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + ) -> Result { + match self.walk_pde_levels(pramin, vfn, |_| None)? { + WalkPdeResult::Complete { pte_table } => { + Self::read_pte_at_level(pramin, vfn, pte_table) + } + WalkPdeResult::Missing { .. } => Ok(WalkResult::PageTableMissing), + } + } + + /// Read the PTE at the PTE level given the PTE table address. + fn read_pte_at_level( + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + pte_table: VramAddress, + ) -> Result { + let va = VirtualAddress::from(vfn); + let pte_level = M::PTE_LEVEL; + let pte_idx = M::level_index(va, pte_level.as_index()); + let pte_addr = Self::entry_addr(pte_table, pte_level, pte_idx); + let pte = M::Pte::read(pramin, pte_addr)?; + + if pte.is_valid() { + return Ok(WalkResult::Mapped { + pte_addr, + pfn: pte.frame_number(), + }); + } + Ok(WalkResult::Unmapped { pte_addr }) + } +} + +macro_rules! pt_walk_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtWalk::V2(inner) => inner.$method($($arg),*), + PtWalk::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table walker dispatch. +pub(in crate::mm) enum PtWalk { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtWalkInner), + /// MMU v3 (Hopper+). + V3(PtWalkInner), +} + +impl PtWalk { + /// Create a new page table walker for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtWalkInner::::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtWalkInner::::new(pdb_addr)), + } + } + + /// Walk to PTE for lookup. + pub(in crate::mm) fn walk_to_pte(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result { + pt_walk_dispatch!(self, walk_to_pte_lookup(mm, vfn)) + } +} -- cgit v1.2.3 From 3c7561e194d573a3926702d1e051a8c8fbc187ea Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:49 +0900 Subject: gpu: nova-core: mm: Add Virtual Memory Manager Add the Virtual Memory Manager (VMM) infrastructure for GPU address space management. Each Vmm instance manages a single address space identified by its Page Directory Base (PDB) address, used for Channel, BAR1 and BAR2 mappings. Mapping APIs and virtual address range tracking are added in later commits. Signed-off-by: Joel Fernandes [ecourtney: pass mutable GpuMm to mapping lookup] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-11-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/vmm.rs | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 drivers/gpu/nova-core/mm/vmm.rs diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 377cc9d44702..2cf37254fbb9 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -65,6 +65,7 @@ pub(super) mod pagetable; mod pramin; mod regs; pub(super) mod tlb; +pub(super) mod vmm; /// GPU Memory Manager - owns all core MM components. /// diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs new file mode 100644 index 000000000000..3738262442d1 --- /dev/null +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Virtual Memory Manager for NVIDIA GPU page table management. +//! +//! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU +//! virtual address spaces (Channels, BAR1, BAR2). It wraps the page table walker +//! and handles TLB flushing after modifications. + +use kernel::{ + gpu::buddy::AllocatedBlocks, + prelude::*, // +}; + +use crate::mm::{ + pagetable::{ + walk::{ + PtWalk, + WalkResult, // + }, + MmuVersion, // + }, + GpuMm, + Pfn, + Vfn, + VramAddress, // +}; + +/// Virtual Memory Manager for a GPU address space. +/// +/// Each [`Vmm`] instance manages a single address space identified by its Page +/// Directory Base (`PDB`) address. The [`Vmm`] is used for Channel, BAR1 and +/// BAR2 mappings. +pub(crate) struct Vmm { + /// Page Directory Base address for this address space. + pdb_addr: VramAddress, + /// MMU version used for page table layout. + mmu_version: MmuVersion, + /// Page table allocations required for mappings. + page_table_allocs: KVec>>, +} + +impl Vmm { + /// Create a new [`Vmm`] for the given Page Directory Base address. + pub(crate) fn new(pdb_addr: VramAddress, mmu_version: MmuVersion) -> Result { + Ok(Self { + pdb_addr, + mmu_version, + page_table_allocs: KVec::new(), + }) + } + + /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. + pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { + let walker = PtWalk::new(self.pdb_addr, self.mmu_version); + + match walker.walk_to_pte(mm, vfn)? { + WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)), + WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None), + } + } +} -- cgit v1.2.3 From 16f573374fd6ff8ff4ab54abceb5d093869de9d2 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:50 +0900 Subject: gpu: nova-core: mm: Add virtual address range tracking to VMM Add virtual address range tracking to the VMM using a maple tree allocator. This enables contiguous virtual address range allocation for mappings. Signed-off-by: Joel Fernandes Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-12-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/vmm.rs | 87 +++++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 3738262442d1..9f4f40109227 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -8,21 +8,30 @@ use kernel::{ gpu::buddy::AllocatedBlocks, + maple_tree::MapleTreeAlloc, prelude::*, // }; -use crate::mm::{ - pagetable::{ - walk::{ - PtWalk, - WalkResult, // +use core::ops::Range; + +use crate::{ + mm::{ + pagetable::{ + walk::{ + PtWalk, + WalkResult, // + }, + MmuVersion, // }, - MmuVersion, // + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // }, - GpuMm, - Pfn, - Vfn, - VramAddress, // }; /// Virtual Memory Manager for a GPU address space. @@ -37,18 +46,74 @@ pub(crate) struct Vmm { mmu_version: MmuVersion, /// Page table allocations required for mappings. page_table_allocs: KVec>>, + /// Maple tree allocator for virtual address range tracking. + virt_alloc: Pin>>, + /// Total number of pages in the virtual address space. + va_pages: usize, } impl Vmm { /// Create a new [`Vmm`] for the given Page Directory Base address. - pub(crate) fn new(pdb_addr: VramAddress, mmu_version: MmuVersion) -> Result { + /// + /// The [`Vmm`] will manage a virtual address space of `va_size` bytes. + pub(crate) fn new( + pdb_addr: VramAddress, + mmu_version: MmuVersion, + va_size: u64, + ) -> Result { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let va_pages: usize = (va_size / page_size).into_safe_cast(); + let virt_alloc = KBox::pin_init(MapleTreeAlloc::<()>::new(), GFP_KERNEL)?; + Ok(Self { pdb_addr, mmu_version, page_table_allocs: KVec::new(), + virt_alloc, + va_pages, }) } + /// Allocate a contiguous virtual frame number range. + /// + /// # Arguments + /// + /// - `num_pages`: Number of pages to allocate. + /// - `va_range`: `None` = allocate anywhere, `Some(range)` = constrain allocation to the given + /// range. + fn alloc_vfn_range(&self, num_pages: usize, va_range: Option>) -> Result { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + + let start_vfn = match va_range { + Some(r) => { + let num_pages_u64: u64 = num_pages.into_safe_cast(); + let size = num_pages_u64.checked_mul(page_size).ok_or(EOVERFLOW)?; + let range_size = r.end.checked_sub(r.start).ok_or(EOVERFLOW)?; + if range_size != size { + return Err(EINVAL); + } + let start_vfn: usize = (r.start / page_size).into_safe_cast(); + let end_vfn: usize = (r.end / page_size).into_safe_cast(); + self.virt_alloc + .insert_range(start_vfn..end_vfn, (), GFP_KERNEL)?; + start_vfn + } + None => self + .virt_alloc + .alloc_range(num_pages, (), ..self.va_pages, GFP_KERNEL)?, + }; + + Ok(Vfn::new(start_vfn.into_safe_cast())) + } + + /// Free a virtual frame number range back to the maple tree. + fn free_vfn(&self, vfn: Vfn) { + let vfn_index: usize = vfn.raw().into_safe_cast(); + if self.virt_alloc.erase(vfn_index).is_none() { + kernel::pr_warn!("free_vfn: VFN {} not found in maple tree\n", vfn_index); + } + } + /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { let walker = PtWalk::new(self.pdb_addr, self.mmu_version); -- cgit v1.2.3 From 4e4182282d4e8d81bfe62ed8ff634526fdd5e238 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:51 +0900 Subject: gpu: nova-core: mm: Add multi-page mapping API to VMM Add the page table mapping and unmapping API to the Virtual Memory Manager, implementing a two-phase prepare/execute model suitable for use both inside and outside the DMA fence signalling critical path. Signed-off-by: Joel Fernandes [ecourtney: pass mutable GpuMm, replace window guards with scoped borrows] [ecourtney: use current raw address helpers, drop stale expects] [ecourtney: zero page table pages through one PRAMIN view] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-13-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/mm/pagetable.rs | 1 + drivers/gpu/nova-core/mm/pagetable/map.rs | 345 ++++++++++++++++++++++++++++++ drivers/gpu/nova-core/mm/vmm.rs | 255 ++++++++++++++++++++-- 3 files changed, 583 insertions(+), 18 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/pagetable/map.rs diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 0e9ffad34c90..63111ff95780 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -8,6 +8,7 @@ #![expect(dead_code)] +pub(super) mod map; pub(super) mod ver2; pub(super) mod ver3; pub(super) mod walk; diff --git a/drivers/gpu/nova-core/mm/pagetable/map.rs b/drivers/gpu/nova-core/mm/pagetable/map.rs new file mode 100644 index 000000000000..77431c509a89 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/map.rs @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table mapping operations for NVIDIA GPUs. + +use core::marker::PhantomData; + +use kernel::{ + gpu::buddy::{ + AllocatedBlocks, + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + io::io_write, + prelude::*, + ptr::Alignment, + rbtree::{ + RBTree, + RBTreeNode, // + }, + sizes::SZ_4K, // +}; + +use super::{ + walk::{ + PtWalkInner, + WalkPdeResult, + WalkResult, // + }, + AperturePde, + AperturePte, + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// A pre-allocated and zeroed page table page. +/// +/// Created during the mapping prepare phase and consumed during the execute phase. +/// Stored in an [`RBTree`] keyed by the PDE slot address (`install_addr`). +pub(in crate::mm) struct PreparedPtPage { + /// The allocated and zeroed page table page. + pub(in crate::mm) alloc: Pin>, + /// Page table level -- needed to determine if this PT page is for a dual PDE. + pub(in crate::mm) level: PageTableLevel, +} + +/// Page table mapper. +pub(in crate::mm) struct PtMapInner { + walker: PtWalkInner, + pdb_addr: VramAddress, + _phantom: PhantomData, +} + +impl PtMapInner { + /// Create a new [`PtMapInner`]. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + walker: PtWalkInner::::new(pdb_addr), + pdb_addr, + _phantom: PhantomData, + } + } + + /// Allocate and zero a physical page table page. + fn alloc_and_zero_page(mm: &mut GpuMm<'_>, level: PageTableLevel) -> Result { + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let page_vram = VramAddress::from_raw(blocks.iter().next().ok_or(ENOMEM)?.offset()); + + // Zero via PRAMIN. + let window = mm + .pramin_mut() + .window_at::<[u64; PAGE_SIZE / 8]>(page_vram)?; + for i in 0..PAGE_SIZE / 8 { + io_write!(window.view(), [build: i], 0); + } + + Ok(PreparedPtPage { + alloc: blocks, + level, + }) + } + + /// Ensure all intermediate page table pages exist for a single VFN. + /// + /// The mutable PRAMIN borrow ends before each allocation. + fn ensure_single_pte_path( + &self, + mm: &mut GpuMm<'_>, + vfn: Vfn, + pt_pages: &mut RBTree, + ) -> Result { + let max_iter = 2 * M::PDE_LEVELS.len(); + + for _ in 0..max_iter { + let result = self + .walker + .walk_pde_levels(mm.pramin_mut(), vfn, |install_addr| { + pt_pages.get(&install_addr).and_then(|p| { + p.alloc + .iter() + .next() + .map(|b| VramAddress::from_raw(b.offset())) + }) + })?; + + match result { + WalkPdeResult::Complete { .. } => { + return Ok(()); + } + WalkPdeResult::Missing { + install_addr, + level, + } => { + let page = Self::alloc_and_zero_page(mm, level)?; + let node = RBTreeNode::new(install_addr, page, GFP_KERNEL)?; + let old = pt_pages.insert(node); + if old.is_some() { + kernel::pr_warn_once!( + "VMM: duplicate install_addr in pt_pages (internal consistency error)\n" + ); + return Err(EIO); + } + } + } + } + + kernel::pr_warn!( + "VMM: ensure_pte_path: loop exhausted after {} iters (VFN {:?})\n", + max_iter, + vfn + ); + Err(EIO) + } + + /// Prepare page table resources for mapping `num_pages` pages starting at `vfn_start`. + /// + /// Reserves capacity in `page_table_allocs`, then walks the hierarchy + /// per-VFN to prepare pages for all missing PDEs. + pub(super) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec>>, + pt_pages: &mut RBTree, + ) -> Result { + // Pre-reserve so install_mappings() can use push_within_capacity (no alloc + // in fence signalling critical path). + let pt_upper_bound = M::pt_pages_upper_bound(num_pages); + page_table_allocs.reserve(pt_upper_bound, GFP_KERNEL)?; + + // Walk the hierarchy per-VFN to prepare pages for all missing PDEs. + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + self.ensure_single_pte_path(mm, vfn, pt_pages)?; + } + Ok(()) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + /// + /// Drains `pt_pages` and moves allocations into `page_table_allocs`. + pub(super) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree, + page_table_allocs: &mut KVec>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + { + let pramin = mm.pramin_mut(); + + // Drain prepared PT pages, install all pending PDEs. + let mut cursor = pt_pages.cursor_front_mut(); + while let Some(c) = cursor { + let (next, node) = c.remove_current(); + let (install_addr, page) = node.to_key_value(); + let page_vram = + VramAddress::from_raw(page.alloc.iter().next().ok_or(ENOMEM)?.offset()); + + if page.level == M::DUAL_PDE_LEVEL { + let new_dpde = M::DualPde::new_small(Pfn::from(page_vram)); + new_dpde.write(pramin, install_addr)?; + } else { + let new_pde = M::Pde::new(AperturePde::VideoMemory, Pfn::from(page_vram)); + new_pde.write(pramin, install_addr)?; + } + + page_table_allocs + .push_within_capacity(page.alloc) + .map_err(|_| ENOMEM)?; + + cursor = next; + } + + // Write PTEs (all PDEs now installed in HW). + for (i, &pfn) in pfns.iter().enumerate() { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Unmapped { pte_addr } | WalkResult::Mapped { pte_addr, .. } => { + let pte = M::Pte::new(AperturePte::VideoMemory, pfn, writable); + pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + kernel::pr_warn_once!("VMM: page table missing for VFN {vfn:?}\n"); + return Err(EIO); + } + } + } + } + + // Flush TLB. + mm.tlb().flush(self.pdb_addr) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(super) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + let invalid_pte = M::Pte::invalid(); + + { + let pramin = mm.pramin_mut(); + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Mapped { pte_addr, .. } | WalkResult::Unmapped { pte_addr } => { + invalid_pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + continue; + } + } + } + } + + mm.tlb().flush(self.pdb_addr) + } +} + +macro_rules! pt_map_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtMap::V2(inner) => inner.$method($($arg),*), + PtMap::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table mapper dispatch. +pub(in crate::mm) enum PtMap { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtMapInner), + /// MMU v3 (Hopper+). + V3(PtMapInner), +} + +impl PtMap { + /// Create a new page table mapper for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtMapInner::::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtMapInner::::new(pdb_addr)), + } + } + + /// Prepare page table resources for a mapping. + pub(in crate::mm) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec>>, + pt_pages: &mut RBTree, + ) -> Result { + pt_map_dispatch!( + self, + prepare_map(mm, vfn_start, num_pages, page_table_allocs, pt_pages) + ) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + pub(in crate::mm) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree, + page_table_allocs: &mut KVec>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + pt_map_dispatch!( + self, + install_mappings(mm, pt_pages, page_table_allocs, vfn_start, pfns, writable) + ) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(in crate::mm) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + pt_map_dispatch!(self, invalidate_ptes(mm, vfn_start, num_pages)) + } +} diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 9f4f40109227..411710d03f7a 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -3,20 +3,26 @@ //! Virtual Memory Manager for NVIDIA GPU page table management. //! //! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU -//! virtual address spaces (Channels, BAR1, BAR2). It wraps the page table walker -//! and handles TLB flushing after modifications. +//! virtual address spaces (Channels, BAR1, BAR2). use kernel::{ gpu::buddy::AllocatedBlocks, maple_tree::MapleTreeAlloc, - prelude::*, // + prelude::*, + rbtree::RBTree, // }; -use core::ops::Range; +use core::{ + cell::Cell, + ops::Range, // +}; use crate::{ mm::{ pagetable::{ + map::{ + PtMap, // + }, walk::{ PtWalk, WalkResult, // @@ -34,22 +40,108 @@ use crate::{ }, }; +/// Multi-page prepared mapping -- VA range allocated, ready for execute. +/// +/// Produced by [`Vmm::prepare_map()`], consumed by [`Vmm::execute_map()`]. +/// The VA space allocation is tracked in the [`Vmm`]'s maple tree and freed +/// on error or via [`Vmm::unmap_pages()`]. +/// +/// Dropping without calling [`Vmm::execute_map()`] logs a warning and leaks +/// the VA range in the maple tree. +pub(crate) struct PreparedMapping { + vfn_start: Vfn, + num_pages: usize, + /// Logs a warning if dropped without executing. + _drop_guard: MustExecuteGuard, +} + +/// Result of a mapping operation -- tracks the active mapped range. +/// +/// Returned by [`Vmm::execute_map()`] and [`Vmm::map_pages()`]. +/// Callers must call [`Vmm::unmap_pages()`] before dropping to invalidate +/// PTEs and free the VA range. Dropping without unmapping logs a warning +/// and leaks the VA range in the maple tree. +pub(crate) struct MappedRange { + pub(super) vfn_start: Vfn, + pub(super) num_pages: usize, + /// Logs a warning if dropped without unmapping. + _drop_guard: MustUnmapGuard, +} + +/// Guard that logs a warning if a [`PreparedMapping`] is dropped without +/// being consumed by [`Vmm::execute_map()`]. +struct MustExecuteGuard { + armed: Cell, +} + +impl MustExecuteGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustExecuteGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("PreparedMapping dropped without calling execute_map()\n"); + } + } +} + +/// Guard that logs a warning if a [`MappedRange`] is dropped without +/// calling [`Vmm::unmap_pages()`]. +struct MustUnmapGuard { + armed: Cell, +} + +impl MustUnmapGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustUnmapGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("MappedRange dropped without calling unmap_pages()\n"); + } + } +} + /// Virtual Memory Manager for a GPU address space. /// /// Each [`Vmm`] instance manages a single address space identified by its Page -/// Directory Base (`PDB`) address. The [`Vmm`] is used for Channel, BAR1 and -/// BAR2 mappings. +/// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings. pub(crate) struct Vmm { /// Page Directory Base address for this address space. pdb_addr: VramAddress, - /// MMU version used for page table layout. - mmu_version: MmuVersion, + /// Page table walker for reading existing mappings. + pt_walk: PtWalk, + /// Page table mapper for prepare/execute operations. + pt_map: PtMap, /// Page table allocations required for mappings. page_table_allocs: KVec>>, /// Maple tree allocator for virtual address range tracking. virt_alloc: Pin>>, /// Total number of pages in the virtual address space. va_pages: usize, + /// Prepared PT pages pending PDE installation, keyed by `install_addr`. + /// + /// Populated during prepare phase and drained in execute phase. Shared by all + /// pending maps, preventing races on the same PDE slot. + pt_pages: RBTree, } impl Vmm { @@ -67,20 +159,16 @@ impl Vmm { Ok(Self { pdb_addr, - mmu_version, + pt_walk: PtWalk::new(pdb_addr, mmu_version), + pt_map: PtMap::new(pdb_addr, mmu_version), page_table_allocs: KVec::new(), virt_alloc, va_pages, + pt_pages: RBTree::new(), }) } /// Allocate a contiguous virtual frame number range. - /// - /// # Arguments - /// - /// - `num_pages`: Number of pages to allocate. - /// - `va_range`: `None` = allocate anywhere, `Some(range)` = constrain allocation to the given - /// range. fn alloc_vfn_range(&self, num_pages: usize, va_range: Option>) -> Result { let page_size: u64 = PAGE_SIZE.into_safe_cast(); @@ -116,11 +204,142 @@ impl Vmm { /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result> { - let walker = PtWalk::new(self.pdb_addr, self.mmu_version); - - match walker.walk_to_pte(mm, vfn)? { + match self.pt_walk.walk_to_pte(mm, vfn)? { WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)), WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None), } } + + /// Prepare resources for mapping `num_pages` pages. + /// + /// Allocates a contiguous VA range, then walks the hierarchy per-VFN to prepare pages + /// for all missing PDEs. Returns a [`PreparedMapping`] with the VA allocation. + /// + /// If `va_range` is not `None`, the VA range is constrained to the given range. Safe + /// to call outside the fence signalling critical path. + pub(crate) fn prepare_map( + &mut self, + mm: &mut GpuMm<'_>, + num_pages: usize, + va_range: Option>, + ) -> Result { + if num_pages == 0 { + return Err(EINVAL); + } + + // Allocate contiguous VA range. + let vfn_start = self.alloc_vfn_range(num_pages, va_range)?; + + if let Err(e) = self.pt_map.prepare_map( + mm, + vfn_start, + num_pages, + &mut self.page_table_allocs, + &mut self.pt_pages, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(PreparedMapping { + vfn_start, + num_pages, + _drop_guard: MustExecuteGuard::new(), + }) + } + + /// Execute a prepared multi-page mapping. + /// + /// Installs all prepared PDEs and writes PTEs into the page table, then flushes TLB. + pub(crate) fn execute_map( + &mut self, + mm: &mut GpuMm<'_>, + prepared: PreparedMapping, + pfns: &[Pfn], + writable: bool, + ) -> Result { + if pfns.len() != prepared.num_pages { + self.free_vfn(prepared.vfn_start); + return Err(EINVAL); + } + + let PreparedMapping { + vfn_start, + num_pages, + _drop_guard, + } = prepared; + _drop_guard.disarm(); + + if let Err(e) = self.pt_map.install_mappings( + mm, + &mut self.pt_pages, + &mut self.page_table_allocs, + vfn_start, + pfns, + writable, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(MappedRange { + vfn_start, + num_pages, + _drop_guard: MustUnmapGuard::new(), + }) + } + + /// Map pages doing prepare and execute in the same call. + /// + /// This is a convenience wrapper for callers outside the fence signalling critical + /// path (e.g., BAR mappings). For DRM usecases, [`Vmm::prepare_map()`] and + /// [`Vmm::execute_map()`] will be called separately. + pub(crate) fn map_pages( + &mut self, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + va_range: Option>, + writable: bool, + ) -> Result { + if pfns.is_empty() { + return Err(EINVAL); + } + + // Check if provided VA range is sufficient (if provided). + if let Some(ref range) = va_range { + let required: u64 = pfns + .len() + .checked_mul(PAGE_SIZE) + .ok_or(EOVERFLOW)? + .into_safe_cast(); + let available = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if available < required { + return Err(EINVAL); + } + } + + let prepared = self.prepare_map(mm, pfns.len(), va_range)?; + self.execute_map(mm, prepared, pfns, writable) + } + + /// Unmap all pages in a [`MappedRange`] with a single TLB flush. + pub(crate) fn unmap_pages(&mut self, mm: &mut GpuMm<'_>, range: MappedRange) -> Result { + let result = self + .pt_map + .invalidate_ptes(mm, range.vfn_start, range.num_pages); + + // TODO: Internal page table pages (PDE, PTE pages) are still kept around. + // This is by design as repeated maps/unmaps will be fast. As a future TODO, + // we can add a reclaimer here to reclaim if VRAM is short. For now, the PT + // pages are dropped once the `Vmm` is dropped. + + // Free the VA range regardless of PTE invalidation success, so that the VA + // range is recovered even on failure (PTEs may be stale, but that is better + // than leaking both PTEs and VA range). + self.free_vfn(range.vfn_start); + + // Unmap complete, safe to drop `MappedRange`. + range._drop_guard.disarm(); + result + } } -- cgit v1.2.3 From 75ad43886a20e17aa5fad9ae71ebb413cadd8d64 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:52 +0900 Subject: gpu: nova-core: Add BAR1 aperture type and size constant Add BAR1_SIZE constant and Bar1 type alias for the 256MB BAR1 aperture. These are prerequisites for BAR1 memory access functionality. Co-developed-by: Zhi Wang Signed-off-by: Zhi Wang Signed-off-by: Joel Fernandes [ecourtney: add the Bar1 lifetime for current PCI BAR ownership] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-14-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 2 ++ drivers/gpu/nova-core/gsp/commands.rs | 4 ++++ drivers/gpu/nova-core/gsp/fw/commands.rs | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 8cef4c284277..5723ff8f71ea 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -38,6 +38,8 @@ const BAR0_SIZE: usize = SZ_16M; pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region; +#[expect(dead_code)] +pub(crate) type Bar1<'a> = pci::Bar<'a>; kernel::pci_device_table!( PCI_TABLE, diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 91f7037edefc..0b399448ee1c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -214,6 +214,9 @@ impl CommandToGsp for GetGspStaticInfo { /// The reply from the GSP to the [`GetGspStaticInfo`] command. pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], + /// BAR1 Page Directory Entry base address. + #[expect(dead_code)] + pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, /// Exclusive end of the FB physical address space. @@ -237,6 +240,7 @@ impl MessageFromGsp for GetGspStaticInfoReply { Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + bar1_pde_base: msg.bar1_pde_base(), usable_fb_regions, total_fb_end, }) diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 6e85442a4b13..32856ff74183 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -131,6 +131,14 @@ impl GspStaticConfigInfo { self.0.gpuNameString } + /// Returns the BAR1 Page Directory Entry base address. + /// + /// This is the root page table address for BAR1 virtual memory, + /// set up by GSP-RM firmware. + pub(crate) fn bar1_pde_base(&self) -> u64 { + self.0.bar1PdeBase + } + /// Returns an iterator over valid FB regions from GSP firmware data. fn fb_regions( &self, -- cgit v1.2.3 From 159427b213524a90853ef24d0a0c2ed1aec1dbf9 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:53 +0900 Subject: gpu: nova-core: mm: Add BAR1 user interface Add the BAR1 user interface for CPU access to GPU virtual memory through the BAR1 aperture. Signed-off-by: Joel Fernandes [ecourtney: map BAR1 in NovaCore, borrow it in Gpu and BarUser, no Devres] [ecourtney: update for the VramAddress raw API and gsp_resources chipset] [ecourtney: drop the owned GpuMm, pass it mutably through map and release] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-15-8dd5d4225d2e@nvidia.com Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/driver.rs | 45 +++++++-- drivers/gpu/nova-core/gpu.rs | 32 ++++++- drivers/gpu/nova-core/gsp/commands.rs | 1 - drivers/gpu/nova-core/mm.rs | 1 + drivers/gpu/nova-core/mm/bar_user.rs | 171 ++++++++++++++++++++++++++++++++++ 5 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 drivers/gpu/nova-core/mm/bar_user.rs diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5723ff8f71ea..0672a0707a71 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -2,7 +2,11 @@ use kernel::{ auxiliary, - device::Core, + device::{ + Bound, + Core, // + }, + io::resource, pci, pci::{ Class, @@ -28,6 +32,7 @@ pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, + bar1: Bar1<'bound>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } @@ -38,9 +43,27 @@ const BAR0_SIZE: usize = SZ_16M; pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; pub(crate) type NovaRegisters = kernel::io::Region; -#[expect(dead_code)] pub(crate) type Bar1<'a> = pci::Bar<'a>; +/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU. +/// +/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single +/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell +/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that +/// consumes two consecutive resource slots: index 0 holds the low 32 bits and +/// index 1 holds the high 32 bits (with no `flags` / or size of its own), +/// shifting BAR1 to index 2. +pub(crate) fn bar1_resource_index(pdev: &pci::Device) -> Result { + // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing + // if BAR0 and hence BAR1 is 64-bit. + let flags0 = pdev.resource_flags(0)?; + if flags0.contains(resource::Flags::IORESOURCE_MEM_64) { + Ok(2) + } else { + Ok(1) + } +} + kernel::pci_device_table!( PCI_TABLE, ::IdInfo, @@ -82,12 +105,18 @@ impl pci::Driver for NovaCoreDriver { Ok(try_pin_init!(NovaCore { bar: pdev.iomap_region_sized::(0, c"nova-core/bar0")?, - // TODO: Use `&bar` self-referential pin-init syntax once available. - // - // SAFETY: `bar` is initialized before this expression is evaluated - // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned - // stable address, and is dropped after `gpu` (struct field drop order). - gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + bar1: { + let bar1_idx = bar1_resource_index(pdev)?; + pdev.iomap_region(bar1_idx, c"nova-core/bar1")? + }, + // TODO: Use self-referential pin-init syntax once available. + gpu <- Gpu::new( + pdev, + // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar) }, + // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar1) }, + ), // Run optional GPU selftests. #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] _: { gpu.run_selftests(pdev) }, diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index e9d3c10708f1..9848e474c9bf 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -16,11 +16,15 @@ use kernel::{ SizeConstants, SZ_4K, // }, + sync::Arc, }; use crate::{ bounded_enum, - driver::Bar0, + driver::{ + Bar0, + Bar1, // + }, falcon::{ gsp::Gsp as GspFalcon, sec2::Sec2 as Sec2Falcon, @@ -35,6 +39,8 @@ use crate::{ GspBootContext, // }, mm::{ + bar_user::BarUser, + pagetable::MmuVersion, GpuMm, VramAddress, // }, @@ -148,6 +154,11 @@ impl Chipset { pub(crate) fn pci_config_mirror_range(self) -> Range { hal::gpu_hal(self).pci_config_mirror_range() } + + /// Returns the MMU version for this chipset. + pub(crate) fn mmu_version(self) -> MmuVersion { + MmuVersion::from(self.arch()) + } } // TODO @@ -297,6 +308,8 @@ pub(crate) struct Gpu<'gpu> { /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while /// the GSP is still operational. mm: GpuMm<'gpu>, + /// BAR1 user interface for CPU access to GPU virtual memory. + bar_user: Arc>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -340,6 +353,7 @@ impl<'gpu> Gpu<'gpu> { pub(crate) fn new<'a>( pdev: &'gpu pci::Device>, bar: Bar0<'gpu>, + bar1: &'gpu Bar1<'gpu>, ) -> impl PinInit + use<'gpu, 'a> { let dev = pdev.as_ref(); @@ -442,6 +456,22 @@ impl<'gpu> Gpu<'gpu> { VramAddress::from_raw(gsp_static_info.total_fb_end), )? }, + + // Create BAR1 user interface for CPU access to GPU virtual memory. + bar_user: { + let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base); + let bar1_idx = crate::driver::bar1_resource_index(pdev)?; + let bar1_size = pdev.resource_len(bar1_idx)?; + Arc::pin_init( + BarUser::new( + pdb_addr, + gsp_resources.spec.chipset, + bar1_size, + bar1, + )?, + GFP_KERNEL, + )? + }, }) } diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index 0b399448ee1c..e087c9e8c35c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -215,7 +215,6 @@ impl CommandToGsp for GetGspStaticInfo { pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], /// BAR1 Page Directory Entry base address. - #[expect(dead_code)] pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec>, diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index 2cf37254fbb9..e04497256519 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -60,6 +60,7 @@ use crate::{ pub(crate) use tlb::Tlb; +pub(crate) mod bar_user; mod hal; pub(super) mod pagetable; mod pramin; diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs new file mode 100644 index 000000000000..ef1d8e6f8c9c --- /dev/null +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD +//! for GPU work submission, and applications to access GPU buffers via mmap(). + +use kernel::{ + io::Io, + new_mutex, + prelude::*, + sync::{ + Arc, + Mutex, // + }, +}; + +use crate::{ + driver::Bar1, + gpu::Chipset, + mm::{ + vmm::{ + MappedRange, + Vmm, // + }, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, + PAGE_SIZE, // + }, + num::IntoSafeCast, +}; + +/// BAR1 user interface for virtual memory mappings. +/// +/// Owns the [`Vmm`] for the BAR1 address space. +#[pin_data] +pub(crate) struct BarUser<'gpu> { + #[pin] + vmm: Mutex, + bar1: &'gpu Bar1<'gpu>, +} + +impl<'gpu> BarUser<'gpu> { + /// Create a pin-initializer for [`BarUser`]. + pub(crate) fn new( + pdb_addr: VramAddress, + chipset: Chipset, + va_size: u64, + bar1: &'gpu Bar1<'gpu>, + ) -> Result + 'gpu> { + let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; + Ok(pin_init!(Self { + vmm <- new_mutex!(vmm, "bar_user_vmm"), + bar1, + })) + } + + /// Map physical pages to a contiguous BAR1 virtual range. + pub(crate) fn map( + self: &Arc, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + writable: bool, + ) -> Result> { + if pfns.is_empty() { + return Err(EINVAL); + } + let mut vmm = self.vmm.lock(); + let mapped = vmm.map_pages(mm, pfns, None, writable)?; + + Ok(BarUserAccess { + bar_user: self.clone(), + mapped: Some(mapped), + }) + } +} + +/// Access object for a mapped BAR1 region. +pub(crate) struct BarUserAccess<'gpu> { + bar_user: Arc>, + /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at + /// drop time means `release()` was never called. + mapped: Option, +} + +impl BarUserAccess<'_> { + /// Tear down the BAR1 mapping. + pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { + let mapped = self.mapped.take().ok_or(EINVAL)?; + let mut vmm = self.bar_user.vmm.lock(); + vmm.unmap_pages(mm, mapped)?; + Ok(()) + } + + /// Returns the active mapping. + fn mapped(&self) -> &MappedRange { + // `mapped` is only `None` after `take()` in `release`; hence unwrap() + // cannot panic here. + self.mapped.as_ref().unwrap() + } + + /// Get the base virtual address of this mapping. + pub(crate) fn base(&self) -> VirtualAddress { + VirtualAddress::from(self.mapped().vfn_start) + } + + /// Get the total size of the mapped region in bytes. + pub(crate) fn size(&self) -> usize { + self.mapped().num_pages * PAGE_SIZE + } + + /// Get the starting virtual frame number. + pub(crate) fn vfn_start(&self) -> Vfn { + self.mapped().vfn_start + } + + /// Get the number of pages in this mapping. + pub(crate) fn num_pages(&self) -> usize { + self.mapped().num_pages + } + + /// Translate an offset within this mapping to a BAR1 aperture offset. + fn bar_offset(&self, offset: usize) -> Result { + if offset >= self.size() { + return Err(EINVAL); + } + + let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast(); + let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + base.checked_add(offset).ok_or(EOVERFLOW) + } + + // Fallible accessors with runtime bounds checking. + + /// Read a 32-bit value at the given offset. + pub(crate) fn try_read32(&self, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read32(off) + } + + /// Write a 32-bit value at the given offset. + pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write32(value, off) + } + + /// Read a 64-bit value at the given offset. + pub(crate) fn try_read64(&self, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read64(off) + } + + /// Write a 64-bit value at the given offset. + pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write64(value, off) + } +} + +impl Drop for BarUserAccess<'_> { + fn drop(&mut self) { + if self.mapped.is_some() { + kernel::pr_warn!( + "BarUserAccess dropped without calling release(). BarUser address space will leak.\n" + ); + } + // The inner `MappedRange`'s own `MustUnmapGuard` will also fire, + // identifying the leaked VA range. + } +} -- cgit v1.2.3 From d669686f8170c234edf12212fac9180ea18b1448 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Wed, 9 Sep 2026 12:59:54 +0900 Subject: gpu: nova-core: mm: Add BAR1 memory management self-tests Add self-tests for BAR1 access during driver probe when CONFIG_NOVA_CORE_SELFTESTS is enabled (default disabled). This results in testing the Vmm, GPU buddy allocator and BAR1 region all of which should function correctly for the tests to pass. Signed-off-by: Joel Fernandes [ecourtney: use existing self-test runner and CONFIG_NOVA_CORE_SELFTESTS] [ecourtney: update for mutable GpuMm, PRAMIN views and VramAddress raw API] [ecourtney: restore the conditional dead_code expect] Signed-off-by: Eliot Courtney Acked-by: Danilo Krummrich Link: https://patch.msgid.link/20260909-mmrebase-v1-16-8dd5d4225d2e@nvidia.com [acourbot: use vertical import style.] Signed-off-by: Alexandre Courbot --- drivers/gpu/nova-core/gpu.rs | 9 +- drivers/gpu/nova-core/mm.rs | 12 +- drivers/gpu/nova-core/mm/bar_user.rs | 253 ++++++++++++++++++++++++++++++++++ drivers/gpu/nova-core/mm/pagetable.rs | 24 ++++ drivers/gpu/nova-core/mm/vmm.rs | 1 + 5 files changed, 295 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index 9848e474c9bf..d763bc8d3827 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -482,7 +482,14 @@ impl<'gpu> Gpu<'gpu> { let dev = pdev.as_ref(); let regions = &this.gsp_static_info.usable_fb_regions; - if let Err(err) = crate::mm::selftest::run(dev, this.mm, regions) { + if let Err(err) = crate::mm::selftest::run( + dev, + this.mm, + regions, + this.bar_user, + this.gsp_static_info.bar1_pde_base, + this.spec.chipset, + ) { dev_err!(dev, "self-tests failed: {:?}\n", err); } } diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs index e04497256519..a5bc4042577b 100644 --- a/drivers/gpu/nova-core/mm.rs +++ b/drivers/gpu/nova-core/mm.rs @@ -3,7 +3,7 @@ //! Memory management subsystems. -#![expect(dead_code)] +#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] /// Implements `From` conversions between a frame-number type and `Bounded`. /// @@ -202,6 +202,7 @@ bitfield! { impl VirtualAddress { /// Create a new virtual address from a raw value. + #[expect(dead_code)] pub(crate) const fn new(addr: u64) -> Self { Self::from_raw(addr) } @@ -297,7 +298,8 @@ pub(crate) mod selftest { use kernel::{ device, - sizes::SizeConstants, // + sizes::SizeConstants, + sync::Arc, // }; use super::*; @@ -307,6 +309,9 @@ pub(crate) mod selftest { dev: &device::Device, mm: &mut GpuMm<'_>, usable_fb_regions: &[Range], + bar_user: &Arc>, + bar1_pdb: u64, + chipset: Chipset, ) -> Result { // VRAM span the self-tests are free to overwrite, from the chosen test base. const SELFTEST_SPAN: u64 = u64::SZ_64M; @@ -325,6 +330,7 @@ pub(crate) mod selftest { return Ok(()); }; - pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base)) + pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base))?; + bar_user::run_self_test(dev, mm, bar_user, bar1_pdb, chipset) } } diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs index ef1d8e6f8c9c..8f4a27c1fd14 100644 --- a/drivers/gpu/nova-core/mm/bar_user.rs +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -31,6 +31,9 @@ use crate::{ num::IntoSafeCast, }; +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +use kernel::device; + /// BAR1 user interface for virtual memory mappings. /// /// Owns the [`Vmm`] for the BAR1 address space. @@ -84,6 +87,7 @@ pub(crate) struct BarUserAccess<'gpu> { mapped: Option, } +#[expect(dead_code)] impl BarUserAccess<'_> { /// Tear down the BAR1 mapping. pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { @@ -169,3 +173,252 @@ impl Drop for BarUserAccess<'_> { // identifying the leaked VA range. } } + +/// Run MM subsystem self-tests during probe. +/// +/// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1` +/// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables +/// and test pages as needed. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) fn run_self_test( + dev: &device::Device, + mm: &mut GpuMm<'_>, + bar_user: &Arc>, + bar1_pdb: u64, + chipset: Chipset, +) -> Result { + use kernel::{ + gpu::buddy::{ + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + ptr::Alignment, + sizes::{ + SZ_16K, + SZ_32K, + SZ_4K, + SZ_64K, // + }, + }; + + // Test patterns. + const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF; + const PATTERN_BAR1: u32 = 0xCAFE_BABE; + + let bar1 = bar_user.bar1; + dev_info!(dev, "MM: Starting self-test...\n"); + + let pdb_addr = VramAddress::from_raw(bar1_pdb); + + // Check if initial page tables are in VRAM. + if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() { + dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n"); + return Ok(()); + } + + // Set up a test page from the buddy allocator. + let test_page_blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset(); + let test_vram = VramAddress::from_raw(test_vram_offset); + let test_pfn = Pfn::from(test_vram); + + // Create a VMM of size 64K to track virtual memory mappings. + let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?; + + // Create a test mapping. + let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?; + let test_vfn = mapped.vfn_start; + + // Pre-compute test addresses for the PRAMIN to BAR1 read test. + let vfn_offset: usize = test_vfn.raw().into_safe_cast(); + let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + let bar1_read_offset: usize = bar1_base_offset + 0x100; + let vram_read_addr = test_vram + 0x100; + + // Test 1: Write via PRAMIN, read via BAR1. + mm.pramin_mut() + .window_at::(vram_read_addr)? + .view() + .write_val(PATTERN_PRAMIN); + + // Read back via BAR1 aperture. + let bar1_value = bar1.try_read32(bar1_read_offset)?; + + let test1_passed = if bar1_value == PATTERN_PRAMIN { + true + } else { + dev_err!( + dev, + "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_PRAMIN, + bar1_value + ); + false + }; + + // Cleanup - invalidate PTE. + vmm.unmap_pages(mm, mapped)?; + + // Test 2: Two-phase prepare/execute API. + let prepared = vmm.prepare_map(mm, 1, None)?; + let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?; + let readback = vmm.read_mapping(mm, mapped2.vfn_start)?; + let test2_passed = if readback == Some(test_pfn) { + true + } else { + dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n"); + false + }; + vmm.unmap_pages(mm, mapped2)?; + + // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven + // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation + // is requested within [base, base+36K). The buddy allocator must split around the + // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total). + // Each block is mapped into BAR1 and verified via PRAMIN read-back. + // + // Address layout (base = 0x10000): + // [ 16K ] [HOLE 4K] [4K] [ 8K ] [4K] + // 0x10000 0x14000 0x15000 0x16000 0x18000 0x19000 + let range_base: u64 = SZ_64K.into_safe_cast(); + let sz_4k: u64 = SZ_4K.into_safe_cast(); + let sz_16k: u64 = SZ_16K.into_safe_cast(); + let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast(); + + // Punch a 4K hole at base+16K so the subsequent 32K allocation must split. + let _hole = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k), + SZ_4K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + // Allocate 32K within [base, base+36K). The hole forces the allocator to return + // split blocks whose sizes are determined by buddy alignment. + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k), + SZ_32K.into_safe_cast(), + Alignment::new::(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let mut test3_passed = true; + let mut total_size = 0usize; + + for block in blocks.iter() { + total_size += IntoSafeCast::::into_safe_cast(block.size()); + + // Map all pages of this block. + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let num_pages: usize = (block.size() / page_size).into_safe_cast(); + + let mut pfns = KVec::new(); + for j in 0..num_pages { + let j_u64: u64 = j.into_safe_cast(); + pfns.push( + Pfn::from(VramAddress::from_raw( + block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?, + )), + GFP_KERNEL, + )?; + } + + let mapped = vmm.map_pages(mm, &pfns, None, true)?; + let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast(); + let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + + for j in 0..num_pages { + let page_bar1_off = bar1_base + j * PAGE_SIZE; + let j_u64: u64 = j.into_safe_cast(); + let page_phys = block.offset() + + j_u64 + .checked_mul(PAGE_SIZE.into_safe_cast()) + .ok_or(EOVERFLOW)?; + + bar1.try_write32(PATTERN_BAR1, page_bar1_off)?; + + let pramin_val = mm + .pramin_mut() + .window_at::(VramAddress::from_raw(page_phys))? + .view() + .read_val(); + + if pramin_val != PATTERN_BAR1 { + dev_err!( + dev, + "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n", + block.offset(), + j, + pramin_val + ); + test3_passed = false; + } + } + + vmm.unmap_pages(mm, mapped)?; + } + + // Verify aggregate: all returned block sizes must sum to allocation size. + if total_size != SZ_32K { + dev_err!( + dev, + "MM: Test 3 FAILED - total size {} != expected {}\n", + total_size, + SZ_32K + ); + test3_passed = false; + } + + // Release Tests 1-3's Vmm before Test 4 constructs a fresh BarUser on + // the same PDB. + drop(vmm); + + // Test 4: Exercise `BarUser::map()` end-to-end. + let bar_user = Arc::pin_init( + BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1)?, + GFP_KERNEL, + )?; + let access = bar_user.map(mm, &[test_pfn], true)?; + + // Write pattern via PRAMIN, read via BarUserAccess. + mm.pramin_mut() + .window_at::(test_vram)? + .view() + .write_val(PATTERN_BAR1); + + let readback = access.try_read32(0)?; + let test4_passed = if readback == PATTERN_BAR1 { + true + } else { + dev_err!( + dev, + "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_BAR1, + readback + ); + false + }; + access.release(mm)?; + + if test1_passed && test2_passed && test3_passed && test4_passed { + dev_info!(dev, "MM: All self-tests PASSED\n"); + Ok(()) + } else { + dev_err!(dev, "MM: Self-tests FAILED\n"); + Err(EIO) + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs index 63111ff95780..63a7e1855caa 100644 --- a/drivers/gpu/nova-core/mm/pagetable.rs +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -398,3 +398,27 @@ impl From for Bounded { Bounded::from_expr(val as u64 & 0x3) } } + +/// Check if the PDB has valid, VRAM-backed page tables. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +fn check_pdb_inner(pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress) -> Result { + let raw = pramin.window_at::(pdb_addr)?.view().read_val(); + + if !M::Pde::from_raw(raw).is_valid_vram() { + return Err(ENOENT); + } + Ok(()) +} + +/// Check if the PDB has valid, VRAM-backed page tables, dispatching by MMU version. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) fn check_pdb_valid( + pramin: &mut pramin::Pramin<'_>, + pdb_addr: VramAddress, + chipset: crate::gpu::Chipset, +) -> Result { + match MmuVersion::from(chipset.arch()) { + MmuVersion::V2 => check_pdb_inner::(pramin, pdb_addr), + MmuVersion::V3 => check_pdb_inner::(pramin, pdb_addr), + } +} diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs index 411710d03f7a..51b500a27233 100644 --- a/drivers/gpu/nova-core/mm/vmm.rs +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -126,6 +126,7 @@ impl Drop for MustUnmapGuard { /// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings. pub(crate) struct Vmm { /// Page Directory Base address for this address space. + #[expect(dead_code)] pdb_addr: VramAddress, /// Page table walker for reading existing mappings. pt_walk: PtWalk, -- cgit v1.2.3 From 66a2c223b620d844fe26c6bd4844d2a6a8c9dffc Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Wed, 9 Sep 2026 15:53:19 +0200 Subject: Documentation: nova: Fix bracket Add missing ')'. Signed-off-by: Manuel Ebner Acked-by: Randy Dunlap Link: https://patch.msgid.link/20260909135319.1364776-3-manuelebnerli@mailbox.org [acourbot: add missing `:` in commit title] Signed-off-by: Alexandre Courbot --- Documentation/gpu/nova/core/todo.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/gpu/nova/core/todo.rst b/Documentation/gpu/nova/core/todo.rst index d5130b2b08fb..a01c362b1be0 100644 --- a/Documentation/gpu/nova/core/todo.rst +++ b/Documentation/gpu/nova/core/todo.rst @@ -33,7 +33,7 @@ A good example from nova-core would be the ``Chipset`` enum type, which defines the value ``AD102``. When probing the GPU the value ``0x192`` can be read from a certain register indication the chipset AD102. Hence, the enum value ``AD102`` should be derived from the number ``0x192``. Currently, nova-core uses a custom -implementation (``Chipset::from_u32`` for this. +implementation (``Chipset::from_u32``) for this. Instead, it would be desirable to have something like the ``FromPrimitive`` trait [1] from the num crate. -- cgit v1.2.3