diff options
Diffstat (limited to 'drivers')
53 files changed, 4441 insertions, 551 deletions
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs index bfb0ba19caff..730b84e37a54 100644 --- a/drivers/gpu/drm/tyr/driver.rs +++ b/drivers/gpu/drm/tyr/driver.rs @@ -46,6 +46,7 @@ use crate::{ }; pub(crate) type IoMem<'a> = kernel::io::mem::IoMem<'a, SZ_2M>; +pub(crate) type TyrRegisters = kernel::io::Region<SZ_2M>; pub(crate) struct TyrDrmDriver; diff --git a/drivers/gpu/drm/tyr/fw.rs b/drivers/gpu/drm/tyr/fw.rs index 47d25c901bd0..7edb5eff1707 100644 --- a/drivers/gpu/drm/tyr/fw.rs +++ b/drivers/gpu/drm/tyr/fw.rs @@ -39,7 +39,8 @@ use kernel::{ use crate::{ driver::{ IoMem, - TyrDrmDevice, // + TyrDrmDevice, + TyrRegisters, // }, fw::parser::{ FwParser, @@ -101,6 +102,8 @@ impl From<CacheMode> for Bounded<u32, 2> { } register! { + base: TyrRegisters; + #[allow(non_upper_case_globals)] pub(super) SectionFlags(u32) @ 0x0 { 0:0 read => bool; diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs index a62724378ced..0c419c4e1186 100644 --- a/drivers/gpu/drm/tyr/regs.rs +++ b/drivers/gpu/drm/tyr/regs.rs @@ -57,7 +57,11 @@ pub(crate) mod gpu_control { uapi, // }; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// GPU identification register. pub(crate) GPU_ID(u32) @ 0x0 { /// Status of the GPU release. @@ -315,6 +319,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU command register. /// /// Use the constructor methods to create commands: @@ -380,6 +386,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU status register. Read only. pub(crate) GPU_STATUS(u32) @ 0x34 { /// GPU active, a 1-bit boolean flag. @@ -463,6 +471,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// GPU fault status register. Read only. pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c { /// Exception type. @@ -768,6 +778,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// Coherency enable. An index of which coherency protocols should be used. /// This register only selects the protocol for coherency messages on the /// interconnect. This is not to enable or disable coherency controlled by MMU. @@ -808,6 +820,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// MCU control. pub(crate) MCU_CONTROL(u32) @ 0x700 { /// Request MCU state change. @@ -849,6 +863,8 @@ pub(crate) mod gpu_control { } register! { + base: TyrRegisters; + /// MCU status. Read only. pub(crate) MCU_STATUS(u32) @ 0x704 { /// Read current state of MCU. @@ -862,7 +878,11 @@ pub(crate) mod gpu_control { pub(crate) mod job_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// Raw status of job interrupts. /// /// Write to this register to trigger these interrupts. @@ -912,7 +932,11 @@ pub(crate) mod job_control { pub(crate) mod mmu_control { use kernel::register; + use crate::driver::TyrRegisters; + register! { + base: TyrRegisters; + /// IRQ sources raw status. /// /// This register contains the raw unmasked interrupt sources for MMU status and exception @@ -966,9 +990,10 @@ pub(crate) mod mmu_control { prelude::*, register, // }; - use pin_init::Zeroable; + use crate::driver::TyrRegisters; + /// Maximum number of hardware address space slots. /// The actual number of slots available is usually lower. pub(crate) const MAX_AS: usize = 16; @@ -977,6 +1002,8 @@ pub(crate) mod mmu_control { const STRIDE: usize = 0x40; register! { + base: TyrRegisters; + /// Translation table base address. A 64-bit pointer. /// /// This field contains the address of the top level of a translation table structure. @@ -1104,6 +1131,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Stage 1 memory attributes (8-bit bitfield). /// /// This is not an actual register, but a bitfield definition used by the MEMATTR @@ -1137,6 +1166,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Memory attributes. /// /// Each address space can configure up to 8 different memory attribute profiles. @@ -1292,6 +1323,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Lock region address for each address space. pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 { /// Lock region size. @@ -1353,6 +1386,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// MMU command register for each address space. Write only. pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 { 7:0 command ?=> MmuCommand; @@ -1480,6 +1515,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Fault status register for each address space. Read only. pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c { /// Exception type. @@ -1705,6 +1742,8 @@ pub(crate) mod mmu_control { } register! { + base: TyrRegisters; + /// Translation configuration and control. pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 { /// Address space mode. @@ -1760,6 +1799,8 @@ pub(crate) mod mmu_control { pub(crate) mod doorbell_block { use kernel::register; + use crate::driver::TyrRegisters; + /// Number of doorbells available. pub(crate) const NUM_DOORBELLS: usize = 64; @@ -1770,6 +1811,8 @@ pub(crate) mod doorbell_block { const STRIDE: usize = 0x10000; register! { + base: TyrRegisters; + /// Doorbell request register. Write-only. pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 { /// Doorbell set. Writing 1 triggers the doorbell. diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig index f918f69e0599..1934f17baa8b 100644 --- a/drivers/gpu/nova-core/Kconfig +++ b/drivers/gpu/nova-core/Kconfig @@ -5,6 +5,7 @@ config NOVA_CORE depends on RUST depends on !CPU_BIG_ENDIAN select AUXILIARY_BUS + select GPU_BUDDY select RUST_FW_LOADER_ABSTRACTIONS default n help @@ -15,3 +16,12 @@ config NOVA_CORE This driver is work in progress and may not be functional. If M is selected, the module will be called nova-core. + +config NOVA_CORE_SELFTESTS + bool "Nova Core driver self-tests" + depends on NOVA_CORE + default n + help + Build the driver self-tests and run them when the GPU is probed. + + If unsure, say N. diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index bbd93959e0b2..0672a0707a71 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -2,7 +2,11 @@ use kernel::{ auxiliary, - device::Core, + device::{ + Bound, + Core, // + }, + io::resource, pci, pci::{ Class, @@ -28,6 +32,7 @@ pub(crate) struct NovaCore<'bound> { #[pin] pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, + bar1: Bar1<'bound>, #[allow(clippy::type_complexity)] _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } @@ -37,6 +42,27 @@ pub(crate) struct NovaCoreDriver; const BAR0_SIZE: usize = SZ_16M; pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; +pub(crate) type NovaRegisters = kernel::io::Region<BAR0_SIZE>; +pub(crate) type Bar1<'a> = pci::Bar<'a>; + +/// Returns the Linux PCI resource index that holds BAR1 for an NVIDIA GPU. +/// +/// On Maxwell through Ada, BAR0 is a 32-bit memory BAR occupying a single +/// Linux PCI resource slot, so BAR1 lives at index 1. Starting with Blackwell +/// (and on some Ampere GA100 / Hopper SKUs) BAR0 is a 64-bit memory BAR that +/// consumes two consecutive resource slots: index 0 holds the low 32 bits and +/// index 1 holds the high 32 bits (with no `flags` / or size of its own), +/// shifting BAR1 to index 2. +pub(crate) fn bar1_resource_index(pdev: &pci::Device<Bound>) -> Result<u32> { + // Probe the `IORESOURCE_MEM_64` flag of BAR0 as a robust way of exposing + // if BAR0 and hence BAR1 is 64-bit. + let flags0 = pdev.resource_flags(0)?; + if flags0.contains(resource::Flags::IORESOURCE_MEM_64) { + Ok(2) + } else { + Ok(1) + } +} kernel::pci_device_table!( PCI_TABLE, @@ -79,12 +105,21 @@ impl pci::Driver for NovaCoreDriver { Ok(try_pin_init!(NovaCore { bar: pdev.iomap_region_sized::<BAR0_SIZE>(0, c"nova-core/bar0")?, - // TODO: Use `&bar` self-referential pin-init syntax once available. - // - // SAFETY: `bar` is initialized before this expression is evaluated - // (`try_pin_init!()` initializes fields in declaration order), lives at a pinned - // stable address, and is dropped after `gpu` (struct field drop order). - gpu <- Gpu::new(pdev, unsafe { &*core::ptr::from_ref(bar) }), + bar1: { + let bar1_idx = bar1_resource_index(pdev)?; + pdev.iomap_region(bar1_idx, c"nova-core/bar1")? + }, + // TODO: Use self-referential pin-init syntax once available. + gpu <- Gpu::new( + pdev, + // SAFETY: `bar` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar) }, + // SAFETY: `bar1` is initialized above, pinned, and outlives `gpu`. + unsafe { &*core::ptr::from_ref(bar1) }, + ), + // Run optional GPU selftests. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + _: { gpu.run_selftests(pdev) }, _reg: auxiliary::Registration::new( pdev.as_ref(), c"nova-drm", diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 65cb12d26e2b..9015de965a53 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -14,13 +14,12 @@ use kernel::{ io::{ io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register::Array, Io, + Mmio, // }, prelude::*, + sizes::SZ_4K, time::Delta, }; @@ -165,18 +164,25 @@ bounded_enum! { } } -/// Type used to represent the `PFALCON` registers address base for a given falcon engine. -pub(crate) struct PFalconBase(()); +const PFALCON_REGION_SIZE: usize = SZ_4K; +const PFALCON2_REGION_SIZE: usize = SZ_4K; -/// Type used to represent the `PFALCON2` registers address base for a given falcon engine. -pub(crate) struct PFalcon2Base(()); +/// Type used to represent the `PFALCON` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalconRegisters([u8; PFALCON_REGION_SIZE]); + +/// Type used to represent the `PFALCON2` registers. +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(crate) struct PFalcon2Registers([u8; PFALCON2_REGION_SIZE]); /// Trait defining the parameters of a given Falcon engine. /// /// Each engine provides one base for `PFALCON` and `PFALCON2` registers. -pub(crate) trait FalconEngine: - Send + Sync + RegisterBase<PFalconBase> + RegisterBase<PFalcon2Base> + Sized -{ +pub(crate) trait FalconEngine: Send + Sync + Sized { + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, PFalconRegisters>; + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, PFalcon2Registers>; } /// Represents a portion of the firmware to be loaded into a particular memory (e.g. IMEM or DMEM) @@ -358,6 +364,9 @@ pub(crate) struct Falcon<'a, E: FalconEngine> { hal: KBox<dyn FalconHal<E>>, dev: &'a device::Device<device::Bound>, bar: Bar0<'a>, + // TODO: make private + pub(crate) pfalcon: Mmio<'a, PFalconRegisters>, + pfalcon2: Mmio<'a, PFalcon2Registers>, } impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { @@ -371,19 +380,19 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { hal: hal::falcon_hal(chipset)?, dev, bar, + pfalcon: E::pfalcon(bar), + pfalcon2: E::pfalcon2(bar), }) } /// Resets DMA-related registers. pub(crate) fn dma_reset(&self) { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. @@ -392,10 +401,10 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.select_core(self)?; self.hal.reset_wait_mem_scrubbing(self)?; - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_RM::from(crate::gpu::boot_0_raw( + self.bar, + ))); Ok(()) } @@ -413,8 +422,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { return Err(EINVAL); } - self.bar.write( - WithBase::of::<E>().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMC::zeroed() .with_secure(load_offsets.secure) .with_aincw(true) @@ -424,14 +433,14 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() { let n = u16::try_from(n)?; let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?; - self.bar.write( - WithBase::of::<E>().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag), ); for word in block.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::<E>().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -450,8 +459,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { return Err(EINVAL); } - self.bar.write( - WithBase::of::<E>().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMC::zeroed() .with_aincw(true) .with_offs(load_offsets.dst_start), @@ -459,8 +468,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for word in load_offsets.data.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - self.bar.write( - WithBase::of::<E>().at(Self::PIO_PORT), + self.pfalcon.write( + Array::at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); } @@ -473,14 +482,12 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { &self, fw: &F, ) -> Result { - self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { + self.pfalcon.update(regs::NV_PFALCON_FBIF_CTL, |v| { v.with_allow_phys_no_ctx(true) }); - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_DMACTL::zeroed(), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMACTL::zeroed()); if let Some(imem_ns) = fw.imem_ns_load_params() { self.pio_wr_imem_slice(imem_ns)?; @@ -492,10 +499,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.program_brom(self, &fw.brom_params()); - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -506,7 +511,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - dma_obj: &Coherent<[u8]>, + dma_obj: &Coherent<'_, [u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, ) -> Result { @@ -547,16 +552,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { // Set up the base source DMA address. - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`. (dma_address >> 8) as u32, - ), - ); - self.bar.write( - WithBase::of::<E>(), + )); + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?, ); @@ -566,23 +568,21 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { for pos in (0..num_transfers).map(|i| i * DMA_LEN) { // Perform a transfer of size `DMA_LEN`. - self.bar.write( - WithBase::of::<E>(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed() .try_with_offs(load_offsets.dst_start + pos)?, ); - self.bar.write( - WithBase::of::<E>(), + self.pfalcon.write_reg( regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos), ); - self.bar.write(WithBase::of::<E>(), cmd); + self.pfalcon.write_reg(cmd); // Wait for the transfer to complete. // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories // should ever take that long. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)), |r| r.idle(), Delta::ZERO, Delta::from_secs(2), @@ -614,8 +614,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { }; self.dma_reset(); - self.bar - .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| { + self.pfalcon + .update(regs::NV_PFALCON_FBIF_TRANSCFG::at(0), |v| { v.with_target(FalconFbifTarget::CoherentSysmem) .with_mem_type(FalconFbifMemType::Physical) }); @@ -626,10 +626,8 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { self.hal.program_brom(self, &fw.brom_params()); // Set `BootVec` to start of non-secure code. - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr())); Ok(()) } @@ -638,7 +636,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { pub(crate) fn wait_till_halted(&self) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( - || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())), + || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL)), |r| r.halted(), Delta::ZERO, Delta::from_secs(2), @@ -649,19 +647,13 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Start the falcon CPU. pub(crate) fn start(&self) -> Result<()> { - match self - .bar - .read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>()) - .alias_en() - { - true => self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true), - ), - false => self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true), - ), + match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() { + true => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true)), + false => self + .pfalcon + .write_reg(regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true)), } Ok(()) @@ -670,32 +662,24 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Writes values to the mailbox registers if provided. pub(crate) fn write_mailboxes(&self, mbox0: Option<u32>, mbox1: Option<u32>) { if let Some(mbox0) = mbox0 { - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0)); } if let Some(mbox1) = mbox1 { - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1)); } } /// Reads the value from `mbox0` register. pub(crate) fn read_mailbox0(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX0).value() } /// Reads the value from `mbox1` register. pub(crate) fn read_mailbox1(&self) -> u32 { - self.bar - .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>()) - .value() + self.pfalcon.read(regs::NV_PFALCON_FALCON_MAILBOX1).value() } /// Reads values from both mailbox registers. @@ -760,9 +744,7 @@ impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Write the application version to the OS register. pub(crate) fn write_os_version(&self, app_version: u32) { - self.bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version)); } } diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 0437180b8829..85f9c8c5d60e 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -8,13 +8,12 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - Array, - RegisterBase, - WithBase, // - }, - Io, // + register, + register::Array, + Io, + Mmio, // }, prelude::*, sizes::SZ_1K, @@ -22,11 +21,13 @@ use kernel::{ }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, num, regs, // @@ -41,15 +42,24 @@ const FSP_EMEM_CHANNEL_0_SIZE: usize = SZ_1K; /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); -impl RegisterBase<PFalconBase> for Fsp { - const BASE: usize = 0x8f2000; -} +register! { + base: NovaRegisters; -impl RegisterBase<PFalcon2Base> for Fsp { - const BASE: usize = 0x8f3000; + PFALCON: super::PFalconRegisters @ 0x8f2000; + PFALCON2: super::PFalcon2Registers @ 0x8f3000; } -impl FalconEngine for Fsp {} +impl FalconEngine for Fsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Fsp> { /// Writes `data` to FSP external memory at offset `0`. @@ -62,19 +72,15 @@ impl<'a> Falcon<'a, Fsp> { } // Begin a write burst at offset `0`, auto-incrementing on each write. - self.bar.write( - WithBase::of::<Fsp>(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true)); for chunk in data.chunks_exact(4) { let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); // Write the next 32-bit `value`; hardware advances the offset. - self.bar.write( - WithBase::of::<Fsp>(), - regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value)); } Ok(()) @@ -90,17 +96,12 @@ impl<'a> Falcon<'a, Fsp> { } // Begin a read burst at offset `0`, auto-incrementing on each read. - self.bar.write( - WithBase::of::<Fsp>(), - regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true)); for chunk in data.chunks_exact_mut(4) { // Read the next 32-bit word; hardware advances the offset. - let value = self - .bar - .read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>()) - .data(); + let value = self.pfalcon.read(regs::NV_PFALCON_FALCON_EMEMD).data(); chunk.copy_from_slice(&value.to_le_bytes()); } diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index ae32f401aeb0..4c96ae325fda 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -2,23 +2,24 @@ use kernel::{ io::{ + io_project, poll::read_poll_timeout, - register::{ - RegisterBase, - WithBase, // - }, + register, Io, + Mmio, // }, prelude::*, time::Delta, // }; use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, falcon::{ Falcon, - FalconEngine, - PFalcon2Base, - PFalconBase, // + FalconEngine, // }, regs, }; @@ -26,24 +27,31 @@ use crate::{ /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); -impl RegisterBase<PFalconBase> for Gsp { - const BASE: usize = 0x00110000; -} +register! { + base: NovaRegisters; -impl RegisterBase<PFalcon2Base> for Gsp { - const BASE: usize = 0x00111000; + PFALCON: super::PFalconRegisters @ 0x00110000; + PFALCON2: super::PFalcon2Registers @ 0x00111000; } -impl FalconEngine for Gsp {} +impl FalconEngine for Gsp { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} impl<'a> Falcon<'a, Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. pub(crate) fn clear_swgen0_intr(&self) { - self.bar.write( - WithBase::of::<Gsp>(), - regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), - ); + self.pfalcon + .write_reg(regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true)); } /// Checks if GSP reload/resume has completed during the boot process. @@ -59,8 +67,8 @@ impl<'a> Falcon<'a, Gsp> { /// Returns whether the RISC-V branch privilege lockdown bit is set. pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool { - self.bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) + self.pfalcon + .read(regs::NV_PFALCON_FALCON_HWCFG2) .riscv_br_priv_lockdown() } @@ -71,10 +79,7 @@ impl<'a> Falcon<'a, Gsp> { const LOCKED_PATTERN: u32 = 0xbadf_4100; const LOCKED_MASK: u32 = 0xffff_ff00; - let hwcfg2 = self - .bar - .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) - .into_raw(); + let hwcfg2 = self.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2).into_raw(); hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN } diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index 7600ee07ca2e..f9a8444cf840 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -6,11 +6,9 @@ use kernel::{ device, io::{ poll::read_poll_timeout, - register::{ - Array, - WithBase, // - }, - Io, // + register::Array, + Io, + Mmio, // }, prelude::*, time::Delta, // @@ -24,6 +22,7 @@ use crate::{ FalconBromParams, FalconEngine, FalconModSelAlgo, + PFalcon2Registers, PeregrineCoreSelect, // }, regs, @@ -31,17 +30,16 @@ use crate::{ use super::FalconHal; -fn select_core_ga102<E: FalconEngine>(bar: Bar0<'_>) -> Result { - let bcr_ctrl = bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>()); +fn select_core_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>) -> Result { + let bcr_ctrl = pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL); if bcr_ctrl.core_select() != PeregrineCoreSelect::Falcon { - bar.write( - WithBase::of::<E>(), + pfalcon2.write_reg( regs::NV_PRISCV_RISCV_BCR_CTRL::zeroed().with_core_select(PeregrineCoreSelect::Falcon), ); // TIMEOUT: falcon core should take less than 10ms to report being enabled. read_poll_timeout( - || Ok(bar.read(regs::NV_PRISCV_RISCV_BCR_CTRL::of::<E>())), + || Ok(pfalcon2.read(regs::NV_PRISCV_RISCV_BCR_CTRL)), |r| r.valid(), Delta::ZERO, Delta::from_millis(10), @@ -86,24 +84,20 @@ fn signature_reg_fuse_version_ga102( Ok(u16::BITS - reg_fuse_version.leading_zeros()) } -fn program_brom_ga102<E: FalconEngine>(bar: Bar0<'_>, params: &FalconBromParams) { - bar.write( - WithBase::of::<E>().at(0), +fn program_brom_ga102(pfalcon2: Mmio<'_, PFalcon2Registers>, params: &FalconBromParams) { + pfalcon2.write( + Array::at(0), regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset), ); - bar.write( - WithBase::of::<E>(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed() .with_value(u32::from(params.engine_id_mask)), ); - bar.write( - WithBase::of::<E>(), + pfalcon2.write_reg( regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed().with_ucode_id(params.ucode_id), ); - bar.write( - WithBase::of::<E>(), - regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k), - ); + pfalcon2 + .write_reg(regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k)); } pub(super) struct Ga102<E: FalconEngine>(PhantomData<E>); @@ -116,7 +110,7 @@ impl<E: FalconEngine> Ga102<E> { impl<E: FalconEngine> FalconHal<E> for Ga102<E> { fn select_core(&self, falcon: &Falcon<'_, E>) -> Result { - select_core_ga102::<E>(falcon.bar) + select_core_ga102(falcon.pfalcon2) } fn signature_reg_fuse_version( @@ -129,27 +123,24 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> { } fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) { - program_brom_ga102::<E>(falcon.bar, params); + program_brom_ga102(falcon.pfalcon2, params); } fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CPUCTL) .active_stat() } fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result<bool> { - Ok(falcon - .bar - .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) - .halted()) + Ok(falcon.pfalcon2.read(regs::NV_PRISCV_RISCV_CPUCTL).halted()) } fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(20), @@ -158,20 +149,18 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - let bar = falcon.bar; - - let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>()); + let _ = falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set // RESET_READY so a non-failing timeout is used. let _ = read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_HWCFG2)), |r| r.reset_ready(), Delta::ZERO, Delta::from_micros(150), ); - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 5291598fedf7..7fc6e83c2566 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -5,7 +5,6 @@ use core::marker::PhantomData; use kernel::{ io::{ poll::read_poll_timeout, - register::WithBase, Io, // }, prelude::*, @@ -50,8 +49,8 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> { fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { falcon - .bar - .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>()) + .pfalcon2 + .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS) .active_stat() } @@ -62,7 +61,7 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> { fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( - || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())), + || Ok(falcon.pfalcon.read(regs::NV_PFALCON_FALCON_DMACTL)), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(10), @@ -71,7 +70,7 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> { } fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(falcon.bar); + regs::NV_PFALCON_FALCON_ENGINE::reset_engine(falcon.pfalcon); self.reset_wait_mem_scrubbing(falcon)?; Ok(()) diff --git a/drivers/gpu/nova-core/falcon/sec2.rs b/drivers/gpu/nova-core/falcon/sec2.rs index 91ec7d49c1f5..6648a397d38a 100644 --- a/drivers/gpu/nova-core/falcon/sec2.rs +++ b/drivers/gpu/nova-core/falcon/sec2.rs @@ -1,22 +1,37 @@ // SPDX-License-Identifier: GPL-2.0 -use kernel::io::register::RegisterBase; +use kernel::io::{ + io_project, + register, + Mmio, // +}; -use crate::falcon::{ - FalconEngine, - PFalcon2Base, - PFalconBase, // +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + falcon::FalconEngine, // }; /// Type specifying the `Sec2` falcon engine. Cannot be instantiated. pub(crate) struct Sec2(()); -impl RegisterBase<PFalconBase> for Sec2 { - const BASE: usize = 0x00840000; -} +register! { + base: NovaRegisters; -impl RegisterBase<PFalcon2Base> for Sec2 { - const BASE: usize = 0x00841000; + PFALCON: super::PFalconRegisters @ 0x00840000; + PFALCON2: super::PFalcon2Registers @ 0x00841000; } -impl FalconEngine for Sec2 {} +impl FalconEngine for Sec2 { + #[inline] + fn pfalcon(io: Bar0<'_>) -> Mmio<'_, super::PFalconRegisters> { + io_project!(io, build: PFALCON) + } + + #[inline] + fn pfalcon2(io: Bar0<'_>) -> Mmio<'_, super::PFalcon2Registers> { + io_project!(io, build: PFALCON2) + } +} diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 1576399389b1..b3a6ab8b57a6 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -49,7 +49,7 @@ pub(crate) struct SysmemFlush<'sys> { device: &'sys device::Device, bar: Bar0<'sys>, /// Keep the page alive as long as we need it. - page: CoherentHandle, + page: CoherentHandle<'sys>, } impl<'sys> SysmemFlush<'sys> { @@ -177,7 +177,7 @@ impl FbRanges { pub(crate) fn new( chipset: Chipset, bar: Bar0<'_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, vgpu_state: VgpuState, ) -> Result<Self> { let hal = hal::fb_hal(chipset); diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index d9e4d62ae632..9fa094939600 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -5,11 +5,10 @@ use kernel::{ io::{ - register::{ - RegisterBase, - WithBase, // - }, - Io, // + io_project, + register, + Io, + Mmio, // }, num::Bounded, prelude::*, @@ -21,7 +20,10 @@ use kernel::{ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, fb::{ hal::FbHal, regs, // @@ -31,17 +33,26 @@ use crate::{ struct Gb100; -impl RegisterBase<regs::Hshub0Base> for Gb100 { - const BASE: usize = 0x0087_0000; +register! { + base: NovaRegisters; + + HSHUB0: regs::Hshub0Registers @ 0x0087_0000; +} + +#[inline] +fn hshub0(bar: Bar0<'_>) -> Mmio<'_, regs::Hshub0Registers> { + io_project!(bar, build: HSHUB0) } -fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { +fn read_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>) -> u64 { let lo = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO) .adr(), ); let hi = u64::from( - bar.read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>()) + hshub0 + .read(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI) .adr(), ); @@ -52,7 +63,7 @@ fn read_sysmem_flush_page_gb100(bar: Bar0<'_>) -> u64 { /// /// Both the primary and EG (egress) register pairs must be programmed to the same address, /// as required by hardware. -fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) { +fn write_sysmem_flush_page_gb100(hshub0: Mmio<'_, regs::Hshub0Registers>, addr: Bounded<u64, 52>) { // CAST: lower 32 bits. Hardware ignores bits 7:0. let addr_lo = *addr as u32; let addr_hi = addr.shr::<32, 20>().cast::<u32>(); @@ -60,24 +71,12 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) { // Write HI first. The hardware will trigger the flush on the LO write. // Primary HSHUB pair. - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(), - regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); // EG (egress) pair -- must match the primary pair. - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb100>(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi), - ); - bar.write( - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb100>(), - regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo), - ); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed().with_adr(addr_hi)); + hshub0.write_reg(regs::NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(addr_lo)); } // This PMU reservation size is r570-specific. @@ -88,13 +87,13 @@ pub(super) const fn pmu_reserved_size_gb100() -> u32 { impl FbHal for Gb100 { fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { - read_sysmem_flush_page_gb100(bar) + read_sysmem_flush_page_gb100(hshub0(bar)) } fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?; - write_sysmem_flush_page_gb100(bar, addr); + write_sysmem_flush_page_gb100(hshub0(bar), addr); Ok(()) } diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs index 95adbe124a30..131787996a24 100644 --- a/drivers/gpu/nova-core/fb/regs.rs +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -2,12 +2,20 @@ use kernel::{ io::register, - sizes::SizeConstants, // + prelude::*, + sizes::{ + SizeConstants, + SZ_4K, // + }, // }; +use crate::driver::NovaRegisters; + // PDISP register! { + base: NovaRegisters; + pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { /// VGA workspace base address divided by 0x10000. 31:8 addr; @@ -30,6 +38,8 @@ impl NV_PDISP_VGA_WORKSPACE_BASE { // PFB register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform sysmembar /// operations (see [`crate::fb::SysmemFlush`]). pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { @@ -59,34 +69,42 @@ register! { } } -/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +const HSHUB0_REGION_SIZE: usize = SZ_4K; + +/// The GB10x HSHUB0 register window (Base defined as `NV_HSHUB0_PRIV_BASE` in Open RM). /// /// The base is provided by the GB10x framebuffer HAL. -pub(super) struct Hshub0Base(()); +#[repr(align(4))] +#[derive(FromBytes, IntoBytes)] +pub(super) struct Hshub0Registers([u8; HSHUB0_REGION_SIZE]); register! { + base: Hshub0Registers; + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar // through a primary and an EG (egress) pair that must both be programmed to the same // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00000e50 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00000e54 { 19:0 adr; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x000006c0 { 31:0 adr => u32; } - pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x000006c4 { 19:0 adr; } } register! { + base: NovaRegisters; + // GB20x FBHUB0 sysmem flush registers. Unlike the older // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an // 8-bit right-shift, these take the raw address split into lower and upper @@ -101,6 +119,8 @@ register! { } register! { + base: NovaRegisters; + /// Low bits of the physical system memory address used by the GPU to perform /// sysmembar operations on Hopper. /// diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index b49613a90bf0..c16fee6e2b2a 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -23,9 +23,9 @@ use crate::{ }; pub(crate) mod booter; -pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; +pub(crate) mod gsp_fmc; pub(crate) mod riscv; pub(crate) mod tlv; diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index dc071edba331..aa4458bb3312 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -186,7 +186,7 @@ impl BooterFirmware { &self, dev: &device::Device<device::Bound>, sec2_falcon: &Falcon<'_, Sec2>, - wpr_meta: &Coherent<T>, + wpr_meta: &Coherent<'_, T>, ) -> Result { sec2_falcon.reset()?; sec2_falcon.load(self)?; diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index ec4d92317a93..a87878fe2aec 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -12,7 +12,10 @@ use kernel::{ Device, // }, dma::Coherent, - io::{register::WithBase, Io}, + io::{ + register::Array, + Io, // + }, prelude::*, ptr::{ Alignable, @@ -23,7 +26,6 @@ use kernel::{ }; use crate::{ - driver::Bar0, falcon::{ self, gsp::Gsp, @@ -98,9 +100,9 @@ unsafe impl AsBytes for BootloaderDmemDescV2 {} /// Wrapper for [`FwsecFirmware`] that includes the bootloader performing the actual load /// operation. -pub(crate) struct FwsecFirmwareWithBl { +pub(crate) struct FwsecFirmwareWithBl<'a> { /// DMA object the bootloader will copy the firmware from. - _firmware_dma: Coherent<[u8]>, + _firmware_dma: Coherent<'a, [u8]>, /// Code of the bootloader to be loaded into non-secure IMEM. ucode: KVec<u8>, /// Descriptor to be loaded into DMEM for the bootloader to read. @@ -113,12 +115,12 @@ pub(crate) struct FwsecFirmwareWithBl { start_tag: u16, } -impl FwsecFirmwareWithBl { +impl<'a> FwsecFirmwareWithBl<'a> { /// Loads the bootloader firmware for `dev` and `chipset`, and wrap `firmware` so it can be /// loaded using it. pub(crate) fn new( firmware: FwsecFirmware, - dev: &Device<device::Bound>, + dev: &'a Device<device::Bound>, chipset: Chipset, ) -> Result<Self> { let fw = request_tlv(dev, chipset, "gen_bootloader")?; @@ -235,12 +237,7 @@ impl FwsecFirmwareWithBl { /// /// The bootloader will load the FWSEC firmware and then execute it. This function returns /// after FWSEC has reached completion. - pub(crate) fn run( - &self, - dev: &Device<device::Bound>, - falcon: &Falcon<'_, Gsp>, - bar: Bar0<'_>, - ) -> Result<()> { + pub(crate) fn run(&self, dev: &Device<device::Bound>, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon .reset() @@ -250,9 +247,8 @@ impl FwsecFirmwareWithBl { .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory. - bar.update( - regs::NV_PFALCON_FBIF_TRANSCFG::of::<Gsp>() - .try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) + falcon.pfalcon.update( + regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(self.dmem_desc.ctx_dma)) .ok_or(EINVAL)?, |v| { v.with_target(FalconFbifTarget::CoherentSysmem) @@ -272,7 +268,7 @@ impl FwsecFirmwareWithBl { } } -impl FalconFirmware for FwsecFirmwareWithBl { +impl FalconFirmware for FwsecFirmwareWithBl<'_> { type Target = Gsp; fn brom_params(&self) -> FalconBromParams { @@ -286,7 +282,7 @@ impl FalconFirmware for FwsecFirmwareWithBl { } } -impl FalconPioLoadable for FwsecFirmwareWithBl { +impl FalconPioLoadable for FwsecFirmwareWithBl<'_> { fn imem_sec_load_params(&self) -> Option<FalconPioImemLoadTarget<'_>> { None } diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index e8f9491e84cc..22d1f9329c9f 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -44,7 +44,7 @@ use crate::{ /// Each page is 4KB, each entry is 8 bytes (64-bit DMA address). /// Also known as "Radix3" firmware. #[pin_data] -pub(crate) struct GspFirmware { +pub(crate) struct GspFirmware<'a> { /// The GSP firmware inside a [`VVec`], device-mapped via a SG table. #[pin] fw: SGTable<Owned<VVec<u8>>>, @@ -55,19 +55,19 @@ pub(crate) struct GspFirmware { #[pin] level1: SGTable<Owned<VVec<u8>>>, /// Level 0 page table (single 4KB page) with one entry: DMA address of first level 1 page. - level0: Coherent<[u64]>, + level0: Coherent<'a, [u64]>, /// Size in bytes of the firmware contained in [`Self::fw`]. pub(crate) size: usize, /// Device-mapped GSP signatures matching the GPU's [`Chipset`]. - pub(crate) signatures: Coherent<[u8]>, + pub(crate) signatures: Coherent<'a, [u8]>, /// GSP bootloader, verifies the GSP firmware before loading and running it. - pub(crate) bootloader: RiscvFirmware, + pub(crate) bootloader: RiscvFirmware<'a>, } -impl GspFirmware { +impl<'a> GspFirmware<'a> { /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. - pub(crate) fn new<'a>( + pub(crate) fn new( dev: &'a device::Device<device::Bound>, chipset: Chipset, ) -> impl PinInit<Self, Error> + 'a { @@ -120,7 +120,7 @@ impl GspFirmware { // Create level 0 page table data and fill its first entry with the level 1 // table. - let mut level0 = CoherentBox::<[u64]>::zeroed_slice( + let mut level0 = CoherentBox::<'_, [u64]>::zeroed_slice( dev, GSP_PAGE_SIZE / size_of::<u64>(), GFP_KERNEL diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/gsp_fmc.rs index 5462e318410a..94fcc86c7dff 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp_fmc.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -//! FSP is a hardware unit that runs FMC firmware. +//! GSP-FMC (First Mutable Code) is loaded by FSP into GSP to serve as the loader and verifier of +//! GSP-RM. use kernel::{ device, @@ -17,16 +18,16 @@ use crate::{ gpu::Chipset, // }; -/// Size of the FSP SHA-384 hash, in bytes. -const FSP_HASH_SIZE: usize = 48; -/// Maximum size of the FSP public key (RSA-3072), in bytes. +/// Size of the GSP-FMC SHA-384 hash, in bytes. +const FMC_HASH_SIZE: usize = 48; +/// Maximum size of the GSP-FMC public key (RSA-3072), in bytes. /// -/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_PKEY_SIZE: usize = 384; -/// Maximum size of the FSP signature (RSA-3072), in bytes. +/// The `PKEY` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_PKEY_SIZE: usize = 384; +/// Maximum size of the GSP-FMC signature (RSA-3072), in bytes. /// -/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded. -const FSP_SIG_SIZE: usize = 384; +/// The `SIGN` tag may be shorter, so the remaining bytes are zero-padded. +const FMC_SIG_SIZE: usize = 384; /// Structure to hold FMC signatures. /// @@ -34,23 +35,27 @@ const FSP_SIG_SIZE: usize = 384; #[derive(Debug, Clone, Copy, Zeroable)] #[repr(C)] pub(crate) struct FmcSignatures { - pub(crate) hash384: [u8; FSP_HASH_SIZE], - pub(crate) public_key: [u8; FSP_PKEY_SIZE], - pub(crate) signature: [u8; FSP_SIG_SIZE], + pub(crate) hash384: [u8; FMC_HASH_SIZE], + pub(crate) public_key: [u8; FMC_PKEY_SIZE], + pub(crate) signature: [u8; FMC_SIG_SIZE], } -pub(crate) struct FspFirmware { +pub(crate) struct GspFmcFirmware<'a> { /// FMC firmware image data - pub(crate) fmc_image: Coherent<[u8]>, + pub(crate) fmc_image: Coherent<'a, [u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox<FmcSignatures>, } -impl FspFirmware { - pub(crate) fn new(dev: &device::Device<device::Bound>, chipset: Chipset) -> Result<Self> { +impl<'a> GspFmcFirmware<'a> { + pub(crate) fn new(dev: &'a device::Device<device::Bound>, chipset: Chipset) -> Result<Self> { let fw = request_tlv(dev, chipset, "fmc")?; let tlv = Tlv::new(fw.data())?; - dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); + dev_dbg!( + dev, + "loaded GSP-FMC firmware v{}\n", + tlv.get_string(b"VERS")? + ); let fmc_image_data = tlv.get_bytes(b"BLOB")?; let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; @@ -70,34 +75,34 @@ impl FspFirmware { let pkey_section = tlv.get_bytes(b"PKEY")?; let sig_section = tlv.get_bytes(b"SIGN")?; - // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. - if hash_section.len() != FSP_HASH_SIZE { + // The hash section is a SHA-384 output: it must be exactly `FMC_HASH_SIZE` bytes. + if hash_section.len() != FMC_HASH_SIZE { dev_err!( dev, "FMC hash section size {} != expected {}\n", hash_section.len(), - FSP_HASH_SIZE + FMC_HASH_SIZE ); return Err(EINVAL); } // The key and signature sections are zero-padded to a fixed maximum, so they may be // shorter, but must not exceed the destination buffers. - if pkey_section.len() > FSP_PKEY_SIZE { + if pkey_section.len() > FMC_PKEY_SIZE { dev_err!( dev, "FMC public key section size {} > maximum {}\n", pkey_section.len(), - FSP_PKEY_SIZE + FMC_PKEY_SIZE ); return Err(EINVAL); } - if sig_section.len() > FSP_SIG_SIZE { + if sig_section.len() > FMC_SIG_SIZE { dev_err!( dev, "FMC signature section size {} > maximum {}\n", sig_section.len(), - FSP_SIG_SIZE + FMC_SIG_SIZE ); return Err(EINVAL); } @@ -106,11 +111,11 @@ impl FspFirmware { // stack, then fill each section from the firmware. let signatures = KBox::init( pin_init::init_zeroed::<FmcSignatures>().chain(|sigs| { - // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). + // PANIC: src and dst lengths are both `FMC_HASH_SIZE` (verified above). sigs.hash384.copy_from_slice(hash_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above). + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_PKEY_SIZE` (verified above). sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above). + // PANIC: dst is sliced to src.len(); src.len() <= `FMC_SIG_SIZE` (verified above). sigs.signature[..sig_section.len()].copy_from_slice(sig_section); Ok(()) }), diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs index 1403f05a7305..f05cfb1c65da 100644 --- a/drivers/gpu/nova-core/firmware/riscv.rs +++ b/drivers/gpu/nova-core/firmware/riscv.rs @@ -13,7 +13,7 @@ use kernel::{ use crate::firmware::tlv::Tlv; /// A parsed firmware for a RISC-V core, ready to be loaded and run. -pub(crate) struct RiscvFirmware { +pub(crate) struct RiscvFirmware<'a> { /// Offset at which the code starts in the firmware image. pub(crate) code_offset: u32, /// Offset at which the data starts in the firmware image. @@ -23,12 +23,12 @@ pub(crate) struct RiscvFirmware { /// Application version. pub(crate) app_version: u32, /// Device-mapped firmware image. - pub(crate) ucode: Coherent<[u8]>, + pub(crate) ucode: Coherent<'a, [u8]>, } -impl RiscvFirmware { +impl<'a> RiscvFirmware<'a> { /// Parses the RISC-V firmware image contained in `fw`. - pub(crate) fn new(dev: &device::Device<device::Bound>, fw: &Firmware) -> Result<Self> { + pub(crate) fn new(dev: &'a device::Device<device::Bound>, fw: &Firmware) -> Result<Self> { let tlv = Tlv::new(fw.data())?; dev_dbg!( dev, diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index ab685fb4168f..b738dcabcdef 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -3,9 +3,12 @@ //! FSP (Foundation Security Processor) interface for Hopper/Blackwell GPUs. //! -//! Hopper/Blackwell use a simplified firmware boot sequence: FMC, then FSP, then GSP. +//! Hopper/Blackwell use a simplified firmware boot sequence: FSP secure-boots independently before +//! the driver starts. The driver then sends FSP a Chain-of-Trust request containing the GSP-FMC +//! image. FSP authenticates the image and launches GSP-FMC on the GSP RISC-V core; GSP-FMC +//! subsequently authenticates and boots GSP-RM. +//! //! Unlike Turing/Ampere/Ada, there is no SEC2 (Security Engine 2) usage. -//! FSP handles secure boot directly using FMC firmware and Chain of Trust. use kernel::{ device, @@ -32,9 +35,9 @@ use crate::{ Falcon, // }, fb::FbSizes, - firmware::fsp::{ + firmware::gsp_fmc::{ FmcSignatures, - FspFirmware, // + GspFmcFirmware, // }, gpu::Chipset, gsp::{ @@ -267,7 +270,7 @@ impl FspCotMessage { /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( fb_info: &FbSizes, - fsp_fw: &'a FspFirmware, + fmc_fw: &'a GspFmcFirmware<'_>, args: &'a FmcBootArgs<'_>, ) -> Result<impl Init<Self> + 'a> { let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; @@ -296,13 +299,13 @@ impl FspCotMessage { .chain(move |msg| { msg.cot.version = version; msg.cot.size = size; - msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address(); + msg.cot.gsp_fmc_sysmem_offset = fmc_fw.fmc_image.dma_address(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address(); - msg.cot.sigs = *fsp_fw.fmc_sigs; + msg.cot.sigs = *fmc_fw.fmc_sigs; Ok(()) })) @@ -345,28 +348,28 @@ impl MessageToFsp for FspPrcMessage { /// Bundled arguments for FMC boot via FSP Chain of Trust. pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, - fmc_boot_params: Coherent<GspFmcBootParams>, + fmc_boot_params: Coherent<'a, GspFmcBootParams>, resume: bool, // Additional dependencies required to be kept alive for FMC boot. - _wpr_meta: Coherent<GspFwWprMeta>, - _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + _wpr_meta: Coherent<'a, GspFwWprMeta>, + _libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, } impl<'a> FmcBootArgs<'a> { /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter /// structure that FSP will read. pub(crate) fn new( - dev: &device::Device<device::Bound>, + dev: &'a device::Device<device::Bound>, chipset: Chipset, - wpr_meta: Coherent<GspFwWprMeta>, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + wpr_meta: Coherent<'a, GspFwWprMeta>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result<Self> { let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address()); Ok(Self { chipset, - fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?, + fmc_boot_params: Coherent::init(dev, GFP_KERNEL, init)?, resume, _wpr_meta: wpr_meta, _libos: libos, @@ -374,19 +377,19 @@ impl<'a> FmcBootArgs<'a> { } /// Returns the FMC boot parameters allocation. - pub(crate) fn boot_params(&self) -> &Coherent<GspFmcBootParams> { + pub(crate) fn boot_params(&self) -> &Coherent<'_, GspFmcBootParams> { &self.fmc_boot_params } } /// FSP interface for Hopper/Blackwell GPUs. /// -/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot -/// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent +/// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot has +/// completed. It owns the FSP falcon and the GSP-FMC firmware, which are used for the subsequent /// Chain of Trust boot. pub(crate) struct Fsp<'a> { falcon: Falcon<'a, FspEngine>, - fsp_fw: FspFirmware, + fmc_fw: GspFmcFirmware<'a>, } impl<'a> Fsp<'a> { @@ -422,7 +425,7 @@ impl<'a> Fsp<'a> { const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; let falcon = Falcon::<FspEngine>::new(dev, chipset, bar)?; - let fsp_fw = FspFirmware::new(dev, chipset)?; + let fmc_fw = GspFmcFirmware::new(dev, chipset)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), @@ -434,7 +437,7 @@ impl<'a> Fsp<'a> { dev_err!(dev, "FSP secure boot completion error: {:?}\n", e); })?; - Ok(Fsp { falcon, fsp_fw }) + Ok(Fsp { falcon, fmc_fw }) } /// Sends a message to FSP and waits for the response. @@ -540,7 +543,7 @@ impl<'a> Fsp<'a> { ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init(FspCotMessage::new(fb_info, &self.fmc_fw, args)?, GFP_KERNEL)?; let _response_buf = self.send_sync_fsp(dev, &*msg)?; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index fd1414004dd0..d763bc8d3827 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -6,16 +6,25 @@ use kernel::{ device, dma::Device, fmt, + gpu::buddy::GpuBuddyParams, io::Io, num::Bounded, pci, prelude::*, - sizes::SizeConstants, // + ptr::Alignment, + sizes::{ + SizeConstants, + SZ_4K, // + }, + sync::Arc, }; use crate::{ bounded_enum, - driver::Bar0, + driver::{ + Bar0, + Bar1, // + }, falcon::{ gsp::Gsp as GspFalcon, sec2::Sec2 as Sec2Falcon, @@ -29,11 +38,17 @@ use crate::{ Gsp, GspBootContext, // }, - regs, + mm::{ + bar_user::BarUser, + pagetable::MmuVersion, + GpuMm, + VramAddress, // + }, vgpu::VgpuManager, // }; mod hal; +mod regs; macro_rules! define_chipset { ({ $($variant:ident = $value:expr),* $(,)* }) => @@ -139,6 +154,11 @@ impl Chipset { pub(crate) fn pci_config_mirror_range(self) -> Range<u32> { hal::gpu_hal(self).pci_config_mirror_range() } + + /// Returns the MMU version for this chipset. + pub(crate) fn mmu_version(self) -> MmuVersion { + MmuVersion::from(self.arch()) + } } // TODO @@ -272,9 +292,9 @@ struct GspResources<'gpu> { vgpu: VgpuManager, /// GSP runtime data. #[pin] - gsp: Gsp, + gsp: Gsp<'gpu>, /// GSP unload firmware bundle, if any. - unload_bundle: Option<gsp::UnloadBundle>, + unload_bundle: Option<gsp::UnloadBundle<'gpu>>, } /// Structure holding the resources required to operate the GPU. @@ -283,6 +303,13 @@ pub(crate) struct Gpu<'gpu> { spec: Spec, /// Static GPU information as provided by the GSP. gsp_static_info: GetGspStaticInfoReply, + /// GPU memory manager owning memory management resources. + /// + /// Must be kept declared *before* `gsp_resources`, so that its components are dropped while + /// the GSP is still operational. + mm: GpuMm<'gpu>, + /// BAR1 user interface for CPU access to GPU virtual memory. + bar_user: Arc<BarUser<'gpu>>, /// GSP and its resources. #[pin] gsp_resources: GspResources<'gpu>, @@ -326,6 +353,7 @@ impl<'gpu> Gpu<'gpu> { pub(crate) fn new<'a>( pdev: &'gpu pci::Device<device::Core<'a>>, bar: Bar0<'gpu>, + bar1: &'gpu Bar1<'gpu>, ) -> impl PinInit<Self, Error> + use<'gpu, 'a> { let dev = pdev.as_ref(); @@ -410,7 +438,64 @@ impl<'gpu> Gpu<'gpu> { } info - } + }, + + // Create GPU memory manager owning memory management resources. + mm: { + let usable_vram = gsp_static_info.usable_fb_regions.first().ok_or(ENODEV)?; + let buddy_params = GpuBuddyParams { + base_offset: usable_vram.start, + size: usable_vram.end - usable_vram.start, + chunk_size: Alignment::new::<SZ_4K>(), + }; + + GpuMm::new( + bar, + gsp_resources.spec.chipset, + buddy_params, + VramAddress::from_raw(gsp_static_info.total_fb_end), + )? + }, + + // Create BAR1 user interface for CPU access to GPU virtual memory. + bar_user: { + let pdb_addr = VramAddress::from_raw(gsp_static_info.bar1_pde_base); + let bar1_idx = crate::driver::bar1_resource_index(pdev)?; + let bar1_size = pdev.resource_len(bar1_idx)?; + Arc::pin_init( + BarUser::new( + pdb_addr, + gsp_resources.spec.chipset, + bar1_size, + bar1, + )?, + GFP_KERNEL, + )? + }, }) } + + /// Runs self-tests on the constructed [`Gpu`], logging failures without failing probe. + #[cfg(CONFIG_NOVA_CORE_SELFTESTS)] + pub(crate) fn run_selftests(self: Pin<&mut Self>, pdev: &pci::Device<device::Bound>) { + let this = self.project(); + let dev = pdev.as_ref(); + let regions = &this.gsp_static_info.usable_fb_regions; + + if let Err(err) = crate::mm::selftest::run( + dev, + this.mm, + regions, + this.bar_user, + this.gsp_static_info.bar1_pde_base, + this.spec.chipset, + ) { + dev_err!(dev, "self-tests failed: {:?}\n", err); + } + } +} + +/// Reads the boot0 register and returns its raw value. +pub(crate) fn boot_0_raw(bar: Bar0<'_>) -> u32 { + bar.read(regs::NV_PMC_BOOT_0).into_raw() } diff --git a/drivers/gpu/nova-core/gpu/regs.rs b/drivers/gpu/nova-core/gpu/regs.rs new file mode 100644 index 000000000000..54e740d847cd --- /dev/null +++ b/drivers/gpu/nova-core/gpu/regs.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::{ + io::register, + prelude::*, // +}; + +use super::{ + Architecture, + Chipset, // +}; + +use crate::driver::NovaRegisters; + +// PMC + +register! { + base: NovaRegisters; + + /// Basic revision information about the GPU. + pub(super) NV_PMC_BOOT_0(u32) @ 0x00000000 { + /// Lower bits of the architecture. + 28:24 architecture_0; + /// Implementation version of the architecture. + 23:20 implementation; + /// MSB of the architecture. + 8:8 architecture_1; + /// Major revision of the chip. + 7:4 major_revision; + /// Minor revision of the chip. + 3:0 minor_revision; + } + + /// Extended architecture information. + pub(super) NV_PMC_BOOT_42(u32) @ 0x00000a00 { + /// Architecture value. + 29:24 architecture ?=> Architecture; + /// Implementation version of the architecture. + 23:20 implementation; + /// Major revision of the chip. + 19:16 major_revision; + /// Minor revision of the chip. + 15:12 minor_revision; + } +} + +impl NV_PMC_BOOT_0 { + pub(super) fn is_older_than_fermi(self) -> bool { + // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : + const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; + + // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than + // GF100, means "older than Fermi". + self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 + } +} + +impl NV_PMC_BOOT_42 { + /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. + pub(super) fn chipset(self) -> Result<Chipset> { + self.architecture() + .map(|arch| { + ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) + | u32::from(self.implementation()) + }) + .and_then(Chipset::try_from) + } + + /// Returns the raw architecture value from the register. + fn architecture_raw(self) -> u8 { + ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) + & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 + } +} + +impl kernel::fmt::Display for NV_PMC_BOOT_42 { + fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { + write!( + f, + "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", + self.inner, + self.architecture_raw(), + self.implementation() + ) + } +} diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 13f361406a6c..25ea43f1cbe9 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -115,11 +115,11 @@ impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { /// then pp points to index into the buffer where the next logging entry will /// be written. Therefore, the logging data is valid if: /// 1 <= pp < sizeof(buffer)/sizeof(u64) -struct LogBuffer(Coherent<[u8; LOG_BUFFER_SIZE]>); +struct LogBuffer<'a>(Coherent<'a, [u8; LOG_BUFFER_SIZE]>); -impl LogBuffer { +impl<'a> LogBuffer<'a> { /// Creates a new `LogBuffer` mapped on `dev`. - fn new(dev: &device::Device<device::Bound>) -> Result<Self> { + fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> { let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?); let start_addr = obj.0.dma_address(); @@ -135,33 +135,33 @@ impl LogBuffer { } } -struct LogBuffers { +struct LogBuffers<'a> { /// Init log buffer. - loginit: LogBuffer, + loginit: LogBuffer<'a>, /// Interrupts log buffer. - logintr: LogBuffer, + logintr: LogBuffer<'a>, /// RM log buffer. - logrm: LogBuffer, + logrm: LogBuffer<'a>, } /// GSP runtime data. #[pin_data] -pub(crate) struct Gsp { +pub(crate) struct Gsp<'gsp> { /// Libos arguments. - pub(crate) libos: Coherent<[LibosMemoryRegionInitArgument]>, + pub(crate) libos: Coherent<'gsp, [LibosMemoryRegionInitArgument]>, /// Log buffers, optionally exposed via debugfs. #[pin] - logs: debugfs::Scope<LogBuffers>, + logs: debugfs::Scope<LogBuffers<'gsp>>, /// Command queue. #[pin] - pub(crate) cmdq: Cmdq, + pub(crate) cmdq: Cmdq<'gsp>, /// RM arguments. - rmargs: Coherent<GspArgumentsPadded>, + rmargs: Coherent<'gsp, GspArgumentsPadded>, } -impl Gsp { +impl<'gsp> Gsp<'gsp> { // Creates an in-place initializer for a `Gsp` manager for `pdev`. - pub(crate) fn new(pdev: &pci::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { + pub(crate) fn new(pdev: &'gsp pci::Device<device::Bound>) -> impl PinInit<Self, Error> + 'gsp { pin_init::pin_init_scope(move || { let dev = pdev.as_ref(); @@ -223,4 +223,4 @@ impl Gsp { } /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`]. -pub(crate) struct UnloadBundle(KBox<dyn hal::UnloadBundle>); +pub(crate) struct UnloadBundle<'a>(KBox<dyn hal::UnloadBundle + 'a>); diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index e03700ee7bea..60bed3dc2f5a 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -22,7 +22,7 @@ use crate::{ }, }; -impl super::Gsp { +impl<'gsp> super::Gsp<'gsp> { /// Attempt to boot the GSP. /// /// This is a GPU-dependent and complex procedure that involves loading firmware files from @@ -33,8 +33,8 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - mut ctx: super::GspBootContext<'_, '_>, - ) -> Result<Option<super::UnloadBundle>> { + mut ctx: super::GspBootContext<'_, 'gsp>, + ) -> Result<Option<super::UnloadBundle<'gsp>>> { let pdev = ctx.pdev; let bar = ctx.bar; let chipset = ctx.chipset; @@ -44,6 +44,11 @@ impl super::Gsp { let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; + self.cmdq + .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; + self.cmdq + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); @@ -73,11 +78,6 @@ impl super::Gsp { dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),); - self.cmdq - .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; - self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; - hal.post_boot(&self, ctx, &gsp_fw)?; // Wait until GSP is fully initialized. @@ -88,7 +88,7 @@ impl super::Gsp { /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, bar: Bar0<'_>, gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, @@ -113,7 +113,7 @@ impl super::Gsp { pub(crate) fn unload( &self, mut ctx: super::GspBootContext<'_, '_>, - unload_bundle: Option<super::UnloadBundle>, + unload_bundle: Option<super::UnloadBundle<'_>>, ) -> Result { let dev = ctx.dev(); diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 6da728201281..9f99e6bbb4fa 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,13 +2,7 @@ mod continuation; -use core::{ - mem, - sync::atomic::{ - fence, - Ordering, // - }, -}; +use core::mem; use kernel::{ device, @@ -26,7 +20,12 @@ use kernel::{ prelude::*, ptr, sync::{ - aref::ARef, + barrier::{ + dma_mb, + Full, + Read, + Write, // + }, Mutex, // }, time::Delta, @@ -230,19 +229,19 @@ unsafe impl FromBytes for GspMem {} /// pointer and the GSP read pointer. This region is returned by [`Self::driver_write_area`]. /// * The driver owns (i.e. can read from) the part of the GSP message queue between the CPU read /// pointer and the GSP write pointer. This region is returned by [`Self::driver_read_area`]. -struct DmaGspMem(Coherent<GspMem>); +struct DmaGspMem<'a>(Coherent<'a, GspMem>); -impl DmaGspMem { +impl<'a> DmaGspMem<'a> { /// Allocate a new instance and map it for `dev`. - fn new(dev: &device::Device<device::Bound>) -> Result<Self> { + fn new(dev: &'a device::Device<device::Bound>) -> Result<Self> { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::<'_, GspMem>::zeroed(dev, GFP_KERNEL)?; gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let gsp_mem: Coherent<_> = gsp_mem.into(); + let gsp_mem: Coherent<'_, _> = gsp_mem.into(); PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; Ok(Self(gsp_mem)) @@ -404,7 +403,12 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_write_ptr(&self) -> u32 { - MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES + let ptr = MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->LOAD ordering needed to order `gsp_write_ptr` read before data read. + dma_mb(Read); + + ptr } // Returns the index of the memory page the GSP will read the next command from. @@ -413,7 +417,12 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES + let ptr = MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES; + + // ORDERING: LOAD->STORE ordering needed to order `gsp_read_ptr` read before data write. + dma_mb(Full); + + ptr } // Returns the index of the memory page the CPU can read the next message from. @@ -427,12 +436,11 @@ impl DmaGspMem { // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { + // ORDERING: LOAD->STORE ordering needed to order `cpu_read_ptr` write after data read. + dma_mb(Full); + let rx = io_project!(self.0, .cpuq.rx); let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - MsgqRxHeader::set_read_ptr(rx, rptr) } @@ -447,12 +455,12 @@ impl DmaGspMem { // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { + // ORDERING: STORE->STORE ordering needed to order `cpu_write_ptr` write after data write. + dma_mb(Write); + let tx = io_project!(self.0, .cpuq.tx); let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; MsgqTxHeader::set_write_ptr(tx, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); } } @@ -469,7 +477,7 @@ struct GspCommand<'a> { /// A message ready to be processed from the message queue. /// -/// This is the type returned by [`Cmdq::wait_for_msg`]. +/// This is the type returned by [`CmdqInner::wait_for_msg`]. struct GspMessage<'a> { // Reference to the header of the message. header: &'a GspMsgElement, @@ -483,15 +491,15 @@ struct GspMessage<'a> { /// Provides the ability to send commands and receive messages from the GSP using a shared memory /// area. #[pin_data] -pub(crate) struct Cmdq { +pub(crate) struct Cmdq<'cmdq> { /// Inner mutex-protected state. #[pin] - inner: Mutex<CmdqInner>, + inner: Mutex<CmdqInner<'cmdq>>, /// DMA address of the command queue's shared memory region. pub(super) dma_addr: DmaAddress, } -impl Cmdq { +impl<'cmdq> Cmdq<'cmdq> { /// Offset of the data after the PTEs. const POST_PTE_OFFSET: usize = core::mem::offset_of!(GspMem, cpuq); @@ -512,14 +520,16 @@ impl Cmdq { pub(super) const RECEIVE_TIMEOUT: Delta = Delta::from_secs(5); /// Creates a new command queue for `dev`. - pub(crate) fn new(dev: &device::Device<device::Bound>) -> impl PinInit<Self, Error> + '_ { + pub(crate) fn new( + dev: &'cmdq device::Device<device::Bound>, + ) -> impl PinInit<Self, Error> + 'cmdq { pin_init_scope(move || { let gsp_mem = DmaGspMem::new(dev)?; Ok(try_pin_init!(Self { dma_addr: gsp_mem.0.dma_address(), inner <- new_mutex!(CmdqInner { - dev: dev.into(), + dev, gsp_mem, seq: 0, }), @@ -610,16 +620,16 @@ impl Cmdq { } /// Inner mutex protected state of [`Cmdq`]. -struct CmdqInner { +struct CmdqInner<'a> { /// Device this command queue belongs to. - dev: ARef<device::Device>, + dev: &'a device::Device, /// Current command sequence number. seq: u32, /// Memory area shared with the GSP for communicating commands and messages. - gsp_mem: DmaGspMem, + gsp_mem: DmaGspMem<'a>, } -impl CmdqInner { +impl CmdqInner<'_> { /// Timeout for waiting for space on the command queue. const ALLOCATE_TIMEOUT: Delta = Delta::from_secs(1); diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index ffc25fd8c47b..e087c9e8c35c 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -187,7 +187,7 @@ impl MessageFromGsp for GspInitDone { } /// Waits for GSP initialization to complete. -pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq) -> Result { +pub(crate) fn wait_gsp_init_done(cmdq: &Cmdq<'_>) -> Result { loop { match cmdq.receive_msg::<GspInitDone>(Cmdq::RECEIVE_TIMEOUT) { Ok(_) => break Ok(()), @@ -214,8 +214,12 @@ impl CommandToGsp for GetGspStaticInfo { /// The reply from the GSP to the [`GetGspStaticInfo`] command. pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], + /// BAR1 Page Directory Entry base address. + pub(crate) bar1_pde_base: u64, /// Usable FB (VRAM) regions for driver memory allocation. pub(crate) usable_fb_regions: KVec<Range<u64>>, + /// Exclusive end of the FB physical address space. + pub(crate) total_fb_end: u64, } impl MessageFromGsp for GetGspStaticInfoReply { @@ -231,10 +235,13 @@ impl MessageFromGsp for GetGspStaticInfoReply { for region in msg.usable_fb_regions() { usable_fb_regions.push(region, GFP_KERNEL)?; } + let total_fb_end = msg.total_fb_end().ok_or(EINVAL)?; Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + bar1_pde_base: msg.bar1_pde_base(), usable_fb_regions, + total_fb_end, }) } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 05f54fee6186..8778c4bf79c0 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -179,7 +179,7 @@ impl GspFwWprMeta { /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the /// framebuffer ranges `ranges`. pub(crate) fn from_ranges<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, ranges: &'a FbRanges, ) -> impl Init<Self> + 'a { let init_inner = init!(bindings::GspFwWprMeta { @@ -231,7 +231,7 @@ impl GspFwWprMeta { /// /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. pub(crate) fn from_sizes<'a>( - gsp_firmware: &'a GspFirmware, + gsp_firmware: &'a GspFirmware<'_>, sizes: &'a FbSizes, ) -> impl Init<Self> + 'a { /// VGA workspace size to reserve at the end of the framebuffer, in bytes. @@ -665,7 +665,7 @@ unsafe impl FromBytes for LibosMemoryRegionInitArgument {} impl LibosMemoryRegionInitArgument { pub(crate) fn new<'a, A: AsBytes + FromBytes + KnownSize + ?Sized>( name: &'static str, - obj: &'a Coherent<A>, + obj: &'a Coherent<'_, A>, ) -> impl Init<Self> + 'a { /// Generates the `ID8` identifier required for some GSP objects. fn id8(name: &str) -> u64 { @@ -897,7 +897,7 @@ pub(crate) struct GspArgumentsCached { impl GspArgumentsCached { /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. - pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> { let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), bDmemStack: 1, @@ -924,7 +924,7 @@ pub(crate) struct GspArgumentsPadded { } impl GspArgumentsPadded { - pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { + pub(crate) fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> { init!(GspArgumentsPadded { inner <- GspArgumentsCached::new(cmdq), ..Zeroable::init_zeroed() @@ -944,7 +944,7 @@ type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. - fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { + fn new<'a, 'b>(cmdq: &'a Cmdq<'b>) -> impl Init<Self> + use<'a, 'b> { init!(MessageQueueInitArguments { sharedMemPhysAddr: cmdq.dma_addr, pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 6dc31d1bf5ae..32856ff74183 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -131,6 +131,14 @@ impl GspStaticConfigInfo { self.0.gpuNameString } + /// Returns the BAR1 Page Directory Entry base address. + /// + /// This is the root page table address for BAR1 virtual memory, + /// set up by GSP-RM firmware. + pub(crate) fn bar1_pde_base(&self) -> u64 { + self.0.bar1PdeBase + } + /// Returns an iterator over valid FB regions from GSP firmware data. fn fb_regions( &self, @@ -165,6 +173,11 @@ impl GspStaticConfigInfo { } }) } + + /// Computes the exclusive end of the FB physical address space. + pub(crate) fn total_fb_end(&self) -> Option<u64> { + self.fb_regions().map(|reg| reg.limit).max()?.checked_add(1) + } } // SAFETY: Padding is explicit and will not contain uninitialized data. diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 5850fa0fe0e9..d8329f6fcc65 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -35,12 +35,12 @@ pub(super) trait GspHal: Send { /// /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one /// could be created. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result<Option<crate::gsp::UnloadBundle>>; + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result<Option<super::UnloadBundle<'gpu>>>; /// Performs HAL-specific post-GSP boot tasks. /// @@ -48,9 +48,9 @@ pub(super) trait GspHal: Send { /// after the initialization commands have been pushed onto its queue. fn post_boot( &self, - _gsp: &Gsp, + _gsp: &Gsp<'_>, _ctx: &mut GspBootContext<'_, '_>, - _gsp_fw: &GspFirmware, + _gsp_fw: &GspFirmware<'_>, ) -> Result { Ok(()) } diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index e283429a95dd..91201b51030e 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -58,7 +58,7 @@ impl GspMbox { fn lockdown_released_or_error( &self, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent<GspFmcBootParams>, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during // boot. If the address is still there, keep polling rather than treating it as an error. @@ -75,7 +75,7 @@ impl GspMbox { fn wait_for_gsp_lockdown_release( dev: &device::Device<device::Bound>, gsp_falcon: &Falcon<'_, GspEngine>, - fmc_boot_params: &Coherent<GspFmcBootParams>, + fmc_boot_params: &Coherent<'_, GspFmcBootParams>, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -141,12 +141,12 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result<Option<crate::gsp::UnloadBundle>> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> { let dev = ctx.dev(); let chipset = ctx.chipset; let gsp_falcon = ctx.gsp_falcon; @@ -159,7 +159,7 @@ impl GspHal for Gh100 { let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; let unload_bundle = crate::gsp::UnloadBundle( - KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle> + KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle + 'gpu> ); // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index a5c0ca355493..e90db1a23032 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -52,34 +52,33 @@ use crate::{ // // Since there are two variants of the prepared firmware (with and without a bootloader), this type // abstracts the difference. -enum FwsecUnloadFirmware { +enum FwsecUnloadFirmware<'a> { WithoutBl(FwsecFirmware), - WithBl(FwsecFirmwareWithBl), + WithBl(FwsecFirmwareWithBl<'a>), } -impl FwsecUnloadFirmware { +impl FwsecUnloadFirmware<'_> { /// Runs the FWSEC SB firmware. fn run( &self, dev: &device::Device<device::Bound>, - bar: Bar0<'_>, gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { match self { Self::WithoutBl(fw) => fw.run(dev, gsp_falcon), - Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithBl(fw) => fw.run(dev, gsp_falcon), } } } // Contains the firmware required to fully reset GSP on chipsets where the GSP is started using // FWSEC/Booter. -struct Sec2UnloadBundle { - fwsec_sb: FwsecUnloadFirmware, +struct Sec2UnloadBundle<'a> { + fwsec_sb: FwsecUnloadFirmware<'a>, booter_unloader: BooterFirmware, } -impl UnloadBundle for Sec2UnloadBundle { +impl UnloadBundle for Sec2UnloadBundle<'_> { fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { let dev = ctx.dev(); let bar = ctx.bar; @@ -88,7 +87,7 @@ impl UnloadBundle for Sec2UnloadBundle { // Log errors but keep going if it fails. let fwsec_sb_res = self .fwsec_sb - .run(dev, bar, ctx.gsp_falcon) + .run(dev, ctx.gsp_falcon) .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. @@ -168,7 +167,7 @@ impl Tu102 { if self.needs_fwsec_bootloader { let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; + fwsec_frts_bl.run(dev, falcon)?; } else { // Load and run FWSEC-FRTS directly. fwsec_frts.run(dev, falcon)?; @@ -213,14 +212,14 @@ impl Tu102 { } /// Load and prepare the resources required to properly reset the GSP after it has been stopped. - fn build_unload_bundle( + fn build_unload_bundle<'gpu>( &self, - dev: &device::Device<device::Bound>, + dev: &'gpu device::Device<device::Bound>, chipset: Chipset, bios: &Vbios, gsp_falcon: &Falcon<'_, GspEngine>, sec2_falcon: &Falcon<'_, Sec2>, - ) -> Result<crate::gsp::UnloadBundle> { + ) -> Result<crate::gsp::UnloadBundle<'gpu>> { // Load the FWSEC SB firmware, as well as its bootloader if required. let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; let fwsec_sb = if self.needs_fwsec_bootloader { @@ -241,18 +240,18 @@ impl Tu102 { }, GFP_KERNEL, ) - .map(|b| crate::gsp::UnloadBundle(b)) + .map(|b| crate::gsp::UnloadBundle(b as KBox<dyn UnloadBundle + 'gpu>)) .map_err(Into::into) } } impl GspHal for Tu102 { - fn boot( + fn boot<'gpu>( &self, - gsp: &Gsp, - ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, - ) -> Result<Option<crate::gsp::UnloadBundle>> { + gsp: &Gsp<'gpu>, + ctx: &mut GspBootContext<'_, 'gpu>, + gsp_fw: &GspFirmware<'gpu>, + ) -> Result<Option<crate::gsp::UnloadBundle<'gpu>>> { let dev = ctx.dev(); let bar = ctx.bar; let chipset = ctx.chipset; @@ -317,9 +316,9 @@ impl GspHal for Tu102 { fn post_boot( &self, - gsp: &Gsp, + gsp: &Gsp<'_>, ctx: &mut GspBootContext<'_, '_>, - gsp_fw: &GspFirmware, + gsp_fw: &GspFirmware<'_>, ) -> Result { GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?; diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs index 9a48aa87e7fb..3c410d65e8e4 100644 --- a/drivers/gpu/nova-core/gsp/regs.rs +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -2,11 +2,16 @@ use kernel::io::register; -use crate::regs::NV_PBUS_SW_SCRATCH; +use crate::{ + driver::NovaRegisters, + regs::NV_PBUS_SW_SCRATCH, // +}; // PGSP register! { + base: NovaRegisters; + pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { 31:0 address; } @@ -15,6 +20,8 @@ register! { // PBUS register! { + base: NovaRegisters; + /// Scratch register 0xe used as FRTS firmware error code. pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { 31:16 frts_err_code; diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index bcad1421953a..dae34c11eb05 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -138,7 +138,7 @@ pub(crate) struct GspSequencer<'a> { /// GSP falcon for core operations. gsp_falcon: &'a Falcon<'a, Gsp>, /// LibOS memory region init arguments. - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. @@ -338,9 +338,9 @@ impl<'a> Iterator for GspSeqIter<'a> { impl<'a> GspSequencer<'a> { pub(crate) fn run( - cmdq: &Cmdq, + cmdq: &Cmdq<'_>, ctx: &'a GspBootContext<'_, '_>, - libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + libos: &'a Coherent<'a, [LibosMemoryRegionInitArgument]>, bootloader_app_version: u32, ) -> Result { let seq_info = loop { diff --git a/drivers/gpu/nova-core/mm.rs b/drivers/gpu/nova-core/mm.rs new file mode 100644 index 000000000000..a5bc4042577b --- /dev/null +++ b/drivers/gpu/nova-core/mm.rs @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management subsystems. + +#![cfg_attr(not(CONFIG_NOVA_CORE_SELFTESTS), expect(dead_code))] + +/// Implements `From` conversions between a frame-number type and `Bounded<u64, N>`. +/// +/// Each MMU version module should invoke this for the specific bit widths used by that version's +/// PTE/PDE bitfield definitions. +macro_rules! impl_frame_number_bounded { + ($type:ty, $bits:literal) => { + impl From<Bounded<u64, $bits>> for $type { + fn from(val: Bounded<u64, $bits>) -> Self { + Self::new(val.get()) + } + } + + impl From<$type> for Bounded<u64, $bits> { + fn from(v: $type) -> Self { + Bounded::from_expr(v.raw() & ::kernel::bits::genmask_u64(0..=($bits - 1))) + } + } + }; +} + +/// Implements `From` conversions between [`Pfn`] and `Bounded<u64, N>` for bitfield interop. +macro_rules! impl_pfn_bounded { + ($bits:literal) => { + impl_frame_number_bounded!(Pfn, $bits); + }; +} + +use core::{ + fmt::LowerHex, + ops, // +}; + +use kernel::{ + bitfield, + fmt, + gpu::buddy::{ + GpuBuddy, + GpuBuddyParams, // + }, + num::Bounded, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::SZ_4K, // +}; + +use crate::{ + driver::Bar0, + gpu::Chipset, // +}; + +pub(crate) use tlb::Tlb; + +pub(crate) mod bar_user; +mod hal; +pub(super) mod pagetable; +mod pramin; +mod regs; +pub(super) mod tlb; +pub(super) mod vmm; + +/// GPU Memory Manager - owns all core MM components. +/// +/// Provides centralized ownership of memory management resources: +/// - [`GpuBuddy`] allocator for VRAM page table allocation. +/// - [`pramin::Pramin`] for direct VRAM access. +/// - [`Tlb`] manager for translation buffer flush operations. +pub(crate) struct GpuMm<'gpu> { + buddy: GpuBuddy, + pramin: pramin::Pramin<'gpu>, + tlb: Pin<KBox<Tlb<'gpu>>>, +} + +impl<'gpu> GpuMm<'gpu> { + /// Creates the GPU memory manager. + pub(crate) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + buddy_params: GpuBuddyParams, + total_fb_end: VramAddress, + ) -> Result<Self> { + // PRAMIN covers all physical VRAM (including GSP-reserved areas + // above the usable region, e.g. the BAR1 page directory). + let vram_region = VramAddress::ZERO..total_fb_end; + + Ok(Self { + buddy: GpuBuddy::new(buddy_params)?, + pramin: pramin::Pramin::new(bar, chipset, vram_region)?, + tlb: KBox::pin_init(Tlb::new(bar), GFP_KERNEL)?, + }) + } + + /// Access the [`GpuBuddy`] allocator. + pub(crate) fn buddy(&self) -> &GpuBuddy { + &self.buddy + } + + /// Access the [`pramin::Pramin`]. + fn pramin_mut(&mut self) -> &mut pramin::Pramin<'gpu> { + &mut self.pramin + } + + /// Access the [`Tlb`] manager. + pub(crate) fn tlb(&self) -> &Tlb<'gpu> { + self.tlb.as_ref().get_ref() + } +} + +/// Page size in bytes (4 KiB). +pub(crate) const PAGE_SIZE: usize = SZ_4K; + +/// Physical VRAM address in GPU video memory. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[repr(transparent)] +pub(crate) struct VramAddress(u64); + +impl VramAddress { + /// The zero address. + pub(crate) const ZERO: Self = Self::from_raw(0); + + /// Creates an address from a raw value. + pub(crate) const fn from_raw(addr: u64) -> Self { + Self(addr) + } + + /// Returns the address as a raw value. + pub(crate) const fn into_raw(self) -> u64 { + self.0 + } + + /// Adds `rhs` to this address, returning [`None`] on overflow. + pub(crate) const fn checked_add(self, rhs: u64) -> Option<Self> { + match self.into_raw().checked_add(rhs) { + Some(addr) => Some(Self::from_raw(addr)), + None => None, + } + } +} + +impl Alignable for VramAddress { + fn align_down(self, alignment: Alignment) -> Self { + Self::from_raw(self.into_raw().align_down(alignment)) + } + + fn align_up(self, alignment: Alignment) -> Option<Self> { + self.into_raw().align_up(alignment).map(Self::from_raw) + } +} + +impl LowerHex for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + LowerHex::fmt(&self.into_raw(), f) + } +} + +impl fmt::Debug for VramAddress { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_fmt(fmt!("{:#x}", self)) + } +} + +impl ops::Add<u64> for VramAddress { + type Output = Self; + + fn add(self, rhs: u64) -> Self::Output { + Self::from_raw(self.into_raw() + rhs) + } +} + +impl ops::Sub for VramAddress { + type Output = u64; + + fn sub(self, rhs: Self) -> Self::Output { + self.into_raw() - rhs.into_raw() + } +} + +impl From<Pfn> for VramAddress { + fn from(pfn: Pfn) -> Self { + Self::from_raw(pfn.raw() << 12) + } +} + +bitfield! { + /// Virtual address in GPU address space. + pub(crate) struct VirtualAddress(u64) { + /// Offset within 4KB page. + 11:0 offset; + /// Virtual frame number. + 63:12 frame_number => Vfn; + } +} + +impl VirtualAddress { + /// Create a new virtual address from a raw value. + #[expect(dead_code)] + pub(crate) const fn new(addr: u64) -> Self { + Self::from_raw(addr) + } +} + +impl From<Vfn> for VirtualAddress { + fn from(vfn: Vfn) -> Self { + Self::zeroed().with_frame_number(vfn) + } +} + +/// Physical Frame Number. +/// +/// Represents a physical page in VRAM. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Pfn(u64); + +impl Pfn { + /// Create a new PFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From<VramAddress> for Pfn { + fn from(addr: VramAddress) -> Self { + Self::new(addr.into_raw() >> 12) + } +} + +impl From<u64> for Pfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From<Pfn> for u64 { + fn from(pfn: Pfn) -> Self { + pfn.0 + } +} + +impl_pfn_bounded!(52); + +/// Virtual Frame Number. +/// +/// Represents a virtual page in GPU address space. +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(crate) struct Vfn(u64); + +impl Vfn { + /// Create a new VFN from a frame number. + pub(crate) const fn new(frame_number: u64) -> Self { + Self(frame_number) + } + + /// Get the raw frame number. + pub(crate) const fn raw(self) -> u64 { + self.0 + } +} + +impl From<VirtualAddress> for Vfn { + fn from(addr: VirtualAddress) -> Self { + addr.frame_number() + } +} + +impl From<u64> for Vfn { + fn from(val: u64) -> Self { + Self(val) + } +} + +impl From<Vfn> for u64 { + fn from(vfn: Vfn) -> Self { + vfn.0 + } +} + +impl_frame_number_bounded!(Vfn, 52); + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) mod selftest { + use core::ops::Range; + + use kernel::{ + device, + sizes::SizeConstants, + sync::Arc, // + }; + + use super::*; + + /// Run MM subsystem self-tests during probe. + pub(crate) fn run( + dev: &device::Device<device::Bound>, + mm: &mut GpuMm<'_>, + usable_fb_regions: &[Range<u64>], + bar_user: &Arc<bar_user::BarUser<'_>>, + bar1_pdb: u64, + chipset: Chipset, + ) -> Result { + // VRAM span the self-tests are free to overwrite, from the chosen test base. + const SELFTEST_SPAN: u64 = u64::SZ_64M; + + let base = usable_fb_regions.iter().find_map(|region| { + // Tests rely on this being 8 byte aligned for checking misalignment handling. + let base = region.start.align_up(Alignment::new::<8>())?; + (base.checked_add(SELFTEST_SPAN)? <= region.end).then_some(base) + }); + let Some(base) = base else { + dev_warn!( + dev, + "PRAMIN: skipping self-tests, no usable VRAM region of {:#x} bytes\n", + SELFTEST_SPAN + ); + return Ok(()); + }; + + pramin::selftest::run(dev, mm.pramin_mut(), VramAddress::from_raw(base))?; + bar_user::run_self_test(dev, mm, bar_user, bar1_pdb, chipset) + } +} diff --git a/drivers/gpu/nova-core/mm/bar_user.rs b/drivers/gpu/nova-core/mm/bar_user.rs new file mode 100644 index 000000000000..8f4a27c1fd14 --- /dev/null +++ b/drivers/gpu/nova-core/mm/bar_user.rs @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! BAR1 user interface for CPU access to GPU virtual memory. Used for USERD +//! for GPU work submission, and applications to access GPU buffers via mmap(). + +use kernel::{ + io::Io, + new_mutex, + prelude::*, + sync::{ + Arc, + Mutex, // + }, +}; + +use crate::{ + driver::Bar1, + gpu::Chipset, + mm::{ + vmm::{ + MappedRange, + Vmm, // + }, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, + PAGE_SIZE, // + }, + num::IntoSafeCast, +}; + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +use kernel::device; + +/// BAR1 user interface for virtual memory mappings. +/// +/// Owns the [`Vmm`] for the BAR1 address space. +#[pin_data] +pub(crate) struct BarUser<'gpu> { + #[pin] + vmm: Mutex<Vmm>, + bar1: &'gpu Bar1<'gpu>, +} + +impl<'gpu> BarUser<'gpu> { + /// Create a pin-initializer for [`BarUser`]. + pub(crate) fn new( + pdb_addr: VramAddress, + chipset: Chipset, + va_size: u64, + bar1: &'gpu Bar1<'gpu>, + ) -> Result<impl PinInit<Self> + 'gpu> { + let vmm = Vmm::new(pdb_addr, chipset.mmu_version(), va_size)?; + Ok(pin_init!(Self { + vmm <- new_mutex!(vmm, "bar_user_vmm"), + bar1, + })) + } + + /// Map physical pages to a contiguous BAR1 virtual range. + pub(crate) fn map( + self: &Arc<Self>, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + writable: bool, + ) -> Result<BarUserAccess<'gpu>> { + if pfns.is_empty() { + return Err(EINVAL); + } + let mut vmm = self.vmm.lock(); + let mapped = vmm.map_pages(mm, pfns, None, writable)?; + + Ok(BarUserAccess { + bar_user: self.clone(), + mapped: Some(mapped), + }) + } +} + +/// Access object for a mapped BAR1 region. +pub(crate) struct BarUserAccess<'gpu> { + bar_user: Arc<BarUser<'gpu>>, + /// [`BarUserAccess::release`] [`Option::take`]s this; `Some` at + /// drop time means `release()` was never called. + mapped: Option<MappedRange>, +} + +#[expect(dead_code)] +impl BarUserAccess<'_> { + /// Tear down the BAR1 mapping. + pub(crate) fn release(mut self, mm: &mut GpuMm<'_>) -> Result { + let mapped = self.mapped.take().ok_or(EINVAL)?; + let mut vmm = self.bar_user.vmm.lock(); + vmm.unmap_pages(mm, mapped)?; + Ok(()) + } + + /// Returns the active mapping. + fn mapped(&self) -> &MappedRange { + // `mapped` is only `None` after `take()` in `release`; hence unwrap() + // cannot panic here. + self.mapped.as_ref().unwrap() + } + + /// Get the base virtual address of this mapping. + pub(crate) fn base(&self) -> VirtualAddress { + VirtualAddress::from(self.mapped().vfn_start) + } + + /// Get the total size of the mapped region in bytes. + pub(crate) fn size(&self) -> usize { + self.mapped().num_pages * PAGE_SIZE + } + + /// Get the starting virtual frame number. + pub(crate) fn vfn_start(&self) -> Vfn { + self.mapped().vfn_start + } + + /// Get the number of pages in this mapping. + pub(crate) fn num_pages(&self) -> usize { + self.mapped().num_pages + } + + /// Translate an offset within this mapping to a BAR1 aperture offset. + fn bar_offset(&self, offset: usize) -> Result<usize> { + if offset >= self.size() { + return Err(EINVAL); + } + + let base_vfn: usize = self.mapped().vfn_start.raw().into_safe_cast(); + let base = base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + base.checked_add(offset).ok_or(EOVERFLOW) + } + + // Fallible accessors with runtime bounds checking. + + /// Read a 32-bit value at the given offset. + pub(crate) fn try_read32(&self, offset: usize) -> Result<u32> { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read32(off) + } + + /// Write a 32-bit value at the given offset. + pub(crate) fn try_write32(&self, value: u32, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write32(value, off) + } + + /// Read a 64-bit value at the given offset. + pub(crate) fn try_read64(&self, offset: usize) -> Result<u64> { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_read64(off) + } + + /// Write a 64-bit value at the given offset. + pub(crate) fn try_write64(&self, value: u64, offset: usize) -> Result { + let off = self.bar_offset(offset)?; + self.bar_user.bar1.try_write64(value, off) + } +} + +impl Drop for BarUserAccess<'_> { + fn drop(&mut self) { + if self.mapped.is_some() { + kernel::pr_warn!( + "BarUserAccess dropped without calling release(). BarUser address space will leak.\n" + ); + } + // The inner `MappedRange`'s own `MustUnmapGuard` will also fire, + // identifying the leaked VA range. + } +} + +/// Run MM subsystem self-tests during probe. +/// +/// Tests page table infrastructure and `BAR1` MMIO access using the `BAR1` +/// address space. Uses the `GpuMm`'s buddy allocator to allocate page tables +/// and test pages as needed. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(crate) fn run_self_test( + dev: &device::Device<device::Bound>, + mm: &mut GpuMm<'_>, + bar_user: &Arc<BarUser<'_>>, + bar1_pdb: u64, + chipset: Chipset, +) -> Result { + use kernel::{ + gpu::buddy::{ + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + ptr::Alignment, + sizes::{ + SZ_16K, + SZ_32K, + SZ_4K, + SZ_64K, // + }, + }; + + // Test patterns. + const PATTERN_PRAMIN: u32 = 0xDEAD_BEEF; + const PATTERN_BAR1: u32 = 0xCAFE_BABE; + + let bar1 = bar_user.bar1; + dev_info!(dev, "MM: Starting self-test...\n"); + + let pdb_addr = VramAddress::from_raw(bar1_pdb); + + // Check if initial page tables are in VRAM. + if crate::mm::pagetable::check_pdb_valid(mm.pramin_mut(), pdb_addr, chipset).is_err() { + dev_info!(dev, "MM: Self-test SKIPPED - no valid VRAM page tables\n"); + return Ok(()); + } + + // Set up a test page from the buddy allocator. + let test_page_blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::<SZ_4K>(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + let test_vram_offset = test_page_blocks.iter().next().ok_or(ENOMEM)?.offset(); + let test_vram = VramAddress::from_raw(test_vram_offset); + let test_pfn = Pfn::from(test_vram); + + // Create a VMM of size 64K to track virtual memory mappings. + let mut vmm = Vmm::new(pdb_addr, chipset.mmu_version(), SZ_64K.into_safe_cast())?; + + // Create a test mapping. + let mapped = vmm.map_pages(mm, &[test_pfn], None, true)?; + let test_vfn = mapped.vfn_start; + + // Pre-compute test addresses for the PRAMIN to BAR1 read test. + let vfn_offset: usize = test_vfn.raw().into_safe_cast(); + let bar1_base_offset = vfn_offset.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + let bar1_read_offset: usize = bar1_base_offset + 0x100; + let vram_read_addr = test_vram + 0x100; + + // Test 1: Write via PRAMIN, read via BAR1. + mm.pramin_mut() + .window_at::<u32>(vram_read_addr)? + .view() + .write_val(PATTERN_PRAMIN); + + // Read back via BAR1 aperture. + let bar1_value = bar1.try_read32(bar1_read_offset)?; + + let test1_passed = if bar1_value == PATTERN_PRAMIN { + true + } else { + dev_err!( + dev, + "MM: Test 1 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_PRAMIN, + bar1_value + ); + false + }; + + // Cleanup - invalidate PTE. + vmm.unmap_pages(mm, mapped)?; + + // Test 2: Two-phase prepare/execute API. + let prepared = vmm.prepare_map(mm, 1, None)?; + let mapped2 = vmm.execute_map(mm, prepared, &[test_pfn], true)?; + let readback = vmm.read_mapping(mm, mapped2.vfn_start)?; + let test2_passed = if readback == Some(test_pfn) { + true + } else { + dev_err!(dev, "MM: Test 2 FAILED - Two-phase map readback mismatch\n"); + false + }; + vmm.unmap_pages(mm, mapped2)?; + + // Test 3: Range-constrained allocation with a hole — exercises block.size()-driven + // BAR1 mapping. A 4K hole is punched at base+16K, then a single 32K allocation + // is requested within [base, base+36K). The buddy allocator must split around the + // hole, returning multiple blocks (expected: {16K, 4K, 8K, 4K} = 32K total). + // Each block is mapped into BAR1 and verified via PRAMIN read-back. + // + // Address layout (base = 0x10000): + // [ 16K ] [HOLE 4K] [4K] [ 8K ] [4K] + // 0x10000 0x14000 0x15000 0x16000 0x18000 0x19000 + let range_base: u64 = SZ_64K.into_safe_cast(); + let sz_4k: u64 = SZ_4K.into_safe_cast(); + let sz_16k: u64 = SZ_16K.into_safe_cast(); + let sz_32k_4k: u64 = (SZ_32K + SZ_4K).into_safe_cast(); + + // Punch a 4K hole at base+16K so the subsequent 32K allocation must split. + let _hole = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base + sz_16k..range_base + sz_16k + sz_4k), + SZ_4K.into_safe_cast(), + Alignment::new::<SZ_4K>(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + // Allocate 32K within [base, base+36K). The hole forces the allocator to return + // split blocks whose sizes are determined by buddy alignment. + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Range(range_base..range_base + sz_32k_4k), + SZ_32K.into_safe_cast(), + Alignment::new::<SZ_4K>(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let mut test3_passed = true; + let mut total_size = 0usize; + + for block in blocks.iter() { + total_size += IntoSafeCast::<usize>::into_safe_cast(block.size()); + + // Map all pages of this block. + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let num_pages: usize = (block.size() / page_size).into_safe_cast(); + + let mut pfns = KVec::new(); + for j in 0..num_pages { + let j_u64: u64 = j.into_safe_cast(); + pfns.push( + Pfn::from(VramAddress::from_raw( + block.offset() + j_u64.checked_mul(page_size).ok_or(EOVERFLOW)?, + )), + GFP_KERNEL, + )?; + } + + let mapped = vmm.map_pages(mm, &pfns, None, true)?; + let bar1_base_vfn: usize = mapped.vfn_start.raw().into_safe_cast(); + let bar1_base = bar1_base_vfn.checked_mul(PAGE_SIZE).ok_or(EOVERFLOW)?; + + for j in 0..num_pages { + let page_bar1_off = bar1_base + j * PAGE_SIZE; + let j_u64: u64 = j.into_safe_cast(); + let page_phys = block.offset() + + j_u64 + .checked_mul(PAGE_SIZE.into_safe_cast()) + .ok_or(EOVERFLOW)?; + + bar1.try_write32(PATTERN_BAR1, page_bar1_off)?; + + let pramin_val = mm + .pramin_mut() + .window_at::<u32>(VramAddress::from_raw(page_phys))? + .view() + .read_val(); + + if pramin_val != PATTERN_BAR1 { + dev_err!( + dev, + "MM: Test 3 FAILED block offset {:#x} page {} (val={:#x})\n", + block.offset(), + j, + pramin_val + ); + test3_passed = false; + } + } + + vmm.unmap_pages(mm, mapped)?; + } + + // Verify aggregate: all returned block sizes must sum to allocation size. + if total_size != SZ_32K { + dev_err!( + dev, + "MM: Test 3 FAILED - total size {} != expected {}\n", + total_size, + SZ_32K + ); + test3_passed = false; + } + + // Release Tests 1-3's Vmm before Test 4 constructs a fresh BarUser on + // the same PDB. + drop(vmm); + + // Test 4: Exercise `BarUser::map()` end-to-end. + let bar_user = Arc::pin_init( + BarUser::new(pdb_addr, chipset, SZ_64K.into_safe_cast(), bar1)?, + GFP_KERNEL, + )?; + let access = bar_user.map(mm, &[test_pfn], true)?; + + // Write pattern via PRAMIN, read via BarUserAccess. + mm.pramin_mut() + .window_at::<u32>(test_vram)? + .view() + .write_val(PATTERN_BAR1); + + let readback = access.try_read32(0)?; + let test4_passed = if readback == PATTERN_BAR1 { + true + } else { + dev_err!( + dev, + "MM: Test 4 FAILED - Expected {:#010x}, got {:#010x}\n", + PATTERN_BAR1, + readback + ); + false + }; + access.release(mm)?; + + if test1_passed && test2_passed && test3_passed && test4_passed { + dev_info!(dev, "MM: All self-tests PASSED\n"); + Ok(()) + } else { + dev_err!(dev, "MM: Self-tests FAILED\n"); + Err(EIO) + } +} diff --git a/drivers/gpu/nova-core/mm/hal.rs b/drivers/gpu/nova-core/mm/hal.rs new file mode 100644 index 000000000000..e7fd1e38bd38 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Memory management HAL. + +use kernel::{ + num::Bounded, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + gpu::{ + Architecture, + Chipset, // + }, + mm::VramAddress, // +}; + +mod gb100; +mod gh100; +mod tu102; + +/// Trait implemented by per-architecture MM HALs. +/// +/// `Sync` is required so that the `&'static dyn MmHal` references can be stored in `Send` +/// structures. +pub(super) trait MmHal: Sync { + /// Positions the PRAMIN window at `base`. + /// + /// This fails if `base` is not aligned to the 64 KiB window alignment or is too large for + /// the receiving register. + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result; +} + +/// Returns the HAL corresponding to `chipset`. +pub(super) fn mm_hal(chipset: Chipset) -> &'static dyn MmHal { + match chipset.arch() { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Hopper => gh100::GH100_HAL, + Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => gb100::GB100_HAL, + } +} + +/// Converts `base` into the value of the window-base register field. +/// +/// Fails with [`EINVAL`] if `base` is not aligned to the window alignment required by the register +/// field's shift, or if the shifted value does not fit within `RES` bits. +fn window_base<const RES: u32>(base: VramAddress) -> Result<Bounded<u64, RES>> { + const WINDOW_BASE_SHIFT: u32 = 16; + + Bounded::<u64, 64>::from(base.into_raw()) + .shr_exact::<WINDOW_BASE_SHIFT, { 64 - WINDOW_BASE_SHIFT }>() + .and_then(Bounded::try_shrink) + .ok_or(EINVAL) +} diff --git a/drivers/gpu/nova-core/mm/hal/gb100.rs b/drivers/gpu/nova-core/mm/hal/gb100.rs new file mode 100644 index 000000000000..3781e143dea7 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gb100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Blackwell GB10x/GB20x memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gb100; + +impl MmHal for Gb100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gb100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GB100: Gb100 = Gb100; +pub(super) const GB100_HAL: &dyn MmHal = &GB100; diff --git a/drivers/gpu/nova-core/mm/hal/gh100.rs b/drivers/gpu/nova-core/mm/hal/gh100.rs new file mode 100644 index 000000000000..8af384db2921 --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/gh100.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Hopper memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Gh100; + +impl MmHal for Gh100 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::gh100::NV_XAL_EP_BAR0_WINDOW::zeroed().with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const GH100: Gh100 = Gh100; +pub(super) const GH100_HAL: &dyn MmHal = &GH100; diff --git a/drivers/gpu/nova-core/mm/hal/tu102.rs b/drivers/gpu/nova-core/mm/hal/tu102.rs new file mode 100644 index 000000000000..e4fe7561223c --- /dev/null +++ b/drivers/gpu/nova-core/mm/hal/tu102.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Turing, Ampere and Ada memory management HAL. + +use kernel::{ + io::Io, + prelude::*, // +}; + +use crate::{ + driver::Bar0, + mm::{ + hal::{ + window_base, + MmHal, // + }, + regs, + VramAddress, // + }, +}; + +struct Tu102; + +impl MmHal for Tu102 { + fn write_pramin_window_base(&self, bar: Bar0<'_>, base: VramAddress) -> Result { + bar.write_reg( + regs::NV_PBUS_BAR0_WINDOW::zeroed() + .with_target(regs::Bar0WindowTarget::VidMem) + .with_base(window_base(base)?.cast()), + ); + Ok(()) + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn MmHal = &TU102; diff --git a/drivers/gpu/nova-core/mm/pagetable.rs b/drivers/gpu/nova-core/mm/pagetable.rs new file mode 100644 index 000000000000..63a7e1855caa --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable.rs @@ -0,0 +1,424 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Common page table types shared between MMU v2 and v3. +//! +//! This module provides foundational types used by both MMU versions: +//! - Page table level hierarchy +//! - Memory aperture types for PDEs and PTEs + +#![expect(dead_code)] + +pub(super) mod map; +pub(super) mod ver2; +pub(super) mod ver3; +pub(super) mod walk; + +use kernel::{ + io::Io, + num::Bounded, + prelude::*, // +}; + +use crate::{ + gpu::Architecture, + mm::{ + pramin, + Pfn, + VirtualAddress, + VramAddress, // + }, +}; + +/// Extracts the page table index at a given level from a virtual address. +pub(super) trait VaLevelIndex { + /// Return the page table index at `level` for this virtual address. + fn level_index(&self, level: u64) -> u64; +} + +/// MMU version enumeration. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MmuVersion { + /// MMU v2 for Turing/Ampere/Ada. + V2, + /// MMU v3 for Hopper and later. + V3, +} + +impl From<Architecture> for MmuVersion { + fn from(arch: Architecture) -> Self { + match arch { + Architecture::Turing | Architecture::Ampere | Architecture::Ada => Self::V2, + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + Self::V3 + } + } + } +} + +/// Page Table Level hierarchy for MMU v2/v3. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum PageTableLevel { + /// Level 0 - Page Directory Base (root). + Pdb, + /// Level 1 - Intermediate page directory. + L1, + /// Level 2 - Intermediate page directory. + L2, + /// Level 3 - Intermediate page directory or dual PDE (version-dependent). + L3, + /// Level 4 - PTE level for v2, intermediate page directory for v3. + L4, + /// Level 5 - PTE level used for MMU v3 only. + L5, +} + +impl PageTableLevel { + /// Number of entries per page table (512 for 4KB pages). + pub(super) const ENTRIES_PER_TABLE: usize = 512; + + /// Get the next level in the hierarchy. + pub(super) const fn next(&self) -> Option<PageTableLevel> { + match self { + Self::Pdb => Some(Self::L1), + Self::L1 => Some(Self::L2), + Self::L2 => Some(Self::L3), + Self::L3 => Some(Self::L4), + Self::L4 => Some(Self::L5), + Self::L5 => None, + } + } + + /// Convert level to index. + pub(super) const fn as_index(&self) -> u64 { + match self { + Self::Pdb => 0, + Self::L1 => 1, + Self::L2 => 2, + Self::L3 => 3, + Self::L4 => 4, + Self::L5 => 5, + } + } +} + +// Trait abstractions for page table operations. + +/// Operations on Page Table Entries (`PTE`s). +pub(super) trait PteOps: Copy + core::fmt::Debug + Into<u64> { + /// Create a `PTE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create an invalid `PTE`. + fn invalid() -> Self; + + /// Create a valid `PTE` for the given memory aperture. + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self; + + /// Check if this `PTE` is valid. + fn is_valid(&self) -> bool; + + /// Get the physical frame number. + fn frame_number(&self) -> Pfn; + + /// Read a `PTE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> { + let val = pramin.window_at::<u64>(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PTE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::<u64>(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } +} + +/// Operations on Page Directory Entries (`PDE`s). +pub(super) trait PdeOps: Copy + core::fmt::Debug + Into<u64> { + /// Create a `PDE` from a raw `u64` value. + fn from_raw(val: u64) -> Self; + + /// Create a valid `PDE` pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self; + + /// Create an invalid `PDE`. + fn invalid() -> Self; + + /// Check if this `PDE` is valid. + fn is_valid(&self) -> bool; + + /// Get the memory aperture of this `PDE`. + fn aperture(&self) -> AperturePde; + + /// Get the VRAM address of the page table. + fn table_vram_address(&self) -> VramAddress; + + /// Read a `PDE` from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> { + let val = pramin.window_at::<u64>(addr)?.view().read_val(); + Ok(Self::from_raw(val)) + } + + /// Write this `PDE` to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::<u64>(addr)? + .view() + .write_val((*self).into()); + Ok(()) + } + + /// Check if this `PDE` is valid and points to video memory. + fn is_valid_vram(&self) -> bool { + self.is_valid() && self.aperture() == AperturePde::VideoMemory + } +} + +/// Operations on Dual Page Directory Entries (128-bit `DualPde`s). +pub(super) trait DualPdeOps: Copy + core::fmt::Debug { + /// Create a `DualPde` from raw 128-bit value (two `u64`s). + fn from_raw(big: u64, small: u64) -> Self; + + /// Create a `DualPde` with only the small page table pointer set. + fn new_small(table_pfn: Pfn) -> Self; + + /// Check if the small page table pointer is valid. + fn has_small(&self) -> bool; + + /// Get the small page table VRAM address. + fn small_vram_address(&self) -> VramAddress; + + /// Get the raw `u64` value of the big PDE. + fn big_raw_u64(&self) -> u64; + + /// Get the raw `u64` value of the small PDE. + fn small_raw_u64(&self) -> u64; + + /// Read a dual PDE (128-bit) from VRAM. + fn read(pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result<Self> { + let lo = pramin.window_at::<u64>(addr)?.view().read_val(); + let hi = pramin.window_at::<u64>(addr + 8)?.view().read_val(); + Ok(Self::from_raw(lo, hi)) + } + + /// Write this dual PDE (128-bit) to VRAM. + fn write(&self, pramin: &mut pramin::Pramin<'_>, addr: VramAddress) -> Result { + pramin + .window_at::<u64>(addr)? + .view() + .write_val(self.big_raw_u64()); + pramin + .window_at::<u64>(addr + 8)? + .view() + .write_val(self.small_raw_u64()); + Ok(()) + } +} + +/// MMU configuration trait -- encodes version-specific constants and types. +pub(super) trait MmuConfig: 'static { + /// Page Table Entry type. + type Pte: PteOps; + /// Page Directory Entry type. + type Pde: PdeOps; + /// Dual Page Directory Entry type (128-bit). + type DualPde: DualPdeOps; + + /// PDE levels (excluding PTE level) for page table walking. + const PDE_LEVELS: &'static [PageTableLevel]; + /// PTE level for this MMU version. + const PTE_LEVEL: PageTableLevel; + /// Dual PDE level (128-bit entries) for this MMU version. + const DUAL_PDE_LEVEL: PageTableLevel; + + /// Get the number of entries per page table page for a given level. + fn entries_per_page(level: PageTableLevel) -> usize; + + /// Extract the page table index at `level` from `va`. + fn level_index(va: VirtualAddress, level: u64) -> u64; + + /// Get the entry size in bytes for a given level. + fn entry_size(level: PageTableLevel) -> usize { + if level == Self::DUAL_PDE_LEVEL { + 16 // 128-bit dual PDE + } else { + 8 // 64-bit PDE/PTE + } + } + + /// Compute upper bound on page table pages needed for `num_virt_pages`. + /// + /// Walks from PTE level up through PDE levels, accumulating the tree. + fn pt_pages_upper_bound(num_virt_pages: usize) -> usize { + let mut total = 0; + + // PTE pages at the leaf level. + let pte_epp = Self::entries_per_page(Self::PTE_LEVEL); + let mut pages_at_level = num_virt_pages.div_ceil(pte_epp); + total += pages_at_level; + + // Walk PDE levels bottom-up (reverse of PDE_LEVELS). + for &level in Self::PDE_LEVELS.iter().rev() { + let epp = Self::entries_per_page(level); + + // How many pages at this level do we need to point to + // the previous pages_at_level? + pages_at_level = pages_at_level.div_ceil(epp); + total += pages_at_level; + } + + total + } +} + +/// Marker struct for MMU v2 (Turing/Ampere/Ada). +pub(super) struct MmuV2; + +impl MmuConfig for MmuV2 { + type Pte = ver2::Pte; + type Pde = ver2::Pde; + type DualPde = ver2::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver2::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver2::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver2::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + // TODO: Calculate these values from the bitfield dynamically + // instead of hardcoding them. + match level { + PageTableLevel::Pdb => 4, // PD3 root: bits [48:47] = 2 bits + PageTableLevel::L3 => 256, // PD0 dual: bits [28:21] = 8 bits + _ => 512, // PD2, PD1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver2::VirtualAddressV2::new(va).level_index(level) + } +} + +/// Marker struct for MMU v3 (Hopper and later). +pub(super) struct MmuV3; + +impl MmuConfig for MmuV3 { + type Pte = ver3::Pte; + type Pde = ver3::Pde; + type DualPde = ver3::DualPde; + + const PDE_LEVELS: &'static [PageTableLevel] = ver3::PDE_LEVELS; + const PTE_LEVEL: PageTableLevel = ver3::PTE_LEVEL; + const DUAL_PDE_LEVEL: PageTableLevel = ver3::DUAL_PDE_LEVEL; + + fn entries_per_page(level: PageTableLevel) -> usize { + match level { + PageTableLevel::Pdb => 2, // PDE4 root: bit [56] = 1 bit, 2 entries + PageTableLevel::L4 => 256, // PDE0 dual: bits [28:21] = 8 bits + _ => 512, // PDE3, PDE2, PDE1, PT: 9 bits each + } + } + + fn level_index(va: VirtualAddress, level: u64) -> u64 { + ver3::VirtualAddressV3::new(va).level_index(level) + } +} + +/// Memory aperture for Page Table Entries (`PTE`s). +/// +/// Determines which memory region the `PTE` points to. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePte { + /// Local video memory (VRAM). + #[default] + VideoMemory = 0, + /// Peer GPU's video memory. + PeerMemory = 1, + /// System memory with cache coherence. + SystemCoherent = 2, + /// System memory without cache coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From<Bounded<u64, 2>> for AperturePte { + fn from(val: Bounded<u64, 2>) -> Self { + match *val { + 0 => Self::VideoMemory, + 1 => Self::PeerMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::VideoMemory, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From<AperturePte> for Bounded<u64, 2> { + fn from(val: AperturePte) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} + +/// Memory aperture for Page Directory Entries (`PDE`s). +/// +/// Note: For `PDE`s, `Invalid` (0) means the entry is not valid. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub(super) enum AperturePde { + /// Invalid/unused entry. + #[default] + Invalid = 0, + /// Page table is in video memory. + VideoMemory = 1, + /// Page table is in system memory with coherence. + SystemCoherent = 2, + /// Page table is in system memory without coherence. + SystemNonCoherent = 3, +} + +// TODO[FPRI]: Replace with `#[derive(FromPrimitive)]` when available. +impl From<Bounded<u64, 2>> for AperturePde { + fn from(val: Bounded<u64, 2>) -> Self { + match *val { + 1 => Self::VideoMemory, + 2 => Self::SystemCoherent, + 3 => Self::SystemNonCoherent, + _ => Self::Invalid, + } + } +} + +// TODO[FPRI]: Replace with `#[derive(ToPrimitive)]` when available. +impl From<AperturePde> for Bounded<u64, 2> { + fn from(val: AperturePde) -> Self { + Bounded::from_expr(val as u64 & 0x3) + } +} + +/// Check if the PDB has valid, VRAM-backed page tables. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +fn check_pdb_inner<M: MmuConfig>(pramin: &mut pramin::Pramin<'_>, pdb_addr: VramAddress) -> Result { + let raw = pramin.window_at::<u64>(pdb_addr)?.view().read_val(); + + if !M::Pde::from_raw(raw).is_valid_vram() { + return Err(ENOENT); + } + Ok(()) +} + +/// Check if the PDB has valid, VRAM-backed page tables, dispatching by MMU version. +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) fn check_pdb_valid( + pramin: &mut pramin::Pramin<'_>, + pdb_addr: VramAddress, + chipset: crate::gpu::Chipset, +) -> Result { + match MmuVersion::from(chipset.arch()) { + MmuVersion::V2 => check_pdb_inner::<MmuV2>(pramin, pdb_addr), + MmuVersion::V3 => check_pdb_inner::<MmuV3>(pramin, pdb_addr), + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable/map.rs b/drivers/gpu/nova-core/mm/pagetable/map.rs new file mode 100644 index 000000000000..77431c509a89 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/map.rs @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table mapping operations for NVIDIA GPUs. + +use core::marker::PhantomData; + +use kernel::{ + gpu::buddy::{ + AllocatedBlocks, + GpuBuddyAllocFlags, + GpuBuddyAllocMode, // + }, + io::io_write, + prelude::*, + ptr::Alignment, + rbtree::{ + RBTree, + RBTreeNode, // + }, + sizes::SZ_4K, // +}; + +use super::{ + walk::{ + PtWalkInner, + WalkPdeResult, + WalkResult, // + }, + AperturePde, + AperturePte, + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// A pre-allocated and zeroed page table page. +/// +/// Created during the mapping prepare phase and consumed during the execute phase. +/// Stored in an [`RBTree`] keyed by the PDE slot address (`install_addr`). +pub(in crate::mm) struct PreparedPtPage { + /// The allocated and zeroed page table page. + pub(in crate::mm) alloc: Pin<KBox<AllocatedBlocks>>, + /// Page table level -- needed to determine if this PT page is for a dual PDE. + pub(in crate::mm) level: PageTableLevel, +} + +/// Page table mapper. +pub(in crate::mm) struct PtMapInner<M: MmuConfig> { + walker: PtWalkInner<M>, + pdb_addr: VramAddress, + _phantom: PhantomData<M>, +} + +impl<M: MmuConfig> PtMapInner<M> { + /// Create a new [`PtMapInner`]. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + walker: PtWalkInner::<M>::new(pdb_addr), + pdb_addr, + _phantom: PhantomData, + } + } + + /// Allocate and zero a physical page table page. + fn alloc_and_zero_page(mm: &mut GpuMm<'_>, level: PageTableLevel) -> Result<PreparedPtPage> { + let blocks = KBox::pin_init( + mm.buddy().alloc_blocks( + GpuBuddyAllocMode::Simple, + SZ_4K.into_safe_cast(), + Alignment::new::<SZ_4K>(), + GpuBuddyAllocFlags::default(), + ), + GFP_KERNEL, + )?; + + let page_vram = VramAddress::from_raw(blocks.iter().next().ok_or(ENOMEM)?.offset()); + + // Zero via PRAMIN. + let window = mm + .pramin_mut() + .window_at::<[u64; PAGE_SIZE / 8]>(page_vram)?; + for i in 0..PAGE_SIZE / 8 { + io_write!(window.view(), [build: i], 0); + } + + Ok(PreparedPtPage { + alloc: blocks, + level, + }) + } + + /// Ensure all intermediate page table pages exist for a single VFN. + /// + /// The mutable PRAMIN borrow ends before each allocation. + fn ensure_single_pte_path( + &self, + mm: &mut GpuMm<'_>, + vfn: Vfn, + pt_pages: &mut RBTree<VramAddress, PreparedPtPage>, + ) -> Result { + let max_iter = 2 * M::PDE_LEVELS.len(); + + for _ in 0..max_iter { + let result = self + .walker + .walk_pde_levels(mm.pramin_mut(), vfn, |install_addr| { + pt_pages.get(&install_addr).and_then(|p| { + p.alloc + .iter() + .next() + .map(|b| VramAddress::from_raw(b.offset())) + }) + })?; + + match result { + WalkPdeResult::Complete { .. } => { + return Ok(()); + } + WalkPdeResult::Missing { + install_addr, + level, + } => { + let page = Self::alloc_and_zero_page(mm, level)?; + let node = RBTreeNode::new(install_addr, page, GFP_KERNEL)?; + let old = pt_pages.insert(node); + if old.is_some() { + kernel::pr_warn_once!( + "VMM: duplicate install_addr in pt_pages (internal consistency error)\n" + ); + return Err(EIO); + } + } + } + } + + kernel::pr_warn!( + "VMM: ensure_pte_path: loop exhausted after {} iters (VFN {:?})\n", + max_iter, + vfn + ); + Err(EIO) + } + + /// Prepare page table resources for mapping `num_pages` pages starting at `vfn_start`. + /// + /// Reserves capacity in `page_table_allocs`, then walks the hierarchy + /// per-VFN to prepare pages for all missing PDEs. + pub(super) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>, + pt_pages: &mut RBTree<VramAddress, PreparedPtPage>, + ) -> Result { + // Pre-reserve so install_mappings() can use push_within_capacity (no alloc + // in fence signalling critical path). + let pt_upper_bound = M::pt_pages_upper_bound(num_pages); + page_table_allocs.reserve(pt_upper_bound, GFP_KERNEL)?; + + // Walk the hierarchy per-VFN to prepare pages for all missing PDEs. + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + self.ensure_single_pte_path(mm, vfn, pt_pages)?; + } + Ok(()) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + /// + /// Drains `pt_pages` and moves allocations into `page_table_allocs`. + pub(super) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree<VramAddress, PreparedPtPage>, + page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + { + let pramin = mm.pramin_mut(); + + // Drain prepared PT pages, install all pending PDEs. + let mut cursor = pt_pages.cursor_front_mut(); + while let Some(c) = cursor { + let (next, node) = c.remove_current(); + let (install_addr, page) = node.to_key_value(); + let page_vram = + VramAddress::from_raw(page.alloc.iter().next().ok_or(ENOMEM)?.offset()); + + if page.level == M::DUAL_PDE_LEVEL { + let new_dpde = M::DualPde::new_small(Pfn::from(page_vram)); + new_dpde.write(pramin, install_addr)?; + } else { + let new_pde = M::Pde::new(AperturePde::VideoMemory, Pfn::from(page_vram)); + new_pde.write(pramin, install_addr)?; + } + + page_table_allocs + .push_within_capacity(page.alloc) + .map_err(|_| ENOMEM)?; + + cursor = next; + } + + // Write PTEs (all PDEs now installed in HW). + for (i, &pfn) in pfns.iter().enumerate() { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Unmapped { pte_addr } | WalkResult::Mapped { pte_addr, .. } => { + let pte = M::Pte::new(AperturePte::VideoMemory, pfn, writable); + pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + kernel::pr_warn_once!("VMM: page table missing for VFN {vfn:?}\n"); + return Err(EIO); + } + } + } + } + + // Flush TLB. + mm.tlb().flush(self.pdb_addr) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(super) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + let invalid_pte = M::Pte::invalid(); + + { + let pramin = mm.pramin_mut(); + for i in 0..num_pages { + let i_u64: u64 = i.into_safe_cast(); + let vfn = Vfn::new(vfn_start.raw() + i_u64); + let result = self.walker.walk_to_pte_lookup_with_window(pramin, vfn)?; + + match result { + WalkResult::Mapped { pte_addr, .. } | WalkResult::Unmapped { pte_addr } => { + invalid_pte.write(pramin, pte_addr)?; + } + WalkResult::PageTableMissing => { + continue; + } + } + } + } + + mm.tlb().flush(self.pdb_addr) + } +} + +macro_rules! pt_map_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtMap::V2(inner) => inner.$method($($arg),*), + PtMap::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table mapper dispatch. +pub(in crate::mm) enum PtMap { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtMapInner<MmuV2>), + /// MMU v3 (Hopper+). + V3(PtMapInner<MmuV3>), +} + +impl PtMap { + /// Create a new page table mapper for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtMapInner::<MmuV2>::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtMapInner::<MmuV3>::new(pdb_addr)), + } + } + + /// Prepare page table resources for a mapping. + pub(in crate::mm) fn prepare_map( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>, + pt_pages: &mut RBTree<VramAddress, PreparedPtPage>, + ) -> Result { + pt_map_dispatch!( + self, + prepare_map(mm, vfn_start, num_pages, page_table_allocs, pt_pages) + ) + } + + /// Install prepared PDEs and write PTEs, then flush TLB. + pub(in crate::mm) fn install_mappings( + &self, + mm: &mut GpuMm<'_>, + pt_pages: &mut RBTree<VramAddress, PreparedPtPage>, + page_table_allocs: &mut KVec<Pin<KBox<AllocatedBlocks>>>, + vfn_start: Vfn, + pfns: &[Pfn], + writable: bool, + ) -> Result { + pt_map_dispatch!( + self, + install_mappings(mm, pt_pages, page_table_allocs, vfn_start, pfns, writable) + ) + } + + /// Invalidate PTEs for a range and flush TLB. + pub(in crate::mm) fn invalidate_ptes( + &self, + mm: &mut GpuMm<'_>, + vfn_start: Vfn, + num_pages: usize, + ) -> Result { + pt_map_dispatch!(self, invalidate_ptes(mm, vfn_start, num_pages)) + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable/ver2.rs b/drivers/gpu/nova-core/mm/pagetable/ver2.rs new file mode 100644 index 000000000000..d7169a0fcff9 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver2.rs @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v2 page table types for Turing, Ampere and Ada GPUs. +//! +//! This module defines MMU version 2 specific types (Turing, Ampere and Ada GPUs). +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/turing/tu102/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 2 Pfn bitfield conversions: +// 25 bits for video memory frame numbers (bits 32:8). +impl_pfn_bounded!(25); +// 46 bits for system memory frame numbers (bits 53:8). +impl_pfn_bounded!(46); + +bitfield! { + /// MMU v2 49-bit virtual address layout. + pub(super) struct VirtualAddressV2(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [48:47]. + 48:47 pde3_idx; + } +} + +impl VirtualAddressV2 { + /// Create a [`VirtualAddressV2`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV2 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde3_idx(), + 1 => *self.pde2_idx(), + 2 => *self.pde1_idx(), + 3 => *self.pde0_idx(), + 4 => *self.pt_idx(), + _ => 0, + } + } +} + +/// `PDE` levels for MMU v2 (5-level hierarchy: `PDB` -> `L1` -> `L2` -> `L3` -> `L4`). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, +]; + +/// `PTE` level for MMU v2. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L4; + +/// Dual `PDE` level for MMU v2 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L3; + +// Page Table Entry (PTE) for MMU v2 - 64-bit entry at level 4. +bitfield! { + /// Page Table Entry for MMU v2. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Encryption enabled (Confidential Computing). + 4:4 encrypted; + /// Privileged access only. + 5:5 privilege; + /// Write protection. + 6:6 read_only; + /// Atomic operations disabled. + 7:7 atomic_disable; + /// Frame number for system memory. + 53:8 frame_number_sys => Pfn; + /// Frame number for video memory. + 32:8 frame_number_vid => Pfn; + /// Peer GPU ID for peer memory (0-7). + 35:33 peer_id; + /// Compression tag line bits. + 53:36 comptagline; + /// Surface kind/format. + 63:56 kind; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let base = Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_read_only(!writable); + match aperture { + AperturePte::VideoMemory => base.with_frame_number_vid(pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePte::SystemCoherent => base.with_frame_number_sys(pfn).with_volatile(true), + AperturePte::PeerMemory | AperturePte::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PTE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + match self.aperture() { + AperturePte::VideoMemory => self.frame_number_vid(), + _ => self.frame_number_sys(), + } + } +} + +// Page Directory Entry (PDE) for MMU v2 - 64-bit entry at levels 0-2. +bitfield! { + /// Page Directory Entry for MMU v2. + pub(in crate::mm) struct Pde(u64) { + /// Valid bit (inverted logic). + 0:0 valid_inverted; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Volatile (bypass L2 cache). + 3:3 volatile; + /// Disable Address Translation Services. + 5:5 no_ats; + /// Table frame number for system memory. + 53:8 table_frame_sys => Pfn; + /// Table frame number for video memory. + 32:8 table_frame_vid => Pfn; + /// Peer GPU ID (0-7). + 35:33 peer_id; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + let base = Self::zeroed() + .with_valid_inverted(false) // 0 = valid + .with_aperture(aperture); + match aperture { + AperturePde::VideoMemory => base.with_table_frame_vid(table_pfn), + // Sysmem PTEs use VOL=1 to bypass L2 for cache coherency. + AperturePde::SystemCoherent => base.with_table_frame_sys(table_pfn).with_volatile(true), + AperturePde::Invalid | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v2 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed() + .with_valid_inverted(true) + .with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + !self.valid_inverted().into_bool() && self.aperture() != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame_vid()) + } +} + +/// Dual `PDE` at Level 3 - 128-bit entry of Large/Small Page Table pointers. +/// +/// The dual `PDE` supports both large (64KB) and small (4KB) page tables. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Large/Big Page Table pointer (lower 64 bits). + pub(super) big: Pde, + /// Small Page Table pointer (upper 64 bits). + pub(super) small: Pde, +} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + PdeOps::is_valid(&self.big) + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: PdeOps::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: PdeOps::from_raw(0), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable/ver3.rs b/drivers/gpu/nova-core/mm/pagetable/ver3.rs new file mode 100644 index 000000000000..47ed3339026b --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/ver3.rs @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! MMU v3 page table types for Hopper and later GPUs. +//! +//! This module defines MMU version 3 specific types (Hopper and later GPUs). +//! +//! Key differences from MMU v2: +//! - Unified 40-bit address field for all apertures (v2 had separate sys/vid fields). +//! - PCF (Page Classification Field) replaces separate privilege/RO/atomic/cache bits. +//! - KIND field is 4 bits (not 8). +//! - IS_PTE bit in PDE to support large pages directly. +//! - No COMPTAGLINE field (compression handled differently in v3). +//! - No separate ENCRYPTED bit. +//! +//! Bit field layouts derived from the NVIDIA OpenRM documentation: +//! `open-gpu-kernel-modules/src/common/inc/swref/published/hopper/gh100/dev_mmu.h` + +#![allow(dead_code)] + +use kernel::{ + bitfield, + num::Bounded, + prelude::*, // +}; + +use pin_init::Zeroable; + +use super::{ + AperturePde, + AperturePte, + DualPdeOps, + PageTableLevel, + PdeOps, + PteOps, + VaLevelIndex, // +}; + +use crate::mm::{ + Pfn, + VirtualAddress, + VramAddress, // +}; + +// Bounded to version 3 Pfn conversion. +impl_pfn_bounded!(40); + +bitfield! { + /// MMU v3 57-bit virtual address layout. + pub(super) struct VirtualAddressV3(u64) { + /// Page offset [11:0]. + 11:0 offset; + /// PT index [20:12]. + 20:12 pt_idx; + /// PDE0 index [28:21]. + 28:21 pde0_idx; + /// PDE1 index [37:29]. + 37:29 pde1_idx; + /// PDE2 index [46:38]. + 46:38 pde2_idx; + /// PDE3 index [55:47]. + 55:47 pde3_idx; + /// PDE4 index [56]. + 56:56 pde4_idx; + } +} + +impl VirtualAddressV3 { + /// Create a [`VirtualAddressV3`] from a [`VirtualAddress`]. + pub(super) fn new(va: VirtualAddress) -> Self { + Self::from_raw(va.into_raw()) + } +} + +impl VaLevelIndex for VirtualAddressV3 { + fn level_index(&self, level: u64) -> u64 { + match level { + 0 => *self.pde4_idx(), + 1 => *self.pde3_idx(), + 2 => *self.pde2_idx(), + 3 => *self.pde1_idx(), + 4 => *self.pde0_idx(), + 5 => *self.pt_idx(), + _ => 0, + } + } +} + +/// PDE levels for MMU v3 (6-level hierarchy). +pub(super) const PDE_LEVELS: &[PageTableLevel] = &[ + PageTableLevel::Pdb, + PageTableLevel::L1, + PageTableLevel::L2, + PageTableLevel::L3, + PageTableLevel::L4, +]; + +/// PTE level for MMU v3. +pub(super) const PTE_LEVEL: PageTableLevel = PageTableLevel::L5; + +/// Dual PDE level for MMU v3 (128-bit entries). +pub(super) const DUAL_PDE_LEVEL: PageTableLevel = PageTableLevel::L4; + +bitfield! { + /// Page Classification Field for PTEs (5 bits) in MMU v3. + pub(in crate::mm) struct PtePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// Access counting disabled (0=enabled, 1=disabled). + 1:1 acd; + /// Read-only access (0=read-write, 1=read-only). + 2:2 read_only; + /// Atomics disabled (0=enabled, 1=disabled). + 3:3 no_atomic; + /// Privileged access only (0=regular, 1=privileged). + 4:4 privileged; + } +} + +impl PtePcf { + /// Create PCF for read-write mapping (cached, no atomics, regular mode). + fn rw() -> Self { + Self::zeroed().with_no_atomic(true) + } + + /// Create PCF for read-only mapping (cached, no atomics, regular mode). + fn ro() -> Self { + Self::zeroed().with_read_only(true).with_no_atomic(true) + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From<Bounded<u64, 5>> for PtePcf { + fn from(val: Bounded<u64, 5>) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From<PtePcf> for Bounded<u64, 5> { + fn from(pcf: PtePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x1F) + } +} + +bitfield! { + /// Page Classification Field for PDEs (3 bits) in MMU v3. + /// + /// Controls Address Translation Services (ATS) and caching. + pub(in crate::mm) struct PdePcf(u8) { + /// Bypass L2 cache (0=cached, 1=bypass). + 0:0 uncached; + /// ATS disabled (0=enabled, 1=disabled). + 1:1 no_ats; + } +} + +impl PdePcf { + /// Create PCF for cached mapping with ATS enabled (default). + fn cached() -> Self { + Self::zeroed() + } + + /// Get the raw `u8` value. + fn raw_u8(&self) -> u8 { + self.into_raw() + } +} + +impl From<Bounded<u64, 3>> for PdePcf { + fn from(val: Bounded<u64, 3>) -> Self { + Self::from_raw(u8::from(val)) + } +} + +impl From<PdePcf> for Bounded<u64, 3> { + fn from(pcf: PdePcf) -> Self { + Bounded::from_expr(u64::from(pcf.into_raw()) & 0x7) + } +} + +bitfield! { + /// Page Table Entry for MMU v3. + pub(in crate::mm) struct Pte(u64) { + /// Entry is valid. + 0:0 valid; + /// Memory aperture type. + 2:1 aperture => AperturePte; + /// Page Classification Field. + 7:3 pcf => PtePcf; + /// Surface kind (4 bits, 0x0=pitch, 0xF=invalid). + 11:8 kind; + /// Physical frame number (for all apertures). + 51:12 frame_number => Pfn; + /// Peer GPU ID for peer memory (0-7). + 63:61 peer_id; + } +} + +impl PteOps for Pte { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn invalid() -> Self { + Self::zeroed() + } + + fn new(aperture: AperturePte, pfn: Pfn, writable: bool) -> Self { + let pcf = match (aperture, writable) { + (AperturePte::VideoMemory, true) => PtePcf::rw(), + (AperturePte::VideoMemory, false) => PtePcf::ro(), + // Sysmem PTEs use uncached+no_atomic PCF for cache coherency. + (AperturePte::SystemCoherent, true) => { + PtePcf::zeroed().with_uncached(true).with_no_atomic(true) + } + (AperturePte::SystemCoherent, false) => PtePcf::zeroed() + .with_uncached(true) + .with_no_atomic(true) + .with_read_only(true), + (AperturePte::PeerMemory | AperturePte::SystemNonCoherent, _) => { + kernel::pr_warn!("MMU v3 PTE aperture {:?} not supported\n", aperture); + return Self::invalid(); + } + }; + Self::zeroed() + .with_valid(true) + .with_aperture(aperture) + .with_pcf(pcf) + .with_frame_number(pfn) + } + + fn is_valid(&self) -> bool { + self.valid().into_bool() + } + + fn frame_number(&self) -> Pfn { + Pte::frame_number(*self) + } +} + +bitfield! { + /// Page Directory Entry for MMU v3 (Hopper+). + /// + /// ## Note + /// + /// v3 uses a unified 40-bit address field (v2 had separate sys/vid address fields). + pub(in crate::mm) struct Pde(u64) { + /// Entry is a PTE (0=PDE, 1=large page PTE). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field (3 bits for PDE). + 5:3 pcf => PdePcf; + /// Table frame number (40-bit unified address). + 51:12 table_frame => Pfn; + } +} + +impl PdeOps for Pde { + fn from_raw(val: u64) -> Self { + Self::from_raw(val) + } + + fn new(aperture: AperturePde, table_pfn: Pfn) -> Self { + match aperture { + AperturePde::VideoMemory => Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_pfn), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 PDE aperture {:?} not supported\n", aperture); + Self::invalid() + } + } + } + + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + fn is_valid(&self) -> bool { + Pde::aperture(*self) != AperturePde::Invalid + } + + fn aperture(&self) -> AperturePde { + Pde::aperture(*self) + } + + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + Pde::aperture(*self) == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM PDE (aperture: {:?})", + Pde::aperture(*self) + ); + VramAddress::from(self.table_frame()) + } +} + +bitfield! { + /// Big Page Table pointer in Dual PDE (MMU v3). + /// + /// 64-bit lower word of the 128-bit Dual PDE. + pub(super) struct DualPdeBig(u64) { + /// Entry is a PTE (for large pages). + 0:0 is_pte; + /// Memory aperture type. + 2:1 aperture => AperturePde; + /// Page Classification Field. + 5:3 pcf => PdePcf; + /// Table frame (table address 256-byte aligned). + 51:8 table_frame; + } +} + +impl DualPdeBig { + /// Create an invalid big page table pointer. + fn invalid() -> Self { + Self::zeroed().with_aperture(AperturePde::Invalid) + } + + /// Create a valid big PDE pointing to a page table in the given aperture. + fn new(aperture: AperturePde, table_addr: VramAddress) -> Result<Self> { + // Big page table addresses must be 256-byte aligned (shift 8). + if table_addr.into_raw() & 0xFF != 0 { + return Err(EINVAL); + } + let table_frame = Bounded::from_expr(table_addr.into_raw() >> 8); + match aperture { + AperturePde::VideoMemory => Ok(Self::zeroed() + .with_is_pte(false) + .with_aperture(aperture) + .with_table_frame(table_frame)), + AperturePde::Invalid | AperturePde::SystemCoherent | AperturePde::SystemNonCoherent => { + kernel::pr_warn!("MMU v3 DualPdeBig aperture {:?} not supported\n", aperture); + Ok(Self::invalid()) + } + } + } + + /// Check if this big PDE is valid. + fn is_valid(&self) -> bool { + self.aperture() != AperturePde::Invalid + } + + /// Get the VRAM address of the big page table. + fn table_vram_address(&self) -> VramAddress { + debug_assert!( + self.aperture() == AperturePde::VideoMemory, + "table_vram_address called on non-VRAM DualPdeBig (aperture: {:?})", + self.aperture() + ); + VramAddress::from_raw(*self.table_frame() << 8) + } +} + +/// Dual PDE at Level 4 for MMU v3 - 128-bit entry. +/// +/// Contains both big (64KB) and small (4KB) page table pointers: +/// - Lower 64 bits: Big Page Table pointer. +/// - Upper 64 bits: Small Page Table pointer. +/// +/// ## Note +/// +/// The big and small page table pointers have different address layouts: +/// - Big address = field value << 8 (256-byte alignment). +/// - Small address = field value << 12 (4KB alignment). +/// +/// This is why `DualPdeBig` is a separate type from `Pde`. +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) struct DualPde { + /// Big Page Table pointer. + pub(super) big: DualPdeBig, + /// Small Page Table pointer. + pub(super) small: Pde, +} + +// SAFETY: Both `DualPdeBig` and `Pde` fields are `Zeroable` (bitfield types are Zeroable). +unsafe impl Zeroable for DualPde {} + +impl DualPde { + /// Check if the big page table pointer is valid. + fn has_big(&self) -> bool { + self.big.is_valid() + } +} + +impl DualPdeOps for DualPde { + fn from_raw(big: u64, small: u64) -> Self { + Self { + big: DualPdeBig::from_raw(big), + small: PdeOps::from_raw(small), + } + } + + fn new_small(table_pfn: Pfn) -> Self { + Self { + big: DualPdeBig::invalid(), + small: PdeOps::new(AperturePde::VideoMemory, table_pfn), + } + } + + fn has_small(&self) -> bool { + PdeOps::is_valid(&self.small) + } + + fn small_vram_address(&self) -> VramAddress { + PdeOps::table_vram_address(&self.small) + } + + fn big_raw_u64(&self) -> u64 { + self.big.into_raw() + } + + fn small_raw_u64(&self) -> u64 { + self.small.into_raw() + } +} diff --git a/drivers/gpu/nova-core/mm/pagetable/walk.rs b/drivers/gpu/nova-core/mm/pagetable/walk.rs new file mode 100644 index 000000000000..76c1729971f5 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pagetable/walk.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Page table walker implementation for NVIDIA GPUs. +//! +//! This module provides page table walking functionality for MMU v2 and v3. +//! The walker traverses the page table hierarchy to resolve virtual addresses +//! to physical addresses or to find PTE locations. +//! +//! # Page Table Hierarchy +//! +//! ## MMU v2 (Turing/Ampere/Ada) - 5 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 Dual |---->| L4 | +//! | (L0) | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE (big+small) PTE +//! ``` +//! +//! ## MMU v3 (Hopper+) - 6 levels +//! +//! ```text +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! | PDB |---->| L1 |---->| L2 |---->| L3 |---->| L4 Dual |---->| L5 | +//! | (L0) | | | | | | | | PDE | | (PTE) | +//! +-------+ +-------+ +-------+ +-------+ +---------+ +-------+ +//! 64-bit 64-bit 64-bit 64-bit 128-bit 64-bit +//! PDE PDE PDE PDE (big+small) PTE +//! ``` +//! +//! # Result of a page table walk +//! +//! The walker returns a [`WalkResult`] indicating the outcome. + +use core::marker::PhantomData; + +use kernel::prelude::*; + +use super::{ + DualPdeOps, + MmuConfig, + MmuV2, + MmuV3, + MmuVersion, + PageTableLevel, + PdeOps, + PteOps, // +}; +use crate::{ + mm::{ + pramin, + GpuMm, + Pfn, + Vfn, + VirtualAddress, + VramAddress, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// Result of walking to a PTE. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkResult { + /// Intermediate page tables are missing (only returned in lookup mode). + PageTableMissing, + /// PTE exists but is invalid (page not mapped). + Unmapped { pte_addr: VramAddress }, + /// PTE exists and is valid (page is mapped). + Mapped { pte_addr: VramAddress, pfn: Pfn }, +} + +/// Result of walking PDE levels only. +/// +/// Returned by [`PtWalkInner::walk_pde_levels()`] to indicate whether all PDE +/// levels resolved or a PDE is missing. +#[derive(Debug, Clone, Copy)] +pub(in crate::mm) enum WalkPdeResult { + /// All PDE levels resolved -- returns PTE page table address. + Complete { + /// VRAM address of the PTE-level page table. + pte_table: VramAddress, + }, + /// A PDE is missing and no prepared page was provided by the closure. + Missing { + /// PDE slot address in the parent page table (where to install). + install_addr: VramAddress, + /// The page table level that is missing. + level: PageTableLevel, + }, +} + +/// Page table walker. +pub(in crate::mm) struct PtWalkInner<M: MmuConfig> { + pdb_addr: VramAddress, + _phantom: PhantomData<M>, +} + +impl<M: MmuConfig> PtWalkInner<M> { + /// Calculate the VRAM address of an entry within a page table. + fn entry_addr(table: VramAddress, level: PageTableLevel, index: u64) -> VramAddress { + let entry_size: u64 = M::entry_size(level).into_safe_cast(); + table + index * entry_size + } + + /// Create a new page table walker. + pub(super) fn new(pdb_addr: VramAddress) -> Self { + Self { + pdb_addr, + _phantom: PhantomData, + } + } + + /// Walk PDE levels with closure-based resolution for missing PDEs. + /// + /// Traverses all PDE levels for the MMU version. At each level, reads the PDE. + /// If valid, extracts the child table address and continues. If missing, calls + /// `resolve_prepared(install_addr)` to resolve the missing PDE. + pub(super) fn walk_pde_levels( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + resolve_prepared: impl Fn(VramAddress) -> Option<VramAddress>, + ) -> Result<WalkPdeResult> { + let va = VirtualAddress::from(vfn); + let mut cur_table = self.pdb_addr; + + for &level in M::PDE_LEVELS { + let idx = M::level_index(va, level.as_index()); + let install_addr = Self::entry_addr(cur_table, level, idx); + + if level == M::DUAL_PDE_LEVEL { + // 128-bit dual PDE with big+small page table pointers. + let dpde = M::DualPde::read(pramin, install_addr)?; + if dpde.has_small() { + cur_table = dpde.small_vram_address(); + continue; + } + } else { + // Regular 64-bit PDE. Use `is_valid_vram()` because + // `table_vram_address()` only reads the VRAM frame-number + // bitfield; system-memory PDEs store the address in a + // different (wider) field and would be silently truncated. + let pde = M::Pde::read(pramin, install_addr)?; + if pde.is_valid_vram() { + cur_table = pde.table_vram_address(); + continue; + } + } + + // PDE missing in HW. Ask caller for resolution. + if let Some(prepared_addr) = resolve_prepared(install_addr) { + cur_table = prepared_addr; + continue; + } + + return Ok(WalkPdeResult::Missing { + install_addr, + level, + }); + } + + Ok(WalkPdeResult::Complete { + pte_table: cur_table, + }) + } + + /// Walk to PTE for lookup only (no allocation). + /// + /// Returns [`WalkResult::PageTableMissing`] if intermediate tables don't exist. + pub(super) fn walk_to_pte_lookup(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<WalkResult> { + self.walk_to_pte_lookup_with_window(mm.pramin_mut(), vfn) + } + + /// Walk to PTE using a caller-provided PRAMIN manager (lookup only). + pub(super) fn walk_to_pte_lookup_with_window( + &self, + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + ) -> Result<WalkResult> { + match self.walk_pde_levels(pramin, vfn, |_| None)? { + WalkPdeResult::Complete { pte_table } => { + Self::read_pte_at_level(pramin, vfn, pte_table) + } + WalkPdeResult::Missing { .. } => Ok(WalkResult::PageTableMissing), + } + } + + /// Read the PTE at the PTE level given the PTE table address. + fn read_pte_at_level( + pramin: &mut pramin::Pramin<'_>, + vfn: Vfn, + pte_table: VramAddress, + ) -> Result<WalkResult> { + let va = VirtualAddress::from(vfn); + let pte_level = M::PTE_LEVEL; + let pte_idx = M::level_index(va, pte_level.as_index()); + let pte_addr = Self::entry_addr(pte_table, pte_level, pte_idx); + let pte = M::Pte::read(pramin, pte_addr)?; + + if pte.is_valid() { + return Ok(WalkResult::Mapped { + pte_addr, + pfn: pte.frame_number(), + }); + } + Ok(WalkResult::Unmapped { pte_addr }) + } +} + +macro_rules! pt_walk_dispatch { + ($self:expr, $method:ident ( $($arg:expr),* $(,)? )) => { + match $self { + PtWalk::V2(inner) => inner.$method($($arg),*), + PtWalk::V3(inner) => inner.$method($($arg),*), + } + }; +} + +/// Page table walker dispatch. +pub(in crate::mm) enum PtWalk { + /// MMU v2 (Turing/Ampere/Ada). + V2(PtWalkInner<MmuV2>), + /// MMU v3 (Hopper+). + V3(PtWalkInner<MmuV3>), +} + +impl PtWalk { + /// Create a new page table walker for the given MMU version. + pub(in crate::mm) fn new(pdb_addr: VramAddress, version: MmuVersion) -> Self { + match version { + MmuVersion::V2 => Self::V2(PtWalkInner::<MmuV2>::new(pdb_addr)), + MmuVersion::V3 => Self::V3(PtWalkInner::<MmuV3>::new(pdb_addr)), + } + } + + /// Walk to PTE for lookup. + pub(in crate::mm) fn walk_to_pte(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<WalkResult> { + pt_walk_dispatch!(self, walk_to_pte_lookup(mm, vfn)) + } +} diff --git a/drivers/gpu/nova-core/mm/pramin.rs b/drivers/gpu/nova-core/mm/pramin.rs new file mode 100644 index 000000000000..7f89c093d591 --- /dev/null +++ b/drivers/gpu/nova-core/mm/pramin.rs @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Utilities for accessing VRAM through the PRAMIN window. + +use core::ops::Range; + +use kernel::{ + io::{ + io_project, + register, + register::OffsetLoc, + Io, + Mmio, // + }, + prelude::*, + ptr::{ + Alignable, + Alignment, // + }, + sizes::{ + SZ_1M, + SZ_64K, // + }, +}; + +use crate::{ + driver::{ + Bar0, + NovaRegisters, // + }, + gpu::Chipset, + mm::{ + hal::{ + self, + MmHal, // + }, + VramAddress, // + }, + num::IntoSafeCast, // +}; + +/// Size of the PRAMIN window (1 MiB). +const WINDOW_SIZE: usize = SZ_1M; + +/// The PRAMIN window, which is a 1 MiB window into VRAM at a fixed BAR0 offset. +#[derive(FromBytes, IntoBytes)] +struct PraminWindow([u8; WINDOW_SIZE]); + +register! { + base: NovaRegisters; + + /// Location of the window inside BAR0. + PRAMIN: PraminWindow @ 0x700000; +} + +/// Owner of the PRAMIN window state. +/// +/// [`Pramin::window_at()`] repositions the window as needed and returns a typed MMIO view into +/// it, holding the manager borrowed for the lifetime of the view. +pub(super) struct Pramin<'gpu> { + bar: Bar0<'gpu>, + hal: &'static dyn MmHal, + /// MMIO view of the PRAMIN window in BAR0. + window: Mmio<'gpu, PraminWindow>, + /// VRAM range to keep the PRAMIN window inside. + vram_range: Range<VramAddress>, + /// Cached window position. + window_range: Range<VramAddress>, +} + +/// Typed view of VRAM through the PRAMIN window. +/// +/// Inserts an ordering point after previous writes through the window on drop. Views returned +/// by [`PraminAccess::view()`] cannot outlive this access, so the ordering point covers every +/// write made through them. +pub(super) struct PraminAccess<'a, T> +where + T: FromBytes + IntoBytes, +{ + view: Mmio<'a, T>, +} + +impl<T> PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + /// Returns the MMIO view of the accessed location. + pub(super) fn view(&self) -> Mmio<'_, T> { + self.view + } +} + +impl<T> Drop for PraminAccess<'_, T> +where + T: FromBytes + IntoBytes, +{ + fn drop(&mut self) { + // Insert an ordering point after previous writes through this window. + self.view.cast::<u8>().read_val(); + } +} + +impl<'gpu> Pramin<'gpu> { + /// Alignment required by the PRAMIN window. + const BASE_ALIGN: Alignment = Alignment::new::<SZ_64K>(); + + /// Creates the window manager for the given VRAM region. + pub(super) fn new( + bar: Bar0<'gpu>, + chipset: Chipset, + vram_range: Range<VramAddress>, + ) -> Result<Self> { + let hal = hal::mm_hal(chipset); + let window = io_project!(bar, build: PRAMIN); + let base = vram_range.start.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + hal.write_pramin_window_base(bar, base)?; + + Ok(Self { + bar, + hal, + window, + vram_range, + window_range, + }) + } + + /// Returns the VRAM range a window based at `base` exposes. + fn window_range(base: VramAddress) -> Result<Range<VramAddress>> { + let end = base + .checked_add(WINDOW_SIZE.into_safe_cast()) + .ok_or(EINVAL)?; + Ok(base..end) + } + + /// Check the window covers `len` bytes at `addr`, moving it if needed. + /// + /// Returns the window offset at which to perform the access. + fn window_offset(&mut self, addr: VramAddress, len: usize) -> Result<usize> { + let end = addr.checked_add(len.into_safe_cast()).ok_or(EINVAL)?; + + let inside = |r: &Range<VramAddress>| r.contains(&addr) && end <= r.end; + if !inside(&self.vram_range) { + return Err(EINVAL); + } + + // Reposition the window if the access falls outside it. + if !inside(&self.window_range) { + let base = addr.align_down(Self::BASE_ALIGN); + let window_range = Self::window_range(base)?; + if !inside(&window_range) { + return Err(EINVAL); + } + self.hal.write_pramin_window_base(self.bar, base)?; + self.window_range = window_range; + } + + Ok((addr - self.window_range.start).into_safe_cast()) + } + + /// Return a typed MMIO view of a `T` at `vram_addr`. + /// + /// Returns an error if `vram_addr` is not aligned to `T`'s alignment, or if + /// a `T` at `vram_addr` does not fit within the VRAM region. + pub(super) fn window_at<'a, T>( + &'a mut self, + vram_addr: VramAddress, + ) -> Result<PraminAccess<'a, T>> + where + T: FromBytes + IntoBytes, + { + let offset = self.window_offset(vram_addr, size_of::<T>())?; + let view = io_project!(self.window, try: OffsetLoc::new(offset)); + + Ok(PraminAccess { view }) + } +} + +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +pub(super) mod selftest { + use kernel::{ + device, + io::io_read, + sizes::SizeConstants, // + }; + + use super::*; + use crate::{ + selftest_assert, + selftest_assert_eq, // + }; + + /// Test read/write at byte granularity, at unaligned addresses. + fn test_byte_readwrite( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + pramin.window_at::<u8>(addr)?.view().write_val(0xA0 + i); + } + + for i in 0u8..4 { + let addr = base + 1 + u64::from(i); + selftest_assert_eq!( + dev, + pramin.window_at::<u8>(addr)?.view().read_val(), + 0xA0 + i + ); + } + Ok(()) + } + + /// Test writing a `u32` and reading back as individual `u8`s. + fn test_u32_as_bytes( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr = base + 0x10; + let val: u32 = 0xDEADBEEF; + pramin.window_at::<u32>(addr)?.view().write_val(val); + + let window = pramin.window_at::<[u8; 4]>(addr)?; + for (i, &expected) in val.to_le_bytes().iter().enumerate() { + selftest_assert_eq!(dev, io_read!(window.view(), [build: i]), expected); + } + Ok(()) + } + + /// Test window repositioning across 1 MiB boundaries. + fn test_window_reposition( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + let addr_a = base; + let addr_b = base + u64::SZ_2M; // base + 2 MiB (different 1 MiB region). + let val_a: u32 = 0x11111111; + let val_b: u32 = 0x22222222; + + pramin.window_at::<u32>(addr_a)?.view().write_val(val_a); + pramin.window_at::<u32>(addr_b)?.view().write_val(val_b); + + selftest_assert_eq!( + dev, + pramin.window_at::<u32>(addr_a)?.view().read_val(), + val_a + ); + selftest_assert_eq!( + dev, + pramin.window_at::<u32>(addr_b)?.view().read_val(), + val_b + ); + Ok(()) + } + + /// Test that offsets outside the VRAM region are rejected. + fn test_invalid_offset( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + vram_end: VramAddress, + ) -> Result { + selftest_assert!(dev, pramin.window_at::<u32>(vram_end).is_err()); + Ok(()) + } + + /// Test that misaligned accesses are rejected. + fn test_misaligned_access( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + // `u16` at odd offset (not 2-byte aligned). + selftest_assert!(dev, pramin.window_at::<u16>(base + 0x21).is_err()); + + // `u32` at 2-byte-aligned (not 4-byte-aligned) offset. + selftest_assert!(dev, pramin.window_at::<u32>(base + 2).is_err()); + + // `u64` at a 4-byte-aligned (not 8-byte-aligned) address. + selftest_assert!(dev, pramin.window_at::<u64>(base + 0x44).is_err()); + + // A `u16` view at an even address is allowed. + pramin.window_at::<u16>(base + 0x22)?; + Ok(()) + } + + /// Run PRAMIN self-tests during probe. + /// + /// `base` is the start of a driver-usable VRAM span that the tests are free to + /// overwrite. + pub(crate) fn run( + dev: &device::Device<device::Bound>, + pramin: &mut Pramin<'_>, + base: VramAddress, + ) -> Result { + dev_dbg!(dev, "PRAMIN: starting self-tests\n"); + + let vram_end = pramin.vram_range.end; + + test_byte_readwrite(dev, pramin, base)?; + test_u32_as_bytes(dev, pramin, base)?; + test_window_reposition(dev, pramin, base)?; + test_invalid_offset(dev, pramin, vram_end)?; + test_misaligned_access(dev, pramin, base)?; + + dev_info!(dev, "PRAMIN: self-tests passed\n"); + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/mm/regs.rs b/drivers/gpu/nova-core/mm/regs.rs new file mode 100644 index 000000000000..82de6dfa4e8b --- /dev/null +++ b/drivers/gpu/nova-core/mm/regs.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Registers used by the memory management subsystems: the BAR0 PRAMIN window. + +use kernel::io::register; + +use crate::{ + bounded_enum, + driver::NovaRegisters, // +}; + +// PRAMIN window + +bounded_enum! { + /// Target memory type for the BAR0 window register. + /// + /// Only VRAM is needed by the driver. Pre-Hopper window registers also define + /// system-memory targets that are unused here; Hopper+ uses a separate register + /// without a target field. + #[derive(Debug, Copy, Clone)] + pub(super) enum Bar0WindowTarget with TryFrom<Bounded<u32, 2>> { + /// Video memory (GPU framebuffer memory). + VidMem = 0, + } +} + +register! { + base: NovaRegisters; + + /// BAR0 window control for PRAMIN access. + pub(super) NV_PBUS_BAR0_WINDOW(u32) @ 0x00001700 { + /// Target memory aperture for the window. + 25:24 target ?=> Bar0WindowTarget; + /// PRAMIN window base bits 39:16. + 23:0 base; + } +} + +pub(super) mod gh100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Hopper register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 37:16. + 21:0 base; + } + } +} + +pub(super) mod gb100 { + use kernel::io::register; + + use crate::driver::NovaRegisters; + + register! { + base: NovaRegisters; + + /// Blackwell GB10x/GB20x register for PRAMIN window. + pub(crate) NV_XAL_EP_BAR0_WINDOW(u32) @ 0x0010fd40 { + /// PRAMIN window base bits 38:16. + 22:0 base; + } + } +} diff --git a/drivers/gpu/nova-core/mm/tlb.rs b/drivers/gpu/nova-core/mm/tlb.rs new file mode 100644 index 000000000000..cc862e8159a1 --- /dev/null +++ b/drivers/gpu/nova-core/mm/tlb.rs @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! TLB (Translation Lookaside Buffer) flush support for GPU MMU. +//! +//! After modifying page table entries, the GPU's TLB must be flushed to +//! ensure the new mappings take effect. This module provides TLB flush +//! functionality for virtual memory managers. +//! +//! # Examples +//! +//! ```ignore +//! use crate::mm::tlb::Tlb; +//! +//! fn page_table_update(tlb: &Tlb, pdb_addr: VramAddress) -> Result<()> { +//! // ... modify page tables ... +//! +//! // Flush TLB to make changes visible (polls for completion). +//! tlb.flush(pdb_addr)?; +//! +//! Ok(()) +//! } +//! ``` + +use kernel::{ + io::poll::read_poll_timeout, + io::Io, + new_mutex, + prelude::*, + sync::Mutex, + time::Delta, // +}; + +use crate::{ + bounded_enum, + driver::Bar0, + mm::VramAddress, + regs, // +}; + +bounded_enum! { + /// TLB invalidation acknowledgment scope. + /// + /// Controls how far the hardware waits for the invalidation to propagate + /// before clearing the `trigger` bit of `NV_TLB_FLUSH_CTRL`. + #[derive(Debug, Copy, Clone, PartialEq, Eq)] + pub(crate) enum TlbAckMode with TryFrom<Bounded<u32, 2>> { + /// Fire-and-forget: no acknowledgment required. + None = 0, + /// Wait for acknowledgment from all consumers, including remote GPUs + /// reachable over NVLink. + /// + /// Globally is strictly required only during unmap or permission + /// tightening, because the backing memory may be reassigned after the + /// flush returns and a stale TLB entry could let the GPU access freed + /// memory. For new mapping or relaxing permissions, a stale entry would + /// merely cause a redundant fault and retry, so [`TlbAckMode::None`] + /// would suffice. + Globally = 1, + /// Wait for acknowledgment from consumers within the local NVLink + /// fabric node only; skip cross-node ack. + Intranode = 2, + } +} + +/// TLB manager for GPU translation buffer operations. +#[pin_data] +pub(crate) struct Tlb<'gpu> { + bar: Bar0<'gpu>, + /// TLB flush serialization lock: This lock is designed to be acquired during + /// the DMA fence signalling critical path. It should NEVER be held across any + /// reclaimable CPU memory allocations because the memory reclaim path can + /// call `dma_fence_wait()` (when implemented), which would deadlock if lock held. + #[pin] + lock: Mutex<()>, +} + +impl<'gpu> Tlb<'gpu> { + /// Create a new TLB manager. + pub(super) fn new(bar: Bar0<'gpu>) -> impl PinInit<Self> { + pin_init!(Self { + bar, + lock <- new_mutex!((), "tlb_flush"), + }) + } + + /// Flush the GPU TLB for a specific page directory base. + /// + /// This invalidates all TLB entries associated with the given PDB address. + /// Must be called after modifying page table entries to ensure the GPU sees + /// the updated mappings. + pub(super) fn flush(&self, pdb_addr: VramAddress) -> Result { + let _guard = self.lock.lock(); + + // Write PDB address. + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_LO::from_pdb_addr( + pdb_addr.into_raw(), + )); + self.bar.write_reg(regs::NV_TLB_FLUSH_PDB_HI::from_pdb_addr( + pdb_addr.into_raw(), + )); + + // Trigger flush. + self.bar.write_reg( + regs::NV_TLB_FLUSH_CTRL::zeroed() + .with_all_va(true) + .with_ack(TlbAckMode::None) + .with_trigger(true), + ); + + // Poll for completion. + read_poll_timeout( + || Ok(self.bar.read(regs::NV_TLB_FLUSH_CTRL)), + |ctrl: ®s::NV_TLB_FLUSH_CTRL| !ctrl.trigger(), + Delta::ZERO, + Delta::from_secs(2), + )?; + + Ok(()) + } +} diff --git a/drivers/gpu/nova-core/mm/vmm.rs b/drivers/gpu/nova-core/mm/vmm.rs new file mode 100644 index 000000000000..51b500a27233 --- /dev/null +++ b/drivers/gpu/nova-core/mm/vmm.rs @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Virtual Memory Manager for NVIDIA GPU page table management. +//! +//! The [`Vmm`] provides high-level page mapping and unmapping operations for GPU +//! virtual address spaces (Channels, BAR1, BAR2). + +use kernel::{ + gpu::buddy::AllocatedBlocks, + maple_tree::MapleTreeAlloc, + prelude::*, + rbtree::RBTree, // +}; + +use core::{ + cell::Cell, + ops::Range, // +}; + +use crate::{ + mm::{ + pagetable::{ + map::{ + PtMap, // + }, + walk::{ + PtWalk, + WalkResult, // + }, + MmuVersion, // + }, + GpuMm, + Pfn, + Vfn, + VramAddress, + PAGE_SIZE, // + }, + num::{ + IntoSafeCast, // + }, +}; + +/// Multi-page prepared mapping -- VA range allocated, ready for execute. +/// +/// Produced by [`Vmm::prepare_map()`], consumed by [`Vmm::execute_map()`]. +/// The VA space allocation is tracked in the [`Vmm`]'s maple tree and freed +/// on error or via [`Vmm::unmap_pages()`]. +/// +/// Dropping without calling [`Vmm::execute_map()`] logs a warning and leaks +/// the VA range in the maple tree. +pub(crate) struct PreparedMapping { + vfn_start: Vfn, + num_pages: usize, + /// Logs a warning if dropped without executing. + _drop_guard: MustExecuteGuard, +} + +/// Result of a mapping operation -- tracks the active mapped range. +/// +/// Returned by [`Vmm::execute_map()`] and [`Vmm::map_pages()`]. +/// Callers must call [`Vmm::unmap_pages()`] before dropping to invalidate +/// PTEs and free the VA range. Dropping without unmapping logs a warning +/// and leaks the VA range in the maple tree. +pub(crate) struct MappedRange { + pub(super) vfn_start: Vfn, + pub(super) num_pages: usize, + /// Logs a warning if dropped without unmapping. + _drop_guard: MustUnmapGuard, +} + +/// Guard that logs a warning if a [`PreparedMapping`] is dropped without +/// being consumed by [`Vmm::execute_map()`]. +struct MustExecuteGuard { + armed: Cell<bool>, +} + +impl MustExecuteGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustExecuteGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("PreparedMapping dropped without calling execute_map()\n"); + } + } +} + +/// Guard that logs a warning if a [`MappedRange`] is dropped without +/// calling [`Vmm::unmap_pages()`]. +struct MustUnmapGuard { + armed: Cell<bool>, +} + +impl MustUnmapGuard { + const fn new() -> Self { + Self { + armed: Cell::new(true), + } + } + + fn disarm(&self) { + self.armed.set(false); + } +} + +impl Drop for MustUnmapGuard { + fn drop(&mut self) { + if self.armed.get() { + kernel::pr_warn!("MappedRange dropped without calling unmap_pages()\n"); + } + } +} + +/// Virtual Memory Manager for a GPU address space. +/// +/// Each [`Vmm`] instance manages a single address space identified by its Page +/// Directory Base (`PDB`) address. Used for Channel, BAR1 and BAR2 mappings. +pub(crate) struct Vmm { + /// Page Directory Base address for this address space. + #[expect(dead_code)] + pdb_addr: VramAddress, + /// Page table walker for reading existing mappings. + pt_walk: PtWalk, + /// Page table mapper for prepare/execute operations. + pt_map: PtMap, + /// Page table allocations required for mappings. + page_table_allocs: KVec<Pin<KBox<AllocatedBlocks>>>, + /// Maple tree allocator for virtual address range tracking. + virt_alloc: Pin<KBox<MapleTreeAlloc<()>>>, + /// Total number of pages in the virtual address space. + va_pages: usize, + /// Prepared PT pages pending PDE installation, keyed by `install_addr`. + /// + /// Populated during prepare phase and drained in execute phase. Shared by all + /// pending maps, preventing races on the same PDE slot. + pt_pages: RBTree<VramAddress, super::pagetable::map::PreparedPtPage>, +} + +impl Vmm { + /// Create a new [`Vmm`] for the given Page Directory Base address. + /// + /// The [`Vmm`] will manage a virtual address space of `va_size` bytes. + pub(crate) fn new( + pdb_addr: VramAddress, + mmu_version: MmuVersion, + va_size: u64, + ) -> Result<Self> { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + let va_pages: usize = (va_size / page_size).into_safe_cast(); + let virt_alloc = KBox::pin_init(MapleTreeAlloc::<()>::new(), GFP_KERNEL)?; + + Ok(Self { + pdb_addr, + pt_walk: PtWalk::new(pdb_addr, mmu_version), + pt_map: PtMap::new(pdb_addr, mmu_version), + page_table_allocs: KVec::new(), + virt_alloc, + va_pages, + pt_pages: RBTree::new(), + }) + } + + /// Allocate a contiguous virtual frame number range. + fn alloc_vfn_range(&self, num_pages: usize, va_range: Option<Range<u64>>) -> Result<Vfn> { + let page_size: u64 = PAGE_SIZE.into_safe_cast(); + + let start_vfn = match va_range { + Some(r) => { + let num_pages_u64: u64 = num_pages.into_safe_cast(); + let size = num_pages_u64.checked_mul(page_size).ok_or(EOVERFLOW)?; + let range_size = r.end.checked_sub(r.start).ok_or(EOVERFLOW)?; + if range_size != size { + return Err(EINVAL); + } + let start_vfn: usize = (r.start / page_size).into_safe_cast(); + let end_vfn: usize = (r.end / page_size).into_safe_cast(); + self.virt_alloc + .insert_range(start_vfn..end_vfn, (), GFP_KERNEL)?; + start_vfn + } + None => self + .virt_alloc + .alloc_range(num_pages, (), ..self.va_pages, GFP_KERNEL)?, + }; + + Ok(Vfn::new(start_vfn.into_safe_cast())) + } + + /// Free a virtual frame number range back to the maple tree. + fn free_vfn(&self, vfn: Vfn) { + let vfn_index: usize = vfn.raw().into_safe_cast(); + if self.virt_alloc.erase(vfn_index).is_none() { + kernel::pr_warn!("free_vfn: VFN {} not found in maple tree\n", vfn_index); + } + } + + /// Read the [`Pfn`] for a mapped [`Vfn`] if one is mapped. + pub(super) fn read_mapping(&self, mm: &mut GpuMm<'_>, vfn: Vfn) -> Result<Option<Pfn>> { + match self.pt_walk.walk_to_pte(mm, vfn)? { + WalkResult::Mapped { pfn, .. } => Ok(Some(pfn)), + WalkResult::Unmapped { .. } | WalkResult::PageTableMissing => Ok(None), + } + } + + /// Prepare resources for mapping `num_pages` pages. + /// + /// Allocates a contiguous VA range, then walks the hierarchy per-VFN to prepare pages + /// for all missing PDEs. Returns a [`PreparedMapping`] with the VA allocation. + /// + /// If `va_range` is not `None`, the VA range is constrained to the given range. Safe + /// to call outside the fence signalling critical path. + pub(crate) fn prepare_map( + &mut self, + mm: &mut GpuMm<'_>, + num_pages: usize, + va_range: Option<Range<u64>>, + ) -> Result<PreparedMapping> { + if num_pages == 0 { + return Err(EINVAL); + } + + // Allocate contiguous VA range. + let vfn_start = self.alloc_vfn_range(num_pages, va_range)?; + + if let Err(e) = self.pt_map.prepare_map( + mm, + vfn_start, + num_pages, + &mut self.page_table_allocs, + &mut self.pt_pages, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(PreparedMapping { + vfn_start, + num_pages, + _drop_guard: MustExecuteGuard::new(), + }) + } + + /// Execute a prepared multi-page mapping. + /// + /// Installs all prepared PDEs and writes PTEs into the page table, then flushes TLB. + pub(crate) fn execute_map( + &mut self, + mm: &mut GpuMm<'_>, + prepared: PreparedMapping, + pfns: &[Pfn], + writable: bool, + ) -> Result<MappedRange> { + if pfns.len() != prepared.num_pages { + self.free_vfn(prepared.vfn_start); + return Err(EINVAL); + } + + let PreparedMapping { + vfn_start, + num_pages, + _drop_guard, + } = prepared; + _drop_guard.disarm(); + + if let Err(e) = self.pt_map.install_mappings( + mm, + &mut self.pt_pages, + &mut self.page_table_allocs, + vfn_start, + pfns, + writable, + ) { + self.free_vfn(vfn_start); + return Err(e); + } + + Ok(MappedRange { + vfn_start, + num_pages, + _drop_guard: MustUnmapGuard::new(), + }) + } + + /// Map pages doing prepare and execute in the same call. + /// + /// This is a convenience wrapper for callers outside the fence signalling critical + /// path (e.g., BAR mappings). For DRM usecases, [`Vmm::prepare_map()`] and + /// [`Vmm::execute_map()`] will be called separately. + pub(crate) fn map_pages( + &mut self, + mm: &mut GpuMm<'_>, + pfns: &[Pfn], + va_range: Option<Range<u64>>, + writable: bool, + ) -> Result<MappedRange> { + if pfns.is_empty() { + return Err(EINVAL); + } + + // Check if provided VA range is sufficient (if provided). + if let Some(ref range) = va_range { + let required: u64 = pfns + .len() + .checked_mul(PAGE_SIZE) + .ok_or(EOVERFLOW)? + .into_safe_cast(); + let available = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if available < required { + return Err(EINVAL); + } + } + + let prepared = self.prepare_map(mm, pfns.len(), va_range)?; + self.execute_map(mm, prepared, pfns, writable) + } + + /// Unmap all pages in a [`MappedRange`] with a single TLB flush. + pub(crate) fn unmap_pages(&mut self, mm: &mut GpuMm<'_>, range: MappedRange) -> Result { + let result = self + .pt_map + .invalidate_ptes(mm, range.vfn_start, range.num_pages); + + // TODO: Internal page table pages (PDE, PTE pages) are still kept around. + // This is by design as repeated maps/unmaps will be fast. As a future TODO, + // we can add a reclaimer here to reclaim if VRAM is short. For now, the PT + // pages are dropped once the `Vmm` is dropped. + + // Free the VA range regardless of PTE invalidation success, so that the VA + // range is recovered even on failure (PTEs may be stale, but that is better + // than leaking both PTEs and VA range). + self.free_vfn(range.vfn_start); + + // Unmap complete, safe to drop `MappedRange`. + range._drop_guard.disarm(); + result + } +} diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 35a8b1214b0e..1133c6ce5c55 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -18,10 +18,13 @@ mod fsp; mod gpu; mod gsp; mod mctp; +mod mm; #[macro_use] mod num; mod regs; mod sbuffer; +#[cfg(CONFIG_NOVA_CORE_SELFTESTS)] +mod selftest; mod vbios; mod vgpu; diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index caeef4d85874..9978fb2803b0 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -4,110 +4,37 @@ use kernel::{ io::{ register, - register::WithBase, - Io, // + Io, + Mmio, // }, - prelude::*, sizes::SizeConstants, time, // }; +use pin_init::Zeroable; use crate::{ - driver::Bar0, + driver::NovaRegisters, falcon::{ DmaTrfCmdSize, FalconCoreRev, FalconCoreRevSubversion, - FalconEngine, FalconFbifMemType, FalconFbifTarget, FalconMem, FalconModSelAlgo, FalconSecurityModel, - PFalcon2Base, - PFalconBase, + PFalcon2Registers, + PFalconRegisters, PeregrineCoreSelect, // }, - gpu::{ - Architecture, - Chipset, // - }, + mm::tlb::TlbAckMode, // }; -// PMC - -register! { - /// Basic revision information about the GPU. - pub(crate) NV_PMC_BOOT_0(u32) @ 0x00000000 { - /// Lower bits of the architecture. - 28:24 architecture_0; - /// Implementation version of the architecture. - 23:20 implementation; - /// MSB of the architecture. - 8:8 architecture_1; - /// Major revision of the chip. - 7:4 major_revision; - /// Minor revision of the chip. - 3:0 minor_revision; - } - - /// Extended architecture information. - pub(crate) NV_PMC_BOOT_42(u32) @ 0x00000a00 { - /// Architecture value. - 29:24 architecture ?=> Architecture; - /// Implementation version of the architecture. - 23:20 implementation; - /// Major revision of the chip. - 19:16 major_revision; - /// Minor revision of the chip. - 15:12 minor_revision; - } -} - -impl NV_PMC_BOOT_0 { - pub(crate) fn is_older_than_fermi(self) -> bool { - // From https://github.com/NVIDIA/open-gpu-doc/tree/master/manuals : - const NV_PMC_BOOT_0_ARCHITECTURE_GF100: u32 = 0xc; - - // Older chips left arch1 zeroed out. That, combined with an arch0 value that is less than - // GF100, means "older than Fermi". - self.architecture_1() == 0 && self.architecture_0() < NV_PMC_BOOT_0_ARCHITECTURE_GF100 - } -} - -impl NV_PMC_BOOT_42 { - /// Combines `architecture` and `implementation` to obtain a code unique to the chipset. - pub(crate) fn chipset(self) -> Result<Chipset> { - self.architecture() - .map(|arch| { - ((arch as u32) << Self::IMPLEMENTATION_RANGE.len()) - | u32::from(self.implementation()) - }) - .and_then(Chipset::try_from) - } - - /// Returns the raw architecture value from the register. - fn architecture_raw(self) -> u8 { - ((self.into_raw() >> Self::ARCHITECTURE_RANGE.start()) - & ((1 << Self::ARCHITECTURE_RANGE.len()) - 1)) as u8 - } -} - -impl kernel::fmt::Display for NV_PMC_BOOT_42 { - fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result { - write!( - f, - "boot42 = 0x{:08x} (architecture 0x{:x}, implementation 0x{:x})", - self.inner, - self.architecture_raw(), - self.implementation() - ) - } -} - // PBUS register! { + base: NovaRegisters; + pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} } @@ -121,6 +48,8 @@ register! { // number. register! { + base: NovaRegisters; + /// Boot Sequence Interface (BSI) register used to determine /// if GSP reload/resume has completed during the boot process. pub(crate) NV_PGC6_BSI_SECURE_SCRATCH_14(u32) @ 0x001180f8 { @@ -175,6 +104,8 @@ impl NV_USABLE_FB_SIZE_IN_MB { pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; register! { + base: NovaRegisters; + pub(crate) NV_FUSE_OPT_FPF_NVDEC_UCODE1_VERSION(u32)[NV_FUSE_OPT_FPF_SIZE] @ 0x00824100 { 15:0 data => u16; } @@ -191,30 +122,32 @@ register! { // PFALCON register! { - pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ PFalconBase + 0x00000004 { + base: PFalconRegisters; + + pub(crate) NV_PFALCON_FALCON_IRQSCLR(u32) @ 0x00000004 { 6:6 swgen0 => bool; 4:4 halt => bool; } - pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ PFalconBase + 0x00000040 { + pub(crate) NV_PFALCON_FALCON_MAILBOX0(u32) @ 0x00000040 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ PFalconBase + 0x00000044 { + pub(crate) NV_PFALCON_FALCON_MAILBOX1(u32) @ 0x00000044 { 31:0 value => u32; } /// Used to store version information about the firmware running /// on the Falcon processor. - pub(crate) NV_PFALCON_FALCON_OS(u32) @ PFalconBase + 0x00000080 { + pub(crate) NV_PFALCON_FALCON_OS(u32) @ 0x00000080 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_RM(u32) @ PFalconBase + 0x00000084 { + pub(crate) NV_PFALCON_FALCON_RM(u32) @ 0x00000084 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ PFalconBase + 0x000000f4 { + pub(crate) NV_PFALCON_FALCON_HWCFG2(u32) @ 0x000000f4 { /// Signal indicating that reset is completed (GA102+). 31:31 reset_ready => bool; /// RISC-V branch privilege lockdown bit. @@ -224,17 +157,17 @@ register! { 10:10 riscv => bool; } - pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ PFalconBase + 0x00000100 { + pub(crate) NV_PFALCON_FALCON_CPUCTL(u32) @ 0x00000100 { 6:6 alias_en => bool; 4:4 halted => bool; 1:1 startcpu => bool; } - pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ PFalconBase + 0x00000104 { + pub(crate) NV_PFALCON_FALCON_BOOTVEC(u32) @ 0x00000104 { 31:0 value => u32; } - pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ PFalconBase + 0x0000010c { + pub(crate) NV_PFALCON_FALCON_DMACTL(u32) @ 0x0000010c { 7:7 secure_stat => bool; 6:3 dmaq_num; 2:2 imem_scrubbing => bool; @@ -242,15 +175,15 @@ register! { 0:0 require_ctx => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ PFalconBase + 0x00000110 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE(u32) @ 0x00000110 { 31:0 base => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ PFalconBase + 0x00000114 { + pub(crate) NV_PFALCON_FALCON_DMATRFMOFFS(u32) @ 0x00000114 { 23:0 offs; } - pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ PFalconBase + 0x00000118 { + pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 { 16:16 set_dmtag; 14:12 ctxdma; 10:8 size ?=> DmaTrfCmdSize; @@ -261,15 +194,15 @@ register! { 0:0 full => bool; } - pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ PFalconBase + 0x0000011c { + pub(crate) NV_PFALCON_FALCON_DMATRFFBOFFS(u32) @ 0x0000011c { 31:0 offs => u32; } - pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ PFalconBase + 0x00000128 { + pub(crate) NV_PFALCON_FALCON_DMATRFBASE1(u32) @ 0x00000128 { 8:0 base; } - pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ PFalconBase + 0x0000012c { + pub(crate) NV_PFALCON_FALCON_HWCFG1(u32) @ 0x0000012c { /// Core revision subversion. 7:6 core_rev_subversion => FalconCoreRevSubversion; /// Security model. @@ -278,12 +211,12 @@ register! { 3:0 core_rev ?=> FalconCoreRev; } - pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ PFalconBase + 0x00000130 { + pub(crate) NV_PFALCON_FALCON_CPUCTL_ALIAS(u32) @ 0x00000130 { 1:1 startcpu => bool; } /// IMEM access control register. Up to 4 ports are available for IMEM access. - pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ PFalconBase + 0x00000180 { + pub(crate) NV_PFALCON_FALCON_IMEMC(u32)[4, stride = 16] @ 0x00000180 { /// Access secure IMEM. 28:28 secure => bool; /// Auto-increment on write. @@ -294,17 +227,17 @@ register! { /// IMEM data register. Reading/writing this register accesses IMEM at the address /// specified by the corresponding IMEMC register. - pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ PFalconBase + 0x00000184 { + pub(crate) NV_PFALCON_FALCON_IMEMD(u32)[4, stride = 16] @ 0x00000184 { 31:0 data; } /// IMEM tag register. Used to set the tag for the current IMEM block. - pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ PFalconBase + 0x00000188 { + pub(crate) NV_PFALCON_FALCON_IMEMT(u32)[4, stride = 16] @ 0x00000188 { 15:0 tag; } /// DMEM access control register. Up to 8 ports are available for DMEM access. - pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ PFalconBase + 0x000001c0 { + pub(crate) NV_PFALCON_FALCON_DMEMC(u32)[8, stride = 8] @ 0x000001c0 { /// Auto-increment on write. 24:24 aincw => bool; /// DMEM block and word offset. @@ -313,29 +246,29 @@ register! { /// DMEM data register. Reading/writing this register accesses DMEM at the address /// specified by the corresponding DMEMC register. - pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ PFalconBase + 0x000001c4 { + pub(crate) NV_PFALCON_FALCON_DMEMD(u32)[8, stride = 8] @ 0x000001c4 { 31:0 data; } /// Actually known as `NV_PSEC_FALCON_ENGINE` and `NV_PGSP_FALCON_ENGINE` depending on the /// falcon instance. - pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ PFalconBase + 0x000003c0 { + pub(crate) NV_PFALCON_FALCON_ENGINE(u32) @ 0x000003c0 { 0:0 reset => bool; } - pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ PFalconBase + 0x00000600 { + pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 { 2:2 mem_type => FalconFbifMemType; 1:0 target ?=> FalconFbifTarget; } - pub(crate) NV_PFALCON_FBIF_CTL(u32) @ PFalconBase + 0x00000624 { + pub(crate) NV_PFALCON_FBIF_CTL(u32) @ 0x00000624 { 7:7 allow_phys_no_ctx => bool; } // Falcon EMEM PIO registers (used by FSP on Hopper/Blackwell). // These provide the falcon external memory communication interface. - pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ PFalconBase + 0x00000ac0 { + pub(crate) NV_PFALCON_FALCON_EMEMC(u32) @ 0x00000ac0 { /// EMEM byte offset (4-byte aligned) within the block. 7:2 offs; /// EMEM block to access. @@ -346,7 +279,7 @@ register! { 25:25 aincr => bool; } - pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ PFalconBase + 0x00000ac4 { + pub(crate) NV_PFALCON_FALCON_EMEMD(u32) @ 0x00000ac4 { 31:0 data => u32; } } @@ -372,13 +305,13 @@ impl NV_PFALCON_FALCON_DMATRFCMD { impl NV_PFALCON_FALCON_ENGINE { /// Resets the falcon - pub(crate) fn reset_engine<E: FalconEngine>(bar: Bar0<'_>) { - bar.update(Self::of::<E>(), |r| r.with_reset(true)); + pub(crate) fn reset_engine(pfalcon: Mmio<'_, PFalconRegisters>) { + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(true)); // TIMEOUT: falcon engine should not take more than 10us to reset. time::delay::fsleep(time::Delta::from_micros(10)); - bar.update(Self::of::<E>(), |r| r.with_reset(false)); + pfalcon.update(NV_PFALCON_FALCON_ENGINE, |r| r.with_reset(false)); } } @@ -392,21 +325,23 @@ impl NV_PFALCON_FALCON_HWCFG2 { /* PFALCON2 */ register! { - pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ PFalcon2Base + 0x00000180 { + base: PFalcon2Registers; + + pub(crate) NV_PFALCON2_FALCON_MOD_SEL(u32) @ 0x00000180 { 7:0 algo ?=> FalconModSelAlgo; } - pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ PFalcon2Base + 0x00000198 { + pub(crate) NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID(u32) @ 0x00000198 { 7:0 ucode_id => u8; } - pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ PFalcon2Base + 0x0000019c { + pub(crate) NV_PFALCON2_FALCON_BROM_ENGIDMASK(u32) @ 0x0000019c { 31:0 value => u32; } /// OpenRM defines this as a register array, but doesn't specify its size and only uses its /// first element. Be conservative until we know the actual size or need to use more registers. - pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ PFalcon2Base + 0x00000210 { + pub(crate) NV_PFALCON2_FALCON_BROM_PARAADDR(u32)[1] @ 0x00000210 { 31:0 value => u32; } } @@ -414,21 +349,23 @@ register! { // PRISCV register! { + base: PFalcon2Registers; + /// RISC-V status register for debug (Turing and GA100 only). /// Reflects current RISC-V core status. - pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ PFalcon2Base + 0x00000240 { + pub(crate) NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS(u32) @ 0x00000240 { /// RISC-V core active/inactive status. 0:0 active_stat => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { + pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ 0x00000388 { 7:7 active_stat => bool; 4:4 halted => bool; } /// GA102 and later. - pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ PFalcon2Base + 0x00000668 { + pub(crate) NV_PRISCV_RISCV_BCR_CTRL(u32) @ 0x00000668 { 8:8 br_fetch => bool; 4:4 core_select => PeregrineCoreSelect; 0:0 valid => bool; @@ -439,6 +376,8 @@ register! { // These registers manage falcon EMEM communication queues. register! { + base: NovaRegisters; + pub(crate) NV_PFSP_QUEUE_HEAD(u32)[8] @ 0x008f2c00 { 31:0 address => u32; } @@ -462,9 +401,13 @@ register! { pub(crate) mod gm107 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00021c04 { 0:0 display_disabled => bool; } @@ -474,9 +417,13 @@ pub(crate) mod gm107 { pub(crate) mod ga100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // FUSE register! { + base: NovaRegisters; + pub(crate) NV_FUSE_STATUS_OPT_DISPLAY(u32) @ 0x00820c04 { 0:0 display_disabled => bool; } @@ -488,9 +435,13 @@ pub(crate) const NV_THERM_I2CS_SCRATCH_FSP_BOOT_COMPLETE_STATUS_SUCCESS: u32 = 0 pub(crate) mod gh100 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x000200bc { 31:0 data; } @@ -505,9 +456,13 @@ pub(crate) mod gh100 { pub(crate) mod gb202 { use kernel::io::register; + use crate::driver::NovaRegisters; + // PTHERM register! { + base: NovaRegisters; + pub(crate) NV_THERM_I2CS_SCRATCH(u32) @ 0x00ad00bc { 31:0 data; } @@ -518,3 +473,69 @@ pub(crate) mod gb202 { } } } + +// MMU TLB + +register! { + base: NovaRegisters; + + /// TLB flush register: PDB address lower bits. + pub(crate) NV_TLB_FLUSH_PDB_LO(u32) @ 0x00b830a0 { + /// PDB address bits [39:8]. + 31:0 pdb_lo => u32; + } + + /// TLB flush register: PDB address higher bits. + pub(crate) NV_TLB_FLUSH_PDB_HI(u32) @ 0x00b830a4 { + /// PDB address bits [47:40]. + 7:0 pdb_hi => u8; + } + + /// TLB flush control register. + pub(crate) NV_TLB_FLUSH_CTRL(u32) @ 0x00b830b0 { + /// Invalidate every VA in the PDB selected by `NV_TLB_FLUSH_PDB_LO/HI`. + 0:0 all_va => bool; + /// Invalidate TLBs for all PDBs (ignores `NV_TLB_FLUSH_PDB_LO/HI`). + 1:1 all_pdb => bool; + /// Restrict the flush to the HUB MMU's TLBs; skip broadcasting to the + /// per-GPC L2 TLBs. + /// + /// The GPU MMU has a two-level TLB hierarchy: + /// 1. The *HUB MMU* sits at the top and serves memory requests from + /// "host-side" engines: the host/channel interface, copy engines, + /// display, and BAR1/BAR2 accesses. + /// 2. Each GPC (Graphics Processing Cluster — the block that houses + /// shader cores / SMs) has its own L2 TLB that serves requests from + /// the compute and graphics engines inside the cluster. + /// + /// When set, only the HUB TLBs are invalidated. This is a performance + /// optimization for flushes that only affect HUB-side mappings (e.g. + /// BAR1/BAR2 windows), where fanning the invalidation out to every + /// GPC's L2 TLB would be wasted work. Must be false when flushing + /// mappings that may be cached by compute/graphics engines. + 2:2 hubtlb_only => bool; + /// Invalidation acknowledgment scope. See [`TlbAckMode`] for details. + 8:7 ack ?=> TlbAckMode; + /// Write 1 to kick off the flush. Hardware clears this bit when the + /// flush completes; reads as 1 while the flush is in progress. + 31:31 trigger => bool; + } +} + +impl NV_TLB_FLUSH_PDB_LO { + /// Create a register value from a PDB address. + /// + /// Extracts bits [39:8] of the address and shifts it right by 8 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_lo(((addr >> 8) & 0xFFFF_FFFF) as u32) + } +} + +impl NV_TLB_FLUSH_PDB_HI { + /// Create a register value from a PDB address. + /// + /// Extracts bits [47:40] of the address and shifts it right by 40 bits. + pub(crate) fn from_pdb_addr(addr: u64) -> Self { + Self::zeroed().with_pdb_hi(((addr >> 40) & 0xFF) as u8) + } +} diff --git a/drivers/gpu/nova-core/selftest.rs b/drivers/gpu/nova-core/selftest.rs new file mode 100644 index 000000000000..f5b5965b7e6a --- /dev/null +++ b/drivers/gpu/nova-core/selftest.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +//! Assertion macros for driver self-tests. +//! +//! Self-tests run against live hardware during probe, so a failed assertion should not panic. These +//! macros log the failure on the device and fail the enclosing test by returning +//! [`EIO`](kernel::error::code::EIO) instead. + +/// Like [`assert!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert!`], a custom message with format arguments can follow the condition. +#[macro_export] +macro_rules! selftest_assert { + ($dev:expr, $cond:expr $(,)?) => { + $crate::selftest_assert!($dev, $cond, "assertion failed: {}", ::core::stringify!($cond)) + }; + ($dev:expr, $cond:expr, $($arg:tt)+) => {{ + if !$cond { + ::kernel::dev_err!( + $dev, + "Selftest: {}:{}: {}\n", + ::core::file!(), + ::core::line!(), + ::kernel::prelude::fmt!($($arg)+) + ); + return Err(::kernel::error::code::EIO); + } + }}; +} + +/// Like [`assert_eq!`], but logs the failure via `dev` and fails the enclosing test instead of +/// panicking. +/// +/// As with [`assert_eq!`], a custom message with format arguments can follow the compared values. +#[macro_export] +macro_rules! selftest_assert_eq { + ($dev:expr, $left:expr, $right:expr $(,)?) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `{} == {}` failed: left {:?}, right {:?}", + ::core::stringify!($left), + ::core::stringify!($right), + left, + right + ), + } + }; + ($dev:expr, $left:expr, $right:expr, $($arg:tt)+) => { + match (&$left, &$right) { + (left, right) => $crate::selftest_assert!( + $dev, + left == right, + "assertion `left == right` failed: {}: left {:?}, right {:?}", + ::kernel::prelude::fmt!($($arg)+), + left, + right + ), + } + }; +} diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c03650ee5226..9c214b9f4dd9 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -16,7 +16,10 @@ use kernel::{ }; use crate::{ - driver::Bar0, + driver::{ + Bar0, + NovaRegisters, // + }, firmware::{ fwsec::Bcrt30Rsa3kSignature, FalconUCodeDesc, @@ -92,12 +95,16 @@ impl<'a> VbiosIterator<'a> { fn rom_offset(dev: &device::Device, bar0: Bar0<'_>) -> Result<usize> { // IFR Header in VBIOS. register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 { 31:0 signature; } } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 { 30:16 fixed_data_size; 15:8 version => u8; @@ -105,6 +112,8 @@ impl<'a> VbiosIterator<'a> { } register! { + base: NovaRegisters; + NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 { 19:0 total_data_size; } |