diff options
Diffstat (limited to 'drivers/gpu/nova-core')
55 files changed, 2411 insertions, 2493 deletions
diff --git a/drivers/gpu/nova-core/.gitignore b/drivers/gpu/nova-core/.gitignore new file mode 100644 index 000000000000..7cc8318c76b1 --- /dev/null +++ b/drivers/gpu/nova-core/.gitignore @@ -0,0 +1 @@ +exports_nova_core_generated.h diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile index 4ae544f808f4..216329760a5b 100644 --- a/drivers/gpu/nova-core/Makefile +++ b/drivers/gpu/nova-core/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 - -obj-$(CONFIG_NOVA_CORE) += nova-core.o -nova-core-y := nova_core.o +# nova-core is built from drivers/gpu/Makefile. +# nova_core.o (rust-analyzer marker - DO NOT REMOVE). diff --git a/drivers/gpu/nova-core/bitfield.rs b/drivers/gpu/nova-core/bitfield.rs deleted file mode 100644 index 660c3911402d..000000000000 --- a/drivers/gpu/nova-core/bitfield.rs +++ /dev/null @@ -1,329 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -//! Bitfield library for Rust structures -//! -//! Support for defining bitfields in Rust structures. Also used by the [`register!`] macro. - -/// Defines a struct with accessors to access bits within an inner unsigned integer. -/// -/// # Syntax -/// -/// ```rust -/// use nova_core::bitfield; -/// -/// #[derive(Debug, Clone, Copy, Default)] -/// enum Mode { -/// #[default] -/// Low = 0, -/// High = 1, -/// Auto = 2, -/// } -/// -/// impl TryFrom<u8> for Mode { -/// type Error = u8; -/// fn try_from(value: u8) -> Result<Self, Self::Error> { -/// match value { -/// 0 => Ok(Mode::Low), -/// 1 => Ok(Mode::High), -/// 2 => Ok(Mode::Auto), -/// _ => Err(value), -/// } -/// } -/// } -/// -/// impl From<Mode> for u8 { -/// fn from(mode: Mode) -> u8 { -/// mode as u8 -/// } -/// } -/// -/// #[derive(Debug, Clone, Copy, Default)] -/// enum State { -/// #[default] -/// Inactive = 0, -/// Active = 1, -/// } -/// -/// impl From<bool> for State { -/// fn from(value: bool) -> Self { -/// if value { State::Active } else { State::Inactive } -/// } -/// } -/// -/// impl From<State> for bool { -/// fn from(state: State) -> bool { -/// match state { -/// State::Inactive => false, -/// State::Active => true, -/// } -/// } -/// } -/// -/// bitfield! { -/// pub struct ControlReg(u32) { -/// 7:7 state as bool => State; -/// 3:0 mode as u8 ?=> Mode; -/// } -/// } -/// ``` -/// -/// This generates a struct with: -/// - Field accessors: `mode()`, `state()`, etc. -/// - Field setters: `set_mode()`, `set_state()`, etc. (supports chaining with builder pattern). -/// Note that the compiler will error out if the size of the setter's arg exceeds the -/// struct's storage size. -/// - Debug and Default implementations. -/// -/// Note: Field accessors and setters inherit the same visibility as the struct itself. -/// In the example above, both `mode()` and `set_mode()` methods will be `pub`. -/// -/// Fields are defined as follows: -/// -/// - `as <type>` simply returns the field value casted to <type>, typically `u32`, `u16`, `u8` or -/// `bool`. Note that `bool` fields must have a range of 1 bit. -/// - `as <type> => <into_type>` calls `<into_type>`'s `From::<<type>>` implementation and returns -/// the result. -/// - `as <type> ?=> <try_into_type>` calls `<try_into_type>`'s `TryFrom::<<type>>` implementation -/// and returns the result. This is useful with fields for which not all values are valid. -macro_rules! bitfield { - // Main entry point - defines the bitfield struct with fields - ($vis:vis struct $name:ident($storage:ty) $(, $comment:literal)? { $($fields:tt)* }) => { - bitfield!(@core $vis $name $storage $(, $comment)? { $($fields)* }); - }; - - // All rules below are helpers. - - // Defines the wrapper `$name` type, as well as its relevant implementations (`Debug`, - // `Default`, and conversion to the value type) and field accessor methods. - (@core $vis:vis $name:ident $storage:ty $(, $comment:literal)? { $($fields:tt)* }) => { - $( - #[doc=$comment] - )? - #[repr(transparent)] - #[derive(Clone, Copy)] - $vis struct $name($storage); - - impl ::core::convert::From<$name> for $storage { - fn from(val: $name) -> $storage { - val.0 - } - } - - bitfield!(@fields_dispatcher $vis $name $storage { $($fields)* }); - }; - - // Captures the fields and passes them to all the implementers that require field information. - // - // Used to simplify the matching rules for implementers, so they don't need to match the entire - // complex fields rule even though they only make use of part of it. - (@fields_dispatcher $vis:vis $name:ident $storage:ty { - $($hi:tt:$lo:tt $field:ident as $type:tt - $(?=> $try_into_type:ty)? - $(=> $into_type:ty)? - $(, $comment:literal)? - ; - )* - } - ) => { - bitfield!(@field_accessors $vis $name $storage { - $( - $hi:$lo $field as $type - $(?=> $try_into_type)? - $(=> $into_type)? - $(, $comment)? - ; - )* - }); - bitfield!(@debug $name { $($field;)* }); - bitfield!(@default $name { $($field;)* }); - }; - - // Defines all the field getter/setter methods for `$name`. - ( - @field_accessors $vis:vis $name:ident $storage:ty { - $($hi:tt:$lo:tt $field:ident as $type:tt - $(?=> $try_into_type:ty)? - $(=> $into_type:ty)? - $(, $comment:literal)? - ; - )* - } - ) => { - $( - bitfield!(@check_field_bounds $hi:$lo $field as $type); - )* - - #[allow(dead_code)] - impl $name { - $( - bitfield!(@field_accessor $vis $name $storage, $hi:$lo $field as $type - $(?=> $try_into_type)? - $(=> $into_type)? - $(, $comment)? - ; - ); - )* - } - }; - - // Boolean fields must have `$hi == $lo`. - (@check_field_bounds $hi:tt:$lo:tt $field:ident as bool) => { - #[allow(clippy::eq_op)] - const _: () = { - ::kernel::build_assert::build_assert!( - $hi == $lo, - concat!("boolean field `", stringify!($field), "` covers more than one bit") - ); - }; - }; - - // Non-boolean fields must have `$hi >= $lo`. - (@check_field_bounds $hi:tt:$lo:tt $field:ident as $type:tt) => { - #[allow(clippy::eq_op)] - const _: () = { - ::kernel::build_assert::build_assert!( - $hi >= $lo, - concat!("field `", stringify!($field), "`'s MSB is smaller than its LSB") - ); - }; - }; - - // Catches fields defined as `bool` and convert them into a boolean value. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool - => $into_type:ty $(, $comment:literal)?; - ) => { - bitfield!( - @leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$into_type>::from(f != 0) } - bool $into_type => $into_type $(, $comment)?; - ); - }; - - // Shortcut for fields defined as `bool` without the `=>` syntax. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as bool - $(, $comment:literal)?; - ) => { - bitfield!( - @field_accessor $vis $name $storage, $hi:$lo $field as bool => bool $(, $comment)?; - ); - }; - - // Catches the `?=>` syntax for non-boolean fields. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - ?=> $try_into_type:ty $(, $comment:literal)?; - ) => { - bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$try_into_type>::try_from(f as $type) } $type $try_into_type => - ::core::result::Result< - $try_into_type, - <$try_into_type as ::core::convert::TryFrom<$type>>::Error - > - $(, $comment)?;); - }; - - // Catches the `=>` syntax for non-boolean fields. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - => $into_type:ty $(, $comment:literal)?; - ) => { - bitfield!(@leaf_accessor $vis $name $storage, $hi:$lo $field - { |f| <$into_type>::from(f as $type) } $type $into_type => $into_type $(, $comment)?;); - }; - - // Shortcut for non-boolean fields defined without the `=>` or `?=>` syntax. - ( - @field_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident as $type:tt - $(, $comment:literal)?; - ) => { - bitfield!( - @field_accessor $vis $name $storage, $hi:$lo $field as $type => $type $(, $comment)?; - ); - }; - - // Generates the accessor methods for a single field. - ( - @leaf_accessor $vis:vis $name:ident $storage:ty, $hi:tt:$lo:tt $field:ident - { $process:expr } $prim_type:tt $to_type:ty => $res_type:ty $(, $comment:literal)?; - ) => { - ::kernel::macros::paste!( - const [<$field:upper _RANGE>]: ::core::ops::RangeInclusive<u8> = $lo..=$hi; - const [<$field:upper _MASK>]: $storage = { - // Generate mask for shifting - match ::core::mem::size_of::<$storage>() { - 1 => ::kernel::bits::genmask_u8($lo..=$hi) as $storage, - 2 => ::kernel::bits::genmask_u16($lo..=$hi) as $storage, - 4 => ::kernel::bits::genmask_u32($lo..=$hi) as $storage, - 8 => ::kernel::bits::genmask_u64($lo..=$hi) as $storage, - _ => ::kernel::build_error!("Unsupported storage type size") - } - }; - const [<$field:upper _SHIFT>]: u32 = $lo; - ); - - $( - #[doc="Returns the value of this field:"] - #[doc=$comment] - )? - #[inline(always)] - $vis fn $field(self) -> $res_type { - ::kernel::macros::paste!( - const MASK: $storage = $name::[<$field:upper _MASK>]; - const SHIFT: u32 = $name::[<$field:upper _SHIFT>]; - ); - let field = ((self.0 & MASK) >> SHIFT); - - $process(field) - } - - ::kernel::macros::paste!( - $( - #[doc="Sets the value of this field:"] - #[doc=$comment] - )? - #[inline(always)] - $vis fn [<set_ $field>](mut self, value: $to_type) -> Self { - const MASK: $storage = $name::[<$field:upper _MASK>]; - const SHIFT: u32 = $name::[<$field:upper _SHIFT>]; - let value = ($storage::from($prim_type::from(value)) << SHIFT) & MASK; - self.0 = (self.0 & !MASK) | value; - - self - } - ); - }; - - // Generates the `Debug` implementation for `$name`. - (@debug $name:ident { $($field:ident;)* }) => { - impl ::kernel::fmt::Debug for $name { - fn fmt(&self, f: &mut ::kernel::fmt::Formatter<'_>) -> ::kernel::fmt::Result { - f.debug_struct(stringify!($name)) - .field("<raw>", &::kernel::prelude::fmt!("{:#x}", &self.0)) - $( - .field(stringify!($field), &self.$field()) - )* - .finish() - } - } - }; - - // Generates the `Default` implementation for `$name`. - (@default $name:ident { $($field:ident;)* }) => { - /// Returns a value for the bitfield where all fields are set to their default value. - impl ::core::default::Default for $name { - fn default() -> Self { - let value = Self(Default::default()); - - ::kernel::macros::paste!( - $( - let value = value.[<set_ $field>](Default::default()); - )* - ); - - value - } - } - }; -} diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs index 5738d4ac521b..bbd93959e0b2 100644 --- a/drivers/gpu/nova-core/driver.rs +++ b/drivers/gpu/nova-core/driver.rs @@ -15,7 +15,7 @@ use kernel::{ Atomic, Relaxed, // }, - types::ForLt, + types::CovariantForLt, }; use crate::gpu::Gpu; @@ -29,7 +29,7 @@ pub(crate) struct NovaCore<'bound> { pub(crate) gpu: Gpu<'bound>, bar: pci::Bar<'bound, BAR0_SIZE>, #[allow(clippy::type_complexity)] - _reg: auxiliary::Registration<'bound, ForLt!(())>, + _reg: auxiliary::Registration<'bound, CovariantForLt!(())>, } pub(crate) struct NovaCoreDriver; @@ -40,7 +40,6 @@ pub(crate) type Bar0<'a> = &'a pci::Bar<'a, BAR0_SIZE>; kernel::pci_device_table!( PCI_TABLE, - MODULE_PCI_TABLE, <NovaCoreDriver as pci::Driver>::IdInfo, [ // Modern NVIDIA GPUs will show up as either VGA or 3D controllers. @@ -70,7 +69,7 @@ impl pci::Driver for NovaCoreDriver { fn probe<'bound>( pdev: &'bound pci::Device<Core<'_>>, - _info: &'bound Self::IdInfo, + _info: Option<&'bound Self::IdInfo>, ) -> impl PinInit<Self::Data<'bound>, Error> + 'bound { pin_init::pin_init_scope(move || { dev_dbg!(pdev, "Probe Nova Core GPU driver.\n"); diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs index 94c7696a6493..65cb12d26e2b 100644 --- a/drivers/gpu/nova-core/falcon.rs +++ b/drivers/gpu/nova-core/falcon.rs @@ -5,17 +5,14 @@ use hal::FalconHal; use kernel::{ - device::{ - self, - Device, // - }, + device, dma::{ Coherent, CoherentBox, - DmaAddress, - DmaMask, // + DmaAddress, // }, io::{ + io_project, poll::read_poll_timeout, register::{ RegisterBase, @@ -24,7 +21,6 @@ use kernel::{ Io, }, prelude::*, - sync::aref::ARef, time::Delta, }; @@ -358,41 +354,47 @@ pub(crate) trait FalconFirmware { } /// Contains the base parameters common to all Falcon instances. -pub(crate) struct Falcon<E: FalconEngine> { +pub(crate) struct Falcon<'a, E: FalconEngine> { hal: KBox<dyn FalconHal<E>>, - dev: ARef<device::Device>, + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, } -impl<E: FalconEngine + 'static> Falcon<E> { +impl<'a, E: FalconEngine + 'static> Falcon<'a, E> { /// Create a new falcon instance. - pub(crate) fn new(dev: &device::Device, chipset: Chipset) -> Result<Self> { + pub(crate) fn new( + dev: &'a device::Device<device::Bound>, + chipset: Chipset, + bar: Bar0<'a>, + ) -> Result<Self> { Ok(Self { hal: hal::falcon_hal(chipset)?, - dev: dev.into(), + dev, + bar, }) } /// Resets DMA-related registers. - pub(crate) fn dma_reset(&self, bar: Bar0<'_>) { - bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { + pub(crate) fn dma_reset(&self) { + self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { v.with_allow_phys_no_ctx(true) }); - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_DMACTL::zeroed(), ); } /// Reset the controller, select the falcon core, and wait for memory scrubbing to complete. - pub(crate) fn reset(&self, bar: Bar0<'_>) -> Result { - self.hal.reset_eng(bar)?; - self.hal.select_core(self, bar)?; - self.hal.reset_wait_mem_scrubbing(bar)?; + pub(crate) fn reset(&self) -> Result { + self.hal.reset_eng(self)?; + self.hal.select_core(self)?; + self.hal.reset_wait_mem_scrubbing(self)?; - bar.write( + self.bar.write( WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_RM::from(bar.read(regs::NV_PMC_BOOT_0).into_raw()), + regs::NV_PFALCON_FALCON_RM::from(self.bar.read(regs::NV_PMC_BOOT_0).into_raw()), ); Ok(()) @@ -404,18 +406,14 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Write a slice to Falcon IMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_imem_slice( - &self, - bar: Bar0<'_>, - load_offsets: FalconPioImemLoadTarget<'_>, - ) -> Result { + fn pio_wr_imem_slice(&self, load_offsets: FalconPioImemLoadTarget<'_>) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { return Err(EINVAL); } - bar.write( + self.bar.write( WithBase::of::<E>().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMC::zeroed() .with_secure(load_offsets.secure) @@ -426,13 +424,13 @@ impl<E: FalconEngine + 'static> Falcon<E> { for (n, block) in load_offsets.data.chunks(MEM_BLOCK_ALIGNMENT).enumerate() { let n = u16::try_from(n)?; let tag: u16 = load_offsets.start_tag.checked_add(n).ok_or(ERANGE)?; - bar.write( + self.bar.write( WithBase::of::<E>().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMT::zeroed().with_tag(tag), ); for word in block.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - bar.write( + self.bar.write( WithBase::of::<E>().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_IMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); @@ -445,18 +443,14 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Write a slice to Falcon DMEM memory using programmed I/O (PIO). /// /// Returns `EINVAL` if `img.len()` is not a multiple of 4. - fn pio_wr_dmem_slice( - &self, - bar: Bar0<'_>, - load_offsets: FalconPioDmemLoadTarget<'_>, - ) -> Result { + fn pio_wr_dmem_slice(&self, load_offsets: FalconPioDmemLoadTarget<'_>) -> Result { // Rejecting misaligned images here allows us to avoid checking // inside the loops. if load_offsets.data.len() % 4 != 0 { return Err(EINVAL); } - bar.write( + self.bar.write( WithBase::of::<E>().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMC::zeroed() .with_aincw(true) @@ -465,7 +459,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { for word in load_offsets.data.chunks_exact(4) { let w = [word[0], word[1], word[2], word[3]]; - bar.write( + self.bar.write( WithBase::of::<E>().at(Self::PIO_PORT), regs::NV_PFALCON_FALCON_DMEMD::zeroed().with_data(u32::from_le_bytes(w)), ); @@ -477,29 +471,28 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// Perform a PIO copy into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>( &self, - bar: Bar0<'_>, fw: &F, ) -> Result { - bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { + self.bar.update(regs::NV_PFALCON_FBIF_CTL::of::<E>(), |v| { v.with_allow_phys_no_ctx(true) }); - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_DMACTL::zeroed(), ); if let Some(imem_ns) = fw.imem_ns_load_params() { - self.pio_wr_imem_slice(bar, imem_ns)?; + self.pio_wr_imem_slice(imem_ns)?; } if let Some(imem_sec) = fw.imem_sec_load_params() { - self.pio_wr_imem_slice(bar, imem_sec)?; + self.pio_wr_imem_slice(imem_sec)?; } - self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?; + self.pio_wr_dmem_slice(fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params()); + self.hal.program_brom(self, &fw.brom_params()); - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), ); @@ -507,33 +500,43 @@ impl<E: FalconEngine + 'static> Falcon<E> { Ok(()) } - /// Perform a DMA write according to `load_offsets` from `dma_handle` into the falcon's + /// Perform a DMA write according to `load_offsets` from `dma_obj` into the falcon's /// `target_mem`. /// /// `sec` is set if the loaded firmware is expected to run in secure mode. fn dma_wr( &self, - bar: Bar0<'_>, dma_obj: &Coherent<[u8]>, target_mem: FalconMem, load_offsets: FalconDmaLoadTarget, ) -> Result { const DMA_LEN: u32 = num::usize_into_u32::<{ MEM_BLOCK_ALIGNMENT }>(); + // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we + // need to perform. + let num_transfers = load_offsets.len.div_ceil(DMA_LEN); + // For IMEM, we want to use the start offset as a virtual address tag for each page, since // code addresses in the firmware (and the boot vector) are virtual. // - // For DMEM we can fold the start offset into the DMA handle. + // For DMEM, the start offset is folded into the DMA address. let (src_start, dma_start) = match target_mem { - FalconMem::ImemSecure | FalconMem::ImemNonSecure => { - (load_offsets.src_start, dma_obj.dma_handle()) - } - FalconMem::Dmem => ( - 0, - dma_obj.dma_handle() + DmaAddress::from(load_offsets.src_start), - ), + FalconMem::ImemSecure | FalconMem::ImemNonSecure => (load_offsets.src_start, 0), + FalconMem::Dmem => (0, usize::from_safe_cast(load_offsets.src_start)), + }; + + let dma_address = { + // Upper limit of transfer is `(num_transfers * DMA_LEN) + load_offsets.src_start`. + let dma_end = num_transfers + .checked_mul(DMA_LEN) + .and_then(|size| size.checked_add(load_offsets.src_start)) + .map(usize::from_safe_cast) + .ok_or(EOVERFLOW)?; + + io_project!(dma_obj, [try: dma_start..dma_end]).dma_address() }; - if dma_start % DmaAddress::from(DMA_LEN) > 0 { + + if dma_address % DmaAddress::from(DMA_LEN) > 0 { dev_err!( self.dev, "DMA transfer start addresses must be a multiple of {}\n", @@ -542,46 +545,19 @@ impl<E: FalconEngine + 'static> Falcon<E> { return Err(EINVAL); } - // The DMATRFBASE/1 register pair only supports a 49-bit address. - if dma_start > DmaMask::new::<49>().value() { - dev_err!(self.dev, "DMA address {:#x} exceeds 49 bits\n", dma_start); - return Err(ERANGE); - } - - // DMA transfers can only be done in units of 256 bytes. Compute how many such transfers we - // need to perform. - let num_transfers = load_offsets.len.div_ceil(DMA_LEN); - - // Check that the area we are about to transfer is within the bounds of the DMA object. - // Upper limit of transfer is `(num_transfers * DMA_LEN) + load_offsets.src_start`. - match num_transfers - .checked_mul(DMA_LEN) - .and_then(|size| size.checked_add(load_offsets.src_start)) - { - None => { - dev_err!(self.dev, "DMA transfer length overflow\n"); - return Err(EOVERFLOW); - } - Some(upper_bound) if usize::from_safe_cast(upper_bound) > dma_obj.size() => { - dev_err!(self.dev, "DMA transfer goes beyond range of DMA object\n"); - return Err(EINVAL); - } - Some(_) => (), - }; - // Set up the base source DMA address. - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base( // CAST: `as u32` is used on purpose since we do want to strip the upper bits, // which will be written to `NV_PFALCON_FALCON_DMATRFBASE1`. - (dma_start >> 8) as u32, + (dma_address >> 8) as u32, ), ); - bar.write( + self.bar.write( WithBase::of::<E>(), - regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_start >> 40)?, + regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(dma_address >> 40)?, ); let cmd = regs::NV_PFALCON_FALCON_DMATRFCMD::zeroed() @@ -590,23 +566,23 @@ impl<E: FalconEngine + 'static> Falcon<E> { for pos in (0..num_transfers).map(|i| i * DMA_LEN) { // Perform a transfer of size `DMA_LEN`. - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed() .try_with_offs(load_offsets.dst_start + pos)?, ); - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed().with_offs(src_start + pos), ); - bar.write(WithBase::of::<E>(), cmd); + self.bar.write(WithBase::of::<E>(), cmd); // Wait for the transfer to complete. // TIMEOUT: arbitrarily large value, no DMA transfer to the falcon's small memories // should ever take that long. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())), + || Ok(self.bar.read(regs::NV_PFALCON_FALCON_DMATRFCMD::of::<E>())), |r| r.idle(), Delta::ZERO, Delta::from_secs(2), @@ -617,12 +593,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it. - fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>( - &self, - dev: &Device<device::Bound>, - bar: Bar0<'_>, - fw: &F, - ) -> Result { + fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) -> Result { // DMA object with firmware content as the source of the DMA engine. let dma_obj = { let fw_slice = fw.as_slice(); @@ -630,7 +601,7 @@ impl<E: FalconEngine + 'static> Falcon<E> { // DMA copies are done in chunks of `MEM_BLOCK_ALIGNMENT`, so pad the length // accordingly and fill with `0`. let mut dma_obj = CoherentBox::zeroed_slice( - dev, + self.dev, fw_slice.len().next_multiple_of(MEM_BLOCK_ALIGNMENT), GFP_KERNEL, )?; @@ -642,24 +613,20 @@ impl<E: FalconEngine + 'static> Falcon<E> { dma_obj.into() }; - self.dma_reset(bar); - bar.update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| { - v.with_target(FalconFbifTarget::CoherentSysmem) - .with_mem_type(FalconFbifMemType::Physical) - }); + self.dma_reset(); + self.bar + .update(regs::NV_PFALCON_FBIF_TRANSCFG::of::<E>().at(0), |v| { + v.with_target(FalconFbifTarget::CoherentSysmem) + .with_mem_type(FalconFbifMemType::Physical) + }); - self.dma_wr( - bar, - &dma_obj, - FalconMem::ImemSecure, - fw.imem_sec_load_params(), - )?; - self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; + self.dma_wr(&dma_obj, FalconMem::ImemSecure, fw.imem_sec_load_params())?; + self.dma_wr(&dma_obj, FalconMem::Dmem, fw.dmem_load_params())?; - self.hal.program_brom(self, bar, &fw.brom_params()); + self.hal.program_brom(self, &fw.brom_params()); // Set `BootVec` to start of non-secure code. - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(fw.boot_addr()), ); @@ -668,10 +635,10 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Wait until the falcon CPU is halted. - pub(crate) fn wait_till_halted(&self, bar: Bar0<'_>) -> Result<()> { + pub(crate) fn wait_till_halted(&self) -> Result<()> { // TIMEOUT: arbitrarily large value, firmwares should complete in less than 2 seconds. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())), + || Ok(self.bar.read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>())), |r| r.halted(), Delta::ZERO, Delta::from_secs(2), @@ -681,16 +648,17 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Start the falcon CPU. - pub(crate) fn start(&self, bar: Bar0<'_>) -> Result<()> { - match bar + pub(crate) fn start(&self) -> Result<()> { + match self + .bar .read(regs::NV_PFALCON_FALCON_CPUCTL::of::<E>()) .alias_en() { - true => bar.write( + true => self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_CPUCTL_ALIAS::zeroed().with_startcpu(true), ), - false => bar.write( + false => self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_CPUCTL::zeroed().with_startcpu(true), ), @@ -700,16 +668,16 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Writes values to the mailbox registers if provided. - pub(crate) fn write_mailboxes(&self, bar: Bar0<'_>, mbox0: Option<u32>, mbox1: Option<u32>) { + pub(crate) fn write_mailboxes(&self, mbox0: Option<u32>, mbox1: Option<u32>) { if let Some(mbox0) = mbox0 { - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_MAILBOX0::zeroed().with_value(mbox0), ); } if let Some(mbox1) = mbox1 { - bar.write( + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_MAILBOX1::zeroed().with_value(mbox1), ); @@ -717,21 +685,23 @@ impl<E: FalconEngine + 'static> Falcon<E> { } /// Reads the value from `mbox0` register. - pub(crate) fn read_mailbox0(&self, bar: Bar0<'_>) -> u32 { - bar.read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>()) + pub(crate) fn read_mailbox0(&self) -> u32 { + self.bar + .read(regs::NV_PFALCON_FALCON_MAILBOX0::of::<E>()) .value() } /// Reads the value from `mbox1` register. - pub(crate) fn read_mailbox1(&self, bar: Bar0<'_>) -> u32 { - bar.read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>()) + pub(crate) fn read_mailbox1(&self) -> u32 { + self.bar + .read(regs::NV_PFALCON_FALCON_MAILBOX1::of::<E>()) .value() } /// Reads values from both mailbox registers. - pub(crate) fn read_mailboxes(&self, bar: Bar0<'_>) -> (u32, u32) { - let mbox0 = self.read_mailbox0(bar); - let mbox1 = self.read_mailbox1(bar); + pub(crate) fn read_mailboxes(&self) -> (u32, u32) { + let mbox0 = self.read_mailbox0(); + let mbox1 = self.read_mailbox1(); (mbox0, mbox1) } @@ -743,54 +713,54 @@ impl<E: FalconEngine + 'static> Falcon<E> { /// /// Wait up to two seconds for the firmware to complete, and return its exit status read from /// the `MBOX0` and `MBOX1` registers. - pub(crate) fn boot( - &self, - bar: Bar0<'_>, - mbox0: Option<u32>, - mbox1: Option<u32>, - ) -> Result<(u32, u32)> { - self.write_mailboxes(bar, mbox0, mbox1); - self.start(bar)?; - self.wait_till_halted(bar)?; - Ok(self.read_mailboxes(bar)) + pub(crate) fn boot(&self, mbox0: Option<u32>, mbox1: Option<u32>) -> Result<(u32, u32)> { + self.write_mailboxes(mbox0, mbox1); + self.start()?; + self.wait_till_halted()?; + Ok(self.read_mailboxes()) } /// Returns the fused version of the signature to use in order to run a HS firmware on this /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. pub(crate) fn signature_reg_fuse_version( &self, - bar: Bar0<'_>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32> { self.hal - .signature_reg_fuse_version(self, bar, engine_id_mask, ucode_id) + .signature_reg_fuse_version(self, engine_id_mask, ucode_id) } /// Check if the RISC-V core is active. /// + /// Note that this does not guarantee that the RISC-V core is halted if it returns `false`. + /// /// Returns `true` if the RISC-V core is active, `false` otherwise. - pub(crate) fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - self.hal.is_riscv_active(bar) + pub(crate) fn is_riscv_active(&self) -> bool { + self.hal.is_riscv_active(self) + } + + /// Checks whether the RISC-V core is halted. + /// + /// Note that this does not guarantee that the RISC-V core is active if it returns `false`. + /// + /// Returns [`ENOTSUPP`] if the status is not available. + pub(crate) fn is_riscv_halted(&self) -> Result<bool> { + self.hal.is_riscv_halted(self) } /// Load a firmware image into Falcon memory, using the preferred method for the current /// chipset. - pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>( - &self, - dev: &Device<device::Bound>, - bar: Bar0<'_>, - fw: &F, - ) -> Result { + pub(crate) fn load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) -> Result { match self.hal.load_method() { - LoadMethod::Dma => self.dma_load(dev, bar, fw), - LoadMethod::Pio => self.pio_load(bar, &fw.try_as_pio_loadable()?), + LoadMethod::Dma => self.dma_load(fw), + LoadMethod::Pio => self.pio_load(&fw.try_as_pio_loadable()?), } } /// Write the application version to the OS register. - pub(crate) fn write_os_version(&self, bar: Bar0<'_>, app_version: u32) { - bar.write( + pub(crate) fn write_os_version(&self, app_version: u32) { + self.bar.write( WithBase::of::<E>(), regs::NV_PFALCON_FALCON_OS::zeroed().with_value(app_version), ); diff --git a/drivers/gpu/nova-core/falcon/fsp.rs b/drivers/gpu/nova-core/falcon/fsp.rs index 52cdb84ef0e8..0437180b8829 100644 --- a/drivers/gpu/nova-core/falcon/fsp.rs +++ b/drivers/gpu/nova-core/falcon/fsp.rs @@ -17,11 +17,11 @@ use kernel::{ Io, // }, prelude::*, + sizes::SZ_1K, time::Delta, }; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconEngine, @@ -35,6 +35,9 @@ use crate::{ /// FSP message timeout in milliseconds. const FSP_MSG_TIMEOUT_MS: i64 = 2000; +/// Size of the FSP EMEM channel 0 that we can use. +const FSP_EMEM_CHANNEL_0_SIZE: usize = SZ_1K; + /// Type specifying the `Fsp` falcon engine. Cannot be instantiated. pub(crate) struct Fsp(()); @@ -48,18 +51,18 @@ impl RegisterBase<PFalcon2Base> for Fsp { impl FalconEngine for Fsp {} -impl Falcon<Fsp> { +impl<'a> Falcon<'a, Fsp> { /// Writes `data` to FSP external memory at offset `0`. /// /// `data` is interpreted as little-endian 32-bit words. Returns `EINVAL` /// if the `data` length is not 4-byte aligned. - fn write_emem(&mut self, bar: Bar0<'_>, data: &[u8]) -> Result { + fn write_emem(&mut self, data: &[u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } // Begin a write burst at offset `0`, auto-incrementing on each write. - bar.write( + self.bar.write( WithBase::of::<Fsp>(), regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincw(true), ); @@ -68,7 +71,7 @@ impl Falcon<Fsp> { let value = u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); // Write the next 32-bit `value`; hardware advances the offset. - bar.write( + self.bar.write( WithBase::of::<Fsp>(), regs::NV_PFALCON_FALCON_EMEMD::zeroed().with_data(value), ); @@ -81,20 +84,23 @@ impl Falcon<Fsp> { /// /// `data` is stored as little-endian 32-bit words. Returns `EINVAL` if /// the `data` length is not 4-byte aligned. - fn read_emem(&mut self, bar: Bar0<'_>, data: &mut [u8]) -> Result { + fn read_emem(&mut self, data: &mut [u8]) -> Result { if data.len() % 4 != 0 { return Err(EINVAL); } // Begin a read burst at offset `0`, auto-incrementing on each read. - bar.write( + self.bar.write( WithBase::of::<Fsp>(), regs::NV_PFALCON_FALCON_EMEMC::zeroed().with_aincr(true), ); for chunk in data.chunks_exact_mut(4) { // Read the next 32-bit word; hardware advances the offset. - let value = bar.read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>()).data(); + let value = self + .bar + .read(regs::NV_PFALCON_FALCON_EMEMD::of::<Fsp>()) + .data(); chunk.copy_from_slice(&value.to_le_bytes()); } @@ -105,37 +111,41 @@ impl Falcon<Fsp> { /// /// Returns the size of available data in bytes, or 0 if no data is available. /// + /// Returns [`EIO`] if the queue pointers are bogus (`tail < head`). + /// /// The FSP message queue is not circular. Pointers are reset to 0 after each /// message exchange, so `tail >= head` is always true when data is present. - fn poll_msgq(&self, bar: Bar0<'_>) -> u32 { - let head = bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); - let tail = bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); + fn poll_msgq(&self) -> Result<u32> { + let head = self.bar.read(regs::NV_PFSP_MSGQ_HEAD::at(0)).val(); + let tail = self.bar.read(regs::NV_PFSP_MSGQ_TAIL::at(0)).val(); if head == tail { - return 0; + Ok(0) + } else { + // TAIL points at the last DWORD written, so the size is `tail - head + 4`. + tail.checked_sub(head) + .and_then(|delta| delta.checked_add(4)) + .ok_or(EIO) } - - // TAIL points at last DWORD written, so add 4 to get total size. - tail.saturating_sub(head).saturating_add(4) } /// Writes `packet` to FSP EMEM and updates the queue pointers to notify FSP. /// /// Returns `EINVAL` if `packet` is empty or its length is not 4-byte aligned. - pub(crate) fn send_msg(&mut self, bar: Bar0<'_>, packet: &[u8]) -> Result { + pub(crate) fn send_msg(&mut self, packet: &[u8]) -> Result { if packet.is_empty() { return Err(EINVAL); } - self.write_emem(bar, packet)?; + self.write_emem(packet)?; // Update queue pointers. TAIL points at the last DWORD written. let tail_offset = u32::try_from(packet.len() - 4).map_err(|_| EINVAL)?; - bar.write( + self.bar.write( Array::at(0), regs::NV_PFSP_QUEUE_TAIL::zeroed().with_address(tail_offset), ); - bar.write( + self.bar.write( Array::at(0), regs::NV_PFSP_QUEUE_HEAD::zeroed().with_address(0), ); @@ -148,23 +158,30 @@ impl Falcon<Fsp> { /// /// Returns `ETIMEDOUT` if no message was available until timeout, or a regular error code if a /// memory allocation error occurred. - pub(crate) fn recv_msg(&mut self, bar: Bar0<'_>) -> Result<KVec<u8>> { + pub(crate) fn recv_msg(&mut self) -> Result<KVec<u8>> { let msg_size = read_poll_timeout( - || Ok(self.poll_msgq(bar)), + || self.poll_msgq(), |&size| size > 0, Delta::from_millis(10), Delta::from_millis(FSP_MSG_TIMEOUT_MS), ) .map(num::u32_as_usize)?; + // Don't blindly allocate more than the maximum we expect from FSP. + if msg_size > FSP_EMEM_CHANNEL_0_SIZE { + return Err(EMSGSIZE); + } + let mut buffer = KVec::<u8>::new(); buffer.resize(msg_size, 0, GFP_KERNEL)?; - self.read_emem(bar, &mut buffer)?; + self.read_emem(&mut buffer)?; // Reset message queue pointers after reading. - bar.write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); - bar.write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); + self.bar + .write(Array::at(0), regs::NV_PFSP_MSGQ_TAIL::zeroed().with_val(0)); + self.bar + .write(Array::at(0), regs::NV_PFSP_MSGQ_HEAD::zeroed().with_val(0)); Ok(buffer) } diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs index d1f6f7fcffff..ae32f401aeb0 100644 --- a/drivers/gpu/nova-core/falcon/gsp.rs +++ b/drivers/gpu/nova-core/falcon/gsp.rs @@ -14,7 +14,6 @@ use kernel::{ }; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconEngine, @@ -24,10 +23,6 @@ use crate::{ regs, }; -/// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU access. -const GSP_TARGET_MASK_LOCKED_PATTERN: u32 = 0xbadf_4100; -const GSP_TARGET_MASK_LOCKED_MASK: u32 = 0xffff_ff00; - /// Type specifying the `Gsp` falcon engine. Cannot be instantiated. pub(crate) struct Gsp(()); @@ -41,20 +36,20 @@ impl RegisterBase<PFalcon2Base> for Gsp { impl FalconEngine for Gsp {} -impl Falcon<Gsp> { +impl<'a> Falcon<'a, Gsp> { /// Clears the SWGEN0 bit in the Falcon's IRQ status clear register to /// allow GSP to signal CPU for processing new messages in message queue. - pub(crate) fn clear_swgen0_intr(&self, bar: Bar0<'_>) { - bar.write( + pub(crate) fn clear_swgen0_intr(&self) { + self.bar.write( WithBase::of::<Gsp>(), regs::NV_PFALCON_FALCON_IRQSCLR::zeroed().with_swgen0(true), ); } /// Checks if GSP reload/resume has completed during the boot process. - pub(crate) fn check_reload_completed(&self, bar: Bar0<'_>, timeout: Delta) -> Result<bool> { + pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result<bool> { read_poll_timeout( - || Ok(bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), + || Ok(self.bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)), |val| val.boot_stage_3_handoff(), Delta::ZERO, timeout, @@ -63,17 +58,24 @@ impl Falcon<Gsp> { } /// Returns whether the RISC-V branch privilege lockdown bit is set. - pub(crate) fn riscv_branch_privilege_lockdown(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) + pub(crate) fn riscv_branch_privilege_lockdown(&self) -> bool { + self.bar + .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) .riscv_br_priv_lockdown() } /// Returns whether GSP registers can be read by the CPU. - pub(crate) fn priv_target_mask_released(&self, bar: Bar0<'_>) -> bool { - let hwcfg2 = bar + pub(crate) fn priv_target_mask_released(&self) -> bool { + /// Pattern returned by GSP register reads while the PRIV target mask still blocks CPU + /// access. The low byte varies; the upper 24 bits are fixed. + const LOCKED_PATTERN: u32 = 0xbadf_4100; + const LOCKED_MASK: u32 = 0xffff_ff00; + + let hwcfg2 = self + .bar .read(regs::NV_PFALCON_FALCON_HWCFG2::of::<Gsp>()) .into_raw(); - hwcfg2 != 0 && (hwcfg2 & GSP_TARGET_MASK_LOCKED_MASK) != GSP_TARGET_MASK_LOCKED_PATTERN + hwcfg2 != 0 && (hwcfg2 & LOCKED_MASK) != LOCKED_PATTERN } } diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs index 89b56823906b..7e532889a1f4 100644 --- a/drivers/gpu/nova-core/falcon/hal.rs +++ b/drivers/gpu/nova-core/falcon/hal.rs @@ -3,7 +3,6 @@ use kernel::prelude::*; use crate::{ - driver::Bar0, falcon::{ Falcon, FalconBromParams, @@ -34,7 +33,7 @@ pub(crate) enum LoadMethod { /// registers. pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync { /// Activates the Falcon core if the engine is a risvc/falcon dual engine. - fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result { + fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result { Ok(()) } @@ -42,24 +41,28 @@ pub(crate) trait FalconHal<E: FalconEngine>: Send + Sync { /// falcon instance. `engine_id_mask` and `ucode_id` are obtained from the firmware header. fn signature_reg_fuse_version( &self, - falcon: &Falcon<E>, - bar: Bar0<'_>, + falcon: &Falcon<'_, E>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32>; /// Program the boot ROM registers prior to starting a secure firmware. - fn program_brom(&self, falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams); + fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams); /// Check if the RISC-V core is active. /// Returns `true` if the RISC-V core is active, `false` otherwise. - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool; + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool; + + /// Checks whether the RISC-V core is halted. + /// + /// Returns [`ENOTSUPP`] if the chipset does not expose RISC-V halt status. + fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result<bool>; /// Wait for memory scrubbing to complete. - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result; + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result; /// Reset the falcon engine. - fn reset_eng(&self, bar: Bar0<'_>) -> Result; + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result; /// Returns the method used to load data into the falcon's memory. /// diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs index cf6ce47e6b25..7600ee07ca2e 100644 --- a/drivers/gpu/nova-core/falcon/hal/ga102.rs +++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs @@ -115,33 +115,41 @@ impl<E: FalconEngine> Ga102<E> { } impl<E: FalconEngine> FalconHal<E> for Ga102<E> { - fn select_core(&self, _falcon: &Falcon<E>, bar: Bar0<'_>) -> Result { - select_core_ga102::<E>(bar) + fn select_core(&self, falcon: &Falcon<'_, E>) -> Result { + select_core_ga102::<E>(falcon.bar) } fn signature_reg_fuse_version( &self, - falcon: &Falcon<E>, - bar: Bar0<'_>, + falcon: &Falcon<'_, E>, engine_id_mask: u16, ucode_id: u8, ) -> Result<u32> { - signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id) + signature_reg_fuse_version_ga102(falcon.dev, falcon.bar, engine_id_mask, ucode_id) } - fn program_brom(&self, _falcon: &Falcon<E>, bar: Bar0<'_>, params: &FalconBromParams) { - program_brom_ga102::<E>(bar, params); + fn program_brom(&self, falcon: &Falcon<'_, E>, params: &FalconBromParams) { + program_brom_ga102::<E>(falcon.bar, params); } - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { + falcon + .bar + .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { + fn is_riscv_halted(&self, falcon: &Falcon<'_, E>) -> Result<bool> { + Ok(falcon + .bar + .read(regs::NV_PRISCV_RISCV_CPUCTL::of::<E>()) + .halted()) + } + + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 20ms. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())), + || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>())), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(20), @@ -149,7 +157,9 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> { .map(|_| ()) } - fn reset_eng(&self, bar: Bar0<'_>) -> Result { + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { + let bar = falcon.bar; + let _ = bar.read(regs::NV_PFALCON_FALCON_HWCFG2::of::<E>()); // According to OpenRM's `kflcnPreResetWait_GA102` documentation, HW sometimes does not set @@ -162,7 +172,7 @@ impl<E: FalconEngine> FalconHal<E> for Ga102<E> { ); regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); - self.reset_wait_mem_scrubbing(bar)?; + self.reset_wait_mem_scrubbing(falcon)?; Ok(()) } diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs index 3aaee3869312..5291598fedf7 100644 --- a/drivers/gpu/nova-core/falcon/hal/tu102.rs +++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs @@ -13,7 +13,6 @@ use kernel::{ }; use crate::{ - driver::Bar0, falcon::{ hal::LoadMethod, Falcon, @@ -34,31 +33,36 @@ impl<E: FalconEngine> Tu102<E> { } impl<E: FalconEngine> FalconHal<E> for Tu102<E> { - fn select_core(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>) -> Result { + fn select_core(&self, _falcon: &Falcon<'_, E>) -> Result { Ok(()) } fn signature_reg_fuse_version( &self, - _falcon: &Falcon<E>, - _bar: Bar0<'_>, + _falcon: &Falcon<'_, E>, _engine_id_mask: u16, _ucode_id: u8, ) -> Result<u32> { Ok(0) } - fn program_brom(&self, _falcon: &Falcon<E>, _bar: Bar0<'_>, _params: &FalconBromParams) {} + fn program_brom(&self, _falcon: &Falcon<'_, E>, _params: &FalconBromParams) {} - fn is_riscv_active(&self, bar: Bar0<'_>) -> bool { - bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>()) + fn is_riscv_active(&self, falcon: &Falcon<'_, E>) -> bool { + falcon + .bar + .read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>()) .active_stat() } - fn reset_wait_mem_scrubbing(&self, bar: Bar0<'_>) -> Result { + fn is_riscv_halted(&self, _falcon: &Falcon<'_, E>) -> Result<bool> { + Err(ENOTSUPP) + } + + fn reset_wait_mem_scrubbing(&self, falcon: &Falcon<'_, E>) -> Result { // TIMEOUT: memory scrubbing should complete in less than 10ms. read_poll_timeout( - || Ok(bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())), + || Ok(falcon.bar.read(regs::NV_PFALCON_FALCON_DMACTL::of::<E>())), |r| r.mem_scrubbing_done(), Delta::ZERO, Delta::from_millis(10), @@ -66,9 +70,9 @@ impl<E: FalconEngine> FalconHal<E> for Tu102<E> { .map(|_| ()) } - fn reset_eng(&self, bar: Bar0<'_>) -> Result { - regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(bar); - self.reset_wait_mem_scrubbing(bar)?; + fn reset_eng(&self, falcon: &Falcon<'_, E>) -> Result { + regs::NV_PFALCON_FALCON_ENGINE::reset_engine::<E>(falcon.bar); + self.reset_wait_mem_scrubbing(falcon)?; Ok(()) } diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs index 725e428154cf..1576399389b1 100644 --- a/drivers/gpu/nova-core/fb.rs +++ b/drivers/gpu/nova-core/fb.rs @@ -24,10 +24,11 @@ use crate::{ gpu::Chipset, gsp, num::FromSafeCast, - regs, // + vgpu::VgpuState, // }; mod hal; +mod regs; /// Type holding the sysmem flush memory page, a page of memory to be written into the /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR*` registers and used to maintain memory coherency. @@ -60,7 +61,7 @@ impl<'sys> SysmemFlush<'sys> { ) -> Result<Self> { let page = CoherentHandle::alloc(dev, kernel::page::PAGE_SIZE, GFP_KERNEL)?; - hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_handle())?; + hal::fb_hal(chipset).write_sysmem_flush_page(bar, page.dma_address())?; Ok(Self { chipset, @@ -75,7 +76,7 @@ impl Drop for SysmemFlush<'_> { fn drop(&mut self) { let hal = hal::fb_hal(self.chipset); - if hal.read_sysmem_flush_page(self.bar) == self.page.dma_handle() { + if hal.read_sysmem_flush_page(self.bar) == self.page.dma_address() { let _ = hal.write_sysmem_flush_page(self.bar, 0).inspect_err(|e| { dev_warn!( &self.device, @@ -148,7 +149,7 @@ impl fmt::Debug for FbRange { /// /// Contains ranges of GPU memory reserved for a given purpose during the GSP boot process. #[derive(Debug)] -pub(crate) struct FbLayout { +pub(crate) struct FbRanges { /// Range of the framebuffer. Starts at `0`. pub(crate) fb: FbRange, /// VGA workspace, small area of reserved memory at the end of the framebuffer. @@ -163,15 +164,22 @@ pub(crate) struct FbLayout { pub(crate) wpr2_heap: FbRange, /// WPR2 region range, starting with an instance of `GspFwWprMeta`. pub(crate) wpr2: FbRange, - pub(crate) heap: FbRange, + /// Non-WPR heap, located just below WPR2. + pub(crate) non_wpr_heap: FbRange, + /// Number of VF partitions. pub(crate) vf_partition_count: u8, /// PMU reserved memory size, in bytes. pub(crate) pmu_reserved_size: u32, } -impl FbLayout { - /// Computes the FB layout for `chipset` required to run the `gsp_fw` GSP firmware. - pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, gsp_fw: &GspFirmware) -> Result<Self> { +impl FbRanges { + /// Computes concrete framebuffer ranges required on non-FSP booting architectures. + pub(crate) fn new( + chipset: Chipset, + bar: Bar0<'_>, + gsp_fw: &GspFirmware, + vgpu_state: VgpuState, + ) -> Result<Self> { let hal = hal::fb_hal(chipset); let fb = { @@ -234,11 +242,15 @@ impl FbLayout { FbRange(elf_addr..elf_addr + elf_size) }; + let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb.end)?; + let wpr2_heap = { const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>(); - let wpr2_heap_size = - gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?; - let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN); + let wpr2_heap_addr = elf + .start + .checked_sub(wpr2_heap_size) + .ok_or(EOVERFLOW)? + .align_down(WPR2_HEAP_DOWN_ALIGN); FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN)) }; @@ -251,9 +263,9 @@ impl FbLayout { FbRange(wpr2_addr..frts.end) }; - let heap = { - let heap_size = u64::from(hal.non_wpr_heap_size()); - FbRange(wpr2.start - heap_size..wpr2.start) + let non_wpr_heap = { + let non_wpr_heap_size = hal.non_wpr_heap_size(); + FbRange(wpr2.start - non_wpr_heap_size..wpr2.start) }; Ok(Self { @@ -264,9 +276,69 @@ impl FbLayout { elf, wpr2_heap, wpr2, - heap, - vf_partition_count: 0, + non_wpr_heap, + vf_partition_count, + pmu_reserved_size: hal.pmu_reserved_size(), + }) + } +} + +/// Reads the WPR2 memory region registers and returns the range if set. +/// Returns `None` if the WPR2 region is not set. +pub(crate) fn wpr2_range(bar: Bar0<'_>) -> Option<Range<u64>> { + let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); + + if !wpr2_hi.is_wpr2_set() { + return None; + } + + let wpr2_lo = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO); + + Some(wpr2_lo.lower_bound()..wpr2_hi.higher_bound()) +} + +/// Computes the number of VF partitions and the WPR2 heap size from the vGPU state. +fn wpr2_heap_params(chipset: Chipset, vgpu_state: VgpuState, fb_size: u64) -> Result<(u8, u64)> { + Ok(match vgpu_state { + VgpuState::Disabled => ( + 0, + gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb_size)?, + ), + VgpuState::Enabled { total_vfs } => ( + u8::try_from(total_vfs.get()).map_err(|_| EINVAL)?, + gsp::LibosParams::vgpu_wpr_heap_size(), + ), + }) +} + +/// Framebuffer region sizes needed for GSP-FMC boot. +#[derive(Debug)] +pub(crate) struct FbSizes { + /// FRTS size, in bytes. + pub(crate) frts_size: u64, + /// WPR2 heap size, in bytes. + pub(crate) wpr2_heap_size: u64, + /// Non-WPR heap size, in bytes. + pub(crate) non_wpr_heap_size: u64, + /// PMU reserved memory size, in bytes. + pub(crate) pmu_reserved_size: u32, + /// Number of VF partitions. + pub(crate) vf_partition_count: u8, +} + +impl FbSizes { + /// Computes the framebuffer region sizes for GSP-FMC boot. + pub(crate) fn new(chipset: Chipset, bar: Bar0<'_>, vgpu_state: VgpuState) -> Result<Self> { + let hal = hal::fb_hal(chipset); + let fb_size = hal.vidmem_size(bar); + let (vf_partition_count, wpr2_heap_size) = wpr2_heap_params(chipset, vgpu_state, fb_size)?; + + Ok(Self { + frts_size: hal.frts_size(), + wpr2_heap_size, + non_wpr_heap_size: hal.non_wpr_heap_size(), pmu_reserved_size: hal.pmu_reserved_size(), + vf_partition_count, }) } } diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs index 714f0b51cd8f..99363cfb116b 100644 --- a/drivers/gpu/nova-core/fb/hal.rs +++ b/drivers/gpu/nova-core/fb/hal.rs @@ -37,7 +37,7 @@ pub(crate) trait FbHal { fn pmu_reserved_size(&self) -> u32; /// Returns the non-WPR heap size for this chipset, in bytes. - fn non_wpr_heap_size(&self) -> u32; + fn non_wpr_heap_size(&self) -> u64; /// Returns the FRTS size, in bytes. fn frts_size(&self) -> u64; diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs index 3cc1caf361c7..a81b3e42ae41 100644 --- a/drivers/gpu/nova-core/fb/hal/ga100.rs +++ b/drivers/gpu/nova-core/fb/hal/ga100.rs @@ -9,8 +9,10 @@ use kernel::{ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; use super::tu102::FLUSH_SYSMEM_ADDR_SHIFT; @@ -41,7 +43,7 @@ pub(super) fn write_sysmem_flush_page_ga100(bar: Bar0<'_>, addr: u64) { } pub(super) fn display_enabled_ga100(bar: Bar0<'_>) -> bool { - !bar.read(regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) + !bar.read(crate::regs::ga100::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } @@ -72,7 +74,7 @@ impl FbHal for Ga100 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { super::tu102::non_wpr_heap_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs index 44a2cf8a00f1..a5eeddb8a1ae 100644 --- a/drivers/gpu/nova-core/fb/hal/ga102.rs +++ b/drivers/gpu/nova-core/fb/hal/ga102.rs @@ -41,7 +41,7 @@ impl FbHal for Ga102 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { super::tu102::non_wpr_heap_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/hal/gb100.rs b/drivers/gpu/nova-core/fb/hal/gb100.rs index 6e0eba101ca1..d9e4d62ae632 100644 --- a/drivers/gpu/nova-core/fb/hal/gb100.rs +++ b/drivers/gpu/nova-core/fb/hal/gb100.rs @@ -22,9 +22,11 @@ use kernel::{ use crate::{ driver::Bar0, - fb::hal::FbHal, + fb::{ + hal::FbHal, + regs, // + }, num::usize_into_u32, - regs, // }; struct Gb100; @@ -78,6 +80,7 @@ fn write_sysmem_flush_page_gb100(bar: Bar0<'_>, addr: Bounded<u64, 52>) { ); } +// This PMU reservation size is r570-specific. pub(super) const fn pmu_reserved_size_gb100() -> u32 { usize_into_u32::<{ const_align_up(SZ_8M + SZ_16M + SZ_4K, Alignment::new::<SZ_128K>()).unwrap() }>( ) @@ -108,9 +111,9 @@ impl FbHal for Gb100 { pmu_reserved_size_gb100() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for GB10x (see Open RM: kgspGetNonWprHeapSize, GB100/GB102). - u32::SZ_2M + u64::SZ_2M } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/gb202.rs b/drivers/gpu/nova-core/fb/hal/gb202.rs index 038d1278c634..4341ecf36188 100644 --- a/drivers/gpu/nova-core/fb/hal/gb202.rs +++ b/drivers/gpu/nova-core/fb/hal/gb202.rs @@ -4,13 +4,7 @@ //! Blackwell GB20x framebuffer HAL. use kernel::{ - io::{ - register::{ - RegisterBase, - WithBase, // - }, - Io, // - }, + io::Io, num::Bounded, prelude::*, sizes::SizeConstants, // @@ -18,41 +12,37 @@ use kernel::{ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; struct Gb202; -impl RegisterBase<regs::Fbhub0Base> for Gb202 { - const BASE: usize = 0x008a_0000; -} - fn read_sysmem_flush_page_gb202(bar: Bar0<'_>) -> u64 { let lo = u64::from( - bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>()) + bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO) .adr(), ); let hi = u64::from( - bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>()) + bar.read(regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI) .adr(), ); - lo | (hi << 32) + (hi << 32) | lo } /// Write the sysmem flush page address through the GB20x FBHUB0 registers. fn write_sysmem_flush_page_gb202(bar: Bar0<'_>, addr: Bounded<u64, 52>) { // Write HI first. The hardware will trigger the flush on the LO write. - bar.write( - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::of::<Gb202>(), - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + bar.write_reg( + regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() .with_adr(addr.shr::<32, 20>().cast::<u32>()), ); - bar.write( - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::of::<Gb202>(), + bar.write_reg( // CAST: lower 32 bits. Hardware ignores bits 7:0. - regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + regs::NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), ); } @@ -81,9 +71,10 @@ impl FbHal for Gb202 { super::gb100::pmu_reserved_size_gb100() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for GB20x (see Open RM: kgspGetNonWprHeapSize, GB202+). - u32::SZ_2M + u32::SZ_128K + // This size is r570-specific. + u64::SZ_2M + u64::SZ_128K } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/gh100.rs b/drivers/gpu/nova-core/fb/hal/gh100.rs index 5450c7254dad..cf8e6403ef98 100644 --- a/drivers/gpu/nova-core/fb/hal/gh100.rs +++ b/drivers/gpu/nova-core/fb/hal/gh100.rs @@ -2,24 +2,51 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. use kernel::{ + io::Io, + num::Bounded, prelude::*, sizes::SizeConstants, // }; use crate::{ driver::Bar0, - fb::hal::FbHal, // + fb::{ + hal::FbHal, + regs, // + }, }; struct Gh100; +fn read_sysmem_flush_page_gh100(bar: Bar0<'_>) -> u64 { + let lo = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO).adr()); + let hi = u64::from(bar.read(regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI).adr()); + + (hi << 32) | lo +} + +/// Write the sysmem flush page address through the Hopper FBHUB registers. +fn write_sysmem_flush_page_gh100(bar: Bar0<'_>, addr: Bounded<u64, 52>) { + // Write HI first. The hardware will trigger the flush on the LO write. + bar.write_reg( + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI::zeroed() + .with_adr(addr.shr::<32, 20>().cast::<u32>()), + ); + bar.write_reg( + // CAST: lower 32 bits. Hardware ignores bits 7:0. + regs::NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO::zeroed().with_adr(*addr as u32), + ); +} + impl FbHal for Gh100 { fn read_sysmem_flush_page(&self, bar: Bar0<'_>) -> u64 { - super::ga100::read_sysmem_flush_page_ga100(bar) + read_sysmem_flush_page_gh100(bar) } fn write_sysmem_flush_page(&self, bar: Bar0<'_>, addr: u64) -> Result { - super::ga100::write_sysmem_flush_page_ga100(bar, addr); + let addr = Bounded::<u64, 52>::try_new(addr).ok_or(EINVAL)?; + + write_sysmem_flush_page_gh100(bar, addr); Ok(()) } @@ -36,9 +63,9 @@ impl FbHal for Gh100 { super::tu102::pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { // Non-WPR heap for Hopper (see Open RM: kgspCalculateFbLayout_GH100). - u32::SZ_2M + u64::SZ_2M } fn frts_size(&self) -> u64 { diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs index f629e8e9d5d5..bf5967cbffd3 100644 --- a/drivers/gpu/nova-core/fb/hal/tu102.rs +++ b/drivers/gpu/nova-core/fb/hal/tu102.rs @@ -9,8 +9,10 @@ use kernel::{ use crate::{ driver::Bar0, - fb::hal::FbHal, - regs, // + fb::{ + hal::FbHal, + regs, // + }, }; /// Shift applied to the sysmem address before it is written into `NV_PFB_NISO_FLUSH_SYSMEM_ADDR`, @@ -31,7 +33,7 @@ pub(super) fn write_sysmem_flush_page_gm107(bar: Bar0<'_>, addr: u64) -> Result } pub(super) fn display_enabled_gm107(bar: Bar0<'_>) -> bool { - !bar.read(regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) + !bar.read(crate::regs::gm107::NV_FUSE_STATUS_OPT_DISPLAY) .display_disabled() } @@ -44,8 +46,8 @@ pub(super) const fn pmu_reserved_size_tu102() -> u32 { 0 } -pub(super) const fn non_wpr_heap_size_tu102() -> u32 { - u32::SZ_1M +pub(super) const fn non_wpr_heap_size_tu102() -> u64 { + u64::SZ_1M } pub(super) const fn frts_size_tu102() -> u64 { @@ -75,7 +77,7 @@ impl FbHal for Tu102 { pmu_reserved_size_tu102() } - fn non_wpr_heap_size(&self) -> u32 { + fn non_wpr_heap_size(&self) -> u64 { non_wpr_heap_size_tu102() } diff --git a/drivers/gpu/nova-core/fb/regs.rs b/drivers/gpu/nova-core/fb/regs.rs new file mode 100644 index 000000000000..95adbe124a30 --- /dev/null +++ b/drivers/gpu/nova-core/fb/regs.rs @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::{ + io::register, + sizes::SizeConstants, // +}; + +// PDISP + +register! { + pub(super) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { + /// VGA workspace base address divided by 0x10000. + 31:8 addr; + /// Set if the `addr` field is valid. + 3:3 status_valid => bool; + } +} + +impl NV_PDISP_VGA_WORKSPACE_BASE { + /// Returns the base address of the VGA workspace, or `None` if none exists. + pub(super) fn vga_workspace_addr(self) -> Option<u64> { + if self.status_valid() { + Some(u64::from(self.addr()) << 16) + } else { + None + } + } +} + +// PFB + +register! { + /// Low bits of the physical system memory address used by the GPU to perform sysmembar + /// operations (see [`crate::fb::SysmemFlush`]). + pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { + 31:0 adr_39_08; + } + + /// High bits of the physical system memory address used by the GPU to perform sysmembar + /// operations. + pub(super) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 { + 23:0 adr_63_40; + } + + pub(super) NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE(u32) @ 0x00100ce0 { + 30:30 ecc_mode_enabled => bool; + 9:4 lower_mag; + 3:0 lower_scale; + } + + pub(super) NV_PFB_PRI_MMU_WPR2_ADDR_LO(u32) @ 0x001fa824 { + /// Bits 12..40 of the lower (inclusive) bound of the WPR2 region. + 31:4 lo_val; + } + + pub(super) NV_PFB_PRI_MMU_WPR2_ADDR_HI(u32) @ 0x001fa828 { + /// Bits 12..40 of the higher (exclusive) bound of the WPR2 region. + 31:4 hi_val; + } +} + +/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). +/// +/// The base is provided by the GB10x framebuffer HAL. +pub(super) struct Hshub0Base(()); + +register! { + // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar + // through a primary and an EG (egress) pair that must both be programmed to the same + // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed + // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { + 19:0 adr; + } + + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { + 19:0 adr; + } +} + +register! { + // GB20x FBHUB0 sysmem flush registers. Unlike the older + // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers, which encode the address with an + // 8-bit right-shift, these take the raw address split into lower and upper + // halves. Hardware ignores bits 7:0 of the LO register. + pub(super) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x008a1d58 { + 31:0 adr => u32; + } + + pub(super) NV_PFB_FBHUB0_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x008a1d5c { + 19:0 adr; + } +} + +register! { + /// Low bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + /// + /// Like the GB20x FBHUB0 registers, and unlike the Ampere + /// `NV_PFB_NISO_FLUSH_SYSMEM_ADDR` registers (which encode the address with an + /// 8-bit right-shift), these take the raw address split into lower and upper + /// halves. Hardware ignores bits 7:0 of the LO register. + pub(super) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ 0x00100a34 { + 31:0 adr => u32; + } + + /// High bits of the physical system memory address used by the GPU to perform + /// sysmembar operations on Hopper. + pub(super) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100a38 { + 19:0 adr; + } +} + +impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { + /// Returns the usable framebuffer size, in bytes. + pub(super) fn usable_fb_size(self) -> u64 { + let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; + + if self.ecc_mode_enabled() { + // Remove the amount of memory reserved for ECC (one per 16 units). + size / 16 * 15 + } else { + size + } + } +} + +impl NV_PFB_PRI_MMU_WPR2_ADDR_LO { + /// Returns the lower (inclusive) bound of the WPR2 region. + pub(super) fn lower_bound(self) -> u64 { + u64::from(self.lo_val()) << 12 + } +} + +impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { + /// Returns the higher (exclusive) bound of the WPR2 region. + /// + /// A value of zero means the WPR2 region is not set. + pub(super) fn higher_bound(self) -> u64 { + u64::from(self.hi_val()) << 12 + } + + /// Returns whether the WPR2 region is currently set. + pub(super) fn is_wpr2_set(self) -> bool { + self.hi_val() != 0 + } +} diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs index 1e89390209f5..b49613a90bf0 100644 --- a/drivers/gpu/nova-core/firmware.rs +++ b/drivers/gpu/nova-core/firmware.rs @@ -8,11 +8,8 @@ use core::marker::PhantomData; use core::ops::Deref; use kernel::{ - device, firmware, - prelude::*, - str::CString, - transmute::FromBytes, // + prelude::*, // }; use crate::{ @@ -21,10 +18,8 @@ use crate::{ FalconFirmware, // }, gpu, - num::{ - FromSafeCast, - IntoSafeCast, // - }, + gsp::boot_firmware_files, + num::IntoSafeCast, // }; pub(crate) mod booter; @@ -32,21 +27,7 @@ pub(crate) mod fsp; pub(crate) mod fwsec; pub(crate) mod gsp; pub(crate) mod riscv; - -pub(crate) const FIRMWARE_VERSION: &str = "570.144"; - -/// Requests the GPU firmware `name` suitable for `chipset`, with version `ver`. -fn request_firmware( - dev: &device::Device, - chipset: gpu::Chipset, - name: &str, - ver: &str, -) -> Result<firmware::Firmware> { - let chip_name = chipset.name(); - - CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{name}-{ver}.bin")) - .and_then(|path| firmware::Firmware::request(&path, dev)) -} +pub(crate) mod tlv; /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] @@ -88,7 +69,7 @@ pub(crate) struct FalconUCodeDescV2 { /// Structure used to describe some firmwares, notably FWSEC-FRTS. #[repr(C)] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] pub(crate) struct FalconUCodeDescV3 { /// Header defined by `NV_BIT_FALCON_UCODE_DESC_HEADER_VDESC*` in OpenRM. hdr: u32, @@ -119,10 +100,6 @@ pub(crate) struct FalconUCodeDescV3 { _reserved: u16, } -// SAFETY: all bit patterns are valid for this type, and it doesn't use -// interior mutability. -unsafe impl FromBytes for FalconUCodeDescV3 {} - /// Enum wrapping the different versions of Falcon microcode descriptors. /// /// This allows handling both V2 and V3 descriptor formats through a @@ -349,60 +326,6 @@ impl<F: FalconFirmware> FirmwareObject<F, Unsigned> { } } -/// Header common to most firmware files. -#[repr(C)] -#[derive(Debug, Clone)] -struct BinHdr { - /// Magic number, must be `0x10de`. - bin_magic: u32, - /// Version of the header. - bin_ver: u32, - /// Size in bytes of the binary (to be ignored). - bin_size: u32, - /// Offset of the start of the application-specific header. - header_offset: u32, - /// Offset of the start of the data payload. - data_offset: u32, - /// Size in bytes of the data payload. - data_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for BinHdr {} - -// A firmware blob starting with a `BinHdr`. -struct BinFirmware<'a> { - hdr: BinHdr, - fw: &'a [u8], -} - -impl<'a> BinFirmware<'a> { - /// Interpret `fw` as a firmware image starting with a [`BinHdr`], and returns the - /// corresponding [`BinFirmware`] that can be used to extract its payload. - fn new(fw: &'a firmware::Firmware) -> Result<Self> { - const BIN_MAGIC: u32 = 0x10de; - let fw = fw.data(); - - fw.get(0..size_of::<BinHdr>()) - // Extract header. - .and_then(BinHdr::from_bytes_copy) - // Validate header. - .filter(|hdr| hdr.bin_magic == BIN_MAGIC) - .map(|hdr| Self { hdr, fw }) - .ok_or(EINVAL) - } - - /// Returns the data payload of the firmware, or `None` if the data range is out of bounds of - /// the firmware image. - fn data(&self) -> Option<&[u8]> { - let fw_start = usize::from_safe_cast(self.hdr.data_offset); - let fw_size = usize::from_safe_cast(self.hdr.data_size); - let fw_end = fw_start.checked_add(fw_size)?; - - self.fw.get(fw_start..fw_end) - } -} - pub(crate) struct ModInfoBuilder<const N: usize>(firmware::ModInfoBuilder<N>); impl<const N: usize> ModInfoBuilder<N> { @@ -413,33 +336,28 @@ impl<const N: usize> ModInfoBuilder<N> { .push("nvidia/") .push(chipset) .push("/gsp/") - .push(fw) - .push("-") - .push(FIRMWARE_VERSION) - .push(".bin"), + .push(fw), ) } const fn make_entry_chipset(self, chipset: gpu::Chipset) -> Self { let name = chipset.name(); - let this = self - .make_entry_file(name, "booter_load") - .make_entry_file(name, "booter_unload") - .make_entry_file(name, "bootloader") - .make_entry_file(name, "gsp"); - - let this = if chipset.needs_fwsec_bootloader() { - this.make_entry_file(name, "gen_bootloader") - } else { - this - }; - - if chipset.uses_fsp() { - this.make_entry_file(name, "fmc") - } else { - this + // GSP firmware files are always present. + let mut this = self + .make_entry_file(name, "gsp_bootloader.tlv") + .make_entry_file(name, "gsp.tlv") + .make_entry_file(name, "gsp.bin"); + + // Add the firmware files specific to the GSP boot method of `chipset`. + let boot_files = boot_firmware_files(chipset); + let mut i = 0; + while i < boot_files.len() { + this = this.make_entry_file(name, boot_files[i]); + i += 1; } + + this } pub(crate) const fn create( @@ -456,210 +374,3 @@ impl<const N: usize> ModInfoBuilder<N> { this.0 } } - -/// Ad-hoc and temporary module to extract sections from ELF images. -/// -/// Some firmware images are currently packaged as ELF files, where sections names are used as keys -/// to specific and related bits of data. Future firmware versions are scheduled to move away from -/// that scheme before nova-core becomes stable, which means this module will eventually be -/// removed. -mod elf { - use core::mem::size_of; - - use kernel::{ - bindings, - str::CStr, - transmute::FromBytes, // - }; - - /// Trait to abstract over ELF header differences. - trait ElfHeader: FromBytes { - fn shnum(&self) -> u16; - fn shoff(&self) -> u64; - fn shstrndx(&self) -> u16; - } - - /// Trait to abstract over ELF section-header differences. - trait ElfSectionHeader: FromBytes { - fn name(&self) -> u32; - fn offset(&self) -> u64; - fn size(&self) -> u64; - } - - /// Trait describing a matching ELF header and section-header format. - trait ElfFormat { - type Header: ElfHeader; - type SectionHeader: ElfSectionHeader; - } - - /// Newtype to provide a [`FromBytes`] implementation. - #[repr(transparent)] - struct Elf64Hdr(bindings::elf64_hdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf64Hdr {} - - impl ElfHeader for Elf64Hdr { - fn shnum(&self) -> u16 { - self.0.e_shnum - } - - fn shoff(&self) -> u64 { - self.0.e_shoff - } - - fn shstrndx(&self) -> u16 { - self.0.e_shstrndx - } - } - - #[repr(transparent)] - struct Elf64SHdr(bindings::elf64_shdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf64SHdr {} - - impl ElfSectionHeader for Elf64SHdr { - fn name(&self) -> u32 { - self.0.sh_name - } - - fn offset(&self) -> u64 { - self.0.sh_offset - } - - fn size(&self) -> u64 { - self.0.sh_size - } - } - - struct Elf64Format; - - impl ElfFormat for Elf64Format { - type Header = Elf64Hdr; - type SectionHeader = Elf64SHdr; - } - - /// Newtype to provide [`FromBytes`] and [`ElfHeader`] implementations for ELF32. - #[repr(transparent)] - struct Elf32Hdr(bindings::elf32_hdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf32Hdr {} - - impl ElfHeader for Elf32Hdr { - fn shnum(&self) -> u16 { - self.0.e_shnum - } - - fn shoff(&self) -> u64 { - u64::from(self.0.e_shoff) - } - - fn shstrndx(&self) -> u16 { - self.0.e_shstrndx - } - } - - /// Newtype to provide [`FromBytes`] and [`ElfSectionHeader`] implementations for ELF32. - #[repr(transparent)] - struct Elf32SHdr(bindings::elf32_shdr); - // SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. - unsafe impl FromBytes for Elf32SHdr {} - - impl ElfSectionHeader for Elf32SHdr { - fn name(&self) -> u32 { - self.0.sh_name - } - - fn offset(&self) -> u64 { - u64::from(self.0.sh_offset) - } - - fn size(&self) -> u64 { - u64::from(self.0.sh_size) - } - } - - struct Elf32Format; - - impl ElfFormat for Elf32Format { - type Header = Elf32Hdr; - type SectionHeader = Elf32SHdr; - } - - /// Returns a NULL-terminated string from the ELF image at `offset`. - fn elf_str(elf: &[u8], offset: u64) -> Option<&str> { - let idx = usize::try_from(offset).ok()?; - let bytes = elf.get(idx..)?; - CStr::from_bytes_until_nul(bytes).ok()?.to_str().ok() - } - - fn elf_section_generic<'a, F>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> - where - F: ElfFormat, - { - let hdr = F::Header::from_bytes(elf.get(0..size_of::<F::Header>())?)?; - - let shdr_num = usize::from(hdr.shnum()); - let shdr_start = usize::try_from(hdr.shoff()).ok()?; - let shdr_end = shdr_num - .checked_mul(size_of::<F::SectionHeader>()) - .and_then(|v| v.checked_add(shdr_start))?; - - // Get all the section headers as an iterator over byte chunks. - let shdr_bytes = elf.get(shdr_start..shdr_end)?; - let mut shdr_iter = shdr_bytes.chunks_exact(size_of::<F::SectionHeader>()); - - // Get the strings table. - let strhdr = shdr_iter - .clone() - .nth(usize::from(hdr.shstrndx())) - .and_then(F::SectionHeader::from_bytes)?; - - // Find the section which name matches `name` and return it. - shdr_iter.find_map(|sh_bytes| { - let sh = F::SectionHeader::from_bytes(sh_bytes)?; - let name_offset = strhdr.offset().checked_add(u64::from(sh.name()))?; - let section_name = elf_str(elf, name_offset)?; - - if section_name != name { - return None; - } - - let start = usize::try_from(sh.offset()).ok()?; - let end = usize::try_from(sh.size()) - .ok() - .and_then(|sz| start.checked_add(sz))?; - - elf.get(start..end) - }) - } - - /// Extract the section with name `name` from the ELF64 image `elf`. - fn elf64_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - elf_section_generic::<Elf64Format>(elf, name) - } - - /// Extract the section with name `name` from the ELF32 image `elf`. - fn elf32_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - elf_section_generic::<Elf32Format>(elf, name) - } - - /// Automatically detects ELF32 vs ELF64 based on the ELF header. - pub(super) fn elf_section<'a>(elf: &'a [u8], name: &str) -> Option<&'a [u8]> { - // ELF identification: a 4-byte magic followed by a class byte (32- vs 64-bit). - const ELFMAG: &[u8] = b"\x7fELF"; - const SELFMAG: usize = ELFMAG.len(); - const EI_CLASS: usize = 4; - const ELFCLASS32: u8 = 1; - const ELFCLASS64: u8 = 2; - - if elf.get(0..SELFMAG) != Some(ELFMAG) { - return None; - } - - match *elf.get(EI_CLASS)? { - ELFCLASS32 => elf32_section(elf, name), - ELFCLASS64 => elf64_section(elf, name), - _ => None, - } - } -} diff --git a/drivers/gpu/nova-core/firmware/booter.rs b/drivers/gpu/nova-core/firmware/booter.rs index d9313ac361af..dc071edba331 100644 --- a/drivers/gpu/nova-core/firmware/booter.rs +++ b/drivers/gpu/nova-core/firmware/booter.rs @@ -10,12 +10,10 @@ use core::marker::PhantomData; use kernel::{ device, dma::Coherent, - prelude::*, - transmute::FromBytes, // + prelude::*, // }; use crate::{ - driver::Bar0, falcon::{ sec2::Sec2, Falcon, @@ -25,224 +23,19 @@ use crate::{ FalconFirmware, // }, firmware::{ - BinFirmware, + tlv::{ + request_tlv, // + Tlv, + }, FirmwareObject, FirmwareSignature, Signed, Unsigned, // }, gpu::Chipset, - num::{ - FromSafeCast, - IntoSafeCast, // - }, + num::IntoSafeCast, }; -/// Local convenience function to return a copy of `S` by reinterpreting the bytes starting at -/// `offset` in `slice`. -fn frombytes_at<S: FromBytes + Sized>(slice: &[u8], offset: usize) -> Result<S> { - let end = offset.checked_add(size_of::<S>()).ok_or(EINVAL)?; - slice - .get(offset..end) - .and_then(S::from_bytes_copy) - .ok_or(EINVAL) -} - -/// Heavy-Secured firmware header. -/// -/// Such firmwares have an application-specific payload that needs to be patched with a given -/// signature. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsHeaderV2 { - /// Offset to the start of the signatures. - sig_prod_offset: u32, - /// Size in bytes of the signatures. - sig_prod_size: u32, - /// Offset to a `u32` containing the location at which to patch the signature in the microcode - /// image. - patch_loc_offset: u32, - /// Offset to a `u32` containing the index of the signature to patch. - patch_sig_offset: u32, - /// Start offset to the signature metadata. - meta_data_offset: u32, - /// Size in bytes of the signature metadata. - meta_data_size: u32, - /// Offset to a `u32` containing the number of signatures in the signatures section. - num_sig_offset: u32, - /// Offset of the application-specific header. - header_offset: u32, - /// Size in bytes of the application-specific header. - header_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsHeaderV2 {} - -/// Heavy-Secured Firmware image container. -/// -/// This provides convenient access to the fields of [`HsHeaderV2`] that are actually indices to -/// read from in the firmware data. -struct HsFirmwareV2<'a> { - hdr: HsHeaderV2, - fw: &'a [u8], -} - -impl<'a> HsFirmwareV2<'a> { - /// Interprets the header of `bin_fw` as a [`HsHeaderV2`] and returns an instance of - /// `HsFirmwareV2` for further parsing. - /// - /// Fails if the header pointed at by `bin_fw` is not within the bounds of the firmware image. - fn new(bin_fw: &BinFirmware<'a>) -> Result<Self> { - frombytes_at::<HsHeaderV2>(bin_fw.fw, bin_fw.hdr.header_offset.into_safe_cast()) - .map(|hdr| Self { hdr, fw: bin_fw.fw }) - } - - /// Returns the location at which the signatures should be patched in the microcode image. - /// - /// Fails if the offset of the patch location is outside the bounds of the firmware - /// image. - fn patch_location(&self) -> Result<u32> { - frombytes_at::<u32>(self.fw, self.hdr.patch_loc_offset.into_safe_cast()) - } - - /// Returns an iterator to the signatures of the firmware. The iterator can be empty if the - /// firmware is unsigned. - /// - /// Fails if the pointed signatures are outside the bounds of the firmware image. - fn signatures_iter(&'a self) -> Result<impl Iterator<Item = BooterSignature<'a>>> { - let num_sig = frombytes_at::<u32>(self.fw, self.hdr.num_sig_offset.into_safe_cast())?; - let iter = match self.hdr.sig_prod_size.checked_div(num_sig) { - // If there are no signatures, return an iterator that will yield zero elements. - None => (&[] as &[u8]).chunks_exact(1), - Some(sig_size) => { - let patch_sig = - frombytes_at::<u32>(self.fw, self.hdr.patch_sig_offset.into_safe_cast())?; - - let signatures_start = self - .hdr - .sig_prod_offset - .checked_add(patch_sig) - .map(usize::from_safe_cast) - .ok_or(EINVAL)?; - - let signatures_end = signatures_start - .checked_add(usize::from_safe_cast(self.hdr.sig_prod_size)) - .ok_or(EINVAL)?; - - self.fw - // Get signatures range. - .get(signatures_start..signatures_end) - .ok_or(EINVAL)? - .chunks_exact(sig_size.into_safe_cast()) - } - }; - - // Map the byte slices into signatures. - Ok(iter.map(BooterSignature)) - } -} - -/// Signature parameters, as defined in the firmware. -#[repr(C)] -struct HsSignatureParams { - /// Fuse version to use. - fuse_ver: u32, - /// Mask of engine IDs this firmware applies to. - engine_id_mask: u32, - /// ID of the microcode. - ucode_id: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsSignatureParams {} - -impl HsSignatureParams { - /// Returns the signature parameters contained in `hs_fw`. - /// - /// Fails if the meta data parameter of `hs_fw` is outside the bounds of the firmware image, or - /// if its size doesn't match that of [`HsSignatureParams`]. - fn new(hs_fw: &HsFirmwareV2<'_>) -> Result<Self> { - let start = usize::from_safe_cast(hs_fw.hdr.meta_data_offset); - let end = start - .checked_add(hs_fw.hdr.meta_data_size.into_safe_cast()) - .ok_or(EINVAL)?; - - hs_fw - .fw - .get(start..end) - .and_then(Self::from_bytes_copy) - .ok_or(EINVAL) - } -} - -/// Header for code and data load offsets. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsLoadHeaderV2 { - // Offset at which the code starts. - os_code_offset: u32, - // Total size of the code, for all apps. - os_code_size: u32, - // Offset at which the data starts. - os_data_offset: u32, - // Size of the data. - os_data_size: u32, - // Number of apps following this header. Each app is described by a [`HsLoadHeaderV2App`]. - num_apps: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsLoadHeaderV2 {} - -impl HsLoadHeaderV2 { - /// Returns the load header contained in `hs_fw`. - /// - /// Fails if the header pointed at by `hs_fw` is not within the bounds of the firmware image. - fn new(hs_fw: &HsFirmwareV2<'_>) -> Result<Self> { - frombytes_at::<Self>(hs_fw.fw, hs_fw.hdr.header_offset.into_safe_cast()) - } -} - -/// Header for app code loader. -#[repr(C)] -#[derive(Debug, Clone)] -struct HsLoadHeaderV2App { - /// Offset at which to load the app code. - offset: u32, - /// Length in bytes of the app code. - len: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for HsLoadHeaderV2App {} - -impl HsLoadHeaderV2App { - /// Returns the [`HsLoadHeaderV2App`] for app `idx` of `hs_fw`. - /// - /// Fails if `idx` is larger than the number of apps declared in `hs_fw`, or if the header is - /// not within the bounds of the firmware image. - fn new(hs_fw: &HsFirmwareV2<'_>, idx: u32) -> Result<Self> { - let load_hdr = HsLoadHeaderV2::new(hs_fw)?; - if idx >= load_hdr.num_apps { - Err(EINVAL) - } else { - frombytes_at::<Self>( - hs_fw.fw, - usize::from_safe_cast(hs_fw.hdr.header_offset) - // Skip the load header... - .checked_add(size_of::<HsLoadHeaderV2>()) - // ... and jump to app header `idx`. - .and_then(|offset| { - offset - .checked_add(usize::from_safe_cast(idx).checked_mul(size_of::<Self>())?) - }) - .ok_or(EINVAL)?, - ) - } - } -} - /// Signature for Booter firmware. Their size is encoded into the header and not known a compile /// time, so we just wrap a byte slices on which we can implement [`FirmwareSignature`]. struct BooterSignature<'a>(&'a [u8]); @@ -292,89 +85,76 @@ impl BooterFirmware { dev: &device::Device<device::Bound>, kind: BooterKind, chipset: Chipset, - ver: &str, - falcon: &Falcon<<Self as FalconFirmware>::Target>, - bar: Bar0<'_>, + falcon: &Falcon<'_, <Self as FalconFirmware>::Target>, ) -> Result<Self> { let fw_name = match kind { BooterKind::Loader => "booter_load", BooterKind::Unloader => "booter_unload", }; - let fw = super::request_firmware(dev, chipset, fw_name, ver)?; - let bin_fw = BinFirmware::new(&fw)?; - - // The binary firmware embeds a Heavy-Secured firmware. - let hs_fw = HsFirmwareV2::new(&bin_fw)?; + let fw = request_tlv(dev, chipset, fw_name)?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded {} firmware v{}\n", + fw_name, + tlv.get_string(b"VERS")? + ); + + let os_data_offset = tlv.get_u32(b"DAOF")?; + let os_data_size = tlv.get_u32(b"DASZ")?; + let os_code_offset = tlv.get_u32(b"CDOF")?; + let os_code_size = tlv.get_u32(b"CDSZ")?; + let patch_loc = tlv.get_u32(b"PLOC")?; + let fuse_version: usize = tlv.get_u32(b"FUSE")?.into_safe_cast(); + let engine_id = tlv.get_u32(b"ENID")?; + let ucode_id = tlv.get_u32(b"UCID")?; + let app0_code_offset = tlv.get_u32(b"A0CO")?; + let app0_code_size = tlv.get_u32(b"A0CS")?; - // The Heavy-Secured firmware embeds a firmware load descriptor. - let load_hdr = HsLoadHeaderV2::new(&hs_fw)?; - - // Offset in `ucode` where to patch the signature. - let patch_loc = hs_fw.patch_location()?; - - let sig_params = HsSignatureParams::new(&hs_fw)?; let brom_params = FalconBromParams { - // `load_hdr.os_data_offset` is an absolute index, but `pkc_data_offset` is from the + // `os_data_offset` is an absolute index, but `pkc_data_offset` is from the // signature patch location. - pkc_data_offset: patch_loc - .checked_sub(load_hdr.os_data_offset) - .ok_or(EINVAL)?, - engine_id_mask: u16::try_from(sig_params.engine_id_mask).map_err(|_| EINVAL)?, - ucode_id: u8::try_from(sig_params.ucode_id).map_err(|_| EINVAL)?, + pkc_data_offset: patch_loc.checked_sub(os_data_offset).ok_or(EINVAL)?, + engine_id_mask: u16::try_from(engine_id).map_err(|_| EINVAL)?, + ucode_id: u8::try_from(ucode_id).map_err(|_| EINVAL)?, }; - let app0 = HsLoadHeaderV2App::new(&hs_fw, 0)?; - // Object containing the firmware microcode to be signature-patched. - let ucode = bin_fw - .data() - .ok_or(EINVAL) + let ucode = tlv + .get_bytes(b"BLOB") .and_then(FirmwareObject::<Self, _>::new_booter)?; - let ucode_signed = { - let mut signatures = hs_fw.signatures_iter()?.peekable(); - - if signatures.peek().is_none() { - // If there are no signatures, then the firmware is unsigned. - ucode.no_patch_signature() - } else { - // Obtain the version from the fuse register, and extract the corresponding - // signature. - let reg_fuse_version = falcon.signature_reg_fuse_version( - bar, - brom_params.engine_id_mask, - brom_params.ucode_id, - )?; - - // `0` means the last signature should be used. - const FUSE_VERSION_USE_LAST_SIG: u32 = 0; - let signature = match reg_fuse_version { - FUSE_VERSION_USE_LAST_SIG => signatures.last(), - // Otherwise hardware fuse version needs to be subtracted to obtain the index. - reg_fuse_version => { - let Some(idx) = sig_params.fuse_ver.checked_sub(reg_fuse_version) else { - dev_err!(dev, "invalid fuse version for Booter firmware\n"); - return Err(EINVAL); - }; - signatures.nth(idx.into_safe_cast()) - } - } - .ok_or(EINVAL)?; - - ucode.patch_signature(&signature, patch_loc.into_safe_cast())? - } + // Obtain the version from the fuse register, and extract the corresponding + // signature. + let reg_fuse_version: usize = falcon + .signature_reg_fuse_version(brom_params.engine_id_mask, brom_params.ucode_id)? + .into_safe_cast(); + + const FUSE_VERSION_USE_LAST_SIG: usize = 0; + + let index = match reg_fuse_version { + // `0` means the last signature should be used. + FUSE_VERSION_USE_LAST_SIG => None, + // Otherwise, hardware fuse version needs to be subtracted to obtain the index. + _ => Some(fuse_version.checked_sub(reg_fuse_version).ok_or(EINVAL)?), }; + // Extract the nth signature. Booter is always signed. + let sig_chunk = tlv.get_signature(index)?; + + let signature = BooterSignature(sig_chunk); + let ucode_signed = ucode.patch_signature(&signature, patch_loc.into_safe_cast())?; + // There are two versions of Booter, one for Turing/GA100, and another for // GA102+. The extraction of the IMEM sections differs between the two // versions. Unfortunately, the file names are the same, and the headers // don't indicate the versions. The only way to differentiate is by the Chipset. let (imem_sec_dst_start, imem_ns_load_target) = if chipset <= Chipset::GA100 { ( - app0.offset, + app0_code_offset, Some(FalconDmaLoadTarget { src_start: 0, - dst_start: load_hdr.os_code_offset, - len: load_hdr.os_code_size, + dst_start: os_code_offset, + len: os_code_size, }), ) } else { @@ -383,15 +163,15 @@ impl BooterFirmware { Ok(Self { imem_sec_load_target: FalconDmaLoadTarget { - src_start: app0.offset, + src_start: app0_code_offset, dst_start: imem_sec_dst_start, - len: app0.len, + len: app0_code_size, }, imem_ns_load_target, dmem_load_target: FalconDmaLoadTarget { - src_start: load_hdr.os_data_offset, + src_start: os_data_offset, dst_start: 0, - len: load_hdr.os_data_size, + len: os_data_size, }, brom_params, ucode: ucode_signed, @@ -405,17 +185,15 @@ impl BooterFirmware { pub(crate) fn run<T>( &self, dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - sec2_falcon: &Falcon<Sec2>, + sec2_falcon: &Falcon<'_, Sec2>, wpr_meta: &Coherent<T>, ) -> Result { - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, self)?; - let wpr_handle = wpr_meta.dma_handle(); + sec2_falcon.reset()?; + sec2_falcon.load(self)?; + let wpr_dma_address = wpr_meta.dma_address(); let (mbox0, mbox1) = sec2_falcon.boot( - bar, - Some(wpr_handle as u32), - Some((wpr_handle >> 32) as u32), + Some(wpr_dma_address as u32), + Some((wpr_dma_address >> 32) as u32), )?; dev_dbg!(dev, "SEC2 MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); diff --git a/drivers/gpu/nova-core/firmware/fsp.rs b/drivers/gpu/nova-core/firmware/fsp.rs index 6eaf1c684b9d..5462e318410a 100644 --- a/drivers/gpu/nova-core/firmware/fsp.rs +++ b/drivers/gpu/nova-core/firmware/fsp.rs @@ -6,12 +6,14 @@ use kernel::{ device, dma::Coherent, - firmware::Firmware, prelude::*, // }; use crate::{ - firmware::elf, + firmware::tlv::{ + request_tlv, // + Tlv, + }, gpu::Chipset, // }; @@ -19,11 +21,11 @@ use crate::{ const FSP_HASH_SIZE: usize = 48; /// Maximum size of the FSP public key (RSA-3072), in bytes. /// -/// The FMC ELF `publickey` section may be shorter, so the remaining bytes are zero-padded. +/// The FMC `PKEY` tag may be shorter, so the remaining bytes are zero-padded. const FSP_PKEY_SIZE: usize = 384; /// Maximum size of the FSP signature (RSA-3072), in bytes. /// -/// The FMC ELF `signature` section may be shorter, so the remaining bytes are zero-padded. +/// The FMC `SIGN` tag may be shorter, so the remaining bytes are zero-padded. const FSP_SIG_SIZE: usize = 384; /// Structure to hold FMC signatures. @@ -38,64 +40,35 @@ pub(crate) struct FmcSignatures { } pub(crate) struct FspFirmware { - /// FMC firmware image data (only the "image" ELF section). + /// FMC firmware image data pub(crate) fmc_image: Coherent<[u8]>, /// FMC firmware signatures. pub(crate) fmc_sigs: KBox<FmcSignatures>, } impl FspFirmware { - pub(crate) fn new( - dev: &device::Device<device::Bound>, - chipset: Chipset, - ver: &str, - ) -> Result<Self> { - let fw = super::request_firmware(dev, chipset, "fmc", ver)?; + pub(crate) fn new(dev: &device::Device<device::Bound>, chipset: Chipset) -> Result<Self> { + let fw = request_tlv(dev, chipset, "fmc")?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!(dev, "loaded fsp firmware v{}\n", tlv.get_string(b"VERS")?); - // FSP expects only the "image" section, not the entire ELF file. - let fmc_image_data = elf::elf_section(fw.data(), "image").ok_or_else(|| { - dev_err!(dev, "FMC ELF file missing 'image' section\n"); - EINVAL - })?; + let fmc_image_data = tlv.get_bytes(b"BLOB")?; let fmc_image = Coherent::from_slice(dev, fmc_image_data, GFP_KERNEL)?; Ok(Self { fmc_image, - fmc_sigs: Self::extract_fmc_signatures(&fw, dev)?, + fmc_sigs: Self::extract_fmc_signatures(&tlv, dev)?, }) } /// Extract FMC firmware signatures for Chain of Trust verification. /// - /// Extracts real cryptographic signatures from FMC ELF32 firmware sections. + /// Extracts real cryptographic signatures from FMC TLV firmware tags. /// Returns signatures in a heap-allocated structure to prevent stack overflow. - fn extract_fmc_signatures( - fmc_fw: &Firmware, - dev: &device::Device, - ) -> Result<KBox<FmcSignatures>> { - let get_section = |name: &str, max_len: usize| { - elf::elf_section(fmc_fw.data(), name) - .ok_or(EINVAL) - .inspect_err(|_| dev_err!(dev, "FMC firmware missing '{}' section\n", name)) - .and_then(|section| { - if section.len() > max_len { - dev_err!( - dev, - "FMC {} section size {} > maximum {}\n", - name, - section.len(), - max_len - ); - Err(EINVAL) - } else { - Ok(section) - } - }) - }; - - let hash_section = get_section("hash", FSP_HASH_SIZE)?; - let pkey_section = get_section("publickey", FSP_PKEY_SIZE)?; - let sig_section = get_section("signature", FSP_SIG_SIZE)?; + fn extract_fmc_signatures(tlv: &Tlv<'_>, dev: &device::Device) -> Result<KBox<FmcSignatures>> { + let hash_section = tlv.get_bytes(b"HASH")?; + let pkey_section = tlv.get_bytes(b"PKEY")?; + let sig_section = tlv.get_bytes(b"SIGN")?; // The hash section is a SHA-384 output: it must be exactly FSP_HASH_SIZE bytes. if hash_section.len() != FSP_HASH_SIZE { @@ -108,15 +81,36 @@ impl FspFirmware { return Err(EINVAL); } + // The key and signature sections are zero-padded to a fixed maximum, so they may be + // shorter, but must not exceed the destination buffers. + if pkey_section.len() > FSP_PKEY_SIZE { + dev_err!( + dev, + "FMC public key section size {} > maximum {}\n", + pkey_section.len(), + FSP_PKEY_SIZE + ); + return Err(EINVAL); + } + if sig_section.len() > FSP_SIG_SIZE { + dev_err!( + dev, + "FMC signature section size {} > maximum {}\n", + sig_section.len(), + FSP_SIG_SIZE + ); + return Err(EINVAL); + } + // Initialize the signatures in place to avoid building the large `FmcSignatures` on the // stack, then fill each section from the firmware. let signatures = KBox::init( pin_init::init_zeroed::<FmcSignatures>().chain(|sigs| { // PANIC: src and dst lengths are both FSP_HASH_SIZE (verified above). sigs.hash384.copy_from_slice(hash_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE per `get_section`. + // PANIC: dst is sliced to src.len(); src.len() <= FSP_PKEY_SIZE (verified above). sigs.public_key[..pkey_section.len()].copy_from_slice(pkey_section); - // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE per `get_section`. + // PANIC: dst is sliced to src.len(); src.len() <= FSP_SIG_SIZE (verified above). sigs.signature[..sig_section.len()].copy_from_slice(sig_section); Ok(()) }), diff --git a/drivers/gpu/nova-core/firmware/fwsec.rs b/drivers/gpu/nova-core/firmware/fwsec.rs index 199ae2adb664..7a931f22f629 100644 --- a/drivers/gpu/nova-core/firmware/fwsec.rs +++ b/drivers/gpu/nova-core/firmware/fwsec.rs @@ -27,7 +27,6 @@ use kernel::{ }; use crate::{ - driver::Bar0, falcon::{ gsp::Gsp, Falcon, @@ -320,8 +319,7 @@ impl FwsecFirmware { /// command. pub(crate) fn new( dev: &Device<device::Bound>, - falcon: &Falcon<Gsp>, - bar: Bar0<'_>, + falcon: &Falcon<'_, Gsp>, bios: &Vbios, cmd: FwsecCommand, ) -> Result<Self> { @@ -337,7 +335,7 @@ impl FwsecFirmware { .ok_or(EINVAL)?; let desc_sig_versions = u32::from(desc.signature_versions()); let reg_fuse_version = - falcon.signature_reg_fuse_version(bar, desc.engine_id_mask(), desc.ucode_id())?; + falcon.signature_reg_fuse_version(desc.engine_id_mask(), desc.ucode_id())?; dev_dbg!( dev, "desc_sig_versions: {:#x}, reg_fuse_version: {}\n", @@ -387,24 +385,18 @@ impl FwsecFirmware { /// Loads the FWSEC firmware into `falcon` and execute it. /// - /// This must only be called on chipsets that do not need the FWSEC bootloader (i.e., where - /// [`Chipset::needs_fwsec_bootloader()`](crate::gpu::Chipset::needs_fwsec_bootloader) returns - /// `false`). On chipsets that do, use [`bootloader::FwsecFirmwareWithBl`] instead. - pub(crate) fn run( - &self, - dev: &Device<device::Bound>, - falcon: &Falcon<Gsp>, - bar: Bar0<'_>, - ) -> Result<()> { + /// This must only be called on chipsets that do not need the FWSEC bootloader. On chipsets + /// where the bootloader is required, use [`bootloader::FwsecFirmwareWithBl`] instead. + pub(crate) fn run(&self, dev: &Device<device::Bound>, falcon: &Falcon<'_, Gsp>) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon - .reset(bar) + .reset() .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; falcon - .load(dev, bar, self) + .load(self) .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; let (mbox0, _) = falcon - .boot(bar, Some(0), None) + .boot(Some(0), None) .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?; if mbox0 != 0 { dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0); diff --git a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs index 039920dc340b..ec4d92317a93 100644 --- a/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs +++ b/drivers/gpu/nova-core/firmware/fwsec/bootloader.rs @@ -7,26 +7,19 @@ //! be loaded using PIO. use kernel::{ - alloc::KVec, device::{ self, Device, // }, dma::Coherent, - io::{ - register::WithBase, // - Io, - }, + io::{register::WithBase, Io}, prelude::*, ptr::{ Alignable, Alignment, // }, sizes, - transmute::{ - AsBytes, - FromBytes, // - }, + transmute::AsBytes, }; use crate::{ @@ -46,38 +39,16 @@ use crate::{ }, firmware::{ fwsec::FwsecFirmware, - request_firmware, - BinHdr, - FIRMWARE_VERSION, // + tlv::{ + request_tlv, // + Tlv, + }, }, gpu::Chipset, - num::FromSafeCast, + num::FromSafeCast, // regs, }; -/// Descriptor used by RM to figure out the requirements of the boot loader. -/// -/// Most of its fields appear to be legacy and carry incorrect values, so they are left unused. -#[repr(C)] -#[derive(Debug, Clone)] -struct BootloaderDesc { - /// Starting tag of bootloader. - start_tag: u32, - /// DMEM load offset - unused here as we always load at offset `0`. - _dmem_load_off: u32, - /// Offset of code section in the image. Unused as there is only one section in the bootloader - /// binary. - _code_off: u32, - /// Size of code section in the image. - code_size: u32, - /// Offset of data section in the image. Unused as we build the data section ourselves. - _data_off: u32, - /// Size of data section in the image. Unused as we build the data section ourselves. - _data_size: u32, -} -// SAFETY: any byte sequence is valid for this struct. -unsafe impl FromBytes for BootloaderDesc {} - /// Structure used by the boot-loader to load the rest of the code. /// /// This has to be filled by the GPU driver and copied into DMEM at offset @@ -150,38 +121,24 @@ impl FwsecFirmwareWithBl { dev: &Device<device::Bound>, chipset: Chipset, ) -> Result<Self> { - let fw = request_firmware(dev, chipset, "gen_bootloader", FIRMWARE_VERSION)?; - let hdr = fw - .data() - .get(0..size_of::<BinHdr>()) - .and_then(BinHdr::from_bytes_copy) - .ok_or(EINVAL)?; - - let desc = { - let desc_offset = usize::from_safe_cast(hdr.header_offset); - - fw.data() - .get(desc_offset..) - .and_then(BootloaderDesc::from_bytes_copy_prefix) - .ok_or(EINVAL)? - .0 - }; + let fw = request_tlv(dev, chipset, "gen_bootloader")?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded generic bootloader firmware v{}\n", + tlv.get_string(b"VERS")? + ); let ucode = { - let ucode_start = usize::from_safe_cast(hdr.data_offset); - let code_size = usize::from_safe_cast(desc.code_size); - // Align to falcon block size (256 bytes). + let blob = tlv.get_bytes(b"BLOB")?; + let code_size = usize::from_safe_cast(tlv.get_u32(b"CDSZ")?); + let code = blob.get(..code_size).ok_or(EINVAL)?; let aligned_code_size = code_size .align_up(Alignment::new::<{ falcon::MEM_BLOCK_ALIGNMENT }>()) .ok_or(EINVAL)?; let mut ucode = KVec::with_capacity(aligned_code_size, GFP_KERNEL)?; - ucode.extend_from_slice( - fw.data() - .get(ucode_start..ucode_start + code_size) - .ok_or(EINVAL)?, - GFP_KERNEL, - )?; + ucode.extend_from_slice(code, GFP_KERNEL)?; ucode.resize(aligned_code_size, 0, GFP_KERNEL)?; ucode @@ -234,7 +191,7 @@ impl FwsecFirmwareWithBl { reserved: [0; 4], signature: [0; 4], ctx_dma: FALCON_DMAIDX_PHYS_SYS_NCOH, - code_dma_base: firmware_dma.dma_handle(), + code_dma_base: firmware_dma.dma_address(), // `dst_start` is also valid as the source offset since the firmware DMA object is // a mirror image of the target IMEM layout. non_sec_code_off: imem_ns.dst_start, @@ -246,7 +203,7 @@ impl FwsecFirmwareWithBl { code_entry_point: 0, // Start of data section is the added padding + the DMEM `src_start` field. data_dma_base: firmware_dma - .dma_handle() + .dma_address() .checked_add(u64::from_safe_cast(align_padding)) .and_then(|offset| offset.checked_add(dmem.src_start.into())) .ok_or(EOVERFLOW)?, @@ -262,13 +219,15 @@ impl FwsecFirmwareWithBl { .checked_sub(ucode.len()) .ok_or(EOVERFLOW)?; + let start_tag = u16::try_from(tlv.get_u32(b"STRT")?)?; + Ok(Self { _firmware_dma: firmware_dma, ucode, dmem_desc, brom_params: firmware.brom_params(), imem_dst_start: u16::try_from(imem_dst_start)?, - start_tag: u16::try_from(desc.start_tag)?, + start_tag, }) } @@ -279,15 +238,15 @@ impl FwsecFirmwareWithBl { pub(crate) fn run( &self, dev: &Device<device::Bound>, - falcon: &Falcon<Gsp>, + falcon: &Falcon<'_, Gsp>, bar: Bar0<'_>, ) -> Result<()> { // Reset falcon, load the firmware, and run it. falcon - .reset(bar) + .reset() .inspect_err(|e| dev_err!(dev, "Failed to reset GSP falcon: {:?}\n", e))?; falcon - .pio_load(bar, self) + .pio_load(self) .inspect_err(|e| dev_err!(dev, "Failed to load FWSEC firmware: {:?}\n", e))?; // Configure DMA index for the bootloader to fetch the FWSEC firmware from system memory. @@ -302,7 +261,7 @@ impl FwsecFirmwareWithBl { ); let (mbox0, _) = falcon - .boot(bar, Some(0), None) + .boot(Some(0), None) .inspect_err(|e| dev_err!(dev, "Failed to boot FWSEC firmware: {:?}\n", e))?; if mbox0 != 0 { dev_err!(dev, "FWSEC firmware returned error {}\n", mbox0); diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs index 99a302bae567..e8f9491e84cc 100644 --- a/drivers/gpu/nova-core/firmware/gsp.rs +++ b/drivers/gpu/nova-core/firmware/gsp.rs @@ -8,22 +8,24 @@ use kernel::{ DataDirection, DmaAddress, // }, + firmware, prelude::*, scatterlist::{ Owned, SGTable, // }, + str::CString, }; use crate::{ firmware::{ - elf, riscv::RiscvFirmware, // + tlv::{ + request_tlv, // + Tlv, + }, }, - gpu::{ - Architecture, - Chipset, // - }, + gpu::Chipset, gsp::GSP_PAGE_SIZE, num::FromSafeCast, }; @@ -63,43 +65,26 @@ pub(crate) struct GspFirmware { } impl GspFirmware { - fn find_gsp_sigs_section(chipset: Chipset) -> &'static str { - match chipset.arch() { - Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => { - ".fwsignature_tu11x" - } - Architecture::Turing => ".fwsignature_tu10x", - Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100", - Architecture::Ampere => ".fwsignature_ga10x", - Architecture::Ada => ".fwsignature_ad10x", - Architecture::Hopper => ".fwsignature_gh10x", - Architecture::BlackwellGB10x => ".fwsignature_gb10x", - Architecture::BlackwellGB20x => ".fwsignature_gb20x", - } - } - /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page /// tables expected by the GSP bootloader to load it. pub(crate) fn new<'a>( dev: &'a device::Device<device::Bound>, chipset: Chipset, - ver: &'a str, ) -> impl PinInit<Self, Error> + 'a { pin_init::pin_init_scope(move || { - let firmware = super::request_firmware(dev, chipset, "gsp", ver)?; + let firmware = request_tlv(dev, chipset, "gsp")?; + let tlv = Tlv::new(firmware.data())?; + dev_dbg!(dev, "loaded gsp firmware v{}\n", tlv.get_string(b"VERS")?); - let fw_section = elf::elf_section(firmware.data(), ".fwimage").ok_or(EINVAL)?; + let size = usize::from_safe_cast(tlv.get_u32(b"SIZE")?); + let mut fw_vvec = VVec::zeroed(size, GFP_KERNEL).map_err(|_| ENOMEM)?; - let size = fw_section.len(); + let chip_name = chipset.name(); + let file = tlv.get_string(b"FILE")?; + let filename = CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{file}"))?; + firmware::request_into_buf(&filename, dev, fw_vvec.as_mut_slice())?; - // Move the firmware into a vmalloc'd vector and map it into the device address - // space. - let fw_vvec = VVec::with_capacity(fw_section.len(), GFP_KERNEL) - .and_then(|mut v| { - v.extend_from_slice(fw_section, GFP_KERNEL)?; - Ok(v) - }) - .map_err(|_| ENOMEM)?; + let signatures = Coherent::from_slice(dev, tlv.get_bytes(b"SIGN")?, GFP_KERNEL)?; Ok(try_pin_init!(Self { fw <- SGTable::new(dev, fw_vvec, DataDirection::ToDevice, GFP_KERNEL), @@ -145,15 +130,9 @@ impl GspFirmware { level0.into() }, size, - signatures: { - let sigs_section = Self::find_gsp_sigs_section(chipset); - - elf::elf_section(firmware.data(), sigs_section) - .ok_or(EINVAL) - .and_then(|data| Coherent::from_slice(dev, data, GFP_KERNEL))? - }, + signatures, bootloader: { - let bl = super::request_firmware(dev, chipset, "bootloader", ver)?; + let bl = request_tlv(dev, chipset, "gsp_bootloader")?; RiscvFirmware::new(dev, &bl)? }, @@ -161,9 +140,9 @@ impl GspFirmware { }) } - /// Returns the DMA handle of the radix3 level 0 page table. - pub(crate) fn radix3_dma_handle(&self) -> DmaAddress { - self.level0.dma_handle() + /// Returns the DMA address of the radix3 level 0 page table. + pub(crate) fn radix3_dma_address(&self) -> DmaAddress { + self.level0.dma_address() } } diff --git a/drivers/gpu/nova-core/firmware/riscv.rs b/drivers/gpu/nova-core/firmware/riscv.rs index 2afa7f36404e..1403f05a7305 100644 --- a/drivers/gpu/nova-core/firmware/riscv.rs +++ b/drivers/gpu/nova-core/firmware/riscv.rs @@ -7,53 +7,10 @@ use kernel::{ device, dma::Coherent, firmware::Firmware, - prelude::*, - transmute::FromBytes, // + prelude::*, // }; -use crate::{ - firmware::BinFirmware, - num::FromSafeCast, // -}; - -/// Descriptor for microcode running on a RISC-V core. -#[repr(C)] -#[derive(Debug)] -struct RmRiscvUCodeDesc { - version: u32, - bootloader_offset: u32, - bootloader_size: u32, - bootloader_param_offset: u32, - bootloader_param_size: u32, - riscv_elf_offset: u32, - riscv_elf_size: u32, - app_version: u32, - manifest_offset: u32, - manifest_size: u32, - monitor_data_offset: u32, - monitor_data_size: u32, - monitor_code_offset: u32, - monitor_code_size: u32, -} - -// SAFETY: all bit patterns are valid for this type, and it doesn't use interior mutability. -unsafe impl FromBytes for RmRiscvUCodeDesc {} - -impl RmRiscvUCodeDesc { - /// Interprets the header of `bin_fw` as a [`RmRiscvUCodeDesc`] and returns it. - /// - /// Fails if the header pointed at by `bin_fw` is not within the bounds of the firmware image. - fn new(bin_fw: &BinFirmware<'_>) -> Result<Self> { - let offset = usize::from_safe_cast(bin_fw.hdr.header_offset); - let end = offset.checked_add(size_of::<Self>()).ok_or(EINVAL)?; - - bin_fw - .fw - .get(offset..end) - .and_then(Self::from_bytes_copy) - .ok_or(EINVAL) - } -} +use crate::firmware::tlv::Tlv; /// A parsed firmware for a RISC-V core, ready to be loaded and run. pub(crate) struct RiscvFirmware { @@ -72,24 +29,26 @@ pub(crate) struct RiscvFirmware { impl RiscvFirmware { /// Parses the RISC-V firmware image contained in `fw`. pub(crate) fn new(dev: &device::Device<device::Bound>, fw: &Firmware) -> Result<Self> { - let bin_fw = BinFirmware::new(fw)?; - - let riscv_desc = RmRiscvUCodeDesc::new(&bin_fw)?; + let tlv = Tlv::new(fw.data())?; + dev_dbg!( + dev, + "loaded gsp bootloader firmware v{}\n", + tlv.get_string(b"VERS")? + ); - let ucode = { - let start = usize::from_safe_cast(bin_fw.hdr.data_offset); - let len = usize::from_safe_cast(bin_fw.hdr.data_size); - let end = start.checked_add(len).ok_or(EINVAL)?; + let code_offset = tlv.get_u32(b"CDOF")?; + let data_offset = tlv.get_u32(b"DAOF")?; + let manifest_offset = tlv.get_u32(b"MFOF")?; + let app_version = tlv.get_u32(b"APPV")?; - Coherent::from_slice(dev, fw.data().get(start..end).ok_or(EINVAL)?, GFP_KERNEL)? - }; + let ucode = Coherent::from_slice(dev, tlv.get_bytes(b"BLOB")?, GFP_KERNEL)?; Ok(Self { ucode, - code_offset: riscv_desc.monitor_code_offset, - data_offset: riscv_desc.monitor_data_offset, - manifest_offset: riscv_desc.manifest_offset, - app_version: riscv_desc.app_version, + code_offset, + data_offset, + manifest_offset, + app_version, }) } } diff --git a/drivers/gpu/nova-core/firmware/tlv.rs b/drivers/gpu/nova-core/firmware/tlv.rs new file mode 100644 index 000000000000..7b879f13a61e --- /dev/null +++ b/drivers/gpu/nova-core/firmware/tlv.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use kernel::{ + device, + firmware, + prelude::*, + str::CString, // +}; + +use crate::{ + gpu, + num::*, // +}; + +/// Requests the GPU firmware TLV `name` suitable for `chipset`. +pub(crate) fn request_tlv( + dev: &device::Device, + chipset: gpu::Chipset, + name: &str, +) -> Result<firmware::Firmware> { + let chip_name = chipset.name(); + + let filename = CString::try_from_fmt(fmt!("nvidia/{chip_name}/gsp/{name}.tlv"))?; + + dev_dbg!(dev, "loading firmware image {:?}\n", &filename); + + firmware::Firmware::request(&filename, dev) +} + +struct TlvBlock<'a> { + tag: [u8; 4], + value: &'a [u8], +} + +/// On-wire TLV block header: 4-byte ASCII tag + little-endian payload length (bytes, excluding +/// padding to a 4-byte boundary). +struct TlvBlockHeader { + tag: [u8; 4], + length: usize, +} + +impl TlvBlockHeader { + const SIZE: usize = size_of::<[u8; 4]>() + size_of::<u32>(); + + /// Parses the first [`Self::SIZE`] bytes of `hdr` (caller may pass a longer slice). + fn parse(hdr: &[u8]) -> Option<Self> { + let hdr = hdr.get(..Self::SIZE)?; + let tag = <[u8; 4]>::try_from(hdr.get(..4)?).ok()?; + if !tag.is_ascii() { + return None; + } + let len_arr = <[u8; 4]>::try_from(hdr.get(4..Self::SIZE)?).ok()?; + let length = u32_as_usize(u32::from_le_bytes(len_arr)); + Some(Self { tag, length }) + } +} + +/// Iterator over the [`TlvBlock`]s of a [`Tlv`]. +/// +/// # Invariants +/// +/// `pos` is a byte offset into `tlv.data` that always lies on a block boundary (in the sense +/// of the [`Tlv`] invariant): it is either the start of a well-formed block, or equal to +/// `tlv.data.len()` (end of iteration). +struct TlvIter<'tlv, 'a> { + tlv: &'tlv Tlv<'a>, + pos: usize, +} + +impl<'tlv, 'a> Iterator for TlvIter<'tlv, 'a> { + type Item = TlvBlock<'a>; + + /// Returns the block starting at `self.pos` and advances the cursor past it, or [`None`] + /// once the cursor reaches the end of the data or encounters an error. + /// + /// Note that errors cannot actually occur because the data is validated in the constructor. + fn next(&mut self) -> Option<Self::Item> { + if self.pos >= self.tlv.data.len() { + return None; + } + + let tail = self.tlv.data.get(self.pos..)?; + + let hdr = tail.get(..TlvBlockHeader::SIZE)?; + let header = TlvBlockHeader::parse(hdr)?; + + let stored_size = header.length.checked_next_multiple_of(4)?; + let advance = TlvBlockHeader::SIZE.checked_add(stored_size)?; + let payload_end = TlvBlockHeader::SIZE.checked_add(header.length)?; + + let value = tail + .get(..advance)? + .get(TlvBlockHeader::SIZE..payload_end)?; + + // INVARIANT: by the `Tlv` invariant the block at `self.pos` occupies exactly `advance` + // bytes, so `self.pos + advance` is the next block boundary (or `data.len()`). + self.pos = self.pos.checked_add(advance)?; + + Some(TlvBlock { + tag: header.tag, + value, + }) + } +} + +/// The post-header part of a validated TLV (type, length, value) firmware image. +/// +/// TLV firmware images start with a 4-byte "NVFW" magic header, followed by a sequence of +/// blocks. Each block has a 4-byte type tag, a 4-byte length field, and a data payload +/// (value) whose stored size is the length rounded up to the nearest multiple of 4. +/// +/// [`Self::new`] checks the magic header and walks every block: tags must be ASCII, +/// lengths and padding must fit without overflow, and the byte stream after `NVFW` must +/// be exactly partitionable into blocks (no trailing partial header or slack). After +/// that, [`TlvIter`] only signals end-of-stream via [`None`], not parse failure. +/// +/// Although the spec forbids duplicate tags, neither the constructor nor the iterator +/// enforces this restriction. Instead, duplicate tags are simply ignored. +/// +/// # Invariants +/// +/// `data` is a validated TLV payload (the bytes *after* the `NVFW` magic): it is the exact +/// concatenation of zero or more well-formed blocks, with no trailing partial header or slack. +/// Consequently, any offset `o` into `data` that is a block boundary and satisfies +/// `o < data.len()` is the start of a complete block whose header parses and whose stored +/// extent (`TlvBlockHeader::SIZE + header.length.next_multiple_of(4)` bytes) lies within +/// `data`. `data.len()` is itself a boundary. +pub(crate) struct Tlv<'a> { + data: &'a [u8], +} + +impl<'a> Tlv<'a> { + const MAGIC: &'static [u8; 4] = b"NVFW"; + + /// Parses `data` as a TLV firmware image, returning [`EINVAL`] if the image is malformed. + pub(crate) fn new(data: &'a [u8]) -> Result<Self> { + // Verify that the magic bytes exist and are the correct value + let magic_len = Self::MAGIC.len(); + if data + .get(..magic_len) + .is_none_or(|magic| magic != Self::MAGIC) + { + return Err(EINVAL); + } + + // The payload is the contiguous sequence of TLV blocks after the magic. + let payload = data.get(magic_len..).ok_or(EINVAL)?; + + // The spec says every TLV must have a VERS tag. + let mut has_vers = false; + + let mut rest = payload; + while !rest.is_empty() { + // Validate and extract the header (type, length). + let Some(header): Option<TlvBlockHeader> = rest + .get(..TlvBlockHeader::SIZE) + .and_then(TlvBlockHeader::parse) + else { + return Err(EINVAL); + }; + + has_vers |= header.tag == *b"VERS"; + + // The `length` field of a TLV block contains the actual byte length of the + // value, but each TLV block is aligned to a 4-byte boundary. + let Some(stored_size) = header.length.checked_next_multiple_of(4) else { + return Err(EINVAL); + }; + + let length = TlvBlockHeader::SIZE + .checked_add(stored_size) + .ok_or(EINVAL)?; + + rest = rest.split_at_checked(length).ok_or(EINVAL)?.1; + } + + if !has_vers { + return Err(EINVAL); + } + + // INVARIANT: the loop above walked `payload` block-by-block. For each block, the + // header is parsed (`TlvBlockHeader::parse` rejects non-ASCII tags), and the + // stored extent (`SIZE + length.next_multiple_of(4)`) is computed without + // overflow and split off `rest` only when it fits. The loop ends only when `rest` + // is empty, so the byte stream is an exact concatenation of blocks with no + // trailing partial header or slack. + Ok(Self { data: payload }) + } + + fn iter(&self) -> TlvIter<'_, 'a> { + // INVARIANT: 0 is a block boundary, either the start of the first block, + // or `data.len()` when `data` is empty. + TlvIter { tlv: self, pos: 0 } + } + + fn find(&self, tag: &[u8; 4]) -> Result<TlvBlock<'a>> { + self.iter().find(|b| b.tag == *tag).ok_or(EINVAL) + } + + /// Return a slice of bytes. + /// + /// Returns `EINVAL` if the value is empty. + pub(crate) fn get_bytes(&self, tag: &[u8; 4]) -> Result<&'a [u8]> { + let tlv = self.find(tag)?; + + // Treat empty value as an error, to avoid trying to parse nothing. + if tlv.value.is_empty() { + return Err(EINVAL); // TODO: Use ENODATA once available. + } + + Ok(tlv.value) + } + + /// Return a little-endian u32. + pub(crate) fn get_u32(&self, tag: &[u8; 4]) -> Result<u32> { + let tlv = self.find(tag)?; + + tlv.value + .try_into() + .ok() + .map(u32::from_le_bytes) + .ok_or(EINVAL) + } + + /// Return a string value. + pub(crate) fn get_string(&self, tag: &[u8; 4]) -> Result<&'a str> { + let tlv = self.find(tag)?; + + let bytes = tlv.value; + + // Strings can only contain printable ASCII characters. + if bytes.iter().any(|&b| !(32..127).contains(&b)) { + return Err(EINVAL); + } + + core::str::from_utf8(bytes).map_err(|_| EINVAL) + } + + /// Obtain the nth signature from a SIGN tag. If `index` is None, + /// then return the last signature. + pub(crate) fn get_signature(&self, index: Option<usize>) -> Result<&'a [u8]> { + let num_sigs: usize = match self.get_u32(b"NSIG")? { + 0 => return Err(EINVAL), + n => n.into_safe_cast(), + }; + + let sig_bytes = self.get_bytes(b"SIGN")?; + + // Ensure that sig_bytes can be divided evenly into chunks. + if sig_bytes.len() % num_sigs != 0 { + return Err(EINVAL); + } + + // num_sigs cannot be 0, and sig_bytes cannot be empty, so this cannot panic. + let sig_size = sig_bytes.len() / num_sigs; + + let index = index.unwrap_or(num_sigs - 1); + + sig_bytes.chunks_exact(sig_size).nth(index).ok_or(EINVAL) + } +} diff --git a/drivers/gpu/nova-core/fsp.rs b/drivers/gpu/nova-core/fsp.rs index 8fc243c66e35..ab685fb4168f 100644 --- a/drivers/gpu/nova-core/fsp.rs +++ b/drivers/gpu/nova-core/fsp.rs @@ -11,6 +11,7 @@ use kernel::{ device, dma::Coherent, io::poll::read_poll_timeout, + num::TryIntoBounded, prelude::*, ptr::{ Alignable, @@ -30,13 +31,17 @@ use crate::{ fsp::Fsp as FspEngine, Falcon, // }, - fb::FbLayout, + fb::FbSizes, firmware::fsp::{ FmcSignatures, FspFirmware, // }, gpu::Chipset, - gsp::GspFmcBootParams, + gsp::{ + GspFmcBootParams, + GspFwWprMeta, + LibosMemoryRegionInitArgument, // + }, mctp::{ MctpHeader, NvdmHeader, @@ -48,6 +53,56 @@ use crate::{ mod hal; +/// PRC message sub-command. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum PrcMessageSubcmd { + /// Read a PRC knob value. + Read = 0x0c, +} + +impl From<PrcMessageSubcmd> for u8 { + fn from(value: PrcMessageSubcmd) -> Self { + value as u8 + } +} + +/// PRC object identifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +enum PrcObjectId { + /// vGPU mode configuration knob. + VgpuMode = 0x29, +} + +impl From<PrcObjectId> for u8 { + fn from(value: PrcObjectId) -> Self { + value as u8 + } +} + +kernel::impl_flags!( + /// PRC request flags. + #[derive(Clone, Copy, Default, PartialEq, Eq)] + struct PrcFlags(u8); + + /// Individual PRC request flag. + #[derive(Clone, Copy, PartialEq, Eq)] + enum PrcFlag { + /// Request the active knob value for the current boot. + Active = 1 << 1, + } +); + +/// vGPU operating mode as reported by FSP via the PRC protocol. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VgpuMode { + /// vGPU support is disabled on this GPU. + Disabled, + /// vGPU support is enabled on this GPU. + Enabled, +} + /// FSP command response payload (`NVDM_PAYLOAD_COMMAND_RESPONSE`). #[repr(C, packed)] #[derive(Clone, Copy)] @@ -57,17 +112,107 @@ struct NvdmPayloadCommandResponse { error_code: u32, } -/// Complete FSP response structure with MCTP and NVDM headers. +/// PRC message payload. +/// +/// Sent to FSP to query or modify a device configuration knob. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct NvdmPayloadPrc { + sub_message_id: u8, + flags: u8, + object_id: u8, + reserved: u8, +} + +impl NvdmPayloadPrc { + /// Constructs a PRC payload from typed protocol fields. + fn new(subcmd: PrcMessageSubcmd, object_id: PrcObjectId, flags: PrcFlags) -> Self { + Self { + sub_message_id: subcmd.into(), + flags: flags.into(), + object_id: object_id.into(), + reserved: 0, + } + } +} + +// SAFETY: NvdmPayloadPrc is a packed C struct with only integral fields. +unsafe impl AsBytes for NvdmPayloadPrc {} + +/// PRC response payload containing the knob state value. #[repr(C, packed)] #[derive(Clone, Copy)] -struct FspResponse { +struct NvdmPayloadPrcResponse { + value_low: u8, + value_high: u8, + reserved1: u8, + reserved2: u8, +} + +impl NvdmPayloadPrcResponse { + /// Returns the PRC knob value as a little-endian 16-bit integer. + fn value(self) -> u16 { + u16::from(self.value_low) | (u16::from(self.value_high) << 8) + } +} + +impl TryFrom<NvdmPayloadPrcResponse> for VgpuMode { + type Error = kernel::error::Error; + + fn try_from(value: NvdmPayloadPrcResponse) -> Result<Self> { + match value.value() { + 0 => Ok(VgpuMode::Disabled), + 1 => Ok(VgpuMode::Enabled), + _ => Err(EINVAL), + } + } +} + +/// Common MCTP and NVDM headers shared by all FSP messages. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspMessageHeader { mctp_header: MctpHeader, nvdm_header: NvdmHeader, +} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl AsBytes for FspMessageHeader {} + +// SAFETY: FspMessageHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspMessageHeader {} + +impl FspMessageHeader { + /// Construct a standard FSP message header for the given NVDM type. + fn new(nvdm_type: NvdmType) -> Self { + Self { + mctp_header: MctpHeader::single_packet(), + nvdm_header: NvdmHeader::new(nvdm_type), + } + } +} + +/// Common FSP response header with MCTP, NVDM and command response payloads. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspResponseHeader { + header: FspMessageHeader, response: NvdmPayloadCommandResponse, } -// SAFETY: FspResponse is a packed C struct with only integral fields. -unsafe impl FromBytes for FspResponse {} +// SAFETY: FspResponseHeader is a packed C struct with only integral fields. +unsafe impl FromBytes for FspResponseHeader {} + +/// Complete FSP PRC response including the knob state payload. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspPrcResponse { + header: FspResponseHeader, + prc_data: NvdmPayloadPrcResponse, +} + +// SAFETY: FspPrcResponse is a packed C struct with only integral fields. +unsafe impl FromBytes for FspPrcResponse {} /// Trait implemented by types representing a message to send to FSP. /// @@ -94,45 +239,56 @@ struct NvdmPayloadCot { gsp_boot_args_sysmem_offset: u64, } -/// Complete FSP message structure with MCTP and NVDM headers. +/// Complete FSP COT (Chain of Trust) message structure. #[repr(C)] #[derive(Clone, Copy)] -struct FspMessage { - mctp_header: MctpHeader, - nvdm_header: NvdmHeader, +struct FspCotMessage { + header: FspMessageHeader, cot: NvdmPayloadCot, } -impl FspMessage { - /// Returns an in-place initializer for [`FspMessage`]. +impl FspCotMessage { + /// Computes the FRTS vidmem offset for the Chain-of-Trust message. It is measured backwards + /// from the end of the framebuffer. + fn frts_vidmem_offset(hal: &dyn hal::FspHal, fb_info: &FbSizes) -> Result<u64> { + let mut offset = hal.fb_end_reserved_size(); + + // As per OpenRM's `kfspPrepareBootCommands_GH100`. + if fb_info.pmu_reserved_size != 0 { + offset = (offset + u64::from(fb_info.pmu_reserved_size)) + // The 2 MiB alignment is r570-specific. + .align_up(Alignment::new::<SZ_2M>()) + .ok_or(EINVAL)?; + } + + Ok(offset) + } + + /// Returns an in-place initializer for [`FspCotMessage`]. fn new<'a>( - fb_layout: &FbLayout, + fb_info: &FbSizes, fsp_fw: &'a FspFirmware, - args: &'a FmcBootArgs, + args: &'a FmcBootArgs<'_>, ) -> Result<impl Init<Self> + 'a> { - // frts_offset is relative to FB end: FRTS_location = FB_END - frts_offset - let frts_vidmem_offset = if !args.resume { - let frts_reserved_size = fb_layout.heap.len() + u64::from(fb_layout.pmu_reserved_size); + let hal = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?; - frts_reserved_size - .align_up(Alignment::new::<SZ_2M>()) - .ok_or(EINVAL)? + let frts_vidmem_offset = if !args.resume { + Self::frts_vidmem_offset(hal, fb_info)? } else { 0 }; let frts_size: u32 = if !args.resume { - fb_layout.frts.len().try_into()? + fb_info.frts_size.try_into()? } else { 0 }; - let version = hal::fsp_hal(args.chipset).ok_or(ENOTSUPP)?.cot_version(); + let version = hal.cot_version(); let size = num::usize_into_u16::<{ core::mem::size_of::<NvdmPayloadCot>() }>(); Ok(init!(Self { - mctp_header: MctpHeader::single_packet(), - nvdm_header: NvdmHeader::new(NvdmType::Cot), + header: FspMessageHeader::new(NvdmType::Cot), // The payload is packed, so we cannot use `init!`. Initialize it member-by-member using // `chain`. cot <- pin_init::init_zeroed(), @@ -140,12 +296,12 @@ impl FspMessage { .chain(move |msg| { msg.cot.version = version; msg.cot.size = size; - msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_handle(); + msg.cot.gsp_fmc_sysmem_offset = fsp_fw.fmc_image.dma_address(); msg.cot.frts_vidmem_offset = frts_vidmem_offset; msg.cot.frts_vidmem_size = frts_size; - // frts_sysmem_* intentionally left at zero for now, but will be needed for e.g. - // systems without VRAM. - msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_handle(); + // frts_sysmem_* are left at zero because this path places FRTS in vidmem. The sysmem + // fields point to an FRTS buffer in sysmem instead, for systems without VRAM. + msg.cot.gsp_boot_args_sysmem_offset = args.fmc_boot_params.dma_address(); msg.cot.sigs = *fsp_fw.fmc_sigs; Ok(()) @@ -153,44 +309,73 @@ impl FspMessage { } } -// SAFETY: `FspMessage` is `#[repr(C)]` with no padding, so all of its +// SAFETY: `FspCotMessage` is `#[repr(C)]` with no padding, so all of its // bytes are initialized. -unsafe impl AsBytes for FspMessage {} +unsafe impl AsBytes for FspCotMessage {} + +/// Complete FSP PRC message. +#[repr(C, packed)] +#[derive(Clone, Copy)] +struct FspPrcMessage { + header: FspMessageHeader, + prc: NvdmPayloadPrc, +} -impl MessageToFsp for FspMessage { +impl FspPrcMessage { + /// Constructs a PRC message. + fn new(subcmd: PrcMessageSubcmd, object_id: PrcObjectId, flags: PrcFlags) -> Self { + Self { + header: FspMessageHeader::new(NvdmType::Prc), + prc: NvdmPayloadPrc::new(subcmd, object_id, flags), + } + } +} + +// SAFETY: FspPrcMessage is a packed C struct with only integral fields. +unsafe impl AsBytes for FspPrcMessage {} + +impl MessageToFsp for FspCotMessage { const NVDM_TYPE: NvdmType = NvdmType::Cot; } +impl MessageToFsp for FspPrcMessage { + const NVDM_TYPE: NvdmType = NvdmType::Prc; +} + /// Bundled arguments for FMC boot via FSP Chain of Trust. -pub(crate) struct FmcBootArgs { +pub(crate) struct FmcBootArgs<'a> { chipset: Chipset, fmc_boot_params: Coherent<GspFmcBootParams>, resume: bool, + // Additional dependencies required to be kept alive for FMC boot. + _wpr_meta: Coherent<GspFwWprMeta>, + _libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, } -impl FmcBootArgs { +impl<'a> FmcBootArgs<'a> { /// Builds FMC boot arguments, allocating the DMA-coherent boot parameter /// structure that FSP will read. pub(crate) fn new( dev: &device::Device<device::Bound>, chipset: Chipset, - wpr_meta_addr: u64, - libos_addr: u64, + wpr_meta: Coherent<GspFwWprMeta>, + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, resume: bool, ) -> Result<Self> { - let init = GspFmcBootParams::new(wpr_meta_addr, libos_addr); + let init = GspFmcBootParams::new(wpr_meta.dma_address(), libos.dma_address()); Ok(Self { chipset, fmc_boot_params: Coherent::<GspFmcBootParams>::init(dev, GFP_KERNEL, init)?, resume, + _wpr_meta: wpr_meta, + _libos: libos, }) } - /// DMA address of the FMC boot parameters, needed after boot for lockdown - /// release polling. - pub(crate) fn boot_params_dma_handle(&self) -> u64 { - self.fmc_boot_params.dma_handle() + /// Returns the FMC boot parameters allocation. + pub(crate) fn boot_params(&self) -> &Coherent<GspFmcBootParams> { + &self.fmc_boot_params } } @@ -199,28 +384,45 @@ impl FmcBootArgs { /// An `Fsp` is produced by [`Fsp::wait_secure_boot`], which only returns once FSP secure boot /// has completed. It owns the FSP falcon and the FMC firmware, which are used for the subsequent /// Chain of Trust boot. -pub(crate) struct Fsp { - falcon: Falcon<FspEngine>, +pub(crate) struct Fsp<'a> { + falcon: Falcon<'a, FspEngine>, fsp_fw: FspFirmware, } -impl Fsp { +impl<'a> Fsp<'a> { + /// Attempts to create a `Fsp` instance. + /// + /// This can involve waiting for FSP secure boot completion, but should be instantaneous in + /// practice. + /// + /// If `chipset` doesn't support FSP, `Ok(None)` is returned. + pub(crate) fn try_new( + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, + chipset: Chipset, + ) -> Result<Option<Self>> { + match hal::fsp_hal(chipset) { + None => Ok(None), + Some(hal) => Self::wait_secure_boot(dev, bar, chipset, hal).map(Option::Some), + } + } + /// Waits for FSP secure boot completion, then returns the [`Fsp`] interface. /// /// Polls the thermal scratch register until FSP signals boot completion or the timeout /// elapses. Returning an [`Fsp`] only on success guarantees, at the API level, that the /// interface is not used before secure boot has completed. - pub(crate) fn wait_secure_boot( - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, + fn wait_secure_boot( + dev: &'a device::Device<device::Bound>, + bar: Bar0<'a>, chipset: Chipset, - fsp_fw: FspFirmware, - ) -> Result<Fsp> { + hal: &'static dyn hal::FspHal, + ) -> Result<Fsp<'a>> { /// FSP secure boot completion timeout in milliseconds. const FSP_SECURE_BOOT_TIMEOUT_MS: i64 = 5000; - let hal = hal::fsp_hal(chipset).ok_or(ENOTSUPP)?; - let falcon = Falcon::<FspEngine>::new(dev, chipset)?; + let falcon = Falcon::<FspEngine>::new(dev, chipset, bar)?; + let fsp_fw = FspFirmware::new(dev, chipset)?; read_poll_timeout( || Ok(hal.fsp_boot_status(bar)), @@ -236,23 +438,25 @@ impl Fsp { } /// Sends a message to FSP and waits for the response. - fn send_sync_fsp<M>(&mut self, dev: &device::Device, bar: Bar0<'_>, msg: &M) -> Result + /// Returns the full response buffer on success. + fn send_sync_fsp<M>(&mut self, dev: &device::Device, msg: &M) -> Result<KVec<u8>> where M: MessageToFsp, { - self.falcon.send_msg(bar, msg.as_bytes())?; + self.falcon.send_msg(msg.as_bytes())?; - let response_buf = self.falcon.recv_msg(bar).inspect_err(|e| { + let response_buf = self.falcon.recv_msg().inspect_err(|e| { dev_err!(dev, "FSP response error: {:?}\n", e); })?; - let (response, _) = FspResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { - dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); - EIO - })?; + let (response, _) = + FspResponseHeader::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "FSP response too small: {}\n", response_buf.len()); + EIO + })?; - let mctp_header = response.mctp_header; - let nvdm_header = response.nvdm_header; + let mctp_header = response.header.mctp_header; + let nvdm_header = response.header.nvdm_header; let command_nvdm_type = response.response.command_nvdm_type; let error_code = response.response.error_code; @@ -274,7 +478,7 @@ impl Fsp { return Err(EIO); } - if command_nvdm_type != u8::from(M::NVDM_TYPE).into() { + if command_nvdm_type.try_into_bounded() != Some(M::NVDM_TYPE.into()) { dev_err!( dev, "Expected NVDM type {:?} in reply, got {:#x}\n", @@ -294,7 +498,34 @@ impl Fsp { return Err(EIO); } - Ok(()) + Ok(response_buf) + } + + /// Reads the active vGPU mode from FSP using the PRC protocol. + /// + /// Queries FSP's Management Partition for the active vGPU mode knob value. + pub(crate) fn read_vgpu_mode( + &mut self, + dev: &device::Device<device::Bound>, + ) -> Result<VgpuMode> { + let msg = FspPrcMessage::new( + PrcMessageSubcmd::Read, + PrcObjectId::VgpuMode, + PrcFlags::from(PrcFlag::Active), + ); + + let response_buf = self.send_sync_fsp(dev, &msg)?; + let (prc_response, _) = + FspPrcResponse::from_bytes_prefix(&response_buf[..]).ok_or_else(|| { + dev_err!(dev, "PRC response too small: {}\n", response_buf.len()); + EIO + })?; + + let prc_data = prc_response.prc_data; + + VgpuMode::try_from(prc_data).inspect_err(|_| { + dev_err!(dev, "Unexpected vGPU mode value: {:#x}\n", prc_data.value()); + }) } /// Boots GSP FMC via FSP Chain of Trust. @@ -304,15 +535,14 @@ impl Fsp { pub(crate) fn boot_fmc( &mut self, dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - fb_layout: &FbLayout, - args: &FmcBootArgs, + fb_info: &FbSizes, + args: &FmcBootArgs<'_>, ) -> Result { dev_dbg!(dev, "Starting FSP boot sequence for {}\n", args.chipset); - let msg = KBox::init(FspMessage::new(fb_layout, &self.fsp_fw, args)?, GFP_KERNEL)?; + let msg = KBox::init(FspCotMessage::new(fb_info, &self.fsp_fw, args)?, GFP_KERNEL)?; - self.send_sync_fsp(dev, bar, &*msg)?; + let _response_buf = self.send_sync_fsp(dev, &*msg)?; dev_dbg!(dev, "FSP Chain of Trust completed successfully\n"); Ok(()) diff --git a/drivers/gpu/nova-core/fsp/hal.rs b/drivers/gpu/nova-core/fsp/hal.rs index b6f2624bb13d..eaf5837ac5a8 100644 --- a/drivers/gpu/nova-core/fsp/hal.rs +++ b/drivers/gpu/nova-core/fsp/hal.rs @@ -19,6 +19,10 @@ pub(super) trait FspHal { /// Returns the FSP Chain of Trust protocol version this chipset advertises. fn cot_version(&self) -> u16; + + // TODO: consider moving this into the TLV firmware metadata when ready + /// Returns the size reserved at the end of the framebuffer, in bytes. + fn fb_end_reserved_size(&self) -> u64; } /// Returns the FSP HAL, or `None` if the architecture doesn't support FSP. diff --git a/drivers/gpu/nova-core/fsp/hal/gb100.rs b/drivers/gpu/nova-core/fsp/hal/gb100.rs index 42f5ecfc6400..7cf53aa3d1ff 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb100.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use kernel::sizes::SizeConstants; + use crate::{ driver::Bar0, fsp::hal::FspHal, // @@ -17,6 +19,10 @@ impl FspHal for Gb100 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB100: Gb100 = Gb100; diff --git a/drivers/gpu/nova-core/fsp/hal/gb202.rs b/drivers/gpu/nova-core/fsp/hal/gb202.rs index 1091b169a645..e380bbc5d58d 100644 --- a/drivers/gpu/nova-core/fsp/hal/gb202.rs +++ b/drivers/gpu/nova-core/fsp/hal/gb202.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -21,6 +24,10 @@ impl FspHal for Gb202 { fn cot_version(&self) -> u16 { 2 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + u64::SZ_128K + } } const GB202: Gb202 = Gb202; diff --git a/drivers/gpu/nova-core/fsp/hal/gh100.rs b/drivers/gpu/nova-core/fsp/hal/gh100.rs index 291acaf2845a..9a8563799da8 100644 --- a/drivers/gpu/nova-core/fsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/fsp/hal/gh100.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -use kernel::io::Io; +use kernel::{ + io::Io, + sizes::SizeConstants, // +}; use crate::{ driver::Bar0, @@ -26,6 +29,10 @@ impl FspHal for Gh100 { fn cot_version(&self) -> u16 { 1 } + + fn fb_end_reserved_size(&self) -> u64 { + u64::SZ_2M + } } const GH100: Gh100 = Gh100; diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs index b3c91731db45..fd1414004dd0 100644 --- a/drivers/gpu/nova-core/gpu.rs +++ b/drivers/gpu/nova-core/gpu.rs @@ -9,7 +9,8 @@ use kernel::{ io::Io, num::Bounded, pci, - prelude::*, // + prelude::*, + sizes::SizeConstants, // }; use crate::{ @@ -21,11 +22,15 @@ use crate::{ Falcon, // }, fb::SysmemFlush, + fsp::Fsp, gsp::{ self, - Gsp, // + commands::GetGspStaticInfoReply, + Gsp, + GspBootContext, // }, regs, + vgpu::VgpuManager, // }; mod hal; @@ -130,22 +135,6 @@ impl Chipset { } } - /// Returns `true` if this chipset requires the PIO-loaded bootloader in order to boot FWSEC. - /// - /// This includes all chipsets < GA102. - pub(crate) const fn needs_fwsec_bootloader(self) -> bool { - matches!(self.arch(), Architecture::Turing) || matches!(self, Self::GA100) - } - - /// Returns `true` if this chipset boots via FSP (Hopper and later), which requires the FMC - /// firmware image. - pub(crate) const fn uses_fsp(self) -> bool { - matches!( - self.arch(), - Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x - ) - } - /// Returns the address range of the PCI config mirror space. pub(crate) fn pci_config_mirror_range(self) -> Range<u32> { hal::gpu_hal(self).pci_config_mirror_range() @@ -262,37 +251,87 @@ impl fmt::Display for Spec { } } -/// Structure holding the resources required to operate the GPU. +/// Self-contained resources to operate and drop the GSP. #[pin_data(PinnedDrop)] -pub(crate) struct Gpu<'gpu> { +struct GspResources<'gpu> { /// Device owning the GPU. - device: &'gpu device::Device<device::Bound>, + device: &'gpu pci::Device<device::Bound>, + /// Details about the chipset. spec: Spec, /// MMIO mapping of PCI BAR 0. bar: Bar0<'gpu>, - /// System memory page required for flushing all pending GPU-side memory writes done through - /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). - sysmem_flush: SysmemFlush<'gpu>, /// GSP falcon instance, used for GSP boot up and cleanup. - gsp_falcon: Falcon<GspFalcon>, + gsp_falcon: Falcon<'gpu, GspFalcon>, /// SEC2 falcon instance, used for GSP boot up and cleanup. - sec2_falcon: Falcon<Sec2Falcon>, - /// GSP runtime data. Temporarily an empty placeholder. + sec2_falcon: Falcon<'gpu, Sec2Falcon>, + /// FSP instance, if on an arch that supports it. + // TODO: use different resource types for each boot method, and make the relevant Gsp methods + // generic against them. + fsp: Option<Fsp<'gpu>>, + /// vGPU state detected before GSP boot. + vgpu: VgpuManager, + /// GSP runtime data. #[pin] gsp: Gsp, /// GSP unload firmware bundle, if any. unload_bundle: Option<gsp::UnloadBundle>, } +/// Structure holding the resources required to operate the GPU. +#[pin_data] +pub(crate) struct Gpu<'gpu> { + spec: Spec, + /// Static GPU information as provided by the GSP. + gsp_static_info: GetGspStaticInfoReply, + /// GSP and its resources. + #[pin] + gsp_resources: GspResources<'gpu>, + /// System memory page required for flushing all pending GPU-side memory writes done through + /// PCIE into system memory, via sysmembar (A GPU-initiated HW memory-barrier operation). + /// + /// Must be kept declared *after* `gsp_resources`, as the latter's `PinnedDrop` implementation + /// requires the sysmem flush page to be in place. + sysmem_flush: SysmemFlush<'gpu>, +} + +#[pinned_drop] +impl PinnedDrop for GspResources<'_> { + fn drop(self: Pin<&mut Self>) { + let this = self.project(); + let device = *this.device; + let bar = *this.bar; + let bundle = this.unload_bundle.take(); + + let _ = this + .gsp + .as_ref() + .get_ref() + .unload( + GspBootContext { + pdev: device, + bar, + chipset: this.spec.chipset, + gsp_falcon: &*this.gsp_falcon, + sec2_falcon: &*this.sec2_falcon, + fsp: this.fsp.as_mut(), + vgpu: &*this.vgpu, + }, + bundle, + ) + .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); + } +} + impl<'gpu> Gpu<'gpu> { - pub(crate) fn new( - pdev: &'gpu pci::Device<device::Core<'_>>, + pub(crate) fn new<'a>( + pdev: &'gpu pci::Device<device::Core<'a>>, bar: Bar0<'gpu>, - ) -> impl PinInit<Self, Error> + 'gpu { + ) -> impl PinInit<Self, Error> + use<'gpu, 'a> { + let dev = pdev.as_ref(); + try_pin_init!(Self { - device: pdev.as_ref(), - spec: Spec::new(pdev.as_ref(), bar).inspect(|spec| { - dev_info!(pdev,"NVIDIA ({})\n", spec); + spec: Spec::new(dev, bar).inspect(|spec| { + dev_info!(dev,"NVIDIA ({})\n", spec); })?, // We must wait for GFW_BOOT completion before doing any significant setup on the GPU. @@ -305,43 +344,73 @@ impl<'gpu> Gpu<'gpu> { unsafe { pdev.dma_set_mask_and_coherent(dma_mask)? }; hal.wait_gfw_boot_completion(bar) - .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?; + .inspect_err(|_| dev_err!(dev, "GFW boot did not complete\n"))?; }, - sysmem_flush: SysmemFlush::register(pdev.as_ref(), bar, spec.chipset)?, + // Initialize this early because `gsp_resources` depends on it. + sysmem_flush: SysmemFlush::register(dev, bar, spec.chipset)?, - gsp_falcon: Falcon::new( - pdev.as_ref(), - spec.chipset, - ) - .inspect(|falcon| falcon.clear_swgen0_intr(bar))?, + gsp_resources <- try_pin_init!(GspResources { + device: pdev, - sec2_falcon: Falcon::new(pdev.as_ref(), spec.chipset)?, + spec: *spec, - gsp <- Gsp::new(pdev), + bar, - // This member must be initialized last, so the `UnloadBundle` can never be dropped from - // outside of the constructed `Gpu`, ensuring that the unload sequence is properly run - // in case of failure. - unload_bundle: gsp.boot(pdev, bar, spec.chipset, gsp_falcon, sec2_falcon)?, - bar, - }) - } -} + gsp_falcon: Falcon::new( + dev, + spec.chipset, + bar + ) + .inspect(|falcon| falcon.clear_swgen0_intr())?, -#[pinned_drop] -impl PinnedDrop for Gpu<'_> { - fn drop(self: Pin<&mut Self>) { - let this = self.project(); - let device = *this.device; - let bar = *this.bar; - let bundle = this.unload_bundle.take(); + sec2_falcon: Falcon::new(dev, spec.chipset, bar)?, - let _ = this - .gsp - .as_ref() - .get_ref() - .unload(device, bar, &*this.gsp_falcon, &*this.sec2_falcon, bundle) - .inspect_err(|e| dev_err!(device, "failed to unload GSP: {:?}\n", e)); + fsp: Fsp::try_new(dev, bar, spec.chipset)?, + + vgpu: VgpuManager::new(pdev, spec.chipset, fsp.as_mut()), + + gsp <- Gsp::new(pdev), + + // This member must be initialized last, so the `UnloadBundle` can never be dropped + // from outside of the constructed `GspResources`, ensuring that the unload sequence + // is properly run in case of failure. + unload_bundle: gsp.boot(GspBootContext { + pdev, + bar, + chipset: spec.chipset, + gsp_falcon, + sec2_falcon, + fsp: fsp.as_mut(), + vgpu, + })?, + }), + + gsp_static_info: { + // Obtain and display basic GPU information. + let info = gsp_resources.gsp.get_static_info(bar)?; + match info.gpu_name() { + Ok(name) => dev_info!(dev, "GPU name: {}\n", name), + Err(e) => dev_warn!(dev, "GPU name unavailable: {:?}\n", e), + } + + if !info.usable_fb_regions.is_empty() { + dev_dbg!(dev, "Usable FB regions:\n"); + for region in &info.usable_fb_regions { + dev_dbg!(dev, " - {:#x?}\n", region); + } + + dev_dbg!( + dev, + "Total usable VRAM: {} MiB\n", + info.usable_fb_regions.iter().fold(0u64, |res, region| res + .saturating_add(region.end - region.start)) + / u64::SZ_1M + ); + } + + info + } + }) } } diff --git a/drivers/gpu/nova-core/gsp.rs b/drivers/gpu/nova-core/gsp.rs index 69175ca3315c..13f361406a6c 100644 --- a/drivers/gpu/nova-core/gsp.rs +++ b/drivers/gpu/nova-core/gsp.rs @@ -9,60 +9,95 @@ use kernel::{ dma::{ Coherent, CoherentBox, + CoherentView, DmaAddress, // }, + io::{ + io_project, + io_write, + Io, // + }, pci, - prelude::*, - transmute::{ - AsBytes, - FromBytes, // - }, // + prelude::*, // }; pub(crate) mod cmdq; pub(crate) mod commands; mod fw; +mod regs; mod sequencer; pub(crate) use fw::{ GspFmcBootParams, GspFwWprMeta, + LibosMemoryRegionInitArgument, LibosParams, // }; +pub(crate) use hal::boot_firmware_files; use crate::{ - gsp::cmdq::Cmdq, - gsp::fw::{ - GspArgumentsPadded, - LibosMemoryRegionInitArgument, // + driver::Bar0, + falcon::{ + gsp::Gsp as GspFalcon, + sec2::Sec2 as Sec2Falcon, + Falcon, // + }, + fsp::Fsp, + gpu::Chipset, + gsp::{ + cmdq::Cmdq, + fw::GspArgumentsPadded, // }, num, + vgpu::VgpuManager, // }; pub(crate) const GSP_PAGE_SHIFT: usize = 12; pub(crate) const GSP_PAGE_SIZE: usize = 1 << GSP_PAGE_SHIFT; +/// Common context for the GSP boot process. +/// +/// It carries two distinct lifetimes: +/// +/// - `'gpu` is the lifetime of the bound GPU device, as captured by the GPU subdevices. +/// - `'ctx` is a shorter lifetime during which this context borrows those subdevices. +pub(crate) struct GspBootContext<'ctx, 'gpu> { + pub(crate) pdev: &'gpu pci::Device<device::Bound>, + pub(crate) bar: Bar0<'gpu>, + pub(crate) chipset: Chipset, + pub(crate) gsp_falcon: &'ctx Falcon<'gpu, GspFalcon>, + pub(crate) sec2_falcon: &'ctx Falcon<'gpu, Sec2Falcon>, + pub(crate) fsp: Option<&'ctx mut Fsp<'gpu>>, + pub(crate) vgpu: &'ctx VgpuManager, +} + +impl<'ctx, 'gpu> GspBootContext<'ctx, 'gpu> { + pub(crate) fn dev(&self) -> &'gpu device::Device<device::Bound> { + self.pdev.as_ref() + } +} + /// Number of GSP pages to use in a RM log buffer. const RM_LOG_BUFFER_NUM_PAGES: usize = 0x10; const LOG_BUFFER_SIZE: usize = RM_LOG_BUFFER_NUM_PAGES * GSP_PAGE_SIZE; /// Array of page table entries, as understood by the GSP bootloader. #[repr(C)] +#[derive(FromBytes, IntoBytes)] struct PteArray<const NUM_ENTRIES: usize>([u64; NUM_ENTRIES]); -/// SAFETY: arrays of `u64` implement `FromBytes` and we are but a wrapper around one. -unsafe impl<const NUM_ENTRIES: usize> FromBytes for PteArray<NUM_ENTRIES> {} - -/// SAFETY: arrays of `u64` implement `AsBytes` and we are but a wrapper around one. -unsafe impl<const NUM_ENTRIES: usize> AsBytes for PteArray<NUM_ENTRIES> {} - impl<const NUM_PAGES: usize> PteArray<NUM_PAGES> { - /// Returns the page table entry for `index`, for a mapping starting at `start`. - // TODO: Replace with `IoView` projection once available. - fn entry(start: DmaAddress, index: usize) -> Result<u64> { - start - .checked_add(num::usize_as_u64(index) << GSP_PAGE_SHIFT) - .ok_or(EOVERFLOW) + /// Initialize a new page table array mapping `NUM_PAGES` GSP pages starting at address `start`. + fn init(view: CoherentView<'_, Self>, start: DmaAddress) -> Result<()> { + for i in 0..NUM_PAGES { + io_write!(view, .0[build: i], + start + .checked_add(num::usize_as_u64(i) << GSP_PAGE_SHIFT) + .ok_or(EOVERFLOW)? + ); + } + + Ok(()) } } @@ -87,19 +122,14 @@ impl LogBuffer { fn new(dev: &device::Device<device::Bound>) -> Result<Self> { let obj = Self(Coherent::zeroed(dev, GFP_KERNEL)?); - let start_addr = obj.0.dma_handle(); - - // SAFETY: `obj` has just been created and we are its sole user. - let pte_region = unsafe { - &mut obj.0.as_mut()[size_of::<u64>()..][..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()] - }; + let start_addr = obj.0.dma_address(); - // Write values one by one to avoid an on-stack instance of `PteArray`. - for (i, chunk) in pte_region.chunks_exact_mut(size_of::<u64>()).enumerate() { - let pte_value = PteArray::<0>::entry(start_addr, i)?; - - chunk.copy_from_slice(&pte_value.to_ne_bytes()); - } + let pte_view = io_project!( + obj.0, + [build: size_of::<u64>()..][build: ..RM_LOG_BUFFER_NUM_PAGES * size_of::<u64>()] + ) + .try_cast::<PteArray<RM_LOG_BUFFER_NUM_PAGES>>()?; + PteArray::init(pte_view, start_addr)?; Ok(obj) } @@ -185,6 +215,11 @@ impl Gsp { })) }) } + + /// Query the GSP for the static GPU information. + pub(crate) fn get_static_info(&self, bar: Bar0<'_>) -> Result<commands::GetGspStaticInfoReply> { + self.cmdq.send_command(bar, commands::GetGspStaticInfo) + } } /// Opaque bundle required to unload the GSP. Created by [`Gsp::boot`], consumed by [`Gsp::unload`]. diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs index 8afb62d689cb..e03700ee7bea 100644 --- a/drivers/gpu/nova-core/gsp/boot.rs +++ b/drivers/gpu/nova-core/gsp/boot.rs @@ -3,10 +3,7 @@ use kernel::{ bits, - device, - dma::Coherent, io::poll::read_poll_timeout, - pci, prelude::*, time::Delta, types::ScopeGuard, // @@ -16,82 +13,15 @@ use crate::{ driver::Bar0, falcon::{ gsp::Gsp, - sec2::Sec2, Falcon, // }, - fb::FbLayout, - firmware::{ - gsp::GspFirmware, - FIRMWARE_VERSION, // - }, - gpu::Chipset, + firmware::gsp::GspFirmware, gsp::{ cmdq::Cmdq, - commands, - GspFwWprMeta, // + commands, // }, }; -/// Arguments required to call [`Gsp::unload`](super::Gsp::unload). -/// -/// Stored as their own type to avoid repeating a long and tedious list in [`BootUnloadGuard`]. -pub(super) struct BootUnloadArgs<'a> { - gsp: &'a super::Gsp, - dev: &'a device::Device<device::Bound>, - bar: Bar0<'a>, - gsp_falcon: &'a Falcon<Gsp>, - sec2_falcon: &'a Falcon<Sec2>, - unload_bundle: Option<super::UnloadBundle>, -} - -/// Guard that calls [`Gsp::unload`](super::Gsp::unload) with a -/// [`UnloadBundle`](super::UnloadBundle) when dropped. -/// -/// Used to ensure the `UnloadBundle` is run during failure paths. -pub(super) struct BootUnloadGuard<'a> { - guard: ScopeGuard<BootUnloadArgs<'a>, fn(BootUnloadArgs<'a>)>, -} - -impl<'a> BootUnloadGuard<'a> { - /// Wraps `unload_bundle` into a guard that executes it when dropped. - pub(super) fn new( - gsp: &'a super::Gsp, - dev: &'a device::Device<device::Bound>, - bar: Bar0<'a>, - gsp_falcon: &'a Falcon<Gsp>, - sec2_falcon: &'a Falcon<Sec2>, - unload_bundle: Option<super::UnloadBundle>, - ) -> Self { - Self { - guard: ScopeGuard::new_with_data( - BootUnloadArgs { - gsp, - dev, - bar, - gsp_falcon, - sec2_falcon, - unload_bundle, - }, - |args| { - let _ = super::Gsp::unload( - args.gsp, - args.dev, - args.bar, - args.gsp_falcon, - args.sec2_falcon, - args.unload_bundle, - ); - }, - ), - } - } - - /// Disarms the guard and returns the [`UnloadBundle`](super::UnloadBundle) it contains. - pub(super) fn dismiss(self) -> Option<super::UnloadBundle> { - self.guard.dismiss().unload_bundle - } -} - impl super::Gsp { /// Attempt to boot the GSP. /// @@ -103,71 +33,64 @@ impl super::Gsp { /// [`Self::unload`]) returned. pub(crate) fn boot( self: Pin<&mut Self>, - pdev: &pci::Device<device::Bound>, - bar: Bar0<'_>, - chipset: Chipset, - gsp_falcon: &Falcon<Gsp>, - sec2_falcon: &Falcon<Sec2>, + mut ctx: super::GspBootContext<'_, '_>, ) -> Result<Option<super::UnloadBundle>> { + let pdev = ctx.pdev; + let bar = ctx.bar; + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; let dev = pdev.as_ref(); let hal = super::hal::gsp_hal(chipset); - let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset, FIRMWARE_VERSION), GFP_KERNEL)?; + let gsp_fw = KBox::pin_init(GspFirmware::new(dev, chipset), GFP_KERNEL)?; - let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?; - dev_dbg!(dev, "{:#x?}\n", fb_layout); + // Perform the chipset-specific boot sequence, and retrieve the unload bundle. + let unload_bundle = hal.boot(&self, &mut ctx, &gsp_fw)?.or_else(|| { + dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); + dev_warn!( + dev, + "The GPU will need to be reset before the driver can bind again.\n" + ); - let wpr_meta = Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::new(&gsp_fw, &fb_layout))?; + None + }); - // Perform the chipset-specific boot sequence, and retrieve the unload bundle. - let unload_guard = hal.boot( - &self, - dev, - bar, - chipset, - &fb_layout, - &wpr_meta, - gsp_falcon, - sec2_falcon, - )?; + let mut unload_guard = + ScopeGuard::new_with_data((ctx, unload_bundle), |(ctx, unload_bundle)| { + let _ = self.unload(ctx, unload_bundle); + }); + let ctx = &mut unload_guard.0; - gsp_falcon.write_os_version(bar, gsp_fw.bootloader.app_version); + gsp_falcon.write_os_version(gsp_fw.bootloader.app_version); // Poll for RISC-V to become active before continuing. read_poll_timeout( - || Ok(gsp_falcon.is_riscv_active(bar)), + || Ok(gsp_falcon.is_riscv_active()), |val: &bool| *val, Delta::from_millis(10), Delta::from_secs(5), )?; - dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(bar),); + dev_dbg!(pdev, "RISC-V active? {}\n", gsp_falcon.is_riscv_active(),); self.cmdq .send_command_no_wait(bar, commands::SetSystemInfo::new(pdev, chipset))?; self.cmdq - .send_command_no_wait(bar, commands::SetRegistry::new())?; + .send_command_no_wait(bar, commands::SetRegistry::new(ctx.vgpu.state())?)?; - hal.post_boot(&self, dev, bar, &gsp_fw, gsp_falcon, sec2_falcon)?; + hal.post_boot(&self, ctx, &gsp_fw)?; // Wait until GSP is fully initialized. commands::wait_gsp_init_done(&self.cmdq)?; - // Obtain and display basic GPU information. - let info = self.cmdq.send_command(bar, commands::GetGspStaticInfo)?; - match info.gpu_name() { - Ok(name) => dev_info!(pdev, "GPU name: {}\n", name), - Err(e) => dev_warn!(pdev, "GPU name unavailable: {:?}\n", e), - } - - Ok(unload_guard.dismiss()) + Ok(unload_guard.dismiss().1) } /// Shut down the GSP and wait until it is offline. fn shutdown_gsp( cmdq: &Cmdq, bar: Bar0<'_>, - gsp_falcon: &Falcon<Gsp>, + gsp_falcon: &Falcon<'_, Gsp>, mode: commands::PowerStateLevel, ) -> Result { // Command to shut the GSP down. @@ -176,7 +99,7 @@ impl super::Gsp { // Wait until GSP signals it is suspended. const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31); read_poll_timeout( - || Ok(gsp_falcon.read_mailbox0(bar)), + || Ok(gsp_falcon.read_mailbox0()), |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0, Delta::from_millis(10), Delta::from_secs(5), @@ -189,17 +112,16 @@ impl super::Gsp { /// This stops all activity on the GSP. pub(crate) fn unload( &self, - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - gsp_falcon: &Falcon<Gsp>, - sec2_falcon: &Falcon<Sec2>, + mut ctx: super::GspBootContext<'_, '_>, unload_bundle: Option<super::UnloadBundle>, ) -> Result { + let dev = ctx.dev(); + // Shut down the GSP. Keep going even in case of error. let mut res = Self::shutdown_gsp( &self.cmdq, - bar, - gsp_falcon, + ctx.bar, + ctx.gsp_falcon, commands::PowerStateLevel::Level0, ) .inspect_err(|e| dev_err!(dev, "GSP shutdown failed: {:?}\n", e)); @@ -209,7 +131,7 @@ impl super::Gsp { res = res.and( unload_bundle .0 - .run(dev, bar, gsp_falcon, sec2_falcon) + .run(&mut ctx) .inspect_err(|e| dev_err!(dev, "Unload bundle failed: {:?}\n", e)), ); } else { diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs index 070de0731e95..6da728201281 100644 --- a/drivers/gpu/nova-core/gsp/cmdq.rs +++ b/drivers/gpu/nova-core/gsp/cmdq.rs @@ -2,16 +2,23 @@ mod continuation; -use core::mem; +use core::{ + mem, + sync::atomic::{ + fence, + Ordering, // + }, +}; use kernel::{ device, dma::{ Coherent, + CoherentBox, DmaAddress, // }, - dma_write, io::{ + io_project, poll::read_poll_timeout, Io, // }, @@ -51,10 +58,11 @@ use crate::{ GSP_PAGE_SIZE, // }, num, - regs, sbuffer::SBufferIter, // }; +use super::regs; + /// Marker type representing the absence of a reply for a command. Commands using this as their /// reply type are sent using [`Cmdq::send_command_no_wait`]. pub(crate) struct NoReply; @@ -171,20 +179,18 @@ static_assert!(align_of::<MsgqData>() == GSP_PAGE_SIZE); #[repr(C)] // There is no struct defined for this in the open-gpu-kernel-source headers. // Instead it is defined by code in `GspMsgQueuesInit()`. -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct Msgq { +struct Msgq { /// Header for sending messages, including the write pointer. - pub(super) tx: MsgqTxHeader, + tx: MsgqTxHeader, /// Header for receiving messages, including the read pointer. - pub(super) rx: MsgqRxHeader, + rx: MsgqRxHeader, /// The message queue proper. msgq: MsgqData, } /// Structure shared between the driver and the GSP and containing the command and message queues. #[repr(C)] -// TODO: Revert to private once `IoView` projections replace the `gsp_mem` module. -pub(super) struct GspMem { +struct GspMem { /// Self-mapping page table entries. ptes: PteArray<{ Self::PTE_ARRAY_SIZE }>, /// CPU queue: the driver writes commands here, and the GSP reads them. It also contains the @@ -192,13 +198,13 @@ pub(super) struct GspMem { /// index into the GSP queue. /// /// This member is read-only for the GSP. - pub(super) cpuq: Msgq, + cpuq: Msgq, /// GSP queue: the GSP writes messages here, and the driver reads them. It also contains the /// write and read pointers that the GSP updates. This means that the read pointer here is an /// index into the CPU queue. /// /// This member is read-only for the driver. - pub(super) gspq: Msgq, + gspq: Msgq, } impl GspMem { @@ -232,20 +238,12 @@ impl DmaGspMem { const MSGQ_SIZE: u32 = num::usize_into_u32::<{ size_of::<Msgq>() }>(); const RX_HDR_OFF: u32 = num::usize_into_u32::<{ mem::offset_of!(Msgq, rx) }>(); - let gsp_mem = Coherent::<GspMem>::zeroed(dev, GFP_KERNEL)?; + let mut gsp_mem = CoherentBox::<GspMem>::zeroed(dev, GFP_KERNEL)?; + gsp_mem.cpuq.tx = MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES); + gsp_mem.cpuq.rx = MsgqRxHeader::new(); - let start = gsp_mem.dma_handle(); - // Write values one by one to avoid an on-stack instance of `PteArray`. - for i in 0..GspMem::PTE_ARRAY_SIZE { - dma_write!(gsp_mem, .ptes.0[build: i], PteArray::<0>::entry(start, i)?); - } - - dma_write!( - gsp_mem, - .cpuq.tx, - MsgqTxHeader::new(MSGQ_SIZE, RX_HDR_OFF, MSGQ_NUM_PAGES) - ); - dma_write!(gsp_mem, .cpuq.rx, MsgqRxHeader::new()); + let gsp_mem: Coherent<_> = gsp_mem.into(); + PteArray::init(io_project!(gsp_mem, .ptes), gsp_mem.dma_address())?; Ok(Self(gsp_mem)) } @@ -406,7 +404,7 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_write_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .gspq.tx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the GSP will read the next command from. @@ -415,7 +413,7 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn gsp_read_ptr(&self) -> u32 { - super::fw::gsp_mem::gsp_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .gspq.rx)) % MSGQ_NUM_PAGES } // Returns the index of the memory page the CPU can read the next message from. @@ -424,12 +422,18 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_read_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_read_ptr(&self.0) + MsgqRxHeader::read_ptr(io_project!(self.0, .cpuq.rx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can send `elem_count` new pages into the message queue. fn advance_cpu_read_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_read_ptr(&self.0, elem_count) + let rx = io_project!(self.0, .cpuq.rx); + let rptr = MsgqRxHeader::read_ptr(rx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + + // Ensure read pointer is properly ordered. + fence(Ordering::SeqCst); + + MsgqRxHeader::set_read_ptr(rx, rptr) } // Returns the index of the memory page the CPU can write the next command to. @@ -438,12 +442,17 @@ impl DmaGspMem { // // - The returned value is within `0..MSGQ_NUM_PAGES`. fn cpu_write_ptr(&self) -> u32 { - super::fw::gsp_mem::cpu_write_ptr(&self.0) + MsgqTxHeader::write_ptr(io_project!(self.0, .cpuq.tx)) % MSGQ_NUM_PAGES } // Informs the GSP that it can process `elem_count` new pages from the command queue. fn advance_cpu_write_ptr(&mut self, elem_count: u32) { - super::fw::gsp_mem::advance_cpu_write_ptr(&self.0, elem_count) + let tx = io_project!(self.0, .cpuq.tx); + let wptr = MsgqTxHeader::write_ptr(tx).wrapping_add(elem_count) % MSGQ_NUM_PAGES; + MsgqTxHeader::set_write_ptr(tx, wptr); + + // Ensure all command data is visible before triggering the GSP read. + fence(Ordering::SeqCst); } } @@ -478,8 +487,8 @@ pub(crate) struct Cmdq { /// Inner mutex-protected state. #[pin] inner: Mutex<CmdqInner>, - /// DMA handle of the command queue's shared memory region. - pub(super) dma_handle: DmaAddress, + /// DMA address of the command queue's shared memory region. + pub(super) dma_addr: DmaAddress, } impl Cmdq { @@ -508,7 +517,7 @@ impl Cmdq { let gsp_mem = DmaGspMem::new(dev)?; Ok(try_pin_init!(Self { - dma_handle: gsp_mem.0.dma_handle(), + dma_addr: gsp_mem.0.dma_address(), inner <- new_mutex!(CmdqInner { dev: dev.into(), gsp_mem, @@ -645,8 +654,8 @@ impl CmdqInner { // SAFETY: `msg_header` and `cmd` are valid references, and not touched if the initializer // fails. unsafe { - msg_element.__init(core::ptr::from_mut(dst.header))?; - command.init().__init(core::ptr::from_mut(cmd))?; + pin_init::raw_try_init(core::ptr::from_mut(dst.header), msg_element)?; + pin_init::raw_try_init(core::ptr::from_mut(cmd), command.init())?; } // Fill the variable-length payload, which may be empty. diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs index f84de9f4f045..ffc25fd8c47b 100644 --- a/drivers/gpu/nova-core/gsp/commands.rs +++ b/drivers/gpu/nova-core/gsp/commands.rs @@ -5,6 +5,7 @@ use core::{ array, convert::Infallible, ffi::FromBytesUntilNulError, + ops::Range, str::Utf8Error, // }; @@ -33,6 +34,7 @@ use crate::{ }, }, sbuffer::SBufferIter, + vgpu::VgpuState, // }; /// The `GspSetSystemInfo` command. @@ -66,37 +68,55 @@ struct RegistryEntry { /// The `SetRegistry` command. pub(crate) struct SetRegistry { - entries: [RegistryEntry; Self::NUM_ENTRIES], + entries: KVec<RegistryEntry>, } impl SetRegistry { - // For now we hard-code the registry entries. Future work will allow others to - // be added as module parameters. - const NUM_ENTRIES: usize = 3; - /// Creates a new `SetRegistry` command, using a set of hardcoded entries. - pub(crate) fn new() -> Self { - Self { - entries: [ - // RMSecBusResetEnable - enables PCI secondary bus reset - RegistryEntry { - key: "RMSecBusResetEnable", - value: 1, - }, - // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on - // any PCI reset. - RegistryEntry { - key: "RMForcePcieConfigSave", - value: 1, - }, - // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found - // in the internal product name database. + pub(crate) fn new(vgpu_state: VgpuState) -> Result<Self> { + let mut entries = KVec::new(); + + // RMSecBusResetEnable - enables PCI secondary bus reset + entries.push( + RegistryEntry { + key: "RMSecBusResetEnable", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration registers on + // any PCI reset. + entries.push( + RegistryEntry { + key: "RMForcePcieConfigSave", + value: 1, + }, + GFP_KERNEL, + )?; + + // RMDevidCheckIgnore - allows GSP-RM to boot even if the PCI dev ID is not found + // in the internal product name database. + entries.push( + RegistryEntry { + key: "RMDevidCheckIgnore", + value: 1, + }, + GFP_KERNEL, + )?; + + if matches!(vgpu_state, VgpuState::Enabled { .. }) { + // RMSetSriovMode - required when vGPU is enabled. + entries.push( RegistryEntry { - key: "RMDevidCheckIgnore", + key: "RMSetSriovMode", value: 1, }, - ], + GFP_KERNEL, + )?; } + + Ok(Self { entries }) } } @@ -107,15 +127,15 @@ impl CommandToGsp for SetRegistry { type InitError = Infallible; fn init(&self) -> impl Init<Self::Command, Self::InitError> { - Self::Command::init(Self::NUM_ENTRIES as u32, self.variable_payload_len() as u32) + Self::Command::init(self.entries.len() as u32, self.size() as u32) } fn variable_payload_len(&self) -> usize { let mut key_size = 0; - for i in 0..Self::NUM_ENTRIES { - key_size += self.entries[i].key.len() + 1; // +1 for NULL terminator + for entry in self.entries.iter() { + key_size += entry.key.len() + 1; // +1 for NULL terminator } - Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>() + key_size + self.entries.len() * size_of::<fw::commands::PackedRegistryEntry>() + key_size } fn init_variable_payload( @@ -123,12 +143,12 @@ impl CommandToGsp for SetRegistry { dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>, ) -> Result { let string_data_start_offset = size_of::<Self::Command>() - + Self::NUM_ENTRIES * size_of::<fw::commands::PackedRegistryEntry>(); + + self.entries.len() * size_of::<fw::commands::PackedRegistryEntry>(); // Array for string data. let mut string_data = KVec::new(); - for entry in self.entries.iter().take(Self::NUM_ENTRIES) { + for entry in self.entries.iter() { dst.write_all( fw::commands::PackedRegistryEntry::new( (string_data_start_offset + string_data.len()) as u32, @@ -191,22 +211,30 @@ impl CommandToGsp for GetGspStaticInfo { } } -/// The reply from the GSP to the [`GetGspInfo`] command. +/// The reply from the GSP to the [`GetGspStaticInfo`] command. pub(crate) struct GetGspStaticInfoReply { gpu_name: [u8; 64], + /// Usable FB (VRAM) regions for driver memory allocation. + pub(crate) usable_fb_regions: KVec<Range<u64>>, } impl MessageFromGsp for GetGspStaticInfoReply { const FUNCTION: MsgFunction = MsgFunction::GetGspStaticInfo; type Message = fw::commands::GspStaticConfigInfo; - type InitError = Infallible; + type InitError = Error; fn read( msg: &Self::Message, _sbuffer: &mut SBufferIter<array::IntoIter<&[u8], 2>>, ) -> Result<Self, Self::InitError> { + let mut usable_fb_regions = KVec::new(); + for region in msg.usable_fb_regions() { + usable_fb_regions.push(region, GFP_KERNEL)?; + } + Ok(GetGspStaticInfoReply { gpu_name: msg.gpu_name_str(), + usable_fb_regions, }) } } diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs index 4db0cfa4dc4d..05f54fee6186 100644 --- a/drivers/gpu/nova-core/gsp/fw.rs +++ b/drivers/gpu/nova-core/gsp/fw.rs @@ -10,7 +10,15 @@ use r570_144 as bindings; use core::ops::Range; use kernel::{ - dma::Coherent, + bitfield, + dma::{ + Coherent, + CoherentView, // + }, + io::{ + io_read, + io_write, // + }, prelude::*, ptr::{ Alignable, @@ -28,7 +36,10 @@ use kernel::{ }; use crate::{ - fb::FbLayout, + fb::{ + FbRanges, + FbSizes, // + }, firmware::gsp::GspFirmware, gpu::{ Architecture, @@ -44,59 +55,6 @@ use crate::{ }, }; -// TODO: Replace with `IoView` projections once available. -pub(super) mod gsp_mem { - use core::sync::atomic::{ - fence, - Ordering, // - }; - - use kernel::{ - dma::Coherent, - dma_read, - dma_write, // - }; - - use crate::gsp::cmdq::{ - GspMem, - MSGQ_NUM_PAGES, // - }; - - pub(in crate::gsp) fn gsp_write_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .gspq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn gsp_read_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .gspq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn cpu_read_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .cpuq.rx.0.readPtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_read_ptr(qs: &Coherent<GspMem>, count: u32) { - let rptr = cpu_read_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - // Ensure read pointer is properly ordered. - fence(Ordering::SeqCst); - - dma_write!(qs, .cpuq.rx.0.readPtr, rptr); - } - - pub(in crate::gsp) fn cpu_write_ptr(qs: &Coherent<GspMem>) -> u32 { - dma_read!(qs, .cpuq.tx.0.writePtr) % MSGQ_NUM_PAGES - } - - pub(in crate::gsp) fn advance_cpu_write_ptr(qs: &Coherent<GspMem>, count: u32) { - let wptr = cpu_write_ptr(qs).wrapping_add(count) % MSGQ_NUM_PAGES; - - dma_write!(qs, .cpuq.tx.0.writePtr, wptr); - - // Ensure all command data is visible before triggering the GSP read. - fence(Ordering::SeqCst); - } -} - /// Maximum size of a single GSP message queue element in bytes. pub(crate) const GSP_MSG_QUEUE_ELEMENT_SIZE_MAX: usize = num::u32_as_usize(bindings::GSP_MSG_QUEUE_ELEMENT_SIZE_MAX); @@ -177,6 +135,11 @@ impl LibosParams { } } + /// Returns the WPR heap size to reserve when vGPU is enabled. + pub(crate) fn vgpu_wpr_heap_size() -> u64 { + u64::from(bindings::GSP_FW_HEAP_SIZE_VGPU_DEFAULT) + } + /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size /// of `fb_size` (in bytes) for `chipset`. pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> { @@ -214,48 +177,89 @@ type GspFwWprMetaBootInfo = bindings::GspFwWprMeta__bindgen_ty_1__bindgen_ty_1; impl GspFwWprMeta { /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the - /// `fb_layout` layout. - pub(crate) fn new<'a>( + /// framebuffer ranges `ranges`. + pub(crate) fn from_ranges<'a>( gsp_firmware: &'a GspFirmware, - fb_layout: &'a FbLayout, + ranges: &'a FbRanges, ) -> impl Init<Self> + 'a { - #[allow(non_snake_case)] let init_inner = init!(bindings::GspFwWprMeta { // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. magic: bindings::GSP_FW_WPR_META_MAGIC as u64, revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), - sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_handle(), + sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), - sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_handle(), + sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), __bindgen_anon_1: GspFwWprMetaBootResumeInfo { __bindgen_anon_1: GspFwWprMetaBootInfo { - sysmemAddrOfSignature: gsp_firmware.signatures.dma_handle(), + sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), }, }, - gspFwRsvdStart: fb_layout.heap.start, - nonWprHeapOffset: fb_layout.heap.start, - nonWprHeapSize: fb_layout.heap.end - fb_layout.heap.start, - gspFwWprStart: fb_layout.wpr2.start, - gspFwHeapOffset: fb_layout.wpr2_heap.start, - gspFwHeapSize: fb_layout.wpr2_heap.end - fb_layout.wpr2_heap.start, - gspFwOffset: fb_layout.elf.start, - bootBinOffset: fb_layout.boot.start, - frtsOffset: fb_layout.frts.start, - frtsSize: fb_layout.frts.end - fb_layout.frts.start, - gspFwWprEnd: fb_layout + gspFwRsvdStart: ranges.non_wpr_heap.start, + nonWprHeapOffset: ranges.non_wpr_heap.start, + nonWprHeapSize: ranges.non_wpr_heap.len(), + gspFwWprStart: ranges.wpr2.start, + gspFwHeapOffset: ranges.wpr2_heap.start, + gspFwHeapSize: ranges.wpr2_heap.len(), + gspFwOffset: ranges.elf.start, + bootBinOffset: ranges.boot.start, + frtsOffset: ranges.frts.start, + frtsSize: ranges.frts.len(), + gspFwWprEnd: ranges .vga_workspace .start .align_down(Alignment::new::<SZ_128K>()), - gspFwHeapVfPartitionCount: fb_layout.vf_partition_count, - fbSize: fb_layout.fb.end - fb_layout.fb.start, - vgaWorkspaceOffset: fb_layout.vga_workspace.start, - vgaWorkspaceSize: fb_layout.vga_workspace.end - fb_layout.vga_workspace.start, - pmuReservedSize: fb_layout.pmu_reserved_size, + gspFwHeapVfPartitionCount: ranges.vf_partition_count, + fbSize: ranges.fb.len(), + vgaWorkspaceOffset: ranges.vga_workspace.start, + vgaWorkspaceSize: ranges.vga_workspace.len(), + pmuReservedSize: ranges.pmu_reserved_size, + ..Zeroable::init_zeroed() + }); + + init!(GspFwWprMeta { + inner <- init_inner, + }) + } + + /// Returns an initializer for a `GspFwWprMeta` suitable for booting `gsp_firmware` using the + /// framebuffer region sizes `sizes`. + /// + /// The region offsets are left at zero: the ACR ucode computes them when it sets up WPR2. + pub(crate) fn from_sizes<'a>( + gsp_firmware: &'a GspFirmware, + sizes: &'a FbSizes, + ) -> impl Init<Self> + 'a { + /// VGA workspace size to reserve at the end of the framebuffer, in bytes. + const VGA_WORKSPACE_SIZE: u64 = u64::SZ_128K; + + let init_inner = init!(bindings::GspFwWprMeta { + // CAST: we want to store the bits of `GSP_FW_WPR_META_MAGIC` unmodified. + magic: bindings::GSP_FW_WPR_META_MAGIC as u64, + revision: u64::from(bindings::GSP_FW_WPR_META_REVISION), + sysmemAddrOfRadix3Elf: gsp_firmware.radix3_dma_address(), + sizeOfRadix3Elf: u64::from_safe_cast(gsp_firmware.size), + sysmemAddrOfBootloader: gsp_firmware.bootloader.ucode.dma_address(), + sizeOfBootloader: u64::from_safe_cast(gsp_firmware.bootloader.ucode.size()), + bootloaderCodeOffset: u64::from(gsp_firmware.bootloader.code_offset), + bootloaderDataOffset: u64::from(gsp_firmware.bootloader.data_offset), + bootloaderManifestOffset: u64::from(gsp_firmware.bootloader.manifest_offset), + __bindgen_anon_1: GspFwWprMetaBootResumeInfo { + __bindgen_anon_1: GspFwWprMetaBootInfo { + sysmemAddrOfSignature: gsp_firmware.signatures.dma_address(), + sizeOfSignature: u64::from_safe_cast(gsp_firmware.signatures.size()), + }, + }, + nonWprHeapSize: sizes.non_wpr_heap_size, + gspFwHeapSize: sizes.wpr2_heap_size, + frtsSize: sizes.frts_size, + gspFwHeapVfPartitionCount: sizes.vf_partition_count, + vgaWorkspaceSize: VGA_WORKSPACE_SIZE, + pmuReservedSize: sizes.pmu_reserved_size, ..Zeroable::init_zeroed() }); @@ -674,10 +678,9 @@ impl LibosMemoryRegionInitArgument { u64::from_ne_bytes(bytes) } - #[allow(non_snake_case)] let init_inner = init!(bindings::LibosMemoryRegionInitArgument { id8: id8(name), - pa: obj.dma_handle(), + pa: obj.dma_address(), size: num::usize_as_u64(obj.size()), kind: num::u32_into_u8::< { bindings::LibosMemoryRegionKind_LIBOS_MEMORY_REGION_CONTIGUOUS }, @@ -720,6 +723,16 @@ impl MsgqTxHeader { entryOff: num::usize_into_u32::<GSP_PAGE_SIZE>(), }) } + + /// Returns the value of the write pointer for this queue. + pub(crate) fn write_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.writePtr) + } + + /// Sets the value of the write pointer for this queue. + pub(crate) fn set_write_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.writePtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -735,6 +748,16 @@ impl MsgqRxHeader { pub(crate) fn new() -> Self { Self(Default::default()) } + + /// Returns the value of the read pointer for this queue. + pub(crate) fn read_ptr(this: CoherentView<'_, Self>) -> u32 { + io_read!(this, .0.readPtr) + } + + /// Sets the value of the read pointer for this queue. + pub(crate) fn set_read_ptr(this: CoherentView<'_, Self>, val: u32) { + io_write!(this, .0.readPtr, val) + } } // SAFETY: Padding is explicit and does not contain uninitialized data. @@ -742,8 +765,8 @@ unsafe impl AsBytes for MsgqRxHeader {} bitfield! { struct MsgHeaderVersion(u32) { - 31:24 major as u8; - 23:16 minor as u8; + 31:24 major; + 23:16 minor; } } @@ -752,9 +775,9 @@ impl MsgHeaderVersion { const MINOR_TOT: u8 = 0; fn new() -> Self { - Self::default() - .set_major(Self::MAJOR_TOT) - .set_minor(Self::MINOR_TOT) + Self::zeroed() + .with_major(Self::MAJOR_TOT) + .with_minor(Self::MINOR_TOT) } } @@ -793,7 +816,6 @@ impl GspMsgElement { /// * `sequence` - Sequence number of the message. /// * `cmd_size` - Size of the command (not including the message element), in bytes. /// * `function` - Function of the message. - #[allow(non_snake_case)] pub(crate) fn init( sequence: u32, cmd_size: usize, @@ -876,7 +898,6 @@ pub(crate) struct GspArgumentsCached { impl GspArgumentsCached { /// Creates the arguments for starting the GSP up using `cmdq` as its command queue. pub(crate) fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { - #[allow(non_snake_case)] let init_inner = init!(bindings::GSP_ARGUMENTS_CACHED { messageQueueInitArguments <- MessageQueueInitArguments::new(cmdq), bDmemStack: 1, @@ -923,10 +944,9 @@ type MessageQueueInitArguments = bindings::MESSAGE_QUEUE_INIT_ARGUMENTS; impl MessageQueueInitArguments { /// Creates a new init arguments structure for `cmdq`. - #[allow(non_snake_case)] fn new(cmdq: &Cmdq) -> impl Init<Self> + '_ { init!(MessageQueueInitArguments { - sharedMemPhysAddr: cmdq.dma_handle, + sharedMemPhysAddr: cmdq.dma_addr, pageTableEntryCount: num::usize_into_u32::<{ Cmdq::NUM_PTES }>(), cmdQueueOffset: num::usize_as_u64(Cmdq::CMDQ_OFFSET), statQueueOffset: num::usize_as_u64(Cmdq::STATQ_OFFSET), @@ -947,7 +967,6 @@ type GspAcrBootGspRmParams = bindings::GSP_ACR_BOOT_GSP_RM_PARAMS; impl GspAcrBootGspRmParams { fn new(target: GspDmaTarget, wpr_meta_addr: u64) -> impl Init<Self> { - #[allow(non_snake_case)] let params = init!(Self { target: target as u32, gspRmDescSize: num::usize_into_u32::<{ size_of::<GspFwWprMeta>() }>(), @@ -966,7 +985,6 @@ type GspRmParams = bindings::GSP_RM_PARAMS; impl GspRmParams { fn new(target: GspDmaTarget, libos_addr: u64) -> impl Init<Self> { - #[allow(non_snake_case)] let params = init!(Self { target: target as u32, bootArgsOffset: libos_addr, @@ -986,7 +1004,6 @@ unsafe impl FromBytes for GspFmcBootParams {} impl GspFmcBootParams { pub(crate) fn new(wpr_meta_addr: u64, libos_addr: u64) -> impl Init<Self> { - #[allow(non_snake_case)] let init = init!(Self { // Blackwell FSP obtains WPR info from other sources, so // wprCarveoutOffset and wprCarveoutSize are left zero. diff --git a/drivers/gpu/nova-core/gsp/fw/commands.rs b/drivers/gpu/nova-core/gsp/fw/commands.rs index 7bcc41fc7fa0..6dc31d1bf5ae 100644 --- a/drivers/gpu/nova-core/gsp/fw/commands.rs +++ b/drivers/gpu/nova-core/gsp/fw/commands.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +use core::ops::Range; + use kernel::{ device, pci, @@ -13,7 +15,8 @@ use kernel::{ use crate::{ gpu::Chipset, - gsp::GSP_PAGE_SIZE, // + gsp::GSP_PAGE_SIZE, + num::IntoSafeCast, // }; use super::bindings; @@ -27,7 +30,6 @@ static_assert!(size_of::<GspSetSystemInfo>() < GSP_PAGE_SIZE); impl GspSetSystemInfo { /// Returns an in-place initializer for the `GspSetSystemInfo` command. - #[allow(non_snake_case)] pub(crate) fn init<'a>( dev: &'a pci::Device<device::Bound>, chipset: Chipset, @@ -99,7 +101,6 @@ pub(crate) struct PackedRegistryTable { } impl PackedRegistryTable { - #[allow(non_snake_case)] pub(crate) fn init(num_entries: u32, size: u32) -> impl Init<Self> { type InnerPackedRegistryTable = bindings::PACKED_REGISTRY_TABLE; let init_inner = init!(InnerPackedRegistryTable { @@ -129,6 +130,41 @@ impl GspStaticConfigInfo { pub(crate) fn gpu_name_str(&self) -> [u8; 64] { self.0.gpuNameString } + + /// Returns an iterator over valid FB regions from GSP firmware data. + fn fb_regions( + &self, + ) -> impl Iterator<Item = &bindings::NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO> { + let fb_info = &self.0.fbRegionInfoParams; + fb_info + .fbRegion + .iter() + .take(fb_info.numFBRegions.into_safe_cast()) + .filter(|reg| reg.limit >= reg.base) + } + + /// Iterates over usable FB regions from GSP firmware data. + /// + /// Each yielded region is a [`Range<u64>`] suitable for driver memory allocation. + /// Usable regions are those that satisfy all the following properties: + /// - Are not reserved for firmware internal use. + /// - Are not protected (hardware-enforced access restrictions). + /// - Support compression (can use GPU memory compression for bandwidth). + /// - Support ISO (isochronous memory for display requiring guaranteed bandwidth). + pub(crate) fn usable_fb_regions(&self) -> impl Iterator<Item = Range<u64>> + '_ { + self.fb_regions().filter_map(|reg| { + // Filter: not reserved, not protected, supports compression and ISO. + if reg.reserved == 0 + && reg.bProtected == 0 + && reg.supportCompressed != 0 + && reg.supportISO != 0 + { + reg.limit.checked_add(1).map(|end| reg.base..end) + } else { + None + } + }) + } } // SAFETY: Padding is explicit and will not contain uninitialized data. diff --git a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs index ea350f9b2cc4..afe3e007f088 100644 --- a/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs +++ b/drivers/gpu/nova-core/gsp/fw/r570_144/bindings.rs @@ -40,6 +40,7 @@ pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_TU10X: u32 = 8388608; pub const GSP_FW_HEAP_PARAM_BASE_RM_SIZE_GH100: u32 = 14680064; pub const GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB: u32 = 98304; pub const GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE: u32 = 100663296; +pub const GSP_FW_HEAP_SIZE_VGPU_DEFAULT: u32 = 609222656; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB: u32 = 64; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB: u32 = 256; pub const GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB: u32 = 88; diff --git a/drivers/gpu/nova-core/gsp/hal.rs b/drivers/gpu/nova-core/gsp/hal.rs index 04f004856c60..5850fa0fe0e9 100644 --- a/drivers/gpu/nova-core/gsp/hal.rs +++ b/drivers/gpu/nova-core/gsp/hal.rs @@ -1,33 +1,21 @@ // SPDX-License-Identifier: GPL-2.0 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +mod ga102; mod gh100; mod tu102; use kernel::prelude::*; -use kernel::{ - device, - dma::Coherent, // -}; - use crate::{ - driver::Bar0, - falcon::{ - gsp::Gsp as GspEngine, - sec2::Sec2, - Falcon, // - }, - fb::FbLayout, firmware::gsp::GspFirmware, gpu::{ Architecture, Chipset, // }, gsp::{ - boot::BootUnloadGuard, Gsp, - GspFwWprMeta, // + GspBootContext, // }, }; @@ -38,33 +26,21 @@ use crate::{ /// required for unloading is prepared at load time, and stored here until it needs to be run. pub(super) trait UnloadBundle: Send { /// Performs the steps required to properly reset the GSP after it has been stopped. - fn run( - &self, - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - gsp_falcon: &Falcon<GspEngine>, - sec2_falcon: &Falcon<Sec2>, - ) -> Result; + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result; } /// Trait implemented by GSP HALs. pub(super) trait GspHal: Send { /// Performs the GSP boot process, loading and running the required firmwares as needed. /// - /// Upon success, returns a guard that runs the GSP unload sequence if GSP boot does not - /// complete. - #[allow(clippy::too_many_arguments)] - fn boot<'a>( + /// Upon success, returns the [`crate::gsp::UnloadBundle`] to use with [`Gsp::unload`], if one + /// could be created. + fn boot( &self, - gsp: &'a Gsp, - dev: &'a device::Device<device::Bound>, - bar: Bar0<'a>, - chipset: Chipset, - fb_layout: &FbLayout, - wpr_meta: &Coherent<GspFwWprMeta>, - gsp_falcon: &'a Falcon<GspEngine>, - sec2_falcon: &'a Falcon<Sec2>, - ) -> Result<BootUnloadGuard<'a>>; + gsp: &Gsp, + ctx: &mut GspBootContext<'_, '_>, + gsp_fw: &GspFirmware, + ) -> Result<Option<crate::gsp::UnloadBundle>>; /// Performs HAL-specific post-GSP boot tasks. /// @@ -73,20 +49,38 @@ pub(super) trait GspHal: Send { fn post_boot( &self, _gsp: &Gsp, - _dev: &device::Device<device::Bound>, - _bar: Bar0<'_>, + _ctx: &mut GspBootContext<'_, '_>, _gsp_fw: &GspFirmware, - _gsp_falcon: &Falcon<GspEngine>, - _sec2_falcon: &Falcon<Sec2>, ) -> Result { Ok(()) } } +/// Returns the names of the firmware files required to boot the GSP of `chipset`, in addition to +/// the "bootloader" and "gsp" images required by all chipsets. +pub(crate) const fn boot_firmware_files(chipset: Chipset) -> &'static [&'static str] { + match chipset.arch() { + // Turing chipsets boot the GSP via the SEC2 Booter, and require the FWSEC bootloader. + Architecture::Turing => &["booter_load.tlv", "booter_unload.tlv", "gen_bootloader.tlv"], + // GA100 also requires the FWSEC bootloader. + Architecture::Ampere if matches!(chipset, Chipset::GA100) => { + &["booter_load.tlv", "booter_unload.tlv", "gen_bootloader.tlv"] + } + // Other Ampere chipsets, as well as Ada chipsets, run FWSEC directly. + Architecture::Ampere | Architecture::Ada => &["booter_load.tlv", "booter_unload.tlv"], + // Hopper and later chipsets boot the GSP via the FMC image loaded by FSP. + Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { + &["fmc.tlv"] + } + } +} + /// Returns the GSP HAL to be used for `chipset`. pub(super) fn gsp_hal(chipset: Chipset) -> &'static dyn GspHal { match chipset.arch() { - Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL, + Architecture::Turing => tu102::TU102_HAL, + Architecture::Ampere if matches!(chipset, Chipset::GA100) => tu102::TU102_HAL, + Architecture::Ampere | Architecture::Ada => ga102::GA102_HAL, Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => { gh100::GH100_HAL } diff --git a/drivers/gpu/nova-core/gsp/hal/ga102.rs b/drivers/gpu/nova-core/gsp/hal/ga102.rs new file mode 100644 index 000000000000..ceb3eb39d138 --- /dev/null +++ b/drivers/gpu/nova-core/gsp/hal/ga102.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::gsp::hal::{ + tu102::Tu102, + GspHal, // +}; + +/// The GA102 HAL is like the TU102 one, except it doesn't use the bootloader. +const GA102: Tu102 = Tu102 { + needs_fwsec_bootloader: false, +}; + +pub(super) const GA102_HAL: &dyn GspHal = &GA102; diff --git a/drivers/gpu/nova-core/gsp/hal/gh100.rs b/drivers/gpu/nova-core/gsp/hal/gh100.rs index 98f5ce197d13..e283429a95dd 100644 --- a/drivers/gpu/nova-core/gsp/hal/gh100.rs +++ b/drivers/gpu/nova-core/gsp/hal/gh100.rs @@ -7,33 +7,26 @@ use kernel::{ device, dma::Coherent, io::poll::read_poll_timeout, - time::Delta, // + time::Delta, + types::ScopeGuard, // }; use crate::{ - driver::Bar0, falcon::{ gsp::Gsp as GspEngine, - sec2::Sec2, Falcon, // }, - fb::FbLayout, - firmware::{ - fsp::FspFirmware, - FIRMWARE_VERSION, // - }, - fsp::{ - FmcBootArgs, - Fsp, // - }, - gpu::Chipset, + fb::FbSizes, + firmware::gsp::GspFirmware, + fsp::FmcBootArgs, gsp::{ - boot::BootUnloadGuard, hal::{ GspHal, UnloadBundle, // }, Gsp, + GspBootContext, + GspFmcBootParams, GspFwWprMeta, // }, }; @@ -46,10 +39,10 @@ struct GspMbox { impl GspMbox { /// Reads both mailboxes from the GSP falcon. - fn read(gsp_falcon: &Falcon<GspEngine>, bar: Bar0<'_>) -> Self { + fn read(gsp_falcon: &Falcon<'_, GspEngine>) -> Self { Self { - mbox0: gsp_falcon.read_mailbox0(bar), - mbox1: gsp_falcon.read_mailbox1(bar), + mbox0: gsp_falcon.read_mailbox0(), + mbox1: gsp_falcon.read_mailbox1(), } } @@ -64,27 +57,25 @@ impl GspMbox { /// either condition should stop the poll loop. fn lockdown_released_or_error( &self, - gsp_falcon: &Falcon<GspEngine>, - bar: Bar0<'_>, - fmc_boot_params_addr: u64, + gsp_falcon: &Falcon<'_, GspEngine>, + fmc_boot_params: &Coherent<GspFmcBootParams>, ) -> bool { // GSP-FMC normally clears the boot parameters address from the mailboxes early during // boot. If the address is still there, keep polling rather than treating it as an error. // Any other non-zero mailbox0 value is a GSP-FMC error code. if self.mbox0 != 0 { - return self.combined_addr() != fmc_boot_params_addr; + return self.combined_addr() != fmc_boot_params.dma_address(); } - !gsp_falcon.riscv_branch_privilege_lockdown(bar) + !gsp_falcon.riscv_branch_privilege_lockdown() } } /// Waits for GSP lockdown to be released after FSP Chain of Trust. fn wait_for_gsp_lockdown_release( dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - gsp_falcon: &Falcon<GspEngine>, - fmc_boot_params_addr: u64, + gsp_falcon: &Falcon<'_, GspEngine>, + fmc_boot_params: &Coherent<GspFmcBootParams>, ) -> Result { dev_dbg!(dev, "Waiting for GSP lockdown release\n"); @@ -92,14 +83,14 @@ fn wait_for_gsp_lockdown_release( || { // While the PRIV target mask is still locked to FSP, GSP register and mailbox reads // are not meaningful. Wait until HWCFG2 says the CPU can read them. - Ok(match gsp_falcon.priv_target_mask_released(bar) { + Ok(match gsp_falcon.priv_target_mask_released() { false => None, - true => Some(GspMbox::read(gsp_falcon, bar)), + true => Some(GspMbox::read(gsp_falcon)), }) }, |mbox| match mbox { None => false, - Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, bar, fmc_boot_params_addr), + Some(mbox) => mbox.lockdown_released_or_error(gsp_falcon, fmc_boot_params), }, Delta::from_millis(10), Delta::from_secs(30), @@ -123,22 +114,23 @@ fn wait_for_gsp_lockdown_release( struct FspUnloadBundle; impl UnloadBundle for FspUnloadBundle { - fn run( - &self, - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - gsp_falcon: &Falcon<GspEngine>, - _sec2_falcon: &Falcon<Sec2>, - ) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { // GSP falcon does most of the work of resetting, so just wait for it to finish. read_poll_timeout( - || Ok(gsp_falcon.is_riscv_active(bar)), - |&active| !active, + || { + // GSP register reads are not meaningful until the PRIV target mask is released. + if !ctx.gsp_falcon.priv_target_mask_released() { + return Ok(false); + } + + ctx.gsp_falcon.is_riscv_halted() + }, + |&halted| halted, Delta::from_millis(10), Delta::from_secs(5), ) .map(|_| ()) - .inspect_err(|_| dev_err!(dev, "GSP falcon failed to halt\n")) + .inspect_err(|_| dev_err!(ctx.dev(), "GSP falcon failed to halt\n")) } } @@ -149,42 +141,44 @@ impl GspHal for Gh100 { /// /// This path uses FSP to establish a chain of trust and boot GSP-FMC. FSP handles /// the GSP boot internally - no manual GSP reset/boot is needed. - fn boot<'a>( + fn boot( &self, - gsp: &'a Gsp, - dev: &'a device::Device<device::Bound>, - bar: Bar0<'a>, - chipset: Chipset, - fb_layout: &FbLayout, - wpr_meta: &Coherent<GspFwWprMeta>, - gsp_falcon: &'a Falcon<GspEngine>, - sec2_falcon: &'a Falcon<Sec2>, - ) -> Result<BootUnloadGuard<'a>> { - let fsp_fw = FspFirmware::new(dev, chipset, FIRMWARE_VERSION)?; + gsp: &Gsp, + ctx: &mut GspBootContext<'_, '_>, + gsp_fw: &GspFirmware, + ) -> Result<Option<crate::gsp::UnloadBundle>> { + let dev = ctx.dev(); + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; + + let fb_sizes = FbSizes::new(chipset, ctx.bar, ctx.vgpu.state())?; + dev_dbg!(dev, "{:#x?}\n", fb_sizes); + + let wpr_meta = + Coherent::init(dev, GFP_KERNEL, GspFwWprMeta::from_sizes(gsp_fw, &fb_sizes))?; + let args = FmcBootArgs::new(dev, chipset, wpr_meta, &gsp.libos, false)?; let unload_bundle = crate::gsp::UnloadBundle( KBox::new(FspUnloadBundle, GFP_KERNEL)? as KBox<dyn UnloadBundle> ); - // Wrap the unload bundle into a drop guard so it is automatically run upon failure. - let unload_guard = - BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, Some(unload_bundle)); - - let mut fsp = Fsp::wait_secure_boot(dev, bar, chipset, fsp_fw)?; + // Wait for the GSP RISC-V core to halt in case of error. We create this guard after `args` + // to make sure that the boot args and the WPR metadata they own are kept alive until halt, + // in case they are still being accessed. + let mut unload_guard = + ScopeGuard::new_with_data((unload_bundle, ctx), |(unload_bundle, ctx)| { + let _ = unload_bundle.0.run(ctx); + }); - let args = FmcBootArgs::new( - dev, - chipset, - wpr_meta.dma_handle(), - gsp.libos.dma_handle(), - false, - )?; + let fsp = unload_guard.1.fsp.as_mut().ok_or(ENODEV)?; - fsp.boot_fmc(dev, bar, fb_layout, &args)?; + fsp.boot_fmc(dev, &fb_sizes, &args)?; - wait_for_gsp_lockdown_release(dev, bar, gsp_falcon, args.boot_params_dma_handle())?; + // Wait for GSP-FMC to release the GSP lockdown, indicating that `args` is not accessed + // anymore. + wait_for_gsp_lockdown_release(dev, gsp_falcon, args.boot_params())?; - Ok(unload_guard) + Ok(Some(unload_guard.dismiss().0)) } } diff --git a/drivers/gpu/nova-core/gsp/hal/tu102.rs b/drivers/gpu/nova-core/gsp/hal/tu102.rs index 2f6301af7113..a5c0ca355493 100644 --- a/drivers/gpu/nova-core/gsp/hal/tu102.rs +++ b/drivers/gpu/nova-core/gsp/hal/tu102.rs @@ -6,7 +6,8 @@ use kernel::prelude::*; use kernel::{ device, dma::Coherent, - io::Io, // + io::Io, + types::ScopeGuard, // }; use crate::{ @@ -16,7 +17,10 @@ use crate::{ sec2::Sec2, Falcon, // }, - fb::FbLayout, + fb::{ + wpr2_range, + FbRanges, // + }, firmware::{ booter::{ BooterFirmware, @@ -27,24 +31,20 @@ use crate::{ FwsecCommand, FwsecFirmware, // }, - gsp::GspFirmware, - FIRMWARE_VERSION, // + gsp::GspFirmware, // }, gpu::Chipset, gsp::{ - boot::BootUnloadGuard, hal::{ GspHal, UnloadBundle, // }, - sequencer::{ - GspSequencer, - GspSequencerParams, // - }, + regs, + sequencer::GspSequencer, Gsp, + GspBootContext, GspFwWprMeta, // }, - regs, vbios::Vbios, // }; @@ -58,32 +58,15 @@ enum FwsecUnloadFirmware { } impl FwsecUnloadFirmware { - /// Loads the FWSEC SB firmware, as well as its bootloader if `chipset` requires it. - fn new( - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - chipset: Chipset, - bios: &Vbios, - gsp_falcon: &Falcon<GspEngine>, - ) -> Result<Self> { - let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bar, bios, FwsecCommand::Sb)?; - - Ok(if chipset.needs_fwsec_bootloader() { - Self::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) - } else { - Self::WithoutBl(fwsec_sb) - }) - } - /// Runs the FWSEC SB firmware. fn run( &self, dev: &device::Device<device::Bound>, bar: Bar0<'_>, - gsp_falcon: &Falcon<GspEngine>, + gsp_falcon: &Falcon<'_, GspEngine>, ) -> Result { match self { - Self::WithoutBl(fw) => fw.run(dev, gsp_falcon, bar), + Self::WithoutBl(fw) => fw.run(dev, gsp_falcon), Self::WithBl(fw) => fw.run(dev, gsp_falcon, bar), } } @@ -96,210 +79,225 @@ struct Sec2UnloadBundle { booter_unloader: BooterFirmware, } -impl Sec2UnloadBundle { - /// Load and prepare the resources required to properly reset the GSP after it has been stopped. - fn build( - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - chipset: Chipset, - bios: &Vbios, - gsp_falcon: &Falcon<GspEngine>, - sec2_falcon: &Falcon<Sec2>, - ) -> Result<KBox<dyn UnloadBundle>> { - KBox::new( - Self { - fwsec_sb: FwsecUnloadFirmware::new(dev, bar, chipset, bios, gsp_falcon)?, - booter_unloader: BooterFirmware::new( - dev, - BooterKind::Unloader, - chipset, - FIRMWARE_VERSION, - sec2_falcon, - bar, - )?, - }, - GFP_KERNEL, - ) - .map(|b| b as KBox<dyn UnloadBundle>) - .map_err(Into::into) - } -} - impl UnloadBundle for Sec2UnloadBundle { - fn run( - &self, - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, - gsp_falcon: &Falcon<GspEngine>, - sec2_falcon: &Falcon<Sec2>, - ) -> Result { + fn run(&self, ctx: &mut GspBootContext<'_, '_>) -> Result { + let dev = ctx.dev(); + let bar = ctx.bar; + // Run FWSEC-SB to reset the GSP falcon to its pre-libos state. - self.fwsec_sb.run(dev, bar, gsp_falcon)?; + // Log errors but keep going if it fails. + let fwsec_sb_res = self + .fwsec_sb + .run(dev, bar, ctx.gsp_falcon) + .inspect_err(|e| dev_err!(dev, "FWSEC-SB failed to run: {:?}\n", e)); // Remove WPR2 region if set. - let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { - sec2_falcon.reset(bar)?; - sec2_falcon.load(dev, bar, &self.booter_unloader)?; + let booter_unloader_res = (|| { + if wpr2_range(bar).is_none() { + return Ok(()); + } + + ctx.sec2_falcon.reset()?; + ctx.sec2_falcon.load(&self.booter_unloader)?; // Sentinel value to confirm that Booter Unloader has run. const MAILBOX_SENTINEL: u32 = 0xff; - let (mbox0, _) = - sec2_falcon.boot(bar, Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; + let (mbox0, _) = ctx + .sec2_falcon + .boot(Some(MAILBOX_SENTINEL), Some(MAILBOX_SENTINEL))?; if mbox0 != 0 { dev_err!(dev, "Booter Unloader returned error 0x{:x}\n", mbox0); return Err(EINVAL); } // Confirm that the WPR2 region has been removed. - let wpr2_hi = bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI); - if wpr2_hi.is_wpr2_set() { + if wpr2_range(bar).is_some() { dev_err!( dev, "WPR2 region still set after Booter Unloader returned\n" ); return Err(EBUSY); } - } - Ok(()) + Ok(()) + })() + .inspect_err(|e| dev_err!(dev, "Booter Unloader failed to run: {:?}\n", e)); + + fwsec_sb_res.and(booter_unloader_res) } } -/// Helper function to load and run the FWSEC-FRTS firmware and confirm that it has properly -/// created the WPR2 region. -fn run_fwsec_frts( - dev: &device::Device<device::Bound>, - chipset: Chipset, - falcon: &Falcon<GspEngine>, - bar: Bar0<'_>, - bios: &Vbios, - fb_layout: &FbLayout, -) -> Result { - // Check that the WPR2 region does not already exist - if it does, we cannot run - // FWSEC-FRTS until the GPU is reset. - if bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound() != 0 { - dev_err!( - dev, - "WPR2 region already exists - GPU needs to be reset to proceed\n" - ); - return Err(EBUSY); - } +pub(super) struct Tu102 { + /// If `true`, then the FWSEC-FRTS bootloader will be used to load the actual firmware. + pub(super) needs_fwsec_bootloader: bool, +} - // FWSEC-FRTS will create the WPR2 region. - let fwsec_frts = FwsecFirmware::new( - dev, - falcon, - bar, - bios, - FwsecCommand::Frts { - frts_addr: fb_layout.frts.start, - frts_size: fb_layout.frts.len(), - }, - )?; - - if chipset.needs_fwsec_bootloader() { - let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; - // Load and run the bootloader, which will load FWSEC-FRTS and run it. - fwsec_frts_bl.run(dev, falcon, bar)?; - } else { - // Load and run FWSEC-FRTS directly. - fwsec_frts.run(dev, falcon, bar)?; - } +impl Tu102 { + /// Helper method to load and run the FWSEC-FRTS firmware and confirm that it has properly + /// created the WPR2 region. + fn run_fwsec_frts( + &self, + dev: &device::Device<device::Bound>, + chipset: Chipset, + falcon: &Falcon<'_, GspEngine>, + bar: Bar0<'_>, + bios: &Vbios, + fb_ranges: &FbRanges, + ) -> Result { + // Check that the WPR2 region does not already exist - if it does, we cannot run + // FWSEC-FRTS until the GPU is reset. + if wpr2_range(bar).is_some() { + dev_err!( + dev, + "WPR2 region already exists - GPU needs to be reset to proceed\n" + ); + return Err(EBUSY); + } - // SCRATCH_E contains the error code for FWSEC-FRTS. - let frts_status = bar - .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) - .frts_err_code(); - if frts_status != 0 { - dev_err!( + // FWSEC-FRTS will create the WPR2 region. + let fwsec_frts = FwsecFirmware::new( dev, - "FWSEC-FRTS returned with error code {:#x}\n", - frts_status - ); + falcon, + bios, + FwsecCommand::Frts { + frts_addr: fb_ranges.frts.start, + frts_size: fb_ranges.frts.len(), + }, + )?; - return Err(EIO); - } + if self.needs_fwsec_bootloader { + let fwsec_frts_bl = FwsecFirmwareWithBl::new(fwsec_frts, dev, chipset)?; + // Load and run the bootloader, which will load FWSEC-FRTS and run it. + fwsec_frts_bl.run(dev, falcon, bar)?; + } else { + // Load and run FWSEC-FRTS directly. + fwsec_frts.run(dev, falcon)?; + } + + // SCRATCH_E contains the error code for FWSEC-FRTS. + let frts_status = bar + .read(regs::NV_PBUS_SW_SCRATCH_0E_FRTS_ERR) + .frts_err_code(); + if frts_status != 0 { + dev_err!( + dev, + "FWSEC-FRTS returned with error code {:#x}\n", + frts_status + ); - // Check that the WPR2 region has been created as we requested. - let (wpr2_lo, wpr2_hi) = ( - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_LO).lower_bound(), - bar.read(regs::NV_PFB_PRI_MMU_WPR2_ADDR_HI).higher_bound(), - ); + return Err(EIO); + } - match (wpr2_lo, wpr2_hi) { - (_, 0) => { + // Check that the WPR2 region has been created as we requested. + let Some(wpr2_range) = wpr2_range(bar) else { dev_err!(dev, "WPR2 region not created after running FWSEC-FRTS\n"); - Err(EIO) - } - (wpr2_lo, _) if wpr2_lo != fb_layout.frts.start => { + return Err(EIO); + }; + + if wpr2_range.start != fb_ranges.frts.start { dev_err!( dev, "WPR2 region created at unexpected address {:#x}; expected {:#x}\n", - wpr2_lo, - fb_layout.frts.start, + wpr2_range.start, + fb_ranges.frts.start, ); - Err(EIO) + return Err(EIO); } - (wpr2_lo, wpr2_hi) => { - dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_lo, wpr2_hi); - dev_dbg!(dev, "GPU instance built\n"); - Ok(()) - } + dev_dbg!(dev, "WPR2: {:#x}-{:#x}\n", wpr2_range.start, wpr2_range.end); + dev_dbg!(dev, "GPU instance built\n"); + + Ok(()) } -} -struct Tu102; + /// Load and prepare the resources required to properly reset the GSP after it has been stopped. + fn build_unload_bundle( + &self, + dev: &device::Device<device::Bound>, + chipset: Chipset, + bios: &Vbios, + gsp_falcon: &Falcon<'_, GspEngine>, + sec2_falcon: &Falcon<'_, Sec2>, + ) -> Result<crate::gsp::UnloadBundle> { + // Load the FWSEC SB firmware, as well as its bootloader if required. + let fwsec_sb = FwsecFirmware::new(dev, gsp_falcon, bios, FwsecCommand::Sb)?; + let fwsec_sb = if self.needs_fwsec_bootloader { + FwsecUnloadFirmware::WithBl(FwsecFirmwareWithBl::new(fwsec_sb, dev, chipset)?) + } else { + FwsecUnloadFirmware::WithoutBl(fwsec_sb) + }; + + KBox::new( + Sec2UnloadBundle { + fwsec_sb, + booter_unloader: BooterFirmware::new( + dev, + BooterKind::Unloader, + chipset, + sec2_falcon, + )?, + }, + GFP_KERNEL, + ) + .map(|b| crate::gsp::UnloadBundle(b)) + .map_err(Into::into) + } +} impl GspHal for Tu102 { - fn boot<'a>( + fn boot( &self, - gsp: &'a Gsp, - dev: &'a device::Device<device::Bound>, - bar: Bar0<'a>, - chipset: Chipset, - fb_layout: &FbLayout, - wpr_meta: &Coherent<GspFwWprMeta>, - gsp_falcon: &'a Falcon<GspEngine>, - sec2_falcon: &'a Falcon<Sec2>, - ) -> Result<BootUnloadGuard<'a>> { + gsp: &Gsp, + ctx: &mut GspBootContext<'_, '_>, + gsp_fw: &GspFirmware, + ) -> Result<Option<crate::gsp::UnloadBundle>> { + let dev = ctx.dev(); + let bar = ctx.bar; + let chipset = ctx.chipset; + let gsp_falcon = ctx.gsp_falcon; + let sec2_falcon = ctx.sec2_falcon; + + let fb_ranges = FbRanges::new(chipset, bar, gsp_fw, ctx.vgpu.state())?; + dev_dbg!(dev, "{:#x?}\n", fb_ranges); + + // Declared before the unload guard so that if Booter fails while running, SEC2 is reset + // by the guard before this allocation is freed. + let wpr_meta = Coherent::init( + dev, + GFP_KERNEL, + GspFwWprMeta::from_ranges(gsp_fw, &fb_ranges), + )?; + let bios = Vbios::new(dev, bar)?; // Try and prepare the unload bundle. // // If the unload bundle creation fails, the GPU will need to be reset before the driver can // be probed again. - let unload_bundle = - Sec2UnloadBundle::build(dev, bar, chipset, &bios, gsp_falcon, sec2_falcon) - .inspect_err(|e| { - dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e); - dev_warn!(dev, "The GSP won't be able to unload properly on unbind.\n"); - dev_warn!( - dev, - "The GPU will need to be reset before the driver can bind again.\n" - ); - }) - .ok() - .map(crate::gsp::UnloadBundle); - - // Wrap the unload bundle into a drop guard so it is automatically run upon failure. - let unload_guard = - BootUnloadGuard::new(gsp, dev, bar, gsp_falcon, sec2_falcon, unload_bundle); + let unload_bundle = self + .build_unload_bundle(dev, chipset, &bios, gsp_falcon, sec2_falcon) + .inspect_err(|e| dev_warn!(dev, "Failed to prepare unload firmware: {:?}\n", e)) + .ok(); + + // Run the unload bundle to try and recover the GSP if an error occurs. + let unload_guard = ScopeGuard::new_with_data(unload_bundle, |unload_bundle| { + if let Some(unload_bundle) = unload_bundle { + let _ = unload_bundle.0.run(ctx); + } + }); // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100). - if !fb_layout.frts.is_empty() { - run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, fb_layout)?; + if !fb_ranges.frts.is_empty() { + self.run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_ranges)?; } - gsp_falcon.reset(bar)?; - let libos_handle = gsp.libos.dma_handle(); + gsp_falcon.reset()?; + let libos_dma_address = gsp.libos.dma_address(); let (mbox0, mbox1) = gsp_falcon.boot( - bar, - Some(libos_handle as u32), - Some((libos_handle >> 32) as u32), + Some(libos_dma_address as u32), + Some((libos_dma_address >> 32) as u32), )?; dev_dbg!(dev, "GSP MBOX0: {:#x}, MBOX1: {:#x}\n", mbox0, mbox1); @@ -308,42 +306,30 @@ impl GspHal for Tu102 { "Using SEC2 to load and run the booter_load firmware...\n" ); - BooterFirmware::new( + BooterFirmware::new(dev, BooterKind::Loader, chipset, sec2_falcon)?.run( dev, - BooterKind::Loader, - chipset, - FIRMWARE_VERSION, sec2_falcon, - bar, - )? - .run(dev, bar, sec2_falcon, wpr_meta)?; + &wpr_meta, + )?; - Ok(unload_guard) + Ok(unload_guard.dismiss()) } fn post_boot( &self, gsp: &Gsp, - dev: &device::Device<device::Bound>, - bar: Bar0<'_>, + ctx: &mut GspBootContext<'_, '_>, gsp_fw: &GspFirmware, - gsp_falcon: &Falcon<GspEngine>, - sec2_falcon: &Falcon<Sec2>, ) -> Result { - // Create and run the GSP sequencer. - let seq_params = GspSequencerParams { - bootloader_app_version: gsp_fw.bootloader.app_version, - libos_dma_handle: gsp.libos.dma_handle(), - gsp_falcon, - sec2_falcon, - dev, - bar, - }; - GspSequencer::run(&gsp.cmdq, seq_params)?; + GspSequencer::run(&gsp.cmdq, ctx, &gsp.libos, gsp_fw.bootloader.app_version)?; Ok(()) } } -const TU102: Tu102 = Tu102; +/// The TU102 HAL requires the use of the FWSEC bootloader. +const TU102: Tu102 = Tu102 { + needs_fwsec_bootloader: true, +}; + pub(super) const TU102_HAL: &dyn GspHal = &TU102; diff --git a/drivers/gpu/nova-core/gsp/regs.rs b/drivers/gpu/nova-core/gsp/regs.rs new file mode 100644 index 000000000000..9a48aa87e7fb --- /dev/null +++ b/drivers/gpu/nova-core/gsp/regs.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +use kernel::io::register; + +use crate::regs::NV_PBUS_SW_SCRATCH; + +// PGSP + +register! { + pub(super) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { + 31:0 address; + } +} + +// PBUS + +register! { + /// Scratch register 0xe used as FRTS firmware error code. + pub(super) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { + 31:16 frts_err_code; + } +} diff --git a/drivers/gpu/nova-core/gsp/sequencer.rs b/drivers/gpu/nova-core/gsp/sequencer.rs index e0850d21adca..bcad1421953a 100644 --- a/drivers/gpu/nova-core/gsp/sequencer.rs +++ b/drivers/gpu/nova-core/gsp/sequencer.rs @@ -6,6 +6,7 @@ use core::array; use kernel::{ device, + dma::Coherent, io::{ poll::read_poll_timeout, Io, // @@ -31,6 +32,8 @@ use crate::{ MessageFromGsp, // }, fw, + GspBootContext, + LibosMemoryRegionInitArgument, // }, num::FromSafeCast, sbuffer::SBufferIter, @@ -128,16 +131,14 @@ impl GspSeqCmd { /// GSP Sequencer for executing firmware commands during boot. pub(crate) struct GspSequencer<'a> { - /// Sequencer information with command data. - seq_info: GspSequence, /// `Bar0` for register access. bar: Bar0<'a>, /// SEC2 falcon for core operations. - sec2_falcon: &'a Falcon<Sec2>, + sec2_falcon: &'a Falcon<'a, Sec2>, /// GSP falcon for core operations. - gsp_falcon: &'a Falcon<Gsp>, - /// LibOS DMA handle address. - libos_dma_handle: u64, + gsp_falcon: &'a Falcon<'a, Gsp>, + /// LibOS memory region init arguments. + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, /// Bootloader application version. bootloader_app_version: u32, /// Device for logging. @@ -213,16 +214,16 @@ impl GspSeqCmd { GspSeqCmd::DelayUs(cmd) => cmd.run(seq), GspSeqCmd::RegStore(cmd) => cmd.run(seq), GspSeqCmd::CoreReset => { - seq.gsp_falcon.reset(seq.bar)?; - seq.gsp_falcon.dma_reset(seq.bar); + seq.gsp_falcon.reset()?; + seq.gsp_falcon.dma_reset(); Ok(()) } GspSeqCmd::CoreStart => { - seq.gsp_falcon.start(seq.bar)?; + seq.gsp_falcon.start()?; Ok(()) } GspSeqCmd::CoreWaitForHalt => { - seq.gsp_falcon.wait_till_halted(seq.bar)?; + seq.gsp_falcon.wait_till_halted()?; Ok(()) } GspSeqCmd::CoreResume => { @@ -231,35 +232,34 @@ impl GspSeqCmd { // sequencer will start both. // Reset the GSP to prepare it for resuming. - seq.gsp_falcon.reset(seq.bar)?; + seq.gsp_falcon.reset()?; - // Write the libOS DMA handle to GSP mailboxes. + let libos_dma_address = seq.libos.dma_address(); + + // Write the libOS DMA address to GSP mailboxes. seq.gsp_falcon.write_mailboxes( - seq.bar, - Some(seq.libos_dma_handle as u32), - Some((seq.libos_dma_handle >> 32) as u32), + Some(libos_dma_address as u32), + Some((libos_dma_address >> 32) as u32), ); // Start the SEC2 falcon which will trigger GSP-RM to resume on the GSP. - seq.sec2_falcon.start(seq.bar)?; + seq.sec2_falcon.start()?; // Poll until GSP-RM reload/resume has completed (up to 2 seconds). - seq.gsp_falcon - .check_reload_completed(seq.bar, Delta::from_secs(2))?; + seq.gsp_falcon.check_reload_completed(Delta::from_secs(2))?; // Verify SEC2 completed successfully by checking its mailbox for errors. - let mbox0 = seq.sec2_falcon.read_mailbox0(seq.bar); + let mbox0 = seq.sec2_falcon.read_mailbox0(); if mbox0 != 0 { dev_err!(seq.dev, "Sequencer: sec2 errors: {:?}\n", mbox0); return Err(EIO); } // Configure GSP with the bootloader version. - seq.gsp_falcon - .write_os_version(seq.bar, seq.bootloader_app_version); + seq.gsp_falcon.write_os_version(seq.bootloader_app_version); // Verify the GSP's RISC-V core is active indicating successful GSP boot. - if !seq.gsp_falcon.is_riscv_active(seq.bar) { + if !seq.gsp_falcon.is_riscv_active() { dev_err!(seq.dev, "Sequencer: RISC-V core is not active\n"); return Err(EIO); } @@ -270,7 +270,7 @@ impl GspSeqCmd { } /// Iterator over GSP sequencer commands. -pub(crate) struct GspSeqIter<'a> { +struct GspSeqIter<'a> { /// Command data buffer. cmd_data: &'a [u8], /// Current position in the buffer. @@ -283,6 +283,18 @@ pub(crate) struct GspSeqIter<'a> { dev: &'a device::Device, } +impl<'a> GspSeqIter<'a> { + fn new(seq: &'a GspSequence, dev: &'a device::Device) -> Self { + Self { + cmd_data: &seq.cmd_data, + current_offset: 0, + total_cmds: seq.cmd_index, + cmds_processed: 0, + dev, + } + } +} + impl<'a> Iterator for GspSeqIter<'a> { type Item = Result<GspSeqCmd>; @@ -325,37 +337,12 @@ impl<'a> Iterator for GspSeqIter<'a> { } impl<'a> GspSequencer<'a> { - fn iter(&self) -> GspSeqIter<'_> { - let cmd_data = &self.seq_info.cmd_data[..]; - - GspSeqIter { - cmd_data, - current_offset: 0, - total_cmds: self.seq_info.cmd_index, - cmds_processed: 0, - dev: self.dev, - } - } -} - -/// Parameters for running the GSP sequencer. -pub(crate) struct GspSequencerParams<'a> { - /// Bootloader application version. - pub(crate) bootloader_app_version: u32, - /// LibOS DMA handle address. - pub(crate) libos_dma_handle: u64, - /// GSP falcon for core operations. - pub(crate) gsp_falcon: &'a Falcon<Gsp>, - /// SEC2 falcon for core operations. - pub(crate) sec2_falcon: &'a Falcon<Sec2>, - /// Device for logging. - pub(crate) dev: &'a device::Device, - /// BAR0 for register access. - pub(crate) bar: Bar0<'a>, -} - -impl<'a> GspSequencer<'a> { - pub(crate) fn run(cmdq: &Cmdq, params: GspSequencerParams<'a>) -> Result { + pub(crate) fn run( + cmdq: &Cmdq, + ctx: &'a GspBootContext<'_, '_>, + libos: &'a Coherent<[LibosMemoryRegionInitArgument]>, + bootloader_app_version: u32, + ) -> Result { let seq_info = loop { match cmdq.receive_msg::<GspSequence>(Cmdq::RECEIVE_TIMEOUT) { Ok(seq_info) => break seq_info, @@ -365,25 +352,24 @@ impl<'a> GspSequencer<'a> { }; let sequencer = GspSequencer { - seq_info, - bar: params.bar, - sec2_falcon: params.sec2_falcon, - gsp_falcon: params.gsp_falcon, - libos_dma_handle: params.libos_dma_handle, - bootloader_app_version: params.bootloader_app_version, - dev: params.dev, + bar: ctx.bar, + sec2_falcon: ctx.sec2_falcon, + gsp_falcon: ctx.gsp_falcon, + libos, + bootloader_app_version, + dev: ctx.dev(), }; dev_dbg!(sequencer.dev, "Running CPU Sequencer commands\n"); - for cmd_result in sequencer.iter() { + for cmd_result in GspSeqIter::new(&seq_info, sequencer.dev) { match cmd_result { Ok(cmd) => cmd.run(&sequencer)?, Err(e) => { dev_err!( sequencer.dev, "Error running command at index {}\n", - sequencer.seq_info.cmd_index + seq_info.cmd_index ); return Err(e); } diff --git a/drivers/gpu/nova-core/mctp.rs b/drivers/gpu/nova-core/mctp.rs index 482786e07bc7..90c642c91a72 100644 --- a/drivers/gpu/nova-core/mctp.rs +++ b/drivers/gpu/nova-core/mctp.rs @@ -7,55 +7,53 @@ //! Data Model) messages between the kernel driver and GPU firmware processors //! such as FSP and GSP. -use kernel::pci::Vendor; +use kernel::{ + bitfield, + pci::Vendor, + prelude::*, // +}; -/// NVDM message type identifiers carried over MCTP. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -#[repr(u8)] -pub(crate) enum NvdmType { - #[default] - /// Chain of Trust boot message. - Cot = 0x14, - /// FSP command response. - FspResponse = 0x15, -} - -impl TryFrom<u8> for NvdmType { - type Error = u8; - - fn try_from(value: u8) -> Result<Self, Self::Error> { - match value { - x if x == u8::from(Self::Cot) => Ok(Self::Cot), - x if x == u8::from(Self::FspResponse) => Ok(Self::FspResponse), - _ => Err(value), - } - } -} +use crate::{ + bounded_enum, + num, // +}; -impl From<NvdmType> for u8 { - fn from(value: NvdmType) -> Self { - value as u8 +bounded_enum! { + /// NVDM message type identifiers carried over MCTP. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum NvdmType with TryFrom<Bounded<u32, 8>> { + /// PRC (Product Reconfiguration Control) message. + Prc = 0x13, + /// Chain of Trust boot message. + Cot = 0x14, + /// FSP command response. + FspResponse = 0x15, } } bitfield! { - pub(crate) struct MctpHeader(u32), "MCTP transport header for NVIDIA firmware messages." { - 31:31 som as bool, "Start-of-message bit."; - 30:30 eom as bool, "End-of-message bit."; - 29:28 seq as u8, "Packet sequence number."; - 23:16 seid as u8, "Source endpoint ID."; + /// MCTP transport header for NVIDIA firmware messages. + pub(crate) struct MctpHeader(u32) { + /// Start-of-message bit. + 31:31 som; + /// End-of-message bit. + 30:30 eom; + /// Packet sequence number. + 29:28 seq; + /// Source endpoint ID. + 23:16 seid; } } impl MctpHeader { /// Builds a single-packet MCTP header (`SOM=1`, `EOM=1`, `SEQ=0`, `SEID=0`). pub(crate) fn single_packet() -> Self { - Self::default().set_som(true).set_eom(true) + Self::zeroed().with_som(true).with_eom(true) } /// Returns whether this is a complete single-packet message (`SOM=1` and `EOM=1`). pub(crate) fn is_single_packet(self) -> bool { - self.som() && self.eom() + self.som().into_bool() && self.eom().into_bool() } } @@ -63,26 +61,30 @@ impl MctpHeader { const MSG_TYPE_VENDOR_PCI: u8 = 0x7e; bitfield! { - pub(crate) struct NvdmHeader(u32), "NVIDIA Vendor-Defined Message header over MCTP." { - 31:24 nvdm_type as u8 ?=> NvdmType, "NVDM message type."; - 23:8 vendor_id as u16, "PCI vendor ID."; - 6:0 msg_type as u8, "MCTP vendor-defined message type."; + /// NVIDIA Vendor-Defined Message header over MCTP. + pub(crate) struct NvdmHeader(u32) { + /// NVDM message type. + 31:24 nvdm_type ?=> NvdmType; + /// PCI vendor ID. + 23:8 vendor_id; + /// MCTP vendor-defined message type. + 6:0 msg_type; } } impl NvdmHeader { /// Builds an NVDM header for the given message type. pub(crate) fn new(nvdm_type: NvdmType) -> Self { - Self::default() - .set_msg_type(MSG_TYPE_VENDOR_PCI) - .set_vendor_id(Vendor::NVIDIA.as_raw()) - .set_nvdm_type(nvdm_type) + Self::zeroed() + .with_const_msg_type::<{ num::u8_as_u32(MSG_TYPE_VENDOR_PCI) }>() + .with_vendor_id(Vendor::NVIDIA.as_raw()) + .with_nvdm_type(nvdm_type) } /// Validates this header against the expected NVIDIA NVDM format and type. pub(crate) fn validate(self, expected_type: NvdmType) -> bool { - self.msg_type() == MSG_TYPE_VENDOR_PCI - && self.vendor_id() == Vendor::NVIDIA.as_raw() + u8::from(self.msg_type()) == MSG_TYPE_VENDOR_PCI + && u16::from(self.vendor_id()) == Vendor::NVIDIA.as_raw() && matches!(self.nvdm_type(), Ok(nvdm_type) if nvdm_type == expected_type) } } diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs index 9f0199f7b38c..35a8b1214b0e 100644 --- a/drivers/gpu/nova-core/nova_core.rs +++ b/drivers/gpu/nova-core/nova_core.rs @@ -10,9 +10,6 @@ use kernel::{ InPlaceModule, // }; -#[macro_use] -mod bitfield; - mod driver; mod falcon; mod fb; @@ -26,6 +23,7 @@ mod num; mod regs; mod sbuffer; mod vbios; +mod vgpu; pub(crate) const MODULE_NAME: &core::ffi::CStr = <LocalModule as kernel::ModuleMetadata>::NAME; @@ -54,7 +52,7 @@ struct NovaCoreModule { impl InPlaceModule for NovaCoreModule { fn init(module: &'static kernel::ThisModule) -> impl PinInit<Self, Error> { - let dir = debugfs::Dir::new(kernel::c_str!("nova-core")); + let dir = debugfs::Dir::new(c"nova-core"); // SAFETY: We are the only driver code running during init, so there // cannot be any concurrent access to `DEBUGFS_ROOT`. diff --git a/drivers/gpu/nova-core/nova_core_exports.c b/drivers/gpu/nova-core/nova_core_exports.c new file mode 100644 index 000000000000..6e80ca9792ee --- /dev/null +++ b/drivers/gpu/nova-core/nova_core_exports.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +/* + * Exports Rust symbols from the `nova_core` crate for use by dependent modules. + * + * This is a workaround until the build system supports Rust cross-module + * dependencies natively. + */ + +#include <linux/export.h> + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym) + +#include "exports_nova_core_generated.h" diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs index 0f49c1ab83ad..caeef4d85874 100644 --- a/drivers/gpu/nova-core/regs.rs +++ b/drivers/gpu/nova-core/regs.rs @@ -109,130 +109,6 @@ impl kernel::fmt::Display for NV_PMC_BOOT_42 { register! { pub(crate) NV_PBUS_SW_SCRATCH(u32)[64] @ 0x00001400 {} - - /// Scratch register 0xe used as FRTS firmware error code. - pub(crate) NV_PBUS_SW_SCRATCH_0E_FRTS_ERR(u32) => NV_PBUS_SW_SCRATCH[0xe] { - 31:16 frts_err_code; - } -} - -// PFB - -register! { - /// Low bits of the physical system memory address used by the GPU to perform sysmembar - /// operations (see [`crate::fb::SysmemFlush`]). - pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR(u32) @ 0x00100c10 { - 31:0 adr_39_08; - } - - /// High bits of the physical system memory address used by the GPU to perform sysmembar - /// operations (see [`crate::fb::SysmemFlush`]). - pub(crate) NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI(u32) @ 0x00100c40 { - 23:0 adr_63_40; - } - - pub(crate) NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE(u32) @ 0x00100ce0 { - 30:30 ecc_mode_enabled => bool; - 9:4 lower_mag; - 3:0 lower_scale; - } - - pub(crate) NV_PFB_PRI_MMU_WPR2_ADDR_LO(u32) @ 0x001fa824 { - /// Bits 12..40 of the lower (inclusive) bound of the WPR2 region. - 31:4 lo_val; - } - - pub(crate) NV_PFB_PRI_MMU_WPR2_ADDR_HI(u32) @ 0x001fa828 { - /// Bits 12..40 of the higher (exclusive) bound of the WPR2 region. - 31:4 hi_val; - } -} - -/// Base of the GB10x HSHUB0 register window (`NV_HSHUB0_PRIV_BASE` in Open RM). -/// -/// The base is provided by the GB10x framebuffer HAL. -pub(crate) struct Hshub0Base(()); - -/// Base of the GB20x FBHUB0 register window (`NV_FBHUB0_PRI_BASE` in Open RM). -/// -/// The base is provided by the GB20x framebuffer HAL. -pub(crate) struct Fbhub0Base(()); - -register! { - // GB10x sysmem flush registers, relative to the HSHUB0 base. GB10x routes sysmembar - // through a primary and an EG (egress) pair that must both be programmed to the same - // address. Hardware ignores bits 7:0 of each LO register. The boot path uses a fixed - // HSHUB0 base, so the multiple runtime-discovered HSHUB bases are not needed here. - pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x00000e50 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_HSHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x00000e54 { - 19:0 adr; - } - - pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Hshub0Base + 0x000006c0 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_HSHUB_EG_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Hshub0Base + 0x000006c4 { - 19:0 adr; - } - - // GB20x sysmem flush registers, relative to the FBHUB0 base. Unlike the older - // NV_PFB_NISO_FLUSH_SYSMEM_ADDR registers which encode the address with an 8-bit - // right-shift, these take the raw address split into lower and upper halves. Hardware - // ignores bits 7:0 of the LO register. - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_LO(u32) @ Fbhub0Base + 0x00001d58 { - 31:0 adr => u32; - } - - pub(crate) NV_PFB_FBHUB_PCIE_FLUSH_SYSMEM_ADDR_HI(u32) @ Fbhub0Base + 0x00001d5c { - 19:0 adr; - } -} - -impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE { - /// Returns the usable framebuffer size, in bytes. - pub(crate) fn usable_fb_size(self) -> u64 { - let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M; - - if self.ecc_mode_enabled() { - // Remove the amount of memory reserved for ECC (one per 16 units). - size / 16 * 15 - } else { - size - } - } -} - -impl NV_PFB_PRI_MMU_WPR2_ADDR_LO { - /// Returns the lower (inclusive) bound of the WPR2 region. - pub(crate) fn lower_bound(self) -> u64 { - u64::from(self.lo_val()) << 12 - } -} - -impl NV_PFB_PRI_MMU_WPR2_ADDR_HI { - /// Returns the higher (exclusive) bound of the WPR2 region. - /// - /// A value of zero means the WPR2 region is not set. - pub(crate) fn higher_bound(self) -> u64 { - u64::from(self.hi_val()) << 12 - } - - /// Returns whether the WPR2 region is currently set. - pub(crate) fn is_wpr2_set(self) -> bool { - self.hi_val() != 0 - } -} - -// PGSP - -register! { - pub(crate) NV_PGSP_QUEUE_HEAD(u32) @ 0x00110c00 { - 31:0 address; - } } // PGC6 register space. @@ -294,28 +170,6 @@ impl NV_USABLE_FB_SIZE_IN_MB { } } -// PDISP - -register! { - pub(crate) NV_PDISP_VGA_WORKSPACE_BASE(u32) @ 0x00625f04 { - /// VGA workspace base address divided by 0x10000. - 31:8 addr; - /// Set if the `addr` field is valid. - 3:3 status_valid => bool; - } -} - -impl NV_PDISP_VGA_WORKSPACE_BASE { - /// Returns the base address of the VGA workspace, or `None` if none exists. - pub(crate) fn vga_workspace_addr(self) -> Option<u64> { - if self.status_valid() { - Some(u64::from(self.addr()) << 16) - } else { - None - } - } -} - // FUSE pub(crate) const NV_FUSE_OPT_FPF_SIZE: usize = 16; @@ -570,7 +424,7 @@ register! { /// GA102 and later. pub(crate) NV_PRISCV_RISCV_CPUCTL(u32) @ PFalcon2Base + 0x00000388 { 7:7 active_stat => bool; - 0:0 halted => bool; + 4:4 halted => bool; } /// GA102 and later. diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs index c6e6bfcd6a1f..c03650ee5226 100644 --- a/drivers/gpu/nova-core/vbios.rs +++ b/drivers/gpu/nova-core/vbios.rs @@ -13,11 +13,8 @@ use kernel::{ register, sizes::SZ_4K, sync::aref::ARef, - transmute::FromBytes, }; -use zerocopy::FromBytes as _; - use crate::{ driver::Bar0, firmware::{ @@ -359,7 +356,7 @@ impl Vbios { } /// PCI Data Structure as defined in PCI Firmware Specification -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] #[repr(C)] struct PcirStruct { /// PCI Data Structure signature ("PCIR" or "NPDS") @@ -388,15 +385,12 @@ struct PcirStruct { max_runtime_image_len: u16, } -// SAFETY: all bit patterns are valid for `PcirStruct`. -unsafe impl FromBytes for PcirStruct {} - impl PcirStruct { /// The bit in `last_image` that indicates the last image. const LAST_IMAGE_BIT_MASK: u8 = 0x80; fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { - let (pcir, _) = PcirStruct::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (pcir, _) = PcirStruct::read_from_prefix(data).map_err(|_| EINVAL)?; // Signature should be "PCIR" (0x52494350) or "NPDS" (0x5344504e). if &pcir.signature != b"PCIR" && &pcir.signature != b"NPDS" { @@ -432,7 +426,7 @@ impl PcirStruct { /// This is the head of the BIT table, that is used to locate the Falcon data. The BIT table (with /// its header) is in the [`PciAtBiosImage`] and the falcon data it is pointing to is in the /// [`FwSecBiosImage`]. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct BitHeader { /// 0h: BIT Header Identifier (BMP=0x7FFF/BIT=0xB8FF) @@ -451,12 +445,9 @@ struct BitHeader { checksum: u8, } -// SAFETY: all bit patterns are valid for `BitHeader`. -unsafe impl FromBytes for BitHeader {} - impl BitHeader { fn new(data: &[u8]) -> Result<Self> { - let (header, _) = BitHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (header, _) = BitHeader::read_from_prefix(data).map_err(|_| EINVAL)?; // Check header ID and signature if header.id != 0xB8FF || &header.signature != b"BIT\0" { @@ -468,7 +459,7 @@ impl BitHeader { } /// BIT Token Entry: Records in the BIT table followed by the BIT header. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct BitToken { /// 00h: Token identifier @@ -481,9 +472,6 @@ struct BitToken { data_offset: u16, } -// SAFETY: all bit patterns are valid for `BitToken`. -unsafe impl FromBytes for BitToken {} - impl BitToken { /// BIT token ID for Falcon data. const ID_FALCON_DATA: u8 = 0x70; @@ -508,7 +496,7 @@ impl BitToken { .and_then(|data| data.get(..entry_size)) .ok_or(EINVAL)?; - let (token, _) = BitToken::from_bytes_copy_prefix(entry).ok_or(EINVAL)?; + let (token, _) = BitToken::read_from_prefix(entry).map_err(|_| EINVAL)?; // Check if this token has the requested ID if token.id == token_id { @@ -525,7 +513,7 @@ impl BitToken { /// /// This header is at the beginning of every image in the set of images in the ROM. It contains a /// pointer to the PCI Data Structure which describes the image. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, FromBytes)] #[repr(C)] struct PciRomHeader { /// 00h: Signature (0xAA55) @@ -536,13 +524,10 @@ struct PciRomHeader { pci_data_struct_offset: u16, } -// SAFETY: all bit patterns are valid for `PciRomHeader`. -unsafe impl FromBytes for PciRomHeader {} - impl PciRomHeader { fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { - let (rom_header, _) = PciRomHeader::from_bytes_copy_prefix(data) - .ok_or(EINVAL) + let (rom_header, _) = PciRomHeader::read_from_prefix(data) + .map_err(|_| EINVAL) .inspect_err(|_| dev_err!(dev, "Not enough data for ROM header\n"))?; // Check for valid ROM signatures. @@ -564,7 +549,7 @@ impl PciRomHeader { /// PCI Data Structure. It contains some fields that are redundant with the PCI Data Structure, but /// are needed for traversing the BIOS images. It is expected to be present in all BIOS images /// except for NBSI images. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, FromBytes)] #[repr(C)] struct NpdeStruct { /// 00h: Signature ("NPDE") @@ -579,15 +564,12 @@ struct NpdeStruct { last_image: u8, } -// SAFETY: all bit patterns are valid for `NpdeStruct`. -unsafe impl FromBytes for NpdeStruct {} - impl NpdeStruct { /// The bit in `last_image` that indicates the last image. const LAST_IMAGE_BIT_MASK: u8 = 0x80; fn new(dev: &device::Device, data: &[u8]) -> Option<Self> { - let (npde, _) = NpdeStruct::from_bytes_copy_prefix(data)?; + let (npde, _) = NpdeStruct::read_from_prefix(data).ok()?; // Signature should be "NPDE" (0x4544504E). if &npde.signature != b"NPDE" { @@ -784,7 +766,7 @@ impl PciAtBiosImage { let data = &self.base.data; let (ptr, _) = data .get(offset..) - .and_then(u32::from_bytes_copy_prefix) + .and_then(|p| u32::read_from_prefix(p).ok()) .ok_or(EINVAL)?; usize::from_safe_cast(ptr) @@ -814,6 +796,7 @@ impl TryFrom<BiosImage> for PciAtBiosImage { /// The [`PmuLookupTableEntry`] structure is a single entry in the [`PmuLookupTable`]. /// /// See the [`PmuLookupTable`] description for more information. +#[derive(FromBytes)] #[repr(C, packed)] struct PmuLookupTableEntry { application_id: u8, @@ -821,9 +804,6 @@ struct PmuLookupTableEntry { data: u32, } -// SAFETY: all bit patterns are valid for `PmuLookupTableEntry`. -unsafe impl FromBytes for PmuLookupTableEntry {} - impl PmuLookupTableEntry { /// PMU lookup table application ID for firmware security license ucode. #[expect(dead_code)] @@ -836,6 +816,7 @@ impl PmuLookupTableEntry { } #[repr(C)] +#[derive(FromBytes)] struct PmuLookupTableHeader { version: u8, header_len: u8, @@ -843,9 +824,6 @@ struct PmuLookupTableHeader { entry_count: u8, } -// SAFETY: all bit patterns are valid for `PmuLookupTableHeader`. -unsafe impl FromBytes for PmuLookupTableHeader {} - /// The [`PmuLookupTableEntry`] structure is used to find the [`PmuLookupTableEntry`] for a given /// application ID. /// @@ -857,7 +835,7 @@ struct PmuLookupTable { impl PmuLookupTable { fn new(dev: &device::Device, data: &[u8]) -> Result<Self> { - let (header, _) = PmuLookupTableHeader::from_bytes_copy_prefix(data).ok_or(EINVAL)?; + let (header, _) = PmuLookupTableHeader::read_from_prefix(data).map_err(|_| EINVAL)?; let header_len = usize::from(header.header_len); let entry_len = usize::from(header.entry_len); @@ -872,8 +850,8 @@ impl PmuLookupTable { let mut entries = KVVec::with_capacity(entry_count, GFP_KERNEL)?; for i in 0..entry_count { - let (entry, _) = PmuLookupTableEntry::from_bytes_copy_prefix(&data[i * entry_len..]) - .ok_or(EINVAL)?; + let (entry, _) = PmuLookupTableEntry::read_from_prefix(&data[i * entry_len..]) + .map_err(|_| EINVAL)?; entries.push(entry, GFP_KERNEL)?; } @@ -929,15 +907,11 @@ impl FwSecBiosImage { let ver = data.get(1).copied().ok_or(EINVAL)?; match ver { 2 => { - let v2 = FalconUCodeDescV2::read_from_prefix(data) - .map_err(|_| EINVAL)? - .0; + let (v2, _) = FalconUCodeDescV2::read_from_prefix(data).map_err(|_| EINVAL)?; Ok(FalconUCodeDesc::V2(v2)) } 3 => { - let v3 = FalconUCodeDescV3::from_bytes_copy_prefix(data) - .ok_or(EINVAL)? - .0; + let (v3, _) = FalconUCodeDescV3::read_from_prefix(data).map_err(|_| EINVAL)?; Ok(FalconUCodeDesc::V3(v3)) } _ => { diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs new file mode 100644 index 000000000000..6b7e045acea8 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0 + +use core::num::NonZero; + +use kernel::{ + device, + pci, + prelude::*, // +}; + +use crate::{ + fsp::{ + Fsp, + VgpuMode, // + }, + gpu::Chipset, // +}; + +mod hal; + +/// vGPU state detected during GPU construction. +#[derive(Debug, Clone, Copy)] +pub(crate) enum VgpuState { + /// vGPU mode is not enabled for this boot. + Disabled, + /// vGPU mode is enabled for this boot. + Enabled { + /// Total number of SR-IOV VFs supported by this device. + total_vfs: NonZero<u16>, + }, +} + +/// vGPU state manager. +pub(crate) struct VgpuManager { + state: VgpuState, +} + +impl VgpuManager { + /// Creates a vGPU manager by querying SR-IOV and the FSP PRC vGPU knob. + pub(crate) fn new( + pdev: &pci::Device<device::Core<'_>>, + chipset: Chipset, + fsp: Option<&mut Fsp<'_>>, + ) -> Self { + let state = Self::detect_state(pdev, chipset, fsp).unwrap_or_else(|e| { + dev_warn!( + pdev, + "vGPU state detection failed: {:?}; disabling vGPU\n", + e + ); + VgpuState::Disabled + }); + dev_dbg!(pdev, "vGPU state: {:?}\n", state); + + Self { state } + } + + /// Detects the vGPU state from the chipset, SR-IOV capability and FSP PRC knob. + fn detect_state( + pdev: &pci::Device<device::Core<'_>>, + chipset: Chipset, + fsp: Option<&mut Fsp<'_>>, + ) -> Result<VgpuState> { + if !hal::vgpu_hal(chipset).supports_vgpu() { + return Ok(VgpuState::Disabled); + } + + let Some(total_vfs) = pdev.sriov_get_totalvfs() else { + return Ok(VgpuState::Disabled); + }; + + if total_vfs.get() < 2 { + // The current vGPU path does not support single-VF SR-IOV devices yet. + // Treat one total VF as vGPU-disabled for now; single-VF support can relax + // this gate once the manager handles that topology. + return Ok(VgpuState::Disabled); + } + + let fsp = fsp.ok_or(ENODEV)?; + + match fsp.read_vgpu_mode(pdev.as_ref())? { + VgpuMode::Enabled => Ok(VgpuState::Enabled { total_vfs }), + VgpuMode::Disabled => Ok(VgpuState::Disabled), + } + } + + /// Returns the detected vGPU state for this boot. + pub(crate) fn state(&self) -> VgpuState { + self.state + } +} diff --git a/drivers/gpu/nova-core/vgpu/hal.rs b/drivers/gpu/nova-core/vgpu/hal.rs new file mode 100644 index 000000000000..456f8fe27582 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +use crate::gpu::{ + Architecture, + Chipset, // +}; + +mod gb202; +mod tu102; + +pub(super) trait VgpuHal { + /// Returns whether this chipset can support vGPU. + fn supports_vgpu(&self) -> bool; +} + +pub(super) fn vgpu_hal(chipset: Chipset) -> &'static dyn VgpuHal { + match chipset.arch() { + Architecture::BlackwellGB20x => gb202::GB202_HAL, + Architecture::Turing + | Architecture::Ampere + | Architecture::Hopper + | Architecture::Ada + | Architecture::BlackwellGB10x => tu102::TU102_HAL, + } +} diff --git a/drivers/gpu/nova-core/vgpu/hal/gb202.rs b/drivers/gpu/nova-core/vgpu/hal/gb202.rs new file mode 100644 index 000000000000..3add8af26616 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal/gb202.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::vgpu::hal::VgpuHal; + +struct Gb202; + +impl VgpuHal for Gb202 { + fn supports_vgpu(&self) -> bool { + true + } +} + +const GB202: Gb202 = Gb202; +pub(super) const GB202_HAL: &dyn VgpuHal = &GB202; diff --git a/drivers/gpu/nova-core/vgpu/hal/tu102.rs b/drivers/gpu/nova-core/vgpu/hal/tu102.rs new file mode 100644 index 000000000000..baeea3ac5754 --- /dev/null +++ b/drivers/gpu/nova-core/vgpu/hal/tu102.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +use crate::vgpu::hal::VgpuHal; + +struct Tu102; + +impl VgpuHal for Tu102 { + fn supports_vgpu(&self) -> bool { + false + } +} + +const TU102: Tu102 = Tu102; +pub(super) const TU102_HAL: &dyn VgpuHal = &TU102; |