diff options
Diffstat (limited to 'rust/kernel/drm/gpuvm/mod.rs')
| -rw-r--r-- | rust/kernel/drm/gpuvm/mod.rs | 328 |
1 files changed, 328 insertions, 0 deletions
diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs new file mode 100644 index 000000000000..ae58f6f667c1 --- /dev/null +++ b/rust/kernel/drm/gpuvm/mod.rs @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT + +#![cfg(CONFIG_RUST_DRM_GPUVM)] + +//! DRM GPUVM in immediate mode +//! +//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated +//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM +//! and the GPU's virtual address space has the same state at all times. +//! +//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h) + +use kernel::{ + alloc::{ + AllocError, + Flags as AllocFlags, // + }, + bindings, + drm, + drm::gem::IntoGEMObject, + error::to_result, + prelude::*, + sync::aref::{ + ARef, + AlwaysRefCounted, // + }, + types::Opaque, // +}; + +use core::{ + cell::UnsafeCell, + marker::PhantomData, + mem::{ + ManuallyDrop, + MaybeUninit, // + }, + ops::{ + Deref, + DerefMut, + Range, // + }, + ptr::{ + self, + NonNull, // + }, // +}; + +mod sm_ops; +pub use self::sm_ops::*; + +mod vm_bo; +pub use self::vm_bo::*; + +mod va; +pub use self::va::*; + +/// A DRM GPU VA manager. +/// +/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed +/// via the special unique handle [`UniqueRefGpuVm`]. +/// +/// # Invariants +/// +/// * Stored in an allocation managed by the refcount in `self.vm`. +/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type. +/// * Does not contain any sparse [`GpuVa<T>`] instances. +#[pin_data] +pub struct GpuVm<T: DriverGpuVm> { + #[pin] + vm: Opaque<bindings::drm_gpuvm>, + /// Accessed only through the [`UniqueRefGpuVm`] reference. + data: UnsafeCell<T>, +} + +// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will +// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait. +unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {} +// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel. +unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {} + +// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. +unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> { + fn inc_ref(&self) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_get(self.vm.get()) }; + } + + unsafe fn dec_ref(obj: NonNull<Self>) { + // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`. + unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) }; + } +} + +impl<T: DriverGpuVm> PartialEq for GpuVm<T> { + #[inline] + fn eq(&self, other: &Self) -> bool { + core::ptr::eq(self.as_raw(), other.as_raw()) + } +} +impl<T: DriverGpuVm> Eq for GpuVm<T> {} + +impl<T: DriverGpuVm> GpuVm<T> { + const fn vtable() -> &'static bindings::drm_gpuvm_ops { + &bindings::drm_gpuvm_ops { + vm_free: Some(Self::vm_free), + op_alloc: None, + op_free: None, + vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN, + vm_bo_free: GpuVmBo::<T>::FREE_FN, + vm_bo_validate: None, + sm_step_map: Some(Self::sm_step_map), + sm_step_unmap: Some(Self::sm_step_unmap), + sm_step_remap: Some(Self::sm_step_remap), + } + } + + /// Creates a GPUVM instance. + #[expect(clippy::new_ret_no_self)] + pub fn new<E>( + name: &'static CStr, + dev: &drm::Device<T::Driver>, + r_obj: &T::Object, + range: Range<u64>, + reserve_range: Range<u64>, + data: T, + ) -> Result<UniqueRefGpuVm<T>, E> + where + E: From<AllocError>, + E: From<core::convert::Infallible>, + { + let obj = KBox::try_pin_init::<E>( + try_pin_init!(Self { + data: UnsafeCell::new(data), + vm <- Opaque::ffi_init(|vm| { + // SAFETY: These arguments are valid. `vm` is valid until refcount drops to + // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag + // below. + unsafe { + bindings::drm_gpuvm_init( + vm, + name.as_char_ptr(), + bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE + | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED, + dev.as_raw(), + r_obj.as_raw(), + range.start, + range.end - range.start, + reserve_range.start, + reserve_range.end - reserve_range.start, + const { Self::vtable() }, + ) + } + }), + }? E), + GFP_KERNEL | __GFP_ZERO, + )?; + // SAFETY: This transfers the initial refcount to the ARef. + let aref = unsafe { + ARef::from_raw(NonNull::new_unchecked(KBox::into_raw( + Pin::into_inner_unchecked(obj), + ))) + }; + // INVARIANT: This reference is unique. + Ok(UniqueRefGpuVm(aref)) + } + + /// Access this [`GpuVm`] from a raw pointer. + /// + /// # Safety + /// + /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains + /// valid for at least `'a`. + #[inline] + pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the + // pointer is valid for 'a. + unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) } + } + + /// Returns a raw pointer to the embedded `struct drm_gpuvm`. + #[inline] + pub fn as_raw(&self) -> *mut bindings::drm_gpuvm { + self.vm.get() + } + + /// The start of the VA space. + #[inline] + pub fn va_start(&self) -> u64 { + // SAFETY: The `mm_start` field is immutable. + unsafe { (*self.as_raw()).mm_start } + } + + /// The length of the GPU's virtual address space. + #[inline] + pub fn va_length(&self) -> u64 { + // SAFETY: The `mm_range` field is immutable. + unsafe { (*self.as_raw()).mm_range } + } + + /// Returns the range of the GPU virtual address space. + #[inline] + pub fn va_range(&self) -> Range<u64> { + let start = self.va_start(); + // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail. + let end = start + self.va_length(); + Range { start, end } + } + + /// Get or create the [`GpuVmBo`] for this gem object. + #[inline] + pub fn obtain( + &self, + obj: &T::Object, + data: impl PinInit<T::VmBoData>, + ) -> Result<ARef<GpuVmBo<T>>, AllocError> { + Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain()) + } + + /// Clean up buffer objects that are no longer used. + #[inline] + pub fn deferred_cleanup(&self) { + // SAFETY: This GPUVM uses immediate mode. + unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) } + } + + /// Check if this GEM object is an external object for this GPUVM. + #[inline] + pub fn is_extobj(&self, obj: &T::Object) -> bool { + // SAFETY: We may call this with any GPUVM and GEM object. + unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) } + } + + /// Free this GPUVM. + /// + /// # Safety + /// + /// Called when refcount hits zero. + unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) { + // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. + let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() }; + // SAFETY: By type invariants we can free it when refcount hits zero. + drop(unsafe { KBox::from_raw(me) }) + } + + #[inline] + fn raw_resv(&self) -> *mut bindings::dma_resv { + // SAFETY: `r_obj` is immutable and valid for duration of GPUVM. + unsafe { (*(*self.as_raw()).r_obj).resv } + } +} + +/// The manager for a GPUVM. +pub trait DriverGpuVm: Sized + Send { + /// Parent `Driver` for this object. + type Driver: drm::Driver<Object = Self::Object>; + + /// The kind of GEM object stored in this GPUVM. + type Object: IntoGEMObject; + + /// Data stored with each [`struct drm_gpuva`](struct@GpuVa). + type VaData; + + /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo). + type VmBoData; + + /// The private data passed to callbacks. + type SmContext<'ctx>; + + /// Indicates that a new mapping should be created. + fn sm_step_map<'op, 'ctx>( + &mut self, + op: OpMap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpMapped<'op, Self>, Error>; + + /// Indicates that an existing mapping should be removed. + fn sm_step_unmap<'op, 'ctx>( + &mut self, + op: OpUnmap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpUnmapped<'op, Self>, Error>; + + /// Indicates that an existing mapping should be split up. + fn sm_step_remap<'op, 'ctx>( + &mut self, + op: OpRemap<'op, Self>, + context: &mut Self::SmContext<'ctx>, + ) -> Result<OpRemapped<'op, Self>, Error>; +} + +/// The core of the DRM GPU VA manager. +/// +/// This object is a unique reference to the VM that can access the interval tree and the Rust +/// `data` field. +/// +/// # Invariants +/// +/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference. +pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>); + +// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and +// concurrent access to `data` is safe due to the `T: Sync` requirement. +unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {} + +impl<T: DriverGpuVm> UniqueRefGpuVm<T> { + /// Access the data owned by this `UniqueRefGpuVm` immutably. + #[inline] + pub fn data_ref(&self) -> &T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &*self.0.data.get() } + } + + /// Access the data owned by this `UniqueRefGpuVm` mutably. + #[inline] + pub fn data(&mut self) -> &mut T { + // SAFETY: By the type invariants we may access `data`. + unsafe { &mut *self.0.data.get() } + } +} + +impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> { + type Target = GpuVm<T>; + + #[inline] + fn deref(&self) -> &GpuVm<T> { + &self.0 + } +} |