summaryrefslogtreecommitdiff
path: root/rust/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'rust/kernel')
-rw-r--r--rust/kernel/bitfield.rs9
-rw-r--r--rust/kernel/debugfs.rs26
-rw-r--r--rust/kernel/debugfs/entry.rs4
-rw-r--r--rust/kernel/debugfs/file_ops.rs29
-rw-r--r--rust/kernel/device_id.rs3
-rw-r--r--rust/kernel/dma.rs141
-rw-r--r--rust/kernel/dma_buf/dma_fence.rs1022
-rw-r--r--rust/kernel/dma_buf/mod.rs14
-rw-r--r--rust/kernel/io.rs220
-rw-r--r--rust/kernel/io/register.rs759
-rw-r--r--rust/kernel/io/resource.rs8
-rw-r--r--rust/kernel/lib.rs3
-rw-r--r--rust/kernel/maple_tree.rs30
-rw-r--r--rust/kernel/mem.rs234
-rw-r--r--rust/kernel/pci.rs14
-rw-r--r--rust/kernel/sync/atomic.rs4
-rw-r--r--rust/kernel/uaccess.rs4
17 files changed, 1745 insertions, 779 deletions
diff --git a/rust/kernel/bitfield.rs b/rust/kernel/bitfield.rs
index a0d089423f21..15c78790e151 100644
--- a/rust/kernel/bitfield.rs
+++ b/rust/kernel/bitfield.rs
@@ -346,6 +346,15 @@ macro_rules! bitfield {
Self::from_raw(val)
}
}
+
+ // SAFETY: `$name` is transparent over `$storage` and `$storage` has no interior mutability.
+ unsafe impl $crate::mem::AsRepr for $name {
+ // Normalize `$storage` to the canonical repr type in case it is signed.
+ type Repr = <$storage as $crate::mem::AsRepr>::Repr;
+ }
+
+ // SAFETY: `$name` is transparent over `$storage`.
+ unsafe impl $crate::mem::AsReprMut for $name {}
};
// Definitions requiring knowledge of individual fields: private and public field accessors,
diff --git a/rust/kernel/debugfs.rs b/rust/kernel/debugfs.rs
index d7b8014a6474..2beb55d444ca 100644
--- a/rust/kernel/debugfs.rs
+++ b/rust/kernel/debugfs.rs
@@ -538,7 +538,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
}
}
- fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &'static FileOps<T>) {
+ fn create_file<T: Sync>(&self, name: &CStr, data: &'data T, vtable: &FileOps<T>) {
#[cfg(CONFIG_DEBUG_FS)]
core::mem::forget(Entry::file(name, &self.entry, data, vtable));
}
@@ -550,7 +550,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
- pub fn read_only_file<T: Writer + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
+ pub fn read_only_file<T: Writer + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}
@@ -560,11 +560,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
- pub fn read_binary_file<T: BinaryWriter + Send + Sync + 'static>(
- &self,
- name: &CStr,
- data: &'data T,
- ) {
+ pub fn read_binary_file<T: BinaryWriter + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}
@@ -596,11 +592,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
- pub fn read_write_file<T: Writer + Reader + Send + Sync + 'static>(
- &self,
- name: &CStr,
- data: &'data T,
- ) {
+ pub fn read_write_file<T: Writer + Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
let vtable = &<T as ReadWriteFile<_>>::FILE_OPS;
self.create_file(name, data, vtable)
}
@@ -612,7 +604,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
- pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync + 'static>(
+ pub fn read_write_binary_file<T: BinaryWriter + BinaryReader + Send + Sync>(
&self,
name: &CStr,
data: &'data T,
@@ -655,7 +647,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
/// This function does not produce an owning handle to the file. The created
/// file is removed when the [`Scope`] that this directory belongs
/// to is dropped.
- pub fn write_only_file<T: Reader + Send + Sync + 'static>(&self, name: &CStr, data: &'data T) {
+ pub fn write_only_file<T: Reader + Send + Sync>(&self, name: &CStr, data: &'data T) {
let vtable = &<T as WriteFile<_>>::FILE_OPS;
self.create_file(name, data, vtable)
}
@@ -666,11 +658,7 @@ impl<'data, 'dir> ScopedDir<'data, 'dir> {
///
/// This function does not produce an owning handle to the file. The created file is removed
/// when the [`Scope`] that this directory belongs to is dropped.
- pub fn write_binary_file<T: BinaryReader + Send + Sync + 'static>(
- &self,
- name: &CStr,
- data: &'data T,
- ) {
+ pub fn write_binary_file<T: BinaryReader + Send + Sync>(&self, name: &CStr, data: &'data T) {
self.create_file(name, data, &T::FILE_OPS)
}
diff --git a/rust/kernel/debugfs/entry.rs b/rust/kernel/debugfs/entry.rs
index 46aad64896ec..88a870d8c295 100644
--- a/rust/kernel/debugfs/entry.rs
+++ b/rust/kernel/debugfs/entry.rs
@@ -74,7 +74,7 @@ impl Entry<'static> {
parent.as_ptr(),
core::ptr::from_ref(data) as *mut c_void,
core::ptr::null(),
- &**file_ops,
+ file_ops.fops(),
)
};
@@ -127,7 +127,7 @@ impl<'a> Entry<'a> {
parent.as_ptr(),
core::ptr::from_ref(data) as *mut c_void,
core::ptr::null(),
- &**file_ops,
+ file_ops.fops(),
)
};
diff --git a/rust/kernel/debugfs/file_ops.rs b/rust/kernel/debugfs/file_ops.rs
index f15908f71c4a..47acc4851d47 100644
--- a/rust/kernel/debugfs/file_ops.rs
+++ b/rust/kernel/debugfs/file_ops.rs
@@ -20,9 +20,6 @@ use crate::{
use core::marker::PhantomData;
-#[cfg(CONFIG_DEBUG_FS)]
-use core::ops::Deref;
-
/// # Invariant
///
/// `FileOps<T>` will always contain an `operations` which is safe to use for a file backed
@@ -30,7 +27,7 @@ use core::ops::Deref;
/// into a reference.
pub(super) struct FileOps<T> {
#[cfg(CONFIG_DEBUG_FS)]
- operations: bindings::file_operations,
+ operations: &'static bindings::file_operations,
#[cfg(CONFIG_DEBUG_FS)]
mode: u16,
_phantom: PhantomData<T>,
@@ -41,7 +38,7 @@ impl<T> FileOps<T> {
///
/// The caller asserts that the provided `operations` is safe to use for a file whose
/// inode has a pointer to `T` in its private data that is safe to convert into a reference.
- const unsafe fn new(operations: bindings::file_operations, mode: u16) -> Self {
+ const unsafe fn new(operations: &'static bindings::file_operations, mode: u16) -> Self {
Self {
#[cfg(CONFIG_DEBUG_FS)]
operations,
@@ -65,11 +62,11 @@ impl<T: Adapter> FileOps<T> {
}
#[cfg(CONFIG_DEBUG_FS)]
-impl<T> Deref for FileOps<T> {
- type Target = bindings::file_operations;
-
- fn deref(&self) -> &Self::Target {
- &self.operations
+impl<T> FileOps<T> {
+ /// Returns a `'static` reference to the inner `file_operations`.
+ #[inline]
+ pub(crate) fn fops(&self) -> &'static bindings::file_operations {
+ self.operations
}
}
@@ -130,7 +127,7 @@ pub(crate) trait ReadFile<T> {
impl<T: Writer + Sync> ReadFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
read: Some(bindings::seq_read),
llseek: Some(bindings::seq_lseek),
release: Some(bindings::single_release),
@@ -181,7 +178,7 @@ pub(crate) trait ReadWriteFile<T> {
impl<T: Writer + Reader + Sync> ReadWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
open: Some(writer_open::<T>),
read: Some(bindings::seq_read),
write: Some(write::<T>),
@@ -238,7 +235,7 @@ pub(crate) trait WriteFile<T> {
impl<T: Reader + Sync> WriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
open: Some(write_only_open),
write: Some(write_only_write::<T>),
llseek: Some(bindings::noop_llseek),
@@ -290,7 +287,7 @@ pub(crate) trait BinaryReadFile<T> {
impl<T: BinaryWriter + Sync> BinaryReadFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
read: Some(blob_read::<T>),
llseek: Some(bindings::default_llseek),
open: Some(bindings::simple_open),
@@ -344,7 +341,7 @@ pub(crate) trait BinaryWriteFile<T> {
impl<T: BinaryReader + Sync> BinaryWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
write: Some(blob_write::<T>),
llseek: Some(bindings::default_llseek),
open: Some(bindings::simple_open),
@@ -368,7 +365,7 @@ pub(crate) trait BinaryReadWriteFile<T> {
impl<T: BinaryWriter + BinaryReader + Sync> BinaryReadWriteFile<T> for T {
const FILE_OPS: FileOps<T> = {
- let operations = bindings::file_operations {
+ let operations = &bindings::file_operations {
read: Some(blob_read::<T>),
write: Some(blob_write::<T>),
llseek: Some(bindings::default_llseek),
diff --git a/rust/kernel/device_id.rs b/rust/kernel/device_id.rs
index c81fca5b4986..f0b9cb84e58e 100644
--- a/rust/kernel/device_id.rs
+++ b/rust/kernel/device_id.rs
@@ -146,8 +146,7 @@ impl<T: RawDeviceId, const N: usize> IdArray<T, (), N> {
/// If the device implements [`RawDeviceIdIndex`], consider using [`IdArray::new`] instead.
pub const fn new_without_index(ids: [T; N]) -> Self {
// SAFETY: `T` is layout-wise compatible with `T::RawType`, so is the array of them.
- let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { core::mem::transmute_copy(&ids) };
- core::mem::forget(ids);
+ let raw_ids: [MaybeUninit<T::RawType>; N] = unsafe { crate::mem::transmute(ids) };
Self {
ids: raw_ids,
diff --git a/rust/kernel/dma.rs b/rust/kernel/dma.rs
index 2ce09f8e90c6..4ce914b7d1da 100644
--- a/rust/kernel/dma.rs
+++ b/rust/kernel/dma.rs
@@ -24,7 +24,6 @@ use crate::{
},
prelude::*,
ptr::KnownSize,
- sync::aref::ARef,
transmute::{
AsBytes,
FromBytes, //
@@ -223,7 +222,7 @@ impl DmaMask {
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let attribs = DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_WARN;
-/// let c: Coherent<[u64]> =
+/// let c: Coherent<'_, [u64]> =
/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, attribs)?;
/// # Ok::<(), Error>(()) }
/// ```
@@ -390,9 +389,9 @@ impl From<DataDirection> for bindings::dma_data_direction {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
-/// let mut dmem: CoherentBox<u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
+/// let mut dmem: CoherentBox<'_, u64> = CoherentBox::zeroed(dev, GFP_KERNEL)?;
/// *dmem = 42;
-/// let dmem: Coherent<u64> = dmem.into();
+/// let dmem: Coherent<'_, u64> = dmem.into();
/// # Ok::<(), Error>(()) }
/// ```
///
@@ -410,18 +409,18 @@ impl From<DataDirection> for bindings::dma_data_direction {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
-/// let mut dmem: CoherentBox<[u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
+/// let mut dmem: CoherentBox<'_, [u64]> = CoherentBox::zeroed_slice(dev, 4, GFP_KERNEL)?;
/// dmem.fill(42);
-/// let dmem: Coherent<[u64]> = dmem.into();
+/// let dmem: Coherent<'_, [u64]> = dmem.into();
/// # Ok::<(), Error>(()) }
/// ```
-pub struct CoherentBox<T: KnownSize + ?Sized>(Coherent<T>);
+pub struct CoherentBox<'a, T: KnownSize + ?Sized>(Coherent<'a, T>);
-impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
+impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, [T]> {
/// [`CoherentBox`] variant of [`Coherent::zeroed_slice_with_attrs`].
#[inline]
pub fn zeroed_slice_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
count: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@@ -432,7 +431,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
/// Same as [CoherentBox::zeroed_slice_with_attrs], but with `dma::Attrs(0)`.
#[inline]
pub fn zeroed_slice(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
count: usize,
gfp_flags: kernel::alloc::Flags,
) -> Result<Self> {
@@ -480,14 +479,14 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
///
/// # fn test(dev: &Device<Bound>) -> Result {
/// let data = [0u8, 1u8, 2u8, 3u8];
- /// let c: CoherentBox<[u8]> =
+ /// let c: CoherentBox<'_, [u8]> =
/// CoherentBox::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
///
/// assert_eq!(c.deref(), &data);
/// # Ok::<(), Error>(()) }
/// ```
pub fn from_slice_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@@ -512,7 +511,7 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
/// `dma_attrs` is 0 by default.
#[inline]
pub fn from_slice(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
) -> Result<Self>
@@ -523,11 +522,11 @@ impl<T: AsBytes + FromBytes> CoherentBox<[T]> {
}
}
-impl<T: AsBytes + FromBytes> CoherentBox<T> {
+impl<'a, T: AsBytes + FromBytes> CoherentBox<'a, T> {
/// Same as [`CoherentBox::zeroed_slice_with_attrs`], but for a single element.
#[inline]
pub fn zeroed_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@@ -536,12 +535,12 @@ impl<T: AsBytes + FromBytes> CoherentBox<T> {
/// Same as [`CoherentBox::zeroed_slice`], but for a single element.
#[inline]
- pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
+ pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
}
}
-impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
+impl<T: KnownSize + ?Sized> Deref for CoherentBox<'_, T> {
type Target = T;
#[inline]
@@ -554,7 +553,7 @@ impl<T: KnownSize + ?Sized> Deref for CoherentBox<T> {
}
}
-impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
+impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<'_, T> {
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
// SAFETY:
@@ -565,9 +564,9 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> DerefMut for CoherentBox<T> {
}
}
-impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coherent<T> {
+impl<'a, T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<'a, T>> for Coherent<'a, T> {
#[inline]
- fn from(value: CoherentBox<T>) -> Self {
+ fn from(value: CoherentBox<'a, T>) -> Self {
value.0
}
}
@@ -588,26 +587,20 @@ impl<T: AsBytes + FromBytes + KnownSize + ?Sized> From<CoherentBox<T>> for Coher
/// to an allocated region of coherent memory and `dma_addr` is the DMA address base of the
/// region.
/// - The size in bytes of the allocation is equal to size information via pointer.
-// TODO
//
-// DMA allocations potentially carry device resources (e.g.IOMMU mappings), hence for soundness
-// reasons DMA allocation would need to be embedded in a `Devres` container, in order to ensure
-// that device resources can never survive device unbind.
-//
-// However, it is neither desirable nor necessary to protect the allocated memory of the DMA
-// allocation from surviving device unbind; it would require RCU read side critical sections to
-// access the memory, which may require subsequent unnecessary copies.
-//
-// Hence, find a way to revoke the device resources of a `Coherent`, but not the
-// entire `Coherent` including the allocated memory itself.
-pub struct Coherent<T: KnownSize + ?Sized> {
- dev: ARef<device::Device>,
+// The lifetime parameter ties DMA allocations to the device's bound scope, ensuring they are freed
+// before the device is unbound under normal circumstances. However, if a `Coherent` is leaked (e.g.
+// via `mem::forget`), device resources such as IOMMU mappings will not be released. Making all
+// constructors `unsafe` to prevent this is considered too restrictive for the common case; this
+// soundness hole is accepted for now.
+pub struct Coherent<'a, T: KnownSize + ?Sized> {
+ dev: &'a device::Device<Bound>,
dma_addr: DmaAddress,
cpu_addr: NonNull<T>,
dma_attrs: Attrs,
}
-impl<T: KnownSize + ?Sized> Coherent<T> {
+impl<T: KnownSize + ?Sized> Coherent<'_, T> {
/// Returns the size in bytes of this allocation.
#[inline]
pub fn size(&self) -> usize {
@@ -663,10 +656,10 @@ impl<T: KnownSize + ?Sized> Coherent<T> {
}
}
-impl<T: AsBytes + FromBytes> Coherent<T> {
+impl<'a, T: AsBytes + FromBytes> Coherent<'a, T> {
/// Allocates a region of `T` of coherent memory.
fn alloc_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@@ -692,9 +685,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
// INVARIANT:
// - We just successfully allocated a coherent region which is adequately sized for `T`,
// hence the cpu address is valid.
- // - We also hold a refcounted reference to the device.
+ // - `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Self {
- dev: dev.into(),
+ dev,
dma_addr,
cpu_addr,
dma_attrs,
@@ -716,13 +709,13 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
- /// let c: Coherent<[u64; 4]> =
+ /// let c: Coherent<'_, [u64; 4]> =
/// Coherent::zeroed_with_attrs(dev, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn zeroed_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
) -> Result<Self> {
@@ -732,14 +725,14 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// Performs the same functionality as [`Coherent::zeroed_with_attrs`], except the
/// `dma_attrs` is 0 by default.
#[inline]
- pub fn zeroed(dev: &device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
+ pub fn zeroed(dev: &'a device::Device<Bound>, gfp_flags: kernel::alloc::Flags) -> Result<Self> {
Self::zeroed_with_attrs(dev, gfp_flags, Attrs(0))
}
/// Same as [`Coherent::zeroed_with_attrs`], but instead of a zero-initialization the memory is
/// initialized with `init`.
pub fn init_with_attrs<E>(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
init: impl Init<T, E>,
@@ -764,7 +757,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// with `init`.
#[inline]
pub fn init<E>(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
gfp_flags: kernel::alloc::Flags,
init: impl Init<T, E>,
) -> Result<Self>
@@ -776,11 +769,11 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// Allocates a region of `[T; len]` of coherent memory.
fn alloc_slice_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
- ) -> Result<Coherent<[T]>> {
+ ) -> Result<Coherent<'a, [T]>> {
const {
assert!(
core::mem::size_of::<T>() > 0,
@@ -809,9 +802,9 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
// INVARIANT:
// - We just successfully allocated a coherent region which is adequately sized for
// `[T; len]`, hence the cpu address is valid.
- // - We also hold a refcounted reference to the device.
+ // - `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Coherent {
- dev: dev.into(),
+ dev,
dma_addr,
cpu_addr,
dma_attrs,
@@ -836,17 +829,17 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// };
///
/// # fn test(dev: &Device<Bound>) -> Result {
- /// let c: Coherent<[u64]> =
+ /// let c: Coherent<'_, [u64]> =
/// Coherent::zeroed_slice_with_attrs(dev, 4, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn zeroed_slice_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
- ) -> Result<Coherent<[T]>> {
+ ) -> Result<Coherent<'a, [T]>> {
Coherent::alloc_slice_with_attrs(dev, len, gfp_flags | __GFP_ZERO, dma_attrs)
}
@@ -854,10 +847,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// `dma_attrs` is 0 by default.
#[inline]
pub fn zeroed_slice(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
len: usize,
gfp_flags: kernel::alloc::Flags,
- ) -> Result<Coherent<[T]>> {
+ ) -> Result<Coherent<'a, [T]>> {
Self::zeroed_slice_with_attrs(dev, len, gfp_flags, Attrs(0))
}
@@ -876,18 +869,18 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// # fn test(dev: &Device<Bound>) -> Result {
/// let data = [0u8, 1u8, 2u8, 3u8];
/// // `c` has the same content as `data`.
- /// let c: Coherent<[u8]> =
+ /// let c: Coherent<'_, [u8]> =
/// Coherent::from_slice_with_attrs(dev, &data, GFP_KERNEL, DMA_ATTR_NO_WARN)?;
///
/// # Ok::<(), Error>(()) }
/// ```
#[inline]
pub fn from_slice_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
- ) -> Result<Coherent<[T]>>
+ ) -> Result<Coherent<'a, [T]>>
where
T: Copy,
{
@@ -898,10 +891,10 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
/// `dma_attrs` is 0 by default.
#[inline]
pub fn from_slice(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
data: &[T],
gfp_flags: kernel::alloc::Flags,
- ) -> Result<Coherent<[T]>>
+ ) -> Result<Coherent<'a, [T]>>
where
T: Copy,
{
@@ -909,7 +902,7 @@ impl<T: AsBytes + FromBytes> Coherent<T> {
}
}
-impl<T> Coherent<[T]> {
+impl<T> Coherent<'_, [T]> {
/// Returns the number of elements `T` in this allocation.
///
/// Note that this is not the size of the allocation in bytes, which is provided by
@@ -922,10 +915,10 @@ impl<T> Coherent<[T]> {
}
/// Note that the device configured to do DMA must be halted before this object is dropped.
-impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
+impl<T: KnownSize + ?Sized> Drop for Coherent<'_, T> {
fn drop(&mut self) {
let size = T::size(self.cpu_addr.as_ptr());
- // SAFETY: Device pointer is guaranteed as valid by the type invariant on `Device`.
+ // SAFETY: Device pointer is guaranteed as valid by the lifetime of this `Coherent`.
// The cpu address, and the dma address are valid due to the type invariants on
// `Coherent`.
unsafe {
@@ -942,15 +935,15 @@ impl<T: KnownSize + ?Sized> Drop for Coherent<T> {
// SAFETY: It is safe to send a `Coherent` to another thread if `T`
// can be sent to another thread.
-unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<T> {}
+unsafe impl<T: KnownSize + Send + ?Sized> Send for Coherent<'_, T> {}
// SAFETY: Sharing `&Coherent` across threads is safe if `T` is `Sync`, because all
// methods that access the buffer contents (`field_read`, `field_write`, `as_slice`,
// `as_slice_mut`) are `unsafe`, and callers are responsible for ensuring no data races occur.
// The safe methods only return metadata or raw pointers whose use requires `unsafe`.
-unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<T> {}
+unsafe impl<T: KnownSize + ?Sized + AsBytes + FromBytes + Sync> Sync for Coherent<'_, T> {}
-impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
+impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<'_, T> {
fn write_to_slice(
&self,
writer: &mut UserSliceWriter,
@@ -996,15 +989,15 @@ impl<T: KnownSize + AsBytes + ?Sized> debugfs::BinaryWriter for Coherent<T> {
/// - `size` is the allocation size in bytes as passed to `dma_alloc_attrs`.
/// - `dma_attrs` contains the attributes used for the allocation, always including
/// `DMA_ATTR_NO_KERNEL_MAPPING`.
-pub struct CoherentHandle {
- dev: ARef<device::Device>,
+pub struct CoherentHandle<'a> {
+ dev: &'a device::Device<Bound>,
dma_addr: DmaAddress,
cpu_handle: NonNull<c_void>,
size: usize,
dma_attrs: Attrs,
}
-impl CoherentHandle {
+impl<'a> CoherentHandle<'a> {
/// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
///
/// Additional DMA attributes may be passed via `dma_attrs`; `DMA_ATTR_NO_KERNEL_MAPPING` is
@@ -1012,7 +1005,7 @@ impl CoherentHandle {
///
/// Returns `EINVAL` if `size` is zero, `ENOMEM` if the allocation fails.
pub fn alloc_with_attrs(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
size: usize,
gfp_flags: kernel::alloc::Flags,
dma_attrs: Attrs,
@@ -1038,9 +1031,9 @@ impl CoherentHandle {
// INVARIANT: `cpu_handle` is the opaque handle from a successful `dma_alloc_attrs` call
// with `DMA_ATTR_NO_KERNEL_MAPPING`, `dma_addr` is the corresponding DMA address,
- // and we hold a refcounted reference to the device.
+ // and `dev` is a valid reference to a bound device that outlives this allocation.
Ok(Self {
- dev: dev.into(),
+ dev,
dma_addr,
cpu_handle,
size,
@@ -1051,7 +1044,7 @@ impl CoherentHandle {
/// Allocates `size` bytes of coherent DMA memory without creating a kernel virtual mapping.
#[inline]
pub fn alloc(
- dev: &device::Device<Bound>,
+ dev: &'a device::Device<Bound>,
size: usize,
gfp_flags: kernel::alloc::Flags,
) -> Result<Self> {
@@ -1073,7 +1066,7 @@ impl CoherentHandle {
}
}
-impl Drop for CoherentHandle {
+impl Drop for CoherentHandle<'_> {
fn drop(&mut self) {
// SAFETY: All values are valid by the type invariants on `CoherentHandle`.
// `cpu_handle` is the opaque handle from `dma_alloc_attrs` and is passed back unchanged.
@@ -1091,12 +1084,12 @@ impl Drop for CoherentHandle {
// SAFETY: `CoherentHandle` only holds a device reference, a DMA address, an opaque CPU handle,
// and a size. None of these are tied to a specific thread.
-unsafe impl Send for CoherentHandle {}
+unsafe impl Send for CoherentHandle<'_> {}
// SAFETY: `CoherentHandle` provides no CPU access to the underlying allocation. The only
// operations on `&CoherentHandle` are reading the DMA address and size, both of which are
// plain `Copy` values.
-unsafe impl Sync for CoherentHandle {}
+unsafe impl Sync for CoherentHandle<'_> {}
/// View type for `Coherent`.
///
@@ -1236,7 +1229,7 @@ impl<'a, T: ?Sized + KnownSize> IoBase<'a> for CoherentView<'a, T> {
}
}
-impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<T> {
+impl<'a, T: ?Sized + KnownSize> IoBase<'a> for &'a Coherent<'_, T> {
type Backend = CoherentIoBackend;
type Target = T;
diff --git a/rust/kernel/dma_buf/dma_fence.rs b/rust/kernel/dma_buf/dma_fence.rs
new file mode 100644
index 000000000000..18a43e1bb442
--- /dev/null
+++ b/rust/kernel/dma_buf/dma_fence.rs
@@ -0,0 +1,1022 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025-2026 Red Hat Inc.
+ * Author: Philipp Stanner <pstanner@redhat.com>
+ */
+
+//! DMA Fence support.
+//!
+//! Reference: <https://docs.kernel.org/driver-api/dma-buf.html#c.dma_fence>
+//!
+//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h)
+
+use crate::{
+ alloc::AllocError,
+ bindings,
+ container_of,
+ error::to_result,
+ prelude::*,
+ types::ForeignOwnable,
+ types::Opaque, //
+};
+
+use core::{
+ marker::PhantomData,
+ mem::ManuallyDrop,
+ ops::Deref,
+ ptr,
+ ptr::{
+ drop_in_place,
+ NonNull, //
+ }, //
+};
+
+use kernel::{
+ str::CString,
+ sync::{
+ aref::{
+ ARef,
+ AlwaysRefCounted, //
+ },
+ atomic::{
+ Atomic,
+ Relaxed, //
+ },
+ rcu::rcu_barrier, //
+ }, //
+};
+
+/// VTable for dma_fence backend_ops callbacks.
+//
+// Mandatory dma_fence backend_ops are implemented implicitly through
+// [`FenceContext`]. Additional ones shall get implemented on this trait.
+pub trait FenceContextOps {
+ /// The generic payload data for [`DriverFence`]s created on this fctx.
+ type FenceDataType: Send + Sync;
+}
+
+/// A dma-fence context. A fence context takes care of associating related fences
+/// with each other, providing each with raising sequence numbers and a common
+/// identifier.
+#[pin_data(PinnedDrop)]
+pub struct FenceContext<T: FenceContextOps + Send + Sync> {
+ /// The fence context number.
+ nr: u64,
+ /// The sequence number for the next fence created.
+ seqno: Atomic<u64>,
+ // The name parameters can be accessed by the dma_fence backend_ops. UAF
+ // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`.
+ /// The name of the driver this FenceContext's fences belong to.
+ driver_name: CString,
+ /// The name of the timeline this FenceContext's fences belong to.
+ timeline_name: CString,
+ /// The number of all unsignaled fences on this context.
+ // Used to prevent bugs due to forgotten fences.
+ //
+ // The lifetime on `DriverFence`s should typically prevent this from
+ // happening.
+ //
+ // However, we cannot fully guarantee in Rust that `DriverFence`s will not
+ // be forgotten, e.g., through `core::mem::forget()`. This could circumvent
+ // the lifetime which intends to enforce that all fences disappear before
+ // their context.
+ nr_of_unsignaled_fences: Atomic<usize>,
+ /// The user's data.
+ #[pin]
+ data: T,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> FenceContext<T> {
+ // This can later be extended as a vtable in case other parties need support
+ // for the more "exotic" callbacks.
+ const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops {
+ get_driver_name: Some(Self::get_driver_name),
+ get_timeline_name: Some(Self::get_timeline_name),
+ enable_signaling: None,
+ signaled: None,
+ // Deprecated.
+ wait: None,
+ // Must never be implemented for these abstractions.
+ release: None,
+ set_deadline: None,
+ };
+
+ /// Create a new `FenceContext`.
+ pub fn new<E>(
+ initial_seqno: u64,
+ driver_name: &CStr,
+ timeline_name: &CStr,
+ data: impl PinInit<T, E>,
+ ) -> impl PinInit<Self, Error>
+ where
+ Error: From<E>,
+ {
+ let driver_name = CString::try_from(driver_name);
+ let timeline_name = CString::try_from(timeline_name);
+ try_pin_init!(Self {
+ // SAFETY: `dma_fence_context_alloc()` merely works on a global
+ // atomic. Parameter `1` is the number of contexts we want to
+ // allocate.
+ nr: unsafe { bindings::dma_fence_context_alloc(1) },
+ seqno: Atomic::new(initial_seqno),
+ driver_name: driver_name?,
+ timeline_name: timeline_name?,
+ nr_of_unsignaled_fences: Atomic::new(0),
+ data <- data,
+ })
+ }
+
+ fn next_seqno(&self) -> u64 {
+ self.seqno.fetch_add(1, Relaxed)
+ }
+
+ /// Allocate the memory for a [`DriverFence`] and already store `data` inside.
+ ///
+ /// This is needed because many times, creation of a [`DriverFence`] must not
+ /// fail, and allocating might deadlock in some situations.
+ ///
+ /// The `data` you pass here must not perform any operations that are illegal
+ /// in atomic context in its [`Drop`] implementation.
+ pub fn new_fence_allocation(
+ &self,
+ data: T::FenceDataType,
+ ) -> Result<DriverFenceAllocation<'_, T>> {
+ let fence_data = DriverFenceData {
+ rcu_head: Default::default(),
+ // `inner` remains uninitialized until a `DriverFence` takes over.
+ inner: Fence {
+ inner: Opaque::uninit(),
+ },
+ fctx: self,
+ data,
+ };
+
+ // In order to support the C dma_fence callbacks, it is necessary for
+ // a `Fence` and a `DriverFence` to live in the same allocation,
+ // because the C backend passes a dma_fence, from which the driver most
+ // likely wants to be able to access its `data` in `DriverFence`.
+ //
+ // Hence, we need the manage the memory manually. It will be freed by the
+ // C backend automatically once the refcount within `Fence` drops to 0.
+ let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?;
+
+ Ok(DriverFenceAllocation {
+ data,
+ ops: &Self::OPS,
+ })
+ }
+
+ extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.driver_name.as_char_ptr()
+ }
+
+ extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.timeline_name.as_char_ptr()
+ }
+
+ /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides
+ /// within a [`Fence`], which in turn resides in a [`DriverFenceData`].
+ unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member.
+ let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast();
+
+ // SAFETY: Safe because of the comments directly above.
+ let fence_data = unsafe { &*fence_data_ptr };
+
+ fence_data.fctx
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceContextOps + Send + Sync> PinnedDrop for FenceContext<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Fence ops callbacks can be called on unsignaled fences. Since these
+ // callbacks can access the fence context and its data, it needs to be
+ // guaranteed that a context only drops after all associated
+ // `DriverFence`s have been dropped. This is unlikely to occur, but
+ // would result in silent UAF. Throw a panic to prevent that.
+ //
+ // TODO:
+ // It would be better if the fence context signals all forgotten fences
+ // itself. To do so, it would keep a list of unsignaled fences. That
+ // list's members would have to be pre-allocated (see
+ // `FenceCallback::new_fence_allocation()`).
+ if self.nr_of_unsignaled_fences.load(Relaxed) != 0 {
+ panic!("Forgotten fences in FenceContext.");
+ }
+
+ // Ensure that the driver cannot unload while there are still dma_fence
+ // callbacks running. At the same time, the RCU barrier addresses the
+ // problem inherited by the C backend, in which backend ops callbacks
+ // might be accessing the fence while it is being signaled (or shortly
+ // after). This could cause UAF access on the fence context's
+ // `fctx.driver_name` and `fctx.timeline_name`.
+ //
+ // Wait for the RCU callbacks in `DriverFence::drop`.
+ rcu_barrier();
+ }
+}
+
+/// Error type for fence callback registration.
+///
+/// Generic over `T` so that `AlreadySignaled` can return the callback to the
+/// caller, allowing it to reclaim any resources owned by the callback (e.g.,
+/// a fence handle that needs to be signaled).
+#[derive(Debug)]
+pub enum CallbackError<T> {
+ /// The fence was already signaled. The callback is returned so the caller
+ /// can extract owned resources without losing them.
+ AlreadySignaled(T),
+ /// Some other error occurred during registration.
+ Other(Error),
+}
+
+impl<T> From<CallbackError<T>> for Error {
+ #[inline]
+ fn from(err: CallbackError<T>) -> Self {
+ match err {
+ CallbackError::AlreadySignaled(_) => ENOENT,
+ CallbackError::Other(e) => e,
+ }
+ }
+}
+
+impl<T> From<AllocError> for CallbackError<T> {
+ #[inline]
+ fn from(e: AllocError) -> Self {
+ CallbackError::Other(Error::from(e))
+ }
+}
+
+/// Trait for callbacks that can be registered on fences.
+///
+/// When the fence signals, the callback will be invoked.
+///
+/// # Example
+///
+/// ```rust
+/// use kernel::dma_buf::FenceCallback;
+///
+/// struct MyCallback {
+/// // Your callback state here
+/// }
+///
+/// impl FenceCallback for MyCallback {
+/// fn on_signal(&mut self) {
+/// pr_info!("Fence signaled!\n");
+/// // Handle fence completion
+/// }
+/// }
+/// ```
+pub trait FenceCallback: Send + 'static {
+ /// Called when the fence is signaled.
+ ///
+ /// This is called from the fence signaling path, which may be in interrupt
+ /// context or with locks held, which is why `self` is only borrowed, so that
+ /// it cannot drop. Implementations must not sleep or perform
+ /// long-running operations.
+ ///
+ /// An implementation likely wants to inform itself (e.g., through a work item)
+ /// within this callback that the associated [`FenceCallbackRegistration`]
+ /// can now be dropped.
+ fn on_signal(&mut self);
+}
+
+/// A callback registration on a fence.
+///
+/// When this object is dropped, the callback is automatically removed if it
+/// hasn't been called yet.
+#[pin_data(PinnedDrop)]
+pub struct FenceCallbackRegistration<T: FenceCallback + 'static> {
+ #[pin]
+ callback_foreign: Opaque<bindings::dma_fence_cb>,
+ callback: ManuallyDrop<T>,
+ fence: ARef<Fence>,
+}
+
+impl<T: FenceCallback> FenceCallbackRegistration<T> {
+ /// Create a [`PinInit`] closure for registering a callback on a fence.
+ ///
+ /// The actual attempt at registering the callback will take place once you
+ /// call an allocator's `pin_init()` function.
+ ///
+ /// On success the callback is pinned in place and will fire when the fence
+ /// signals. On `AlreadySignaled` the callback is returned to the caller so
+ /// that owned resources can be reclaimed.
+ pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit<Self, CallbackError<T>> + 'a
+ where
+ T: 'a,
+ {
+ try_pin_init!(Self {
+ // We need to fully initialize the fence because after
+ // `dma_fence_add_callback()` ran, the callback might immediately
+ // get invoked.
+ callback: ManuallyDrop::new(callback),
+ fence: ARef::from(fence),
+ callback_foreign <- Opaque::try_ffi_init(|ptr| {
+ // SAFETY: `fence.inner.get()` is a valid, initialized `struct
+ // dma_fence`. `ptr` points to the `struct dma_fence_cb` field
+ // within the pinned allocation, so it remains valid until
+ // `dma_fence_remove_callback()` in `PinnedDrop` or until the
+ // callback fires.
+ let ret = unsafe {
+ to_result(bindings::dma_fence_add_callback(
+ fence.inner.get(),
+ ptr,
+ Some(Self::dma_fence_callback),
+ ))
+ };
+ match ret {
+ Ok(()) => Ok(()),
+ Err(e) => {
+ // SAFETY: We could not register the callback. Thus,
+ // C will not use it. So we can just take it back
+ // and pass it to the user again.
+ let cb_back = unsafe { ManuallyDrop::take(callback) };
+ if e == ENOENT {
+ Err(CallbackError::AlreadySignaled(cb_back))
+ } else {
+ Err(CallbackError::Other(e))
+ }
+ },
+ }
+ }),
+ }? CallbackError<T>)
+ }
+
+ /// Raw dma fence callback that is called by the C code.
+ ///
+ /// # Safety
+ ///
+ /// This is only called by the dma_fence subsystem with valid pointers.
+ unsafe extern "C" fn dma_fence_callback(
+ _fence: *mut bindings::dma_fence,
+ callback_foreign: *mut bindings::dma_fence_cb,
+ ) {
+ let ptr = Opaque::cast_from(callback_foreign).cast_mut();
+
+ // SAFETY: All callbacks we can receive here have been created in such a way that they are
+ // embedded into a `FenceCallbackRegistration`.
+ let reg: *mut Self = unsafe { container_of!(ptr, Self, callback_foreign) };
+
+ // SAFETY: `reg` is a valid `Self` pointer.
+ //
+ // The backend ensures synchronisation so whoever holds the registration object cannot drop
+ // it while this code is running. See `FenceCallbackRegistration::drop`.
+ unsafe { (*reg).callback.on_signal() };
+ }
+
+ /// Returns a reference to the fence this callback is registered on.
+ #[inline]
+ pub fn fence(&self) -> &Fence {
+ &self.fence
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceCallback> PinnedDrop for FenceCallbackRegistration<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Always call `dma_fence_remove_callback()`, even if the callback
+ // already ran. This is necessary for synchronization:
+ // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures
+ // that any in-flight `dma_fence_signal()` (which calls our callback
+ // while holding the same lock) has completed before we free the struct.
+ //
+ // Without this, Drop can race with a concurrent signal:
+ // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress)
+ // CPU1 (drop): skips lock -> frees struct
+ // CPU0: accesses fence_ref -> use-after-free
+ //
+ // When the callback has already fired, the signal path detached the
+ // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just
+ // sees an empty node and returns false — the lock acquisition is the
+ // only thing that matters.
+ //
+ // SAFETY: The fence pointer is valid and the cb was initialized by
+ // `dma_fence_add_callback()` during construction.
+ unsafe {
+ bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get())
+ };
+
+ // SAFETY: This is literally the drop implementation, so no one has
+ // dropped this so far; so we can do it now.
+ unsafe { ManuallyDrop::<T>::drop(self.project().callback) };
+ }
+}
+
+// SAFETY: FenceCallbackRegistration can be sent between threads.
+unsafe impl<T: FenceCallback> Send for FenceCallbackRegistration<T> {}
+
+// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can.
+unsafe impl<T: FenceCallback> Sync for FenceCallbackRegistration<T> where T: Sync {}
+
+/// The receiving counterpart of a [`DriverFence`].
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+#[repr(transparent)]
+pub struct Fence {
+ /// The actual dma_fence passed to C.
+ inner: Opaque<bindings::dma_fence>,
+}
+
+/// Guard helper for locking within this module.
+///
+/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles.
+/// It is never used outside of this module.
+// TODO: This should be made more canonical, probably by basing it on a
+// SpinLockIrqGuard once available.
+struct FenceGuard<'a> {
+ inner: &'a Fence,
+ flags: usize,
+}
+
+impl<'a> Deref for FenceGuard<'a> {
+ type Target = &'a Fence;
+
+ fn deref(&self) -> &Self::Target {
+ &self.inner
+ }
+}
+
+impl Drop for FenceGuard<'_> {
+ fn drop(&mut self) {
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, which lives as long as this function.
+ // When a `FenceGuard` exists, the lock has been taken by definition.
+ unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) };
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Send for Fence {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Sync for Fence {}
+
+impl Fence {
+ /// Check whether the fence was signaled at the moment of the function call.
+ ///
+ /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`]
+ /// has not yet been dropped. The reason is that the fence ops callbacks can
+ /// cause the fence to get signaled by the C backend.
+ #[inline]
+ pub fn is_signaled(&self) -> bool {
+ // We should not use `dma_fence_is_signaled_locked()` here, because
+ // according to the C backend's recommendations, that function is
+ // problematic and we should avoid calling that function with a lock
+ // held.
+
+ // SAFETY: Inner `fence` is valid because `self` is valid.
+ let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) };
+
+ // To be as robust as possible for the future we guarantee that an API
+ // caller can 100% rely on the signaling being completed (i.e., all
+ // fence callbacks ran), so we have to take the lock.
+ //
+ // The reason is that the C dma_fence backend currently does not
+ // carefully synchronize the `dma_fence_is_signaled()` function with the
+ // proper spinlock. This can lead to the function returning `true` while
+ // fence callbacks are still being executed. This can be mitigated by
+ // guarding the entire function with the spinlock.
+ //
+ // The fundamental reason is that the C backend currently does guard
+ // setting of the fence's signaled-bit with the fence's spinlock, but
+ // reading is done locklessly.
+ //
+ // See commit c8a5d5ea3ba6a.
+ let _ = self.lock();
+
+ ret
+ }
+
+ /// Lock the fence. A helper only to be used internally in this module.
+ fn lock(&self) -> FenceGuard<'_> {
+ let mut guard = FenceGuard {
+ inner: self,
+ flags: 0,
+ };
+
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, whose lifetime is tied to the guard
+ // object.
+ unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) };
+
+ guard
+ }
+
+ /// Get the fence's sequence number.
+ #[inline]
+ pub fn seqno(&self) -> u64 {
+ // SAFETY: Valid because `self` is valid.
+ unsafe { (*self.as_raw()).seqno }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.inner.get()
+ }
+
+ /// Create a [`Fence`] from a raw C [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must point to an initialized fence that is embedded into a [`Fence`].
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self {
+ // SAFETY: Safe as per the function's overall safety requirements.
+ unsafe { &*ptr.cast() }
+ }
+}
+
+// SAFETY: These implement the C backends refcounting methods which are proven
+// to work correctly.
+unsafe impl AlwaysRefCounted for Fence {
+ fn inc_ref(&self) {
+ // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`.
+ unsafe { bindings::dma_fence_get(self.as_raw()) }
+ }
+
+ unsafe fn dec_ref(ptr: NonNull<Self>) {
+ // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called
+ // the fence is by definition still valid.
+ let fence = unsafe { (*ptr.as_ptr()).inner.get() };
+
+ // SAFETY: `fence` was created validly above. When `dec_ref()` is called,
+ // there is by definition still a reference alive that can be put.
+ unsafe { bindings::dma_fence_put(fence) }
+ }
+}
+
+// Necessary to guarantee that `inner` always comes first and can be freed by C.
+// Also useful for using casts instead of container_of().
+#[repr(C)]
+#[pin_data]
+struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> {
+ #[pin]
+ /// The inner fence.
+ // Must always be the first member so that unsafe casting works; but also
+ // necessary so that the C backend can free the allocation (coming from our
+ // Rust code) with kfree_rcu().
+ inner: Fence,
+ /// Callback head for dropping this in a deferred manner through RCU.
+ rcu_head: bindings::callback_head,
+ /// Reference to access the FenceContext.
+ fctx: &'a FenceContext<T>,
+ /// The API user's data. It is essential that the data only performs
+ /// operations legal in atomic context in its [`Drop`] implementation.
+ #[pin]
+ data: T::FenceDataType,
+}
+
+/// A synchronization primitive mainly for GPU drivers.
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can be shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+///
+/// It is crucial that a [`DriverFence`] always correctly represents the state
+/// of the associated job on the hardware. Especially, it is strictly necessary
+/// that the owner ensures that all [`DriverFence`]s eventually get signaled.
+/// As a last resort, a [`DriverFence`] will signal itself if it drops
+/// unsignaled and print a warning.
+///
+/// This design intends to implement the [`bindings::dma_fence_ops`] in such a
+/// way that the driver-data necessary to implement the callback's functionality
+/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a
+/// reference to the context, which can be accessed in the callbacks. The
+/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its
+/// [`FenceContext`]. Unfortunately, this can be circumvented under certain
+/// circumstances in Rust (e.g., usage of [`core::mem::forget`]).
+///
+/// In the unlikely case of such violations, a panic is thrown.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::{
+/// dma_buf::{
+/// DriverFence,
+/// FenceContext,
+/// FenceContextOps,
+/// FenceCallback,
+/// FenceCallbackRegistration,
+/// },
+/// str::CString,
+/// sync::aref::ARef, //
+/// };
+/// use core::fmt::Display;
+///
+/// struct CallbackData { }
+///
+/// impl FenceCallback for CallbackData {
+/// fn on_signal(&mut self) {
+/// pr_info!("DmaFence callback executed.\n");
+/// }
+/// }
+///
+/// #[pin_data]
+/// struct FenceContextData {}
+///
+/// impl FenceContextData {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {})
+/// }
+/// }
+///
+/// impl FenceContextOps for FenceContextData {
+/// type FenceDataType = FenceData;
+/// }
+///
+/// let fctx_data = FenceContextData::new();
+///
+///
+/// let mut fctx = KBox::pin_init(
+/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data),
+/// GFP_KERNEL
+/// )?;
+///
+/// struct FenceData {
+/// data: CString,
+/// }
+///
+/// let fence_data = FenceData { data: c"dummy_data".try_into()? };
+///
+/// let fence_alloc = fctx.new_fence_allocation(fence_data)?;
+/// let mut fence = fence_alloc.new_fence();
+///
+/// let cb_data = CallbackData { };
+/// let waiting_fence = ARef::from(fence.as_fence());
+/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data);
+/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?;
+///
+/// // TODO signalling guards
+/// assert_eq!(waiting_fence.is_signaled(), false);
+/// fence.signal(Ok(()));
+/// assert_eq!(waiting_fence.is_signaled(), true);
+///
+/// Ok::<(), Error>(())
+/// ```
+pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> {
+ /// The actual content of the fence. Lives in a [`NonNull`] so that its
+ /// memory can be managed independently. Valid until both the [`DriverFence`]
+ /// and all associated [`Fence`]s have disappeared.
+ data: NonNull<DriverFenceData<'a, T>>,
+}
+
+/// A pre-prepared DMA fence, carrying the user's data and the memory it and the
+/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting
+/// allocation and full initialization is necessary because fences cannot be
+/// allocated dynamically in some circumstances (deadlock).
+pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> {
+ /// The memory for the actual content of the fence.
+ /// Handed over to a [`DriverFence`], or deallocated once the
+ /// [`DriverFenceAllocation`] drops.
+ data: KBox<DriverFenceData<'a, T>>,
+ /// Reference for the ops for the associated [`FenceContext`]
+ ops: &'static bindings::dma_fence_ops,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> {
+ /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`].
+ ///
+ /// This increments the sequence number in the associated [`FenceContext`].
+ pub fn new_fence(self) -> DriverFence<'a, T> {
+ // We feed the C dma_fence backend a NULL for the spinlock so that it
+ // uses per-fence locks automatically.
+ let null_ptr: *mut bindings::spinlock = ptr::null_mut();
+ let seqno = self.data.fctx.next_seqno();
+ let fence_ptr = self.as_raw();
+ // SAFETY: `fence_ptr` has been created directly above. It will live
+ // at least as long as `Self`. The same applies to `&Self::OPS`.
+ unsafe {
+ bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno)
+ };
+
+ self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed);
+
+ // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so
+ // that `DriverFence`s can always be created without allocating. In this
+ // method, ownership over that memory is transferred to the new
+ // `DriverFence` and managed through refcounting. The C dma_fence
+ // backend will ultimately free the memory once the refcount reaches 0.
+ let ptr = KBox::into_raw(self.data);
+ // SAFETY: `ptr` was just created validly directly above.
+ let ptr = unsafe { NonNull::new_unchecked(ptr) };
+
+ DriverFence { data: ptr }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.data.inner.inner.get()
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> {
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ // SAFETY: Valid because `self` is valid.
+ let fence_data = unsafe { &*self.data.as_ptr() };
+
+ fence_data.inner.inner.get()
+ }
+
+ /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through
+ /// a [`DriverFence`] with matching generic data for both fence and associated
+ /// [`FenceContext`].
+ unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // DriverFenceData is `repr(C)` and a Fence is its first member.
+ let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>;
+
+ // SAFETY: `fence_data_ptr` was created validly above.
+ let data = unsafe { NonNull::new_unchecked(fence_data_ptr) };
+
+ Self { data }
+ }
+
+ /// Return the underlying [`Fence`].
+ #[inline]
+ pub fn as_fence(&self) -> &Fence {
+ // SAFETY: `self` is by definition still valid, and it cannot drop until
+ // this new reference is gone.
+ unsafe { Fence::from_raw(self.as_raw()) }
+ }
+
+ /// Signal the fence. This will invoke all registered callbacks.
+ pub fn signal(self, res: Result) {
+ let fence = self.as_fence().lock();
+
+ // SAFETY: `fence` is valid because `self` is valid. The lock must be
+ // held, which we acquired directly above.
+ if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } {
+ if let Err(err) = res {
+ // SAFETY: `fence` is valid because `self` is valid. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) };
+ }
+ // SAFETY: `fence` is valid because `self` is valid. The lock must
+ // be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) };
+ }
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> {
+ type Target = T::FenceDataType;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is.
+ let data = unsafe { &*self.data.as_ptr() };
+
+ &data.data
+ }
+}
+
+/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`].
+pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> {
+ driver_fence: ManuallyDrop<DriverFence<'a, T>>,
+ _lifetime: PhantomData<&'a T>,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> {
+ type Target = DriverFence<'a, T>;
+
+ fn deref(&self) -> &Self::Target {
+ self.driver_fence.deref()
+ }
+}
+
+// SAFETY: The Rust dma_fence abstractions are already designed around the inner
+// C `dma_fence`, which can serve safely as the identification point when being
+// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by
+// only allowing operations without side effects on the Borrowed type.
+unsafe impl<T: Send + Sync + FenceContextOps> ForeignOwnable for DriverFence<'_, T> {
+ type Borrowed<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+ type BorrowedMut<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+
+ const FOREIGN_ALIGN: usize = core::mem::align_of::<bindings::dma_fence>();
+
+ fn into_foreign(self) -> *mut c_void {
+ let fence = self;
+
+ let ptr = fence.as_raw();
+
+ // DriverFence must not drop.
+ let _ = ManuallyDrop::new(fence);
+
+ ptr.cast()
+ }
+
+ unsafe fn from_foreign(ptr: *mut c_void) -> Self {
+ // SAFETY: Safe because the trait implementation only invokes this with
+ // a valid `ptr`, associated to a `DriverFence` with matching generic data.
+ unsafe { Self::from_raw(ptr.cast()) }
+ }
+
+ unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+
+ unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>
+ // FIXME: The bound below and the one above in `borrow` should actually be
+ // unnecessary since the compiler should be able to completely derive all
+ // necessary information automatically. There is currently a compiler bug
+ // preventing that, though:
+ //
+ // https://github.com/rust-lang/rust/issues/155430.
+ //
+ // (Help to) fix the compiler bug and remove the bounds afterwards.
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> {
+ fn drop(&mut self) {
+ let guard = self.as_fence().lock();
+
+ // Use dma_fence_test_signaled_flag() instead of
+ // dma_fence_is_signaled_locked() because the C backend wants to get rid
+ // of the latter.
+
+ // SAFETY: `guard` is valid until the `call_rcu()` below.
+ let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) };
+ if !signaled {
+ pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n");
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) };
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The lock
+ // must be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) };
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+ drop(guard);
+
+ // `DriverFenceData` could be accessed through some dma_fence
+ // callbacks right now. Access is being revoked in principle above by
+ // signaling the fence, but since the C backend does not guarantee
+ // perfect full synchronization, we have to wait for one grace period to
+ // ensure that all accessors of `DriverFenceData` (through the
+ // dma_fence_ops accessible through a `Fence`) are gone.
+
+ if !core::mem::needs_drop::<T::FenceDataType>() {
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is
+ // valid and initialized. It is valid until the refcount drops to 0,
+ // which can earliest happen once we drop the `DriverFence`'s
+ // reference here.
+ unsafe { bindings::dma_fence_put(self.as_raw()) };
+ return;
+ }
+
+ // SAFETY: Valid because `self` is valid.
+ let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head };
+
+ // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was
+ // created validly above. The module must perform a `synchronize_rcu()`
+ // or `rcu_barrier()` call to guard against module unload.
+ unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::<T>)) };
+ }
+}
+
+// TODO:
+// The entire call_rcu() mechanism in the drop above and the code below would be
+// unnecessary if C's dma_fence_signal() could be reworked in a way that after it
+// ran, the caller knows that no fence_ops callbacks can be running anymore.
+// In other words, if the dma_fence backend would use its spinlock for full
+// synchronization.
+//
+// Then we could move the drop_in_place() and dma_fence_put() upwards into the
+// drop() implementation and call it a day.
+
+/// Finally really drop this `DriverFence<T>`
+///
+/// # Safety
+///
+/// `head` references the `rcu_head` field of an `DriverFenceData<T>`. All
+/// accessors to that `DriverFenceData<T>` must be gone by now. This must be
+/// ensured by signalling the associated `DriverFence<T>` and then waiting
+/// for a grace period until calling this function here.
+unsafe extern "C" fn drop_driver_fence_data<T: Send + Sync + FenceContextOps>(
+ head: *mut bindings::callback_head,
+) {
+ // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData<C>`.
+ let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) };
+
+ // SAFETY: `fence_data` was created validly above. All the fence's data will
+ // only drop below, but the raw pointer to the raw C `dma_fence` remains
+ // valid because the reference count is only decremented at the end of the
+ // function.
+ let fence = unsafe { (*fence_data).inner.inner.get() };
+
+ // SAFETY: `fence_data` was created validly above. The user has already
+ // dropped the only conventional accessor to the user data, the `DriverFence`,
+ // one grace period ago. All accessors are gone now.
+ unsafe { drop_in_place(&raw mut (*fence_data).data) };
+
+ // The inner `Fence` explicitly does not get dropped because there may be
+ // many more users / consumers, each holding their own reference.
+
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid
+ // and initialized. It is valid until the refcount drops to 0, which can
+ // earliest happen once we drop the `DriverFence`'s reference here.
+ unsafe { bindings::dma_fence_put(fence) };
+
+ // The actual memory the data associated with a `DriverFence` lives in
+ // gets freed by the C dma_fence backend once the fence's refcount reaches 0.
+}
diff --git a/rust/kernel/dma_buf/mod.rs b/rust/kernel/dma_buf/mod.rs
new file mode 100644
index 000000000000..4764a828642e
--- /dev/null
+++ b/rust/kernel/dma_buf/mod.rs
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+//! DMA-buf subsystem abstractions.
+
+pub mod dma_fence;
+
+pub use self::dma_fence::{
+ DriverFence,
+ Fence,
+ FenceCallback,
+ FenceCallbackRegistration,
+ FenceContext,
+ FenceContextOps, //
+};
diff --git a/rust/kernel/io.rs b/rust/kernel/io.rs
index 5ce9fd129068..de8ef8e2aec4 100644
--- a/rust/kernel/io.rs
+++ b/rust/kernel/io.rs
@@ -11,6 +11,10 @@ use core::{
use crate::{
bindings,
+ mem::{
+ AsRepr,
+ AsReprMut, //
+ },
prelude::*,
ptr::{
Alignment,
@@ -226,6 +230,17 @@ fn io_view<'a, IO: Io<'a>, U>(
Ok(unsafe { IO::Backend::project_view(view, projected_ptr) })
}
+/// Returns the primitive view of a I/O view.
+#[inline]
+fn io_view_as_repr<'a, IO: Io<'a, Target = T>, T: AsRepr>(
+ this: IO,
+) -> <IO::Backend as IoBackend>::View<'a, T::Repr> {
+ let view = this.as_view();
+
+ // SAFETY: `AsRepr` guarantees layout compatibility.
+ unsafe { IO::Backend::project_view(view, IO::Backend::as_ptr(view).cast::<T::Repr>()) }
+}
+
/// I/O backends.
///
/// This is an abstract representation to be implemented by arbitrary I/O
@@ -353,15 +368,12 @@ pub trait IoCopyable: IoBackend {
///
/// - The valid `Base` to operate on. For most registers, this should be [`Region`].
/// - The offset to access (returned by [`IoLoc::offset`]),
-/// - The width of the access (determined by [`IoLoc::IoType`]),
-/// - The type `T` in which the raw data is returned or provided.
+/// - The type `T` in which the data is returned or provided.
///
-/// `T` and `IoLoc::IoType` may differ: for instance, a typed register has `T` = the register type
-/// with its bitfields, and `IoType` = its backing primitive (e.g. `u32`).
+/// `T` is not necessarily the type for underlying I/O operation. Methods that take `IoLoc` have `T:
+/// AsRepr` bound and the `<T as AsRepr>::Repr` type would be used to perform I/O and converted to
+/// `T` instead.
pub trait IoLoc<Base: ?Sized, T> {
- /// Size ([`u8`], [`u16`], etc) of the I/O performed on the returned [`offset`](IoLoc::offset).
- type IoType: Into<T> + From<T>;
-
/// Consumes `self` and returns the offset of this location.
fn offset(self) -> usize;
}
@@ -372,8 +384,6 @@ macro_rules! impl_usize_ioloc {
($($ty:ty),*) => {
$(
impl<const SIZE: usize> IoLoc<Region<SIZE>, $ty> for usize {
- type IoType = $ty;
-
#[inline(always)]
fn offset(self) -> usize {
self
@@ -437,6 +447,45 @@ pub trait Io<'a>: IoBase<'a> {
self.len() == 0
}
+ /// Convert into a different typed I/O view.
+ ///
+ /// The target type must be known (statically) to be of the same or smaller size to current
+ /// type, and the current view must be properly aligned for the target type.
+ ///
+ /// # Examples
+ ///
+ /// ```no_run
+ /// use kernel::io::{
+ /// io_project,
+ /// Mmio,
+ /// Io,
+ /// Region,
+ /// };
+ /// #[derive(FromBytes, IntoBytes)]
+ /// #[repr(C)]
+ /// struct MyStruct { field: u32, }
+ ///
+ /// # fn test(mmio: &Mmio<'_, Region<0x1000>>) {
+ /// // let mmio: Mmio<'_, Region<0x1000>>;
+ /// let whole: Mmio<'_, MyStruct> = mmio.cast();
+ /// # }
+ /// ```
+ #[inline]
+ fn cast<U>(self) -> <Self::Backend as IoBackend>::View<'a, U>
+ where
+ Self::Target: FromBytes + IntoBytes,
+ U: FromBytes + IntoBytes,
+ {
+ let view = self.as_view();
+ let ptr = Self::Backend::as_ptr(view);
+
+ const_assert!(size_of::<U>() <= Self::Target::MIN_SIZE);
+ const_assert!(align_of::<U>() <= Self::Target::MIN_ALIGN.as_usize());
+
+ // SAFETY: We have checked bounds and alignment, so this is a valid projection.
+ unsafe { Self::Backend::project_view(view, ptr.cast()) }
+ }
+
/// Try to convert into a different typed I/O view.
///
/// A runtime check is performed to ensure that the target type is of same or smaller size to
@@ -498,10 +547,10 @@ pub trait Io<'a>: IoBase<'a> {
#[inline]
fn read_val(self) -> Self::Target
where
- Self::Backend: IoCapable<Self::Target>,
- Self::Target: Sized,
+ Self::Target: AsReprMut,
+ Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
{
- Self::Backend::io_read(self.as_view())
+ Self::Target::from_repr(Self::Backend::io_read(io_view_as_repr(self)))
}
/// Write a value to I/O.
@@ -520,10 +569,10 @@ pub trait Io<'a>: IoBase<'a> {
#[inline]
fn write_val(self, value: Self::Target)
where
- Self::Backend: IoCapable<Self::Target>,
- Self::Target: Sized,
+ Self::Target: AsRepr,
+ Self::Backend: IoCapable<<Self::Target as AsRepr>::Repr>,
{
- Self::Backend::io_write(self.as_view(), value)
+ Self::Backend::io_write(io_view_as_repr(self), Self::Target::into_repr(value))
}
/// Copy-read from I/O memory.
@@ -645,7 +694,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_read8(self, offset: usize) -> Result<u8>
where
- usize: IoLoc<Self::Target, u8, IoType = u8>,
+ usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.try_read(offset)
@@ -655,7 +704,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_read16(self, offset: usize) -> Result<u16>
where
- usize: IoLoc<Self::Target, u16, IoType = u16>,
+ usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.try_read(offset)
@@ -665,7 +714,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_read32(self, offset: usize) -> Result<u32>
where
- usize: IoLoc<Self::Target, u32, IoType = u32>,
+ usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.try_read(offset)
@@ -675,7 +724,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_read64(self, offset: usize) -> Result<u64>
where
- usize: IoLoc<Self::Target, u64, IoType = u64>,
+ usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.try_read(offset)
@@ -685,7 +734,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write8(self, value: u8, offset: usize) -> Result
where
- usize: IoLoc<Self::Target, u8, IoType = u8>,
+ usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.try_write(offset, value)
@@ -695,7 +744,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write16(self, value: u16, offset: usize) -> Result
where
- usize: IoLoc<Self::Target, u16, IoType = u16>,
+ usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.try_write(offset, value)
@@ -705,7 +754,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write32(self, value: u32, offset: usize) -> Result
where
- usize: IoLoc<Self::Target, u32, IoType = u32>,
+ usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.try_write(offset, value)
@@ -715,7 +764,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write64(self, value: u64, offset: usize) -> Result
where
- usize: IoLoc<Self::Target, u64, IoType = u64>,
+ usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.try_write(offset, value)
@@ -727,7 +776,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn read8(self, offset: usize) -> u8
where
- usize: IoLoc<Self::Target, u8, IoType = u8>,
+ usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.read(offset)
@@ -739,7 +788,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn read16(self, offset: usize) -> u16
where
- usize: IoLoc<Self::Target, u16, IoType = u16>,
+ usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.read(offset)
@@ -751,7 +800,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn read32(self, offset: usize) -> u32
where
- usize: IoLoc<Self::Target, u32, IoType = u32>,
+ usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.read(offset)
@@ -763,7 +812,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn read64(self, offset: usize) -> u64
where
- usize: IoLoc<Self::Target, u64, IoType = u64>,
+ usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.read(offset)
@@ -775,7 +824,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write8(self, value: u8, offset: usize)
where
- usize: IoLoc<Self::Target, u8, IoType = u8>,
+ usize: IoLoc<Self::Target, u8>,
Self::Backend: IoCapable<u8>,
{
self.write(offset, value)
@@ -787,7 +836,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write16(self, value: u16, offset: usize)
where
- usize: IoLoc<Self::Target, u16, IoType = u16>,
+ usize: IoLoc<Self::Target, u16>,
Self::Backend: IoCapable<u16>,
{
self.write(offset, value)
@@ -799,7 +848,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write32(self, value: u32, offset: usize)
where
- usize: IoLoc<Self::Target, u32, IoType = u32>,
+ usize: IoLoc<Self::Target, u32>,
Self::Backend: IoCapable<u32>,
{
self.write(offset, value)
@@ -811,7 +860,7 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write64(self, value: u64, offset: usize)
where
- usize: IoLoc<Self::Target, u64, IoType = u64>,
+ usize: IoLoc<Self::Target, u64>,
Self::Backend: IoCapable<u64>,
{
self.write(offset, value)
@@ -843,11 +892,11 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_read<T, L>(self, location: L) -> Result<T>
where
+ T: AsReprMut,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
- let view = io_view::<Self, L::IoType>(self, location.offset())?;
- Ok(Self::Backend::io_read(view).into())
+ Ok(io_read!(self, try: location))
}
/// Generic fallible write with runtime bounds check.
@@ -876,12 +925,11 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write<T, L>(self, location: L, value: T) -> Result
where
+ T: AsRepr,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
- let view = io_view::<Self, L::IoType>(self, location.offset())?;
- let io_value = value.into();
- Self::Backend::io_write(view, io_value);
+ io_write!(self, try: location, value);
Ok(())
}
@@ -900,6 +948,8 @@ pub trait Io<'a>: IoBase<'a> {
/// };
///
/// register! {
+ /// base: Region;
+ ///
/// VERSION(u32) @ 0x100 {
/// 15:8 major;
/// 7:0 minor;
@@ -920,9 +970,10 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_write_reg<T, L, V>(self, value: V) -> Result
where
+ T: AsRepr,
L: IoLoc<Self::Target, T>,
V: LocatedRegister<Self::Target, Location = L, Value = T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let (location, value) = value.into_io_op();
@@ -954,16 +1005,13 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn try_update<T, L, F>(self, location: L, f: F) -> Result
where
+ T: AsReprMut,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
F: FnOnce(T) -> T,
{
- let view = io_view::<Self, L::IoType>(self, location.offset())?;
-
- let value: T = Self::Backend::io_read(view).into();
- let io_value = f(value).into();
- Self::Backend::io_write(view, io_value);
-
+ let view = io_project!(self, try: location);
+ view.write_val(f(view.read_val()));
Ok(())
}
@@ -991,11 +1039,11 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn read<T, L>(self, location: L) -> T
where
+ T: AsReprMut,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
- let view = io_view_assert::<Self, L::IoType>(self, location.offset());
- Self::Backend::io_read(view).into()
+ io_read!(self, build: location)
}
/// Generic infallible write with compile-time bounds check.
@@ -1022,12 +1070,11 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write<T, L>(self, location: L, value: T)
where
+ T: AsRepr,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
- let view = io_view_assert::<Self, L::IoType>(self, location.offset());
- let io_value = value.into();
- Self::Backend::io_write(view, io_value);
+ io_write!(self, build: location, value);
}
/// Generic infallible write of a fully-located register value.
@@ -1045,6 +1092,8 @@ pub trait Io<'a>: IoBase<'a> {
/// };
///
/// register! {
+ /// base: Region<0x1000>;
+ ///
/// VERSION(u32) @ 0x100 {
/// 15:8 major;
/// 7:0 minor;
@@ -1064,9 +1113,10 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn write_reg<T, L, V>(self, value: V)
where
+ T: AsRepr,
L: IoLoc<Self::Target, T>,
V: LocatedRegister<Self::Target, Location = L, Value = T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
{
let (location, value) = value.into_io_op();
@@ -1098,14 +1148,13 @@ pub trait Io<'a>: IoBase<'a> {
#[inline(always)]
fn update<T, L, F>(self, location: L, f: F)
where
+ T: AsReprMut,
L: IoLoc<Self::Target, T>,
- Self::Backend: IoCapable<L::IoType>,
+ Self::Backend: IoCapable<<T as AsRepr>::Repr>,
F: FnOnce(T) -> T,
{
- let view = io_view_assert::<Self, L::IoType>(self, location.offset());
- let value: T = Self::Backend::io_read(view).into();
- let io_value = f(value).into();
- Self::Backend::io_write(view, io_value);
+ let view = io_project!(self, build: location);
+ view.write_val(f(view.read_val()));
}
}
@@ -1649,6 +1698,25 @@ where
// SAFETY: Per safety requirement.
unsafe { T::Backend::project_view::<T::Target, _>(self.0, ptr) }
}
+
+ #[inline(always)]
+ pub fn try_project_loc<U, L>(
+ self,
+ location: L,
+ ) -> Result<<T::Backend as IoBackend>::View<'a, U>>
+ where
+ L: IoLoc<T::Target, U>,
+ {
+ io_view::<_, U>(self.0, location.offset())
+ }
+
+ #[inline(always)]
+ pub fn project_loc<U, L>(self, location: L) -> <T::Backend as IoBackend>::View<'a, U>
+ where
+ L: IoLoc<T::Target, U>,
+ {
+ io_view_assert::<_, U>(self.0, location.offset())
+ }
}
/// Project an I/O type to a subview of it.
@@ -1656,26 +1724,54 @@ where
/// The syntax is of form `io_project!(io, proj)` where `io` is an expression to a type that
/// implements [`Io`] and `proj` is a [projection specification](kernel::ptr::project!).
///
+/// `io_project!` can also project to a subview of registers defined with [`register!`] macro.
+/// Register projection has syntax `io_project!(io, try: REGISTER)` for fallible projection and
+/// `io_project!(io, build: REGISTER)` for infallible projection.
+///
/// # Examples
///
/// ```
/// use kernel::io::{
/// io_project,
+/// register,
/// Mmio,
/// };
/// #[repr(C)]
/// struct MyStruct { field: u32, }
///
+/// register! {
+/// base: MyStruct;
+/// FIELD(u32) @ 0 {
+/// 31:0 val;
+/// }
+/// }
+///
/// # fn test(mmio: Mmio<'_, [MyStruct]>) -> Result {
/// // let mmio: Mmio<[MyStruct]>;
/// let field: Mmio<'_, u32> = io_project!(mmio, [try: 1].field);
/// let whole: Mmio<'_, MyStruct> = io_project!(mmio, [try: 2]);
/// let nested: Mmio<'_, u32> = io_project!(whole, .field);
+/// let reg: Mmio<'_, FIELD> = io_project!(whole, build: FIELD);
/// # Ok::<(), Error>(()) }
/// ```
#[macro_export]
#[doc(hidden)]
macro_rules! io_project {
+ // Register projection
+ ($io:expr, try: $ioloc:expr) => {{
+ #[allow(unused)]
+ use $crate::io::IoBase as _;
+ let view = $crate::io::ProjectHelper($io.as_view());
+ view.try_project_loc($ioloc)?
+ }};
+ ($io:expr, build: $ioloc:expr) => {{
+ #[allow(unused)]
+ use $crate::io::IoBase as _;
+ let view = $crate::io::ProjectHelper($io.as_view());
+ view.project_loc($ioloc)
+ }};
+
+ // Field or index projection
($io:expr, $($proj:tt)*) => {{
#[allow(unused)]
use $crate::io::IoBase as _;
@@ -1746,6 +1842,12 @@ macro_rules! io_write {
(@parse [$io:expr] [$($proj:tt)*] [[$flavor:ident: $index:expr] $($rest:tt)*]) => {
$crate::io_write!(@parse [$io] [$($proj)* [$flavor: $index]] [$($rest)*])
};
+ (@parse [$io:expr] [] [try: $ioloc:expr, $($rest:tt)*]) => {
+ $crate::io_write!(@parse [$io] [try: $ioloc] [, $($rest)*])
+ };
+ (@parse [$io:expr] [] [build: $ioloc:expr, $($rest:tt)*]) => {
+ $crate::io_write!(@parse [$io] [build: $ioloc] [, $($rest)*])
+ };
($io:expr, $($rest:tt)*) => {
$crate::io_write!(@parse [$io] [] [$($rest)*])
};
diff --git a/rust/kernel/io/register.rs b/rust/kernel/io/register.rs
index 03dfd2ff48c7..b6513fa0f412 100644
--- a/rust/kernel/io/register.rs
+++ b/rust/kernel/io/register.rs
@@ -8,14 +8,19 @@
//!
//! Note: most of the items in this module are public so they can be referenced by the macro, but
//! most are not to be used directly by users. Outside of the `register!` macro itself, the only
-//! items you might want to import from this module are [`WithBase`] and [`Array`].
+//! item you might want to import from this module is [`Array`].
//!
//! # Simple example
//!
//! ```no_run
-//! use kernel::io::register;
+//! use kernel::io::{
+//! register,
+//! Region,
+//! };
//!
//! register! {
+//! base: Region<0x1000>;
+//!
//! /// Basic information about the chip.
//! pub BOOT_0(u32) @ 0x00000100 {
//! /// Vendor ID.
@@ -55,11 +60,14 @@
//! register,
//! Io,
//! IoLoc,
+//! Region,
//! },
//! num::Bounded,
//! };
-//! # use kernel::io::{Mmio, Region};
+//! # use kernel::io::Mmio;
//! # register! {
+//! # base: Region<0x1000>;
+//! #
//! # pub BOOT_0(u32) @ 0x00000100 {
//! # 15:8 vendor_id;
//! # 7:4 major_revision;
@@ -113,149 +121,50 @@ use crate::{
io::IoLoc, //
};
-use super::Region;
-
-/// Trait implemented by all registers.
-pub trait Register: Sized {
- /// Backing primitive type of the register.
- type Storage: Into<Self> + From<Self>;
-
- /// Start offset of the register.
- ///
- /// The interpretation of this offset depends on the type of the register.
- const OFFSET: usize;
-}
-
-/// Trait implemented by registers with a fixed offset.
-pub trait FixedRegister: Register {}
-
/// Allows `()` to be used as the `location` parameter of [`Io::write`](super::Io::write) when
-/// passing a [`FixedRegister`] value.
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for ()
+/// passing a [`FixedIoLoc`] value.
+impl<Base: ?Sized, T> IoLoc<Base, T> for ()
where
- T: FixedRegister,
+ T: FixedIoLoc<Base>,
{
- type IoType = T::Storage;
-
#[inline(always)]
fn offset(self) -> usize {
- T::OFFSET
+ T::LOCATION.offset()
}
}
-/// A [`FixedRegister`] carries its location in its type. Thus `FixedRegister` values can be used
-/// as an [`IoLoc`].
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for T
-where
- T: FixedRegister,
-{
- type IoType = T::Storage;
-
- #[inline(always)]
- fn offset(self) -> usize {
- T::OFFSET
- }
-}
-
-/// Location of a fixed register.
-pub struct FixedRegisterLoc<T: FixedRegister>(PhantomData<T>);
-
-impl<T: FixedRegister> FixedRegisterLoc<T> {
- /// Returns the location of `T`.
- #[inline(always)]
- // We do not implement `Default` so we can be const.
- #[expect(clippy::new_without_default)]
- pub const fn new() -> Self {
- Self(PhantomData)
- }
-}
-
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for FixedRegisterLoc<T>
-where
- T: FixedRegister,
-{
- type IoType = T::Storage;
-
- #[inline(always)]
- fn offset(self) -> usize {
- T::OFFSET
- }
-}
-
-/// Trait providing a base address to be added to the offset of a relative register to obtain
-/// its actual offset.
-///
-/// The `T` generic argument is used to distinguish which base to use, in case a type provides
-/// several bases. It is given to the `register!` macro to restrict the use of the register to
-/// implementors of this particular variant.
-pub trait RegisterBase<T> {
- /// Base address to which register offsets are added.
- const BASE: usize;
-}
-
-/// Trait implemented by all registers that are relative to a base.
-pub trait WithBase {
- /// Family of bases applicable to this register.
- type BaseFamily;
+// Provides a `IoLoc` impl that for a fixed offset.
+#[doc(hidden)]
+pub struct OffsetLoc<Base: ?Sized, T>(usize, PhantomData<(T, Base)>);
- /// Returns the absolute location of this type when using `B` as its base.
- #[inline(always)]
- fn of<B: RegisterBase<Self::BaseFamily>>() -> RelativeRegisterLoc<Self, B>
- where
- Self: Register,
- {
- RelativeRegisterLoc::new()
- }
-}
-
-/// Trait implemented by relative registers.
-pub trait RelativeRegister: Register + WithBase {}
-
-/// Location of a relative register.
-///
-/// This can either be an immediately accessible regular [`RelativeRegister`], or a
-/// [`RelativeRegisterArray`] that needs one additional resolution through
-/// [`RelativeRegisterLoc::at`].
-pub struct RelativeRegisterLoc<T: WithBase, B: ?Sized>(PhantomData<T>, PhantomData<B>);
-
-impl<T, B> RelativeRegisterLoc<T, B>
-where
- T: Register + WithBase,
- B: RegisterBase<T::BaseFamily> + ?Sized,
-{
- /// Returns the location of a relative register or register array.
- #[inline(always)]
- // We do not implement `Default` so we can be const.
- #[expect(clippy::new_without_default)]
- pub const fn new() -> Self {
- Self(PhantomData, PhantomData)
+impl<Base: ?Sized, T> OffsetLoc<Base, T> {
+ #[inline]
+ pub const fn new(offset: usize) -> Self {
+ Self(offset, PhantomData)
}
- // Returns the absolute offset of the relative register using base `B`.
- //
- // This is implemented as a private const method so it can be reused by the [`IoLoc`]
- // implementations of both [`RelativeRegisterLoc`] and [`RelativeRegisterArrayLoc`].
#[inline]
- const fn offset(self) -> usize {
- B::BASE + T::OFFSET
+ pub const fn const_offset(self) -> usize {
+ self.0
}
}
-impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterLoc<T, B>
-where
- T: RelativeRegister,
- B: RegisterBase<T::BaseFamily> + ?Sized,
-{
- type IoType = T::Storage;
-
+impl<Base: ?Sized, T> IoLoc<Base, T> for OffsetLoc<Base, T> {
#[inline(always)]
fn offset(self) -> usize {
- RelativeRegisterLoc::offset(self)
+ self.0
}
}
/// Trait implemented by arrays of registers.
-pub trait RegisterArray: Register {
+pub trait RegisterArray: Sized {
+ /// Base type for this register.
+ type Base: ?Sized;
+
+ /// Start offset of the register.
+ ///
+ /// The interpretation of this offset depends on the type of the register.
+ const OFFSET: usize;
/// Number of elements in the registers array.
const SIZE: usize;
/// Number of bytes between the start of elements in the registers array.
@@ -285,12 +194,10 @@ impl<T: RegisterArray> RegisterArrayLoc<T> {
}
}
-impl<const SIZE: usize, T> IoLoc<Region<SIZE>, T> for RegisterArrayLoc<T>
+impl<Base: ?Sized, T> IoLoc<Base, T> for RegisterArrayLoc<T>
where
- T: RegisterArray,
+ T: RegisterArray<Base = Base>,
{
- type IoType = T::Storage;
-
#[inline(always)]
fn offset(self) -> usize {
T::OFFSET + self.0 * T::STRIDE
@@ -318,71 +225,15 @@ pub trait Array {
}
}
-/// Trait implemented by arrays of relative registers.
-pub trait RelativeRegisterArray: RegisterArray + WithBase {}
-
-/// Location of a relative array register.
-pub struct RelativeRegisterArrayLoc<
- T: RelativeRegisterArray,
- B: RegisterBase<T::BaseFamily> + ?Sized,
->(RelativeRegisterLoc<T, B>, usize);
-
-impl<T, B> RelativeRegisterArrayLoc<T, B>
-where
- T: RelativeRegisterArray,
- B: RegisterBase<T::BaseFamily> + ?Sized,
-{
- /// Returns the location of register `T` from the base `B` at index `idx`, with build-time
- /// validation.
- #[inline(always)]
- pub fn new(idx: usize) -> Self {
- build_assert!(idx < T::SIZE);
-
- Self(RelativeRegisterLoc::new(), idx)
- }
-
- /// Attempts to return the location of register `T` from the base `B` at index `idx`, with
- /// runtime validation.
- #[inline(always)]
- pub fn try_new(idx: usize) -> Option<Self> {
- if idx < T::SIZE {
- Some(Self(RelativeRegisterLoc::new(), idx))
- } else {
- None
- }
- }
-}
-
-/// Methods exclusive to [`RelativeRegisterLoc`]s created with a [`RelativeRegisterArray`].
-impl<T, B> RelativeRegisterLoc<T, B>
-where
- T: RelativeRegisterArray,
- B: RegisterBase<T::BaseFamily> + ?Sized,
-{
- /// Returns the location of the register at position `idx`, with build-time validation.
- #[inline(always)]
- pub fn at(self, idx: usize) -> RelativeRegisterArrayLoc<T, B> {
- RelativeRegisterArrayLoc::new(idx)
- }
-
- /// Returns the location of the register at position `idx`, with runtime validation.
- #[inline(always)]
- pub fn try_at(self, idx: usize) -> Option<RelativeRegisterArrayLoc<T, B>> {
- RelativeRegisterArrayLoc::try_new(idx)
- }
-}
-
-impl<const SIZE: usize, T, B> IoLoc<Region<SIZE>, T> for RelativeRegisterArrayLoc<T, B>
-where
- T: RelativeRegisterArray,
- B: RegisterBase<T::BaseFamily> + ?Sized,
-{
- type IoType = T::Storage;
+/// Trait implemented by types that indicate there is a fixed I/O location for this given type.
+///
+/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
+pub trait FixedIoLoc<Base: ?Sized>: Sized {
+ /// Type of [`FixedIoLoc::LOCATION`].
+ type Location: IoLoc<Base, Self>;
- #[inline(always)]
- fn offset(self) -> usize {
- self.0.offset() + self.1 * T::STRIDE
- }
+ /// Location of this type within given base.
+ const LOCATION: Self::Location;
}
/// Trait implemented by items that contain both a register value and the absolute I/O location at
@@ -390,8 +241,8 @@ where
///
/// Implementors can be used with [`Io::write_reg`](super::Io::write_reg).
pub trait LocatedRegister<Base: ?Sized> {
- /// Register value to write.
- type Value: Register;
+ /// Value to write.
+ type Value;
/// Full location information at which to write the value.
type Location: IoLoc<Base, Self::Value>;
@@ -400,27 +251,38 @@ pub trait LocatedRegister<Base: ?Sized> {
fn into_io_op(self) -> (Self::Location, Self::Value);
}
-impl<const SIZE: usize, T> LocatedRegister<Region<SIZE>> for T
+impl<Base: ?Sized, T> LocatedRegister<Base> for T
where
- T: FixedRegister,
+ T: FixedIoLoc<Base>,
{
- type Location = FixedRegisterLoc<Self::Value>;
+ type Location = T::Location;
type Value = T;
#[inline(always)]
- fn into_io_op(self) -> (FixedRegisterLoc<T>, T) {
- (FixedRegisterLoc::new(), self)
+ fn into_io_op(self) -> (T::Location, T) {
+ (T::LOCATION, self)
}
}
+/// Helper function for register element alias implementation.
+///
+/// This is used to enforce base matching and provide bounds checking.
+#[doc(hidden)]
+#[inline(always)] // for const eval only
+pub const fn element_alias_offset<Base: ?Sized, Alias: RegisterArray<Base = Base>>(
+ idx: usize,
+) -> usize {
+ assert!(idx < Alias::SIZE);
+ Alias::OFFSET + idx * Alias::STRIDE
+}
+
/// Defines a dedicated type for a register, including getter and setter methods for its fields and
/// methods to read and write it from an [`Io`](kernel::io::Io) region.
///
/// This documentation focuses on how to declare registers. See the [module-level
/// documentation](mod@kernel::io::register) for examples of how to access them.
///
-/// There are 4 possible kinds of registers: fixed offset registers, relative registers, arrays of
-/// registers, and relative arrays of registers.
+/// Registers can either be fixed offset registers or arrays of registers.
///
/// ## Fixed offset registers
///
@@ -444,11 +306,14 @@ where
/// io::{
/// register,
/// Io,
+/// Region,
/// },
/// };
-/// # use kernel::io::{Mmio, Region};
+/// # use kernel::io::Mmio;
///
/// register! {
+/// base: Region<0x1000>;
+///
/// FIXED_REG(u32) @ 0x100 {
/// 15:8 high_byte;
/// 7:0 low_byte;
@@ -479,9 +344,14 @@ where
/// the context:
///
/// ```no_run
-/// use kernel::io::register;
+/// use kernel::io::{
+/// register,
+/// Region,
+/// };
///
/// register! {
+/// base: Region<0x1000>;
+///
/// /// Scratch register.
/// pub SCRATCH(u32) @ 0x00000200 {
/// 31:0 value;
@@ -497,113 +367,45 @@ where
/// In this example, `SCRATCH_BOOT_STATUS` uses the same I/O address as `SCRATCH`, while providing
/// its own `completed` field.
///
-/// ## Relative registers
-///
-/// Relative registers can be instantiated several times at a relative offset of a group of bases.
-/// For instance, imagine the following I/O space:
+/// If you do not wish to have a bitfield defined, you can also create a register using an existing
+/// type.
///
-/// ```text
-/// +-----------------------------+
-/// | ... |
-/// | |
-/// 0x100--->+------------CPU0-------------+
-/// | |
-/// 0x110--->+-----------------------------+
-/// | CPU_CTL |
-/// +-----------------------------+
-/// | ... |
-/// | |
-/// | |
-/// 0x200--->+------------CPU1-------------+
-/// | |
-/// 0x210--->+-----------------------------+
-/// | CPU_CTL |
-/// +-----------------------------+
-/// | ... |
-/// +-----------------------------+
-/// ```
-///
-/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
-/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
-/// them twice and would prefer a way to select which one to use from a single definition.
-///
-/// This can be done using the `Base + Offset` syntax when specifying the register's address:
-///
-/// ```ignore
+/// ```no_run
+/// # use kernel::io::*;
/// register! {
-/// pub RELATIVE_REG(u32) @ Base + 0x80 {
-/// ...
-/// }
+/// base: Region<0x1000>;
+///
+/// /// UART RX register.
+/// pub UART_RX: u8 @ 0x100;
/// }
/// ```
///
-/// This creates a register with an offset of `0x80` from a given base.
+/// In case there is a fixed register associated with a specific type in the base, you can apply
+/// `#[unique]` attribute which enables `write_reg` shorthand. This is automatically applied to
+/// bitfields instantiated via the `register!` macro.
///
-/// `Base` is an arbitrary type (typically a ZST) to be used as a generic parameter of the
-/// [`RegisterBase`] trait to provide the base as a constant, i.e. each type providing a base for
-/// this register needs to implement `RegisterBase<Base>`.
-///
-/// The location of relative registers can be built using the [`WithBase::of`] method to specify
-/// its base. All relative registers implement [`WithBase`].
-///
-/// Here is the above layout translated into code:
+/// This should only be used when types meaningfully represent a register. For example, in the
+/// previous `UART_RX` example, even if only a single register is defined with `u8` type, it is a
+/// bad idea to annotate it with `#[unique]`.
///
/// ```no_run
-/// use kernel::{
-/// io::{
-/// register,
-/// register::{
-/// RegisterBase,
-/// WithBase,
-/// },
-/// Io,
-/// },
-/// };
-/// # use kernel::io::{Mmio, Region};
-///
-/// // Type used to identify the base.
-/// pub struct CpuCtlBase;
-///
-/// // ZST describing `CPU0`.
-/// struct Cpu0;
-/// impl RegisterBase<CpuCtlBase> for Cpu0 {
-/// const BASE: usize = 0x100;
-/// }
-///
-/// // ZST describing `CPU1`.
-/// struct Cpu1;
-/// impl RegisterBase<CpuCtlBase> for Cpu1 {
-/// const BASE: usize = 0x200;
-/// }
+/// # use kernel::{bitfield, io::*};
///
-/// // This makes `CPU_CTL` accessible from all implementors of `RegisterBase<CpuCtlBase>`.
-/// register! {
-/// /// CPU core control.
-/// pub CPU_CTL(u32) @ CpuCtlBase + 0x10 {
-/// 0:0 start;
+/// bitfield! {
+/// pub struct Reset(u32) {
+/// 0:0 reset;
/// }
/// }
///
-/// # fn test(io: Mmio<'_, Region<0x1000>>) {
-/// // Read the status of `Cpu0`.
-/// let cpu0_started = io.read(CPU_CTL::of::<Cpu0>());
-///
-/// // Stop `Cpu0`.
-/// io.write(WithBase::of::<Cpu0>(), CPU_CTL::zeroed());
-/// # }
-///
-/// // Aliases can also be defined for relative register.
/// register! {
-/// /// Alias to CPU core control.
-/// pub CPU_CTL_ALIAS(u32) => CpuCtlBase + CPU_CTL {
-/// /// Start the aliased CPU core.
-/// 1:1 alias_start;
-/// }
+/// base: Region<0x1000>;
+///
+/// pub RESET: #[unique] Reset @ 0x100;
/// }
///
-/// # fn test2(io: Mmio<'_, Region<0x1000>>) {
-/// // Start the aliased `CPU0`, leaving its other fields untouched.
-/// io.update(CPU_CTL_ALIAS::of::<Cpu0>(), |r| r.with_alias_start(true));
+/// # fn test(mmio: Mmio<'_, Region<0x1000>>) {
+/// // let mmio: Mmio<'_, Region<0x1000>>;
+/// mmio.write_reg(Reset::zeroed().with_const_reset::<1>());
/// # }
/// ```
///
@@ -636,15 +438,18 @@ where
/// register,
/// register::Array,
/// Io,
+/// Region,
/// },
/// };
-/// # use kernel::io::{Mmio, Region};
+/// # use kernel::io::Mmio;
/// # fn get_scratch_idx() -> usize {
/// # 0x15
/// # }
///
/// // Array of 64 consecutive registers with the same layout starting at offset `0x80`.
/// register! {
+/// base: Region<0x1000>;
+///
/// /// Scratch registers.
/// pub SCRATCH(u32)[64] @ 0x00000080 {
/// 31:0 value;
@@ -670,6 +475,8 @@ where
/// // Alias to a specific register in an array.
/// // Here `SCRATCH[8]` is used to convey the firmware exit code.
/// register! {
+/// base: Region<0x1000>;
+///
/// /// Firmware exit status code.
/// pub FIRMWARE_STATUS(u32) => SCRATCH[8] {
/// 7:0 status;
@@ -682,6 +489,8 @@ where
/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
/// // registers of the two declarations below are interleaved.
/// register! {
+/// base: Region<0x1000>;
+///
/// /// Scratch registers bank 0.
/// pub SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ 0x000000c0 {
/// 31:0 value;
@@ -696,332 +505,88 @@ where
/// # }
/// ```
///
-/// ## Relative arrays of registers
+/// ## Relative registers
///
-/// Combining the two features described in the sections above, arrays of registers accessible from
-/// a base can also be defined:
+/// There are cases where a register region is subdivided into small subregions, and you may wish to
+/// have your register definition be relative to these subregions. This may be needed, for example,
+/// if these subregions are instantiated several times, or you just want it for encapsulation
+/// purpose.
///
-/// ```ignore
-/// register! {
-/// pub RELATIVE_REGISTER_ARRAY(u8)[10, stride = 4] @ Base + 0x100 {
-/// ...
-/// }
-/// }
+/// For instance, imagine the following I/O space:
+///
+/// ```text
+/// +-----------------------------+
+/// | ... |
+/// | |
+/// 0x100--->+------------CPU0-------------+
+/// | |
+/// 0x110--->+-----------------------------+
+/// | CPU_CTL |
+/// +-----------------------------+
+/// | ... |
+/// | |
+/// | |
+/// 0x200--->+------------CPU1-------------+
+/// | |
+/// 0x210--->+-----------------------------+
+/// | CPU_CTL |
+/// +-----------------------------+
+/// | ... |
+/// +-----------------------------+
/// ```
///
-/// Like relative registers, they implement the [`WithBase`] trait. However the return value of
-/// [`WithBase::of`] cannot be used directly as a location and must be further specified using the
-/// [`at`](RelativeRegisterLoc::at) method.
+/// `CPU0` and `CPU1` both have a `CPU_CTL` register that starts at offset `0x10` of their I/O
+/// space segment. Since both instances of `CPU_CTL` share the same layout, we don't want to define
+/// them twice and would prefer a way to select which one to use from a single definition.
+///
+/// This can be done by defining a new type for the subregion, and then defining registers that use
+/// the new type as the base:
///
/// ```no_run
/// use kernel::{
/// io::{
+/// io_project,
/// register,
-/// register::{
-/// RegisterBase,
-/// WithBase,
-/// },
/// Io,
+/// Region,
/// },
/// };
-/// # use kernel::io::{Mmio, Region};
-/// # fn get_scratch_idx() -> usize {
-/// # 0x15
-/// # }
+/// # use kernel::io::Mmio;
///
-/// // Type used as parameter of `RegisterBase` to specify the base.
-/// pub struct CpuCtlBase;
-///
-/// // ZST describing `CPU0`.
-/// struct Cpu0;
-/// impl RegisterBase<CpuCtlBase> for Cpu0 {
-/// const BASE: usize = 0x100;
-/// }
-///
-/// // ZST describing `CPU1`.
-/// struct Cpu1;
-/// impl RegisterBase<CpuCtlBase> for Cpu1 {
-/// const BASE: usize = 0x200;
-/// }
+/// // Subregion type. Make sure it has adequate size and alignment.
+/// #[repr(align(4))]
+/// #[derive(FromBytes, IntoBytes)]
+/// pub struct CpuCtl([u8; 0x100]);
///
-/// // 64 per-cpu scratch registers, arranged as a contiguous array.
/// register! {
-/// /// Per-CPU scratch registers.
-/// pub CPU_SCRATCH(u32)[64] @ CpuCtlBase + 0x00000080 {
-/// 31:0 value;
-/// }
-/// }
-///
-/// # fn test(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
-/// // Read scratch register 0 of CPU0.
-/// let scratch = io.read(CPU_SCRATCH::of::<Cpu0>().at(0));
+/// base: Region<0x1000>;
///
-/// // Write the retrieved value into scratch register 15 of CPU1.
-/// io.write(WithBase::of::<Cpu1>().at(15), scratch);
-///
-/// // This won't build.
-/// // let cpu0_scratch_128 = io.read(CPU_SCRATCH::of::<Cpu0>().at(128)).value();
-///
-/// // Runtime-obtained array index.
-/// let scratch_idx = get_scratch_idx();
-/// // Access on a runtime index returns an error if it is out-of-bounds.
-/// let cpu0_scratch = io.read(
-/// CPU_SCRATCH::of::<Cpu0>().try_at(scratch_idx).ok_or(EINVAL)?
-/// ).value();
-/// # Ok(())
-/// # }
-///
-/// // Alias to `SCRATCH[8]` used to convey the firmware exit code.
-/// register! {
-/// /// Per-CPU firmware exit status code.
-/// pub CPU_FIRMWARE_STATUS(u32) => CpuCtlBase + CPU_SCRATCH[8] {
-/// 7:0 status;
-/// }
+/// // Subregions can just be defined like normal registers.
+/// CPU0: CpuCtl @ 0x100;
+/// CPU1: CpuCtl @ 0x200;
/// }
///
-/// // Non-contiguous relative register arrays can be defined by adding a stride parameter.
-/// // Here, each of the 16 registers of the array is separated by 8 bytes, meaning that the
-/// // registers of the two declarations below are interleaved.
+/// // Then you can define new registers on the subregion.
/// register! {
-/// /// Scratch registers bank 0.
-/// pub CPU_SCRATCH_INTERLEAVED_0(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d00 {
-/// 31:0 value;
-/// }
+/// base: CpuCtl;
///
-/// /// Scratch registers bank 1.
-/// pub CPU_SCRATCH_INTERLEAVED_1(u32)[16, stride = 8] @ CpuCtlBase + 0x00000d04 {
-/// 31:0 value;
+/// /// CPU core control.
+/// pub CPU_CTL(u32) @ 0x10 {
+/// 0:0 start;
/// }
/// }
///
-/// # fn test2(io: Mmio<'_, Region<0x1000>>) -> Result<(), Error> {
-/// let cpu0_status = io.read(CPU_FIRMWARE_STATUS::of::<Cpu0>()).status();
-/// # Ok(())
+/// # fn test(io: Mmio<'_, Region<0x1000>>) {
+/// // Read the status of `Cpu0`.
+/// let cpu0_started = io_project!(io, build: CPU0).read(CPU_CTL);
+///
+/// // Stop `Cpu0`.
+/// io_project!(io, build: CPU0).write_reg(CPU_CTL::zeroed());
/// # }
/// ```
#[macro_export]
macro_rules! register {
- // Entry point for the macro, allowing multiple registers to be defined in one call.
- // It matches all possible register declaration patterns to dispatch them to corresponding
- // `@reg` rule that defines a single register.
- //
- // TODO: change `alias:ident` to `alias:path` once relative registers are replaced by I/O
- // projections.
- (
- $(
- $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
- $([ $size:expr $(, stride = $stride:expr)? ])?
- $(@ $($base:ident +)? $offset:literal)?
- $(=> $alias:ident $(+ $alias_offset:ident)? $([$alias_idx:expr])? )?
- { $($fields:tt)* }
- )*
- ) => {
- $(
- $crate::register!(
- @reg $(#[$attr])* $vis $name ($storage) $([$size $(, stride = $stride)?])?
- $(@ $($base +)? $offset)?
- $(=> $alias $(+ $alias_offset)? $([$alias_idx])? )?
- { $($fields)* }
- );
- )*
- };
-
- // All the rules below are private helpers.
-
- // Creates a register at a fixed offset of the MMIO space.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $offset:literal
- { $($fields:tt)* }
- ) => {
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(@io_base $name($storage) @ $offset);
- $crate::register!(@io_fixed $(#[$attr])* $vis $name);
- };
-
- // Creates an alias register of fixed offset register `alias` with its own fields.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path
- { $($fields:tt)* }
- ) => {
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(
- @io_base $name($storage) @
- <$alias as $crate::io::register::Register>::OFFSET
- );
- $crate::register!(@io_fixed $(#[$attr])* $vis $name);
- };
-
- // Creates a register at a relative offset from a base address provider.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) @ $base:ident + $offset:literal
- { $($fields:tt)* }
- ) => {
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(@io_base $name($storage) @ $offset);
- $crate::register!(@io_relative $name @ $base);
- };
-
- // Creates an alias register of relative offset register `alias` with its own fields.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $base:ident + $alias:ident
- { $($fields:tt)* }
- ) => {
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(
- @io_base $name($storage) @ <$alias as $crate::io::register::Register>::OFFSET
- );
- $crate::register!(@io_relative $name @ $base);
- };
-
- // Creates an array of registers at a fixed offset of the MMIO space.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
- [ $size:expr, stride = $stride:expr ] @ $offset:literal { $($fields:tt)* }
- ) => {
- $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
-
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(@io_base $name($storage) @ $offset);
- $crate::register!(@io_array $name [ $size, stride = $stride ]);
- };
-
- // Shortcut for contiguous array of registers (stride == size of element).
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ] @ $offset:literal
- { $($fields:tt)* }
- ) => {
- $crate::register!(
- @reg $(#[$attr])* $vis $name($storage)
- [ $size, stride = ::core::mem::size_of::<$storage>() ]
- @ $offset { $($fields)* }
- );
- };
-
- // Creates an alias of register `idx` of array of registers `alias` with its own fields.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) => $alias:path [ $idx:expr ]
- { $($fields:tt)* }
- ) => {
- $crate::build_assert::static_assert!(
- $idx < <$alias as $crate::io::register::RegisterArray>::SIZE
- );
-
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(
- @io_base $name($storage) @
- <$alias as $crate::io::register::Register>::OFFSET
- + $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
- );
- $crate::register!(@io_fixed $(#[$attr])* $vis $name);
- };
-
- // Creates an array of registers at a relative offset from a base address provider.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
- [ $size:expr, stride = $stride:expr ]
- @ $base:ident + $offset:literal { $($fields:tt)* }
- ) => {
- $crate::build_assert::static_assert!(::core::mem::size_of::<$storage>() <= $stride);
-
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(@io_base $name($storage) @ $offset);
- $crate::register!(@io_relative_array $name [ $size, stride = $stride ] @ $base);
- };
-
- // Shortcut for contiguous array of relative registers (stride == size of element).
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty) [ $size:expr ]
- @ $base:ident + $offset:literal { $($fields:tt)* }
- ) => {
- $crate::register!(
- @reg $(#[$attr])* $vis $name($storage)
- [ $size, stride = ::core::mem::size_of::<$storage>() ]
- @ $base + $offset { $($fields)* }
- );
- };
-
- // Creates an alias of register `idx` of relative array of registers `alias` with its own
- // fields.
- (
- @reg $(#[$attr:meta])* $vis:vis $name:ident ($storage:ty)
- => $base:ident + $alias:ident [ $idx:expr ] { $($fields:tt)* }
- ) => {
- $crate::build_assert::static_assert!(
- $idx < <$alias as $crate::io::register::RegisterArray>::SIZE
- );
-
- $crate::register!(@bitfield $(#[$attr])* $vis struct $name($storage) { $($fields)* });
- $crate::register!(
- @io_base $name($storage) @
- <$alias as $crate::io::register::Register>::OFFSET +
- $idx * <$alias as $crate::io::register::RegisterArray>::STRIDE
- );
- $crate::register!(@io_relative $name @ $base);
- };
-
- // Generates the bitfield for the register.
- //
- // `#[allow(non_camel_case_types)]` is added since register names typically use
- // `SCREAMING_CASE`.
- (
- @bitfield $(#[$attr:meta])* $vis:vis struct $name:ident($storage:ty) { $($fields:tt)* }
- ) => {
- $crate::bitfield!(
- #[allow(non_camel_case_types)]
- $(#[$attr])* $vis struct $name($storage) { $($fields)* }
- );
- };
-
- // Implementations shared by all registers types.
- (@io_base $name:ident($storage:ty) @ $offset:expr) => {
- impl $crate::io::register::Register for $name {
- type Storage = $storage;
-
- const OFFSET: usize = $offset;
- }
- };
-
- // Implementations of fixed registers.
- (@io_fixed $(#[$attr:meta])* $vis:vis $name:ident) => {
- impl $crate::io::register::FixedRegister for $name {}
-
- $(#[$attr])*
- $vis const $name: $crate::io::register::FixedRegisterLoc<$name> =
- $crate::io::register::FixedRegisterLoc::<$name>::new();
- };
-
- // Implementations of relative registers.
- (@io_relative $name:ident @ $base:ident) => {
- impl $crate::io::register::WithBase for $name {
- type BaseFamily = $base;
- }
-
- impl $crate::io::register::RelativeRegister for $name {}
- };
-
- // Implementations of register arrays.
- (@io_array $name:ident [ $size:expr, stride = $stride:expr ]) => {
- impl $crate::io::register::Array for $name {}
-
- impl $crate::io::register::RegisterArray for $name {
- const SIZE: usize = $size;
- const STRIDE: usize = $stride;
- }
- };
-
- // Implementations of relative array registers.
- (
- @io_relative_array $name:ident [ $size:expr, stride = $stride:expr ] @ $base:ident
- ) => {
- impl $crate::io::register::WithBase for $name {
- type BaseFamily = $base;
- }
-
- impl $crate::io::register::RegisterArray for $name {
- const SIZE: usize = $size;
- const STRIDE: usize = $stride;
- }
-
- impl $crate::io::register::RelativeRegisterArray for $name {}
+ ($($tt:tt)*) => {
+ $crate::macros::register!($($tt)*);
};
}
diff --git a/rust/kernel/io/resource.rs b/rust/kernel/io/resource.rs
index 17b0c174cfc5..0d3b34f83334 100644
--- a/rust/kernel/io/resource.rs
+++ b/rust/kernel/io/resource.rs
@@ -226,10 +226,18 @@ impl Flags {
/// Resource represents a memory region that must be ioremaped using `ioremap_np`.
pub const IORESOURCE_MEM_NONPOSTED: Flags = Flags::new(bindings::IORESOURCE_MEM_NONPOSTED);
+ /// Memory region uses a 64-bit address (consumes two consecutive PCI resource slots).
+ pub const IORESOURCE_MEM_64: Flags = Flags::new(bindings::IORESOURCE_MEM_64);
+
// Always inline to optimize out error path of `build_assert`.
#[inline(always)]
const fn new(value: u32) -> Self {
build_assert!(value as u64 <= c_ulong::MAX as u64);
Flags(value as c_ulong)
}
+
+ /// Wrap a raw `c_ulong` value returned by a C API into [`Flags`].
+ pub(crate) const fn from_raw(value: c_ulong) -> Self {
+ Flags(value)
+ }
}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 4d5c96ddc49c..f9ef36217bb5 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -67,6 +67,8 @@ pub mod device;
pub mod device_id;
pub mod devres;
pub mod dma;
+#[cfg(CONFIG_DMA_SHARED_BUFFER)]
+pub mod dma_buf;
pub mod driver;
#[cfg(CONFIG_DRM = "y")]
pub mod drm;
@@ -98,6 +100,7 @@ pub mod jump_label;
pub mod kunit;
pub mod list;
pub mod maple_tree;
+pub mod mem;
pub mod miscdevice;
pub mod mm;
pub mod module;
diff --git a/rust/kernel/maple_tree.rs b/rust/kernel/maple_tree.rs
index 265d6396a78a..7abe41228cb7 100644
--- a/rust/kernel/maple_tree.rs
+++ b/rust/kernel/maple_tree.rs
@@ -16,7 +16,11 @@ use kernel::{
alloc::Flags,
error::to_result,
prelude::*,
- types::{ForeignOwnable, Opaque},
+ types::{
+ ForeignOwnable,
+ NotThreadSafe,
+ Opaque, //
+ },
};
/// A maple tree optimized for storing non-overlapping ranges.
@@ -240,7 +244,10 @@ impl<T: ForeignOwnable> MapleTree<T> {
unsafe { bindings::spin_lock(self.ma_lock()) };
// INVARIANT: We just took the spinlock.
- MapleGuard(self)
+ MapleGuard {
+ tree: self,
+ _not_send: NotThreadSafe,
+ }
}
#[inline]
@@ -302,19 +309,30 @@ impl<T: ForeignOwnable> PinnedDrop for MapleTree<T> {
}
}
+// SAFETY: `MapleTree<T>` is `Send` if `T` is `Send` because `MapleTree` owns its elements.
+unsafe impl<T: ForeignOwnable + Send> Send for MapleTree<T> {}
+
+// SAFETY: `&MapleTree<T>` allows inserting and erasing entries from any thread, so `T: Send` is
+// required, and shared borrows of entries require `T: Sync`.
+unsafe impl<T: ForeignOwnable + Send + Sync> Sync for MapleTree<T> {}
+
/// A reference to a [`MapleTree`] that owns the inner lock.
///
/// # Invariants
///
/// This guard owns the inner spinlock.
#[must_use = "if unused, the lock will be immediately unlocked"]
-pub struct MapleGuard<'tree, T: ForeignOwnable>(&'tree MapleTree<T>);
+pub struct MapleGuard<'tree, T: ForeignOwnable> {
+ tree: &'tree MapleTree<T>,
+ // A held spinlock must be released on the same CPU that acquired it.
+ _not_send: NotThreadSafe,
+}
impl<'tree, T: ForeignOwnable> Drop for MapleGuard<'tree, T> {
#[inline]
fn drop(&mut self) {
// SAFETY: By the type invariants, we hold this spinlock.
- unsafe { bindings::spin_unlock(self.0.ma_lock()) };
+ unsafe { bindings::spin_unlock(self.tree.ma_lock()) };
}
}
@@ -323,7 +341,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
pub fn ma_state(&mut self, first: usize, end: usize) -> MaState<'_, T> {
// SAFETY: The `MaState` borrows this `MapleGuard`, so it can also borrow the `MapleGuard`s
// read/write permissions to the maple tree.
- unsafe { MaState::new_raw(self.0, first, end) }
+ unsafe { MaState::new_raw(self.tree, first, end) }
}
/// Load the value at the given index.
@@ -375,7 +393,7 @@ impl<'tree, T: ForeignOwnable> MapleGuard<'tree, T> {
#[inline]
pub fn load(&mut self, index: usize) -> Option<T::BorrowedMut<'_>> {
// SAFETY: `self.tree` contains a valid maple tree.
- let ret = unsafe { bindings::mtree_load(self.0.tree.get(), index) };
+ let ret = unsafe { bindings::mtree_load(self.tree.tree.get(), index) };
if ret.is_null() {
return None;
}
diff --git a/rust/kernel/mem.rs b/rust/kernel/mem.rs
new file mode 100644
index 000000000000..f2d4cdf87d00
--- /dev/null
+++ b/rust/kernel/mem.rs
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Basic utilities for dealing with memory, values, and types.
+
+use crate::prelude::*;
+
+/// Transmute between two types.
+///
+/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
+/// cannot be proven by the compiler.
+///
+/// This is equivalent to Rust's `transmute_unchecked` intrinsics.
+///
+/// # Safety
+///
+/// All safety requirements of [`core::mem::transmute`] apply, plus that the size `Src` and `Dst`
+/// must match.
+///
+/// # Examples
+///
+/// This can be used when types are known to have the same size, but only at runtime.
+///
+/// ```no_run
+/// # use core::any::TypeId;
+/// fn to_u32<T: 'static>(v: T) -> Option<u32> {
+/// if TypeId::of::<T>() != TypeId::of::<u32>() {
+/// return None;
+/// }
+///
+/// // `core::mem::transmute` won't work here.
+/// // SAFETY: We've checked that `T` is `u32`!
+/// Some(unsafe { kernel::mem::transmute_unchecked(v) })
+/// }
+///
+/// to_u32(1u32);
+/// ```
+#[inline(always)]
+pub const unsafe fn transmute_unchecked<Src, Dst>(val: Src) -> Dst {
+ // SAFETY: This is identical to `transmute` except that we bypassed the size check; which is
+ // true per safety requirement.
+ unsafe { core::mem::transmute_copy(&core::mem::ManuallyDrop::new(val)) }
+}
+
+/// Version of `transmute` that performs size check at monomorphization-time.
+///
+/// Use this instead of [`core::mem::transmute`] when it is known that sizes are identical but this
+/// cannot be proven by the compiler during type checking and can be proven during monomorphization.
+///
+/// The signature is equivalent to Rust standard library's unstable `transmute_neo` and that of
+/// [RFC 3844](https://github.com/rust-lang/rfcs/pull/3844).
+///
+/// # Safety
+///
+/// Same as [`core::mem::transmute`].
+///
+/// # Examples
+///
+/// This is typically used in generic code where it's known that type will have the same size, but
+/// the compiler cannot prove it generically.
+///
+/// ```no_run
+/// trait IsU32 {}
+/// impl IsU32 for u32 {}
+///
+/// fn to_u32<T: IsU32>(v: T) -> u32 {
+/// // `core::mem::transmute` won't work here.
+/// // SAFETY: We know that `v` is u32!
+/// unsafe { kernel::mem::transmute(v) }
+/// }
+///
+/// to_u32(1u32);
+/// ```
+#[inline(always)]
+pub const unsafe fn transmute<Src, Dst>(val: Src) -> Dst {
+ const_assert!(size_of::<Src>() == size_of::<Dst>());
+
+ // SAFETY: Size is checked above. Other safety requirements follow those of the function.
+ unsafe { transmute_unchecked(val) }
+}
+
+/// Safely transmutes a value of one type to a value of another type of the same size.
+///
+/// The sizes are checked during monomorphization.
+///
+/// This can be considered as generic version of [`zerocopy::transmute!`] macro that defers the size
+/// check and thus can be used in more cases.
+///
+/// # Examples
+///
+/// ```no_run
+/// fn to_u32<T: FromBytes + IntoBytes>(v: T) -> u32 {
+/// // `zerocopy::transmute!` won't work here.
+/// kernel::mem::safe_transmute(v)
+/// }
+///
+/// to_u32(1i32);
+/// ```
+#[inline(always)]
+pub const fn safe_transmute<Src: IntoBytes, Dst: FromBytes>(val: Src) -> Dst {
+ // SAFETY: `transmute` is safe with `IntoBytes` and `FromBytes` bounds.
+ unsafe { transmute(val) }
+}
+
+/// Type that is layout-compatible with a primitive representation.
+///
+/// # Safety
+///
+/// - [`Self`] must have the same size and alignment as [`Self::Repr`].
+/// - [`Self`] must be [transmutable] to [`Self::Repr`].
+/// - Neither [`Self`] nor [`Self::Repr`] contains interior mutability.
+///
+/// The above basically says that `&Self` can be transmuted to `&Self::Repr`.
+///
+/// [transmutable]: core::mem::transmute
+pub unsafe trait AsRepr: Sized {
+ /// Primitive representation of this type.
+ type Repr;
+
+ /// Convert from [`&Self`](Self) to [`&Self::Repr`](AsRepr::Repr).
+ #[inline(always)]
+ fn as_repr(this: &Self) -> &Self::Repr {
+ // SAFETY: Per safety requirement of the trait.
+ unsafe { core::mem::transmute(this) }
+ }
+
+ /// Convert from [`Self`] to [`Self::Repr`].
+ #[inline(always)]
+ fn into_repr(this: Self) -> Self::Repr {
+ // SAFETY: Per safety requirement of the trait.
+ unsafe { transmute(this) }
+ }
+
+ /// Convert from [`Self::Repr`] to [`Self`].
+ ///
+ /// # Safety
+ ///
+ /// `repr` must be a valid bit pattern of [`Self`] and satisfy type-specific invariants of it.
+ ///
+ /// Alternatively, if `repr` is previously obtained using [`Self::into_repr`], and each
+ /// `from_repr_unchecked` should correspond to a unique `into_repr` call, then it is safe to
+ /// call as well (this means that we're undoing a `into_repr` call getting the exact bytes
+ /// back).
+ ///
+ /// No guarantee is made if the result of a `into_repr` is passed to multiple
+ /// `from_repr_unchecked` (i.e. copies are made), to allow for cases where `Repr` is a pointer
+ /// and the user of the API wants ownership transfer. Users that want the ability to call
+ /// `from_repr_unchecked` after copying can require `Copy` bound explicitly.
+ #[inline(always)]
+ unsafe fn from_repr_unchecked(repr: Self::Repr) -> Self {
+ // SAFETY: Per safety requirement, `repr` is valid repr of `Self`, or it is previously from
+ // `into_repr`, in which case we're undoing the transmute so it is also safe.
+ unsafe { transmute(repr) }
+ }
+}
+
+/// Type that is bi-directionally transmutable with a primitive representation.
+///
+/// # Safety
+///
+/// - [`Self`] must be [transmutable] from [`Self::Repr`].
+///
+/// [transmutable]: core::mem::transmute
+/// [`Self::Repr`]: AsRepr::Repr
+pub unsafe trait AsReprMut: AsRepr {
+ /// Convert from `&mut Self` to [`&mut Self::Repr`](AsRepr::Repr).
+ #[inline(always)]
+ fn as_repr_mut(this: &mut Self) -> &mut Self::Repr {
+ // SAFETY: Per safety requirement of the trait.
+ unsafe { core::mem::transmute(this) }
+ }
+
+ /// Convert from [`Self::Repr`](AsRepr::Repr) to `Self`.
+ #[inline(always)]
+ fn from_repr(repr: Self::Repr) -> Self {
+ // SAFETY: Per safety requirement of the trait.
+ unsafe { transmute(repr) }
+ }
+}
+
+// SAFETY: `bool` has the same size and alignment as `u8`, and Rust guarantees that `bool` has
+// only two valid bit patterns: 0 (`false`) and 1 (`true`). Thus `bool` can be transmuted to `u8`.
+// Neither types contain interior mutability.
+unsafe impl AsRepr for bool {
+ type Repr = u8;
+}
+
+// SAFETY: `*mut T` has the same size and alignment with `*const c_void`, and thus `*mut T` is
+// transmutable to `*const c_void`. Neither types contain interior mutability.
+unsafe impl<T> AsRepr for *mut T {
+ type Repr = *const c_void;
+}
+
+// SAFETY: `*mut T` is transmutable from `*const c_void`.
+unsafe impl<T> AsReprMut for *mut T {}
+
+// SAFETY: `*const T` has the same size and alignment with `*const c_void`, and is transmutable to
+// `*const c_void`. Neither types contain interior mutability.
+unsafe impl<T> AsRepr for *const T {
+ type Repr = *const c_void;
+}
+
+// SAFETY: `*const T` is transmutable from `*const c_void`.
+unsafe impl<T> AsReprMut for *const T {}
+
+macro_rules! int_impl {
+ ($($unsigned:ident $signed:ident ,)*) => {$(
+ // SAFETY: `$unsigned` has the same size and alignment with itself, and is transmutable to
+ // itself. It does not contain interior mutability.
+ unsafe impl AsRepr for $unsigned {
+ type Repr = $unsigned;
+ }
+
+ // SAFETY: `$unsigned` is transmutable from itself.
+ unsafe impl AsReprMut for $unsigned {}
+
+ // SAFETY: `$signed` has the same size and alignment with `$unsigned`, and is transmutable
+ // to it Neither types contain interior mutability.
+ unsafe impl AsRepr for $signed {
+ type Repr = $unsigned;
+ }
+
+ // SAFETY: `$signed` is transmutable from `$unsigned`.
+ unsafe impl AsReprMut for $signed {}
+ )*};
+}
+
+int_impl! {
+ u8 i8,
+ u16 i16,
+ u32 i32,
+ u64 i64,
+ // `usize` is not normalized to particular integer for portability.
+ usize isize,
+}
diff --git a/rust/kernel/pci.rs b/rust/kernel/pci.rs
index 3ec897709e89..19a219847c17 100644
--- a/rust/kernel/pci.rs
+++ b/rust/kernel/pci.rs
@@ -17,6 +17,7 @@ use crate::{
from_result,
to_result, //
},
+ io::resource,
prelude::*,
str::CStr,
types::Opaque,
@@ -439,6 +440,19 @@ impl Device {
Ok(unsafe { bindings::pci_resource_len(self.as_raw(), bar.try_into()?) })
}
+ /// Returns the resource flags (`IORESOURCE_*`) of the given PCI BAR.
+ pub fn resource_flags(&self, bar: u32) -> Result<resource::Flags> {
+ if !Bar::index_is_valid(bar) {
+ return Err(EINVAL);
+ }
+
+ // SAFETY:
+ // - `bar` is a valid bar number, as guaranteed by the above call to `Bar::index_is_valid`,
+ // - by its type invariant `self.as_raw` is always a valid pointer to a `struct pci_dev`.
+ let raw = unsafe { bindings::pci_resource_flags(self.as_raw(), bar.try_into()?) };
+ Ok(resource::Flags::from_raw(raw))
+ }
+
/// Returns the PCI class as a `Class` struct.
#[inline]
pub fn pci_class(&self) -> Class {
diff --git a/rust/kernel/sync/atomic.rs b/rust/kernel/sync/atomic.rs
index 9cd009d57e35..6d27898add42 100644
--- a/rust/kernel/sync/atomic.rs
+++ b/rust/kernel/sync/atomic.rs
@@ -140,7 +140,7 @@ pub unsafe trait AtomicAdd<Rhs = Self>: AtomicType {
const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
// SAFETY: Per the safety requirement of `AtomicType`, `T` is round-trip transmutable to
// `T::Repr`, therefore the transmute operation is sound.
- unsafe { core::mem::transmute_copy(&v) }
+ unsafe { crate::mem::transmute(v) }
}
/// # Safety
@@ -149,7 +149,7 @@ const fn into_repr<T: AtomicType>(v: T) -> T::Repr {
#[inline(always)]
const unsafe fn from_repr<T: AtomicType>(r: T::Repr) -> T {
// SAFETY: Per the safety requirement of the function, the transmute operation is sound.
- unsafe { core::mem::transmute_copy(&r) }
+ unsafe { crate::mem::transmute(r) }
}
impl<T: AtomicType> Atomic<T> {
diff --git a/rust/kernel/uaccess.rs b/rust/kernel/uaccess.rs
index 5f6c4d7a1a51..f09078228c53 100644
--- a/rust/kernel/uaccess.rs
+++ b/rust/kernel/uaccess.rs
@@ -520,14 +520,14 @@ impl UserSliceWriter {
///
/// fn copy_dma_to_user(
/// mut writer: UserSliceWriter,
- /// alloc: &Coherent<[u8]>,
+ /// alloc: &Coherent<'_, [u8]>,
/// ) -> Result {
/// writer.write_dma(alloc, 0, 256)
/// }
/// ```
pub fn write_dma<T: KnownSize + AsBytes + ?Sized>(
&mut self,
- alloc: &Coherent<T>,
+ alloc: &Coherent<'_, T>,
offset: usize,
count: usize,
) -> Result {