diff options
Diffstat (limited to 'rust/zerocopy/src/ref.rs')
| -rw-r--r-- | rust/zerocopy/src/ref.rs | 762 |
1 files changed, 651 insertions, 111 deletions
diff --git a/rust/zerocopy/src/ref.rs b/rust/zerocopy/src/ref.rs index e49f2a887ffa..7a8876a0587b 100644 --- a/rust/zerocopy/src/ref.rs +++ b/rust/zerocopy/src/ref.rs @@ -10,7 +10,7 @@ // those terms. use super::*; use crate::pointer::{ - BecauseInvariantsEq, BecauseMutationCompatible, MutationCompatible, TransmuteFromPtr, + BecauseMutationCompatible, BecauseSharedCompatible, MutationCompatible, TransmuteFromPtr, }; mod def { @@ -18,7 +18,7 @@ mod def { use crate::{ ByteSlice, ByteSliceMut, CloneableByteSlice, CopyableByteSlice, IntoByteSlice, - IntoByteSliceMut, + IntoByteSliceMut, KnownLayout, PointerMetadata, }; /// A typed reference derived from a byte slice. @@ -59,30 +59,83 @@ mod def { /// } /// } /// ``` - pub struct Ref<B, T: ?Sized>( - // INVARIANTS: The referent (via `.deref`, `.deref_mut`, `.into`) byte - // slice is aligned to `T`'s alignment and its size corresponds to a - // valid size for `T`. - B, - PhantomData<T>, - ); - - impl<B, T: ?Sized> Ref<B, T> { + pub struct Ref<B, T: ?Sized> { + // INVARIANTS: + // - The referent (via `.deref`, `.deref_mut`, `.into`) byte slice is + // aligned to `T`'s alignment, and its size is valid for `T`. + // - If `T: KnownLayout`, let `metadata` be + // `T::PointerMetadata::from_elem_count` applied to `elem_count`. + // Then `T::size_for_metadata(metadata)` is `Some(referent.len())`. + bytes: B, + // Ignored when `T` does not implement `KnownLayout`. + elem_count: usize, + _marker: PhantomData<T>, + } + + impl<B, T> Ref<B, T> { + /// Constructs a new `Ref` to a sized `T`. + /// + /// # Safety + /// + /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to + /// a byte slice which is aligned to `T`'s alignment and whose size is + /// `size_of::<T>()`. + /// + /// [`deref`]: core::ops::Deref::deref + /// [`deref_mut`]: core::ops::DerefMut::deref_mut + /// [`into`]: core::convert::Into::into + pub(super) unsafe fn new_sized_unchecked(bytes: B) -> Ref<B, T> { + // INVARIANTS: The caller has promised that `bytes`'s referent is + // validly aligned and has size `size_of::<T>()`. If `T: + // KnownLayout`, then, since `T` is sized, `T::PointerMetadata` is + // `()`. `from_elem_count` returns `()` for any element count, and + // `T::size_for_metadata(()) == Some(size_of::<T>())`. `size_of` + // reports this size in bytes [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + Ref { bytes, elem_count: 0, _marker: PhantomData } + } + } + + impl<B, T: KnownLayout + ?Sized> Ref<B, T> { /// Constructs a new `Ref`. /// /// # Safety /// /// `bytes` dereferences (via [`deref`], [`deref_mut`], and [`into`]) to - /// a byte slice which is aligned to `T`'s alignment and whose size is a - /// valid size for `T`. + /// a byte slice which is aligned to `T`'s alignment and satisfies + /// `T::size_for_metadata(metadata) == Some(referent.len())`. /// /// [`deref`]: core::ops::Deref::deref /// [`deref_mut`]: core::ops::DerefMut::deref_mut /// [`into`]: core::convert::Into::into - pub(crate) unsafe fn new_unchecked(bytes: B) -> Ref<B, T> { + pub(super) unsafe fn new_unchecked(bytes: B, metadata: T::PointerMetadata) -> Ref<B, T> { // INVARIANTS: The caller has promised that `bytes`'s referent is - // validly-aligned and has a valid size. - Ref(bytes, PhantomData) + // validly aligned and has exactly the size described by `metadata`. + // `KnownLayout::PointerMetadata` is `()` for sized types and + // `usize` for slice DSTs. For both permitted metadata types, + // `from_elem_count(metadata.to_elem_count()) == metadata`: `()` + // round-trips through zero, while `usize` round-trips through + // itself. Thus, storing the element count preserves the metadata + // whose size the caller validated. This agrees with Rust's + // language-level definition of slice pointer metadata [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/reference/dynamically-sized-types.html: + // + // Pointers to slices also store the number of elements of the + // slice. + Ref { bytes, elem_count: metadata.to_elem_count(), _marker: PhantomData } + } + + /// Gets the pointer metadata represented by this `Ref`. + #[inline(always)] + pub(super) fn pointer_metadata(&self) -> T::PointerMetadata { + // INVARIANTS: `KnownLayout::PointerMetadata` is either `()` or + // `usize`. For `()`, `from_elem_count` ignores the element count; + // for `usize`, it returns the element count unchanged. + T::PointerMetadata::from_elem_count(self.elem_count) } } @@ -96,16 +149,18 @@ mod def { /// `Any::downcast_ref`). /// /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// validly aligned for `T` and has a valid size for `T`. If `T: + /// KnownLayout`, then `T::size_for_metadata(self.pointer_metadata())` + /// returns `Some(referent.len())`. pub(crate) unsafe fn as_byte_slice(&self) -> &impl ByteSlice { // INVARIANTS: The caller promises not to call methods other than // those on `ByteSlice`. Since `B: ByteSlice`, dereference stability // guarantees that calling `ByteSlice` methods will not change the - // address or length of `self.0`'s referent. + // address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - &self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + &self.bytes } } @@ -118,17 +173,20 @@ mod def { /// [`ByteSliceMut`] other than `ByteSliceMut` methods (for example, via /// `Any::downcast_mut`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `as_byte_slice_mut` promises to return a `ByteSliceMut` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn as_byte_slice_mut(&mut self) -> &mut impl ByteSliceMut { // INVARIANTS: The caller promises not to call methods other than // those on `ByteSliceMut`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - &mut self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + &mut self.bytes } } @@ -141,17 +199,20 @@ mod def { /// [`IntoByteSlice`] other than `IntoByteSlice` methods (for example, /// via `Any::downcast_ref`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `into_byte_slice` promises to return an `IntoByteSlice` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn into_byte_slice(self) -> impl IntoByteSlice<'a> { // INVARIANTS: The caller promises not to call methods other than // those on `IntoByteSlice`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + self.bytes } } @@ -164,33 +225,38 @@ mod def { /// [`IntoByteSliceMut`] other than `IntoByteSliceMut` methods (for /// example, via `Any::downcast_mut`). /// - /// `as_byte_slice` promises to return a `ByteSlice` whose referent is - /// validly-aligned for `T` and has a valid size for `T`. + /// `into_byte_slice_mut` promises to return an `IntoByteSliceMut` whose + /// referent is validly aligned for `T` and has a valid size for `T`. If + /// `T: KnownLayout`, then + /// `T::size_for_metadata(self.pointer_metadata())` returns + /// `Some(referent.len())`. pub(crate) unsafe fn into_byte_slice_mut(self) -> impl IntoByteSliceMut<'a> { // INVARIANTS: The caller promises not to call methods other than // those on `IntoByteSliceMut`. Since `B: ByteSlice`, dereference // stability guarantees that calling `ByteSlice` methods will not - // change the address or length of `self.0`'s referent. + // change the address or length of `self.bytes`'s referent. // - // SAFETY: By invariant on `self.0`, the alignment and size - // post-conditions are upheld. - self.0 + // SAFETY: By invariant on `self`, the alignment, size, and metadata + // postconditions are upheld. + self.bytes } } impl<B: CloneableByteSlice + Clone, T: ?Sized> Clone for Ref<B, T> { #[inline] fn clone(&self) -> Ref<B, T> { - // INVARIANTS: Since `B: CloneableByteSlice`, `self.0.clone()` has - // the same address and length as `self.0`. Since `self.0` upholds - // the field invariants, so does `self.0.clone()`. - Ref(self.0.clone(), PhantomData) + // INVARIANTS: Since `B: CloneableByteSlice`, `self.bytes.clone()` has + // the same address and length as `self.bytes`. We copy + // `self.elem_count`, so the clone also has the same metadata. Since + // `self` upholds the field invariants, so does the clone. + Ref { bytes: self.bytes.clone(), elem_count: self.elem_count, _marker: PhantomData } } } - // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `.0` has the - // same address and length as the original `Ref`'s `.0`. Since the original - // upholds the field invariants, so does the copy. + // INVARIANTS: Since `B: CopyableByteSlice`, the copied `Ref`'s `bytes` has the + // same address and length as the original `Ref`'s `bytes`. The copied + // `elem_count` has the same metadata. Since the original upholds the field + // invariants, so does the copy. impl<B: CopyableByteSlice + Copy, T: ?Sized> Copy for Ref<B, T> {} } @@ -198,7 +264,7 @@ mod def { pub use def::Ref; use crate::pointer::{ - invariant::{Aligned, BecauseExclusive, Initialized, Unaligned, Valid}, + invariant::{Aligned, BecauseExclusive, Initialized, Safe, Unaligned}, BecauseRead, PtrInner, }; @@ -215,8 +281,14 @@ where return Err(err.with_src(bytes).into()); } - // SAFETY: We just validated size and alignment. - Ok(unsafe { Ref::new_unchecked(bytes) }) + // SAFETY: `size_of` returns `T`'s size in bytes [1]. The equality check + // above therefore establishes that `bytes` has exactly that byte + // length, and `validate_aligned_to` establishes alignment. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + Ok(unsafe { Ref::new_sized_unchecked(bytes) }) } } @@ -238,10 +310,15 @@ where )?; // SAFETY: We just validated alignment and that `bytes` is at least as // large as `T`. `bytes.split_at(mem::size_of::<T>())?` ensures that the - // new `bytes` is exactly the size of `T`. By safety postcondition on + // new `bytes` is exactly the size of `T`, measured in bytes as specified + // for `size_of` [1]. By safety postcondition on // `SplitByteSlice::split_at` we can rely on `split_at` to produce the // correct `bytes` and `suffix`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::new_sized_unchecked(bytes) }; Ok((r, suffix)) } @@ -259,11 +336,16 @@ where } // SAFETY: Since `split_at` is defined as `bytes_len - size_of::<T>()`, // the `bytes` which results from `let (prefix, bytes) = - // bytes.split_at(split_at)?` has length `size_of::<T>()`. After - // constructing `bytes`, we validate that it has the proper alignment. - // By safety postcondition on `SplitByteSlice::split_at` we can rely on - // `split_at` to produce the correct `prefix` and `bytes`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // bytes.split_at(split_at)?` has length `size_of::<T>()`, measured in + // bytes as specified for `size_of` [1]. After constructing `bytes`, we + // validate that it has the proper alignment. By safety postcondition on + // `SplitByteSlice::split_at` we can rely on `split_at` to produce the + // correct `prefix` and `bytes`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::new_sized_unchecked(bytes) }; Ok((prefix, r)) } } @@ -306,17 +388,64 @@ where /// /// let _ = Ref::<_, ZSTy>::from_bytes(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_bytes(source: B) -> Result<Ref<B, T>, CastError<B, T>> { static_assert_dst_is_not_zst!(T); - if let Err(e) = - Ptr::from_ref(source.deref()).try_cast_into_no_leftover::<T, BecauseImmutable>(None) + let metadata = match Ptr::from_ref(source.deref()) + .try_cast_into_no_leftover::<T, BecauseImmutable>(None) { - return Err(e.with_src(()).with_src(source)); - } - // SAFETY: `try_cast_into_no_leftover` validates size and alignment. - Ok(unsafe { Ref::new_unchecked(source) }) + Ok(ptr) => T::pointer_to_metadata(ptr.as_inner().as_ptr()), + Err(e) => return Err(e.with_src(()).with_src(source)), + }; + // SAFETY: By its safety postcondition, `try_cast_into_no_leftover` + // returned a `Ptr<T>` aligned as required for references [2], over the + // same byte range as `source.deref()`. `pointer_to_metadata` returns + // the metadata stored in that pointer, and `size_for_metadata` computes + // the corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(source.len())`. + // `ByteSlice` and its applicable subtraits guarantee that the stored + // `source` continues to dereference or convert to that address and + // length. These facts establish both preconditions of `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + Ok(unsafe { Ref::new_unchecked(source, metadata) }) } } @@ -360,14 +489,37 @@ where /// /// let _ = Ref::<_, ZSTy>::from_prefix(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_prefix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_prefix(source: B) -> Result<(Ref<B, T>, B), CastError<B, T>> { static_assert_dst_is_not_zst!(T); - let remainder = match Ptr::from_ref(source.deref()) + let (metadata, remainder) = match Ptr::from_ref(source.deref()) .try_cast_into::<T, BecauseImmutable>(CastType::Prefix, None) { - Ok((_, remainder)) => remainder, + Ok((ptr, remainder)) => (T::pointer_to_metadata(ptr.as_inner().as_ptr()), remainder), Err(e) => { return Err(e.with_src(()).with_src(source)); } @@ -381,11 +533,35 @@ where #[allow(unstable_name_collisions)] let split_at = unsafe { source.len().unchecked_sub(remainder.len()) }; let (bytes, suffix) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; - // SAFETY: `try_cast_into` validates size and alignment, and returns a - // `split_at` that indicates how many bytes of `source` correspond to a - // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we - // can rely on `split_at` to produce the correct `source` and `suffix`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // SAFETY: By its safety postcondition, `try_cast_into` returned an + // `Ptr<T>` aligned as required for references [2], and a remainder + // which are non-overlapping, cover `source.deref()`, and place the + // `Ptr<T>` at `source.deref()`'s address. + // Therefore, `split_at` is the typed pointer's byte length. By the + // safety postcondition of `SplitByteSlice::split_at`, `bytes` has that + // length and address. `pointer_to_metadata` returns the metadata stored + // in the typed pointer, and `size_for_metadata` computes the + // corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(bytes.len())`. The + // `SplitByteSlice` and `ByteSlice` contracts preserve the established + // address and length, satisfying both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + let r = unsafe { Ref::new_unchecked(bytes, metadata) }; Ok((r, suffix)) } @@ -425,14 +601,37 @@ where /// /// let _ = Ref::<_, ZSTy>::from_suffix(&b"UU"[..]); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_suffix", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline] pub fn from_suffix(source: B) -> Result<(B, Ref<B, T>), CastError<B, T>> { static_assert_dst_is_not_zst!(T); - let remainder = match Ptr::from_ref(source.deref()) + let (metadata, remainder) = match Ptr::from_ref(source.deref()) .try_cast_into::<T, BecauseImmutable>(CastType::Suffix, None) { - Ok((_, remainder)) => remainder, + Ok((ptr, remainder)) => (T::pointer_to_metadata(ptr.as_inner().as_ptr()), remainder), Err(e) => { let e = e.with_src(()); return Err(e.with_src(source)); @@ -441,11 +640,39 @@ where let split_at = remainder.len(); let (prefix, bytes) = source.split_at(split_at).map_err(|b| SizeError::new(b).into())?; - // SAFETY: `try_cast_into` validates size and alignment, and returns a - // `split_at` that indicates how many bytes of `source` correspond to a - // valid `T`. By safety postcondition on `SplitByteSlice::split_at` we - // can rely on `split_at` to produce the correct `prefix` and `bytes`. - let r = unsafe { Ref::new_unchecked(bytes) }; + // SAFETY: By its safety postcondition, `try_cast_into` returned an + // `Ptr<T>` aligned as required for references [2], and a remainder + // which are non-overlapping, cover `source.deref()`, and place the + // remainder at `source.deref()`'s address. More precisely, + // `PtrInner::try_cast_into` constructs the typed suffix from the + // `r_slice` produced by `split_at_unchecked(split_at)` and returns its + // `l_slice` as `remainder`. Thus, even when `T` is zero-sized, the typed + // pointer and the `bytes` produced by splitting `source` at + // `remainder.len()` have the same address and length. By the safety + // postcondition of `SplitByteSlice::split_at`, `bytes` retains that + // address and length. `pointer_to_metadata` returns the metadata stored + // in the typed pointer, and `size_for_metadata` computes the + // corresponding dynamically-known byte size [1]. Thus, + // `T::size_for_metadata(metadata)` is `Some(bytes.len())`. The + // `SplitByteSlice` and `ByteSlice` contracts preserve the established + // address and length, satisfying both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + let r = unsafe { Ref::new_unchecked(bytes, metadata) }; Ok((prefix, r)) } } @@ -487,6 +714,24 @@ where /// /// let _ = Ref::<_, ZSTy>::from_bytes_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_bytes_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_bytes_with_elems(source: B, count: usize) -> Result<Ref<B, T>, CastError<B, T>> { static_assert_dst_is_not_zst!(T); @@ -497,7 +742,37 @@ where if source.len() != expected_len { return Err(SizeError::new(source).into()); } - Self::from_bytes(source) + if let Err(e) = Ptr::from_ref(source.deref()) + .try_cast_into_no_leftover::<T, BecauseImmutable>(Some(count)) + { + return Err(e.with_src(()).with_src(source)); + } + // SAFETY: `size_for_metadata(count)` returned `expected_len`, and the + // equality check established that this is `source.len()`. This is the + // dynamically-known byte size described by `size_of_val` [1]. By its + // safety postcondition, `try_cast_into_no_leftover(Some(count))` only + // succeeded after producing a `Ptr<T>` aligned as required for + // references [2], with that metadata over the same byte range as + // `source.deref()`. `ByteSlice` and its applicable subtraits guarantee + // that the stored `source` continues to dereference or convert to that + // address and length. These facts establish both preconditions of + // `new_unchecked`. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of_val.html: + // + // Returns the size of the pointed-to value in bytes. + // + // ... when `T` has no statically-known size ... `size_of_val` can + // be used to get the dynamically-known size. + // + // [2] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.align_of_val.html: + // + // Returns the ABI-required minimum alignment of the type of the + // value that `val` points to. + // + // Every reference to a value of the type `T` must be a multiple of + // this number. + Ok(unsafe { Ref::new_unchecked(source, count) }) } } @@ -538,6 +813,24 @@ where /// /// let _ = Ref::<_, ZSTy>::from_prefix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_prefix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_prefix_with_elems( source: B, @@ -549,7 +842,7 @@ where None => return Err(SizeError::new(source).into()), }; let (prefix, bytes) = source.split_at(expected_len).map_err(SizeError::new)?; - Self::from_bytes(prefix).map(move |l| (l, bytes)) + Self::from_bytes_with_elems(prefix, count).map(move |l| (l, bytes)) } /// Constructs a `Ref` from the suffix of the given bytes with DST length @@ -584,6 +877,24 @@ where /// /// let _ = Ref::<_, ZSTy>::from_suffix_with_elems(&b"UU"[..], 42); // ⚠ Compile Error! /// ``` + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_from_suffix_with_elems", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 2 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn from_suffix_with_elems( source: B, @@ -602,7 +913,7 @@ where // SAFETY: The preceding `source.len().checked_sub(expected_len)` // guarantees that `split_at` is in-bounds. let (bytes, suffix) = unsafe { source.split_at_unchecked(split_at) }; - Self::from_bytes(suffix).map(move |l| (bytes, l)) + Self::from_bytes_with_elems(suffix, count).map(move |l| (bytes, l)) } } @@ -618,12 +929,37 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::into_ref(r)` instead of `r.into_ref()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_into_ref", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[must_use = "has no side effects"] #[inline(always)] pub fn into_ref(r: Self) -> &'a T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = r.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `IntoByteSlice`. let b = unsafe { r.into_byte_slice() }; @@ -642,12 +978,12 @@ where return ptr.as_ref(); } - // PANICS: By post-condition on `into_byte_slice`, `b`'s size and - // alignment are valid for `T`. By post-condition, `b.into_byte_slice()` - // produces a byte slice with identical address and length to that - // produced by `b.deref()`. + // PANICS: By post-condition on `into_byte_slice`, `b` is aligned for + // `T`, and its size is exactly the size encoded by `metadata`. By + // post-condition, `b.into_byte_slice()` produces a byte slice with + // identical address and length to that produced by `b.deref()`. let ptr = Ptr::from_ref(b.into_byte_slice()) - .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .try_cast_into_no_leftover::<T, BecauseImmutable>(Some(metadata)) .expect("zerocopy internal error: into_ref should be infallible"); let ptr = ptr.recall_validity(); ptr.as_ref() @@ -666,12 +1002,32 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::into_mut(r)` instead of `r.into_mut()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_into_mut", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] #[must_use = "has no side effects"] #[inline(always)] pub fn into_mut(r: Self) -> &'a mut T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = r.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `IntoByteSliceMut`. let b = unsafe { r.into_byte_slice_mut() }; @@ -686,7 +1042,7 @@ where T, _, (BecauseRead, BecauseExclusive), - (BecauseMutationCompatible, BecauseInvariantsEq), + (BecauseMutationCompatible, BecauseSharedCompatible), >(ptr) }; @@ -697,13 +1053,13 @@ where return ptr.as_mut(); } - // PANICS: By post-condition on `into_byte_slice_mut`, `b`'s size and - // alignment are valid for `T`. By post-condition, - // `b.into_byte_slice_mut()` produces a byte slice with identical - // address and length to that produced by `b.deref_mut()`. + // PANICS: By post-condition on `into_byte_slice_mut`, `b` is aligned + // for `T`, and its size is exactly the size encoded by `metadata`. By + // post-condition, `b.into_byte_slice_mut()` produces a byte slice with + // identical address and length to that produced by `b.deref_mut()`. let ptr = Ptr::from_mut(b.into_byte_slice_mut()) - .try_cast_into_no_leftover::<T, BecauseExclusive>(None) - .expect("zerocopy internal error: into_ref should be infallible"); + .try_cast_into_no_leftover::<T, BecauseExclusive>(Some(metadata)) + .expect("zerocopy internal error: into_mut should be infallible"); let ptr = ptr.recall_validity::<_, (_, (_, _))>(); ptr.as_mut() } @@ -719,6 +1075,29 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::bytes(r)` instead of `r.bytes()`. This is so that there is /// no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_bytes", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn bytes(r: &Self) -> &[u8] { // SAFETY: We don't call any methods on `b` other than those provided by @@ -737,6 +1116,29 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::bytes_mut(r)` instead of `r.bytes_mut()`. This is so that /// there is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_bytes_mut", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] pub fn bytes_mut(r: &mut Self) -> &mut [u8] { // SAFETY: We don't call any methods on `b` other than those provided by @@ -755,6 +1157,12 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::read(r)` instead of `r.read()`. This is so that there is no /// conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_read_static_size", + format = "coco_static_size", + )] #[must_use = "has no side effects"] #[inline] pub fn read(r: &Self) -> T { @@ -780,14 +1188,20 @@ where /// Note: this is an associated function, which means that you have to call /// it as `Ref::write(r, t)` instead of `r.write(t)`. This is so that there /// is no conflict with a method on the inner type. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_write_static_size", + format = "coco_static_size", + )] #[inline] pub fn write(r: &mut Self, t: T) { // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSliceMut`. let b = unsafe { r.as_byte_slice_mut() }; - // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is - // a valid size and alignment for `T`. By safety invariant on + // SAFETY: By postcondition on `as_byte_slice_mut`, we know that `b` is a + // valid size and alignment for `T`. By safety invariant on // `ByteSlice`, we know that this is preserved via `.deref()`. Writing // `t` to the buffer will allow all of the bytes of `t` to be accessed // as a `[u8]`, but because `T: IntoBytes`, we know that this is sound. @@ -801,11 +1215,38 @@ where T: FromBytes + KnownLayout + Immutable + ?Sized, { type Target = T; + + /// Dereferences the value. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_deref", + format = "coco", + arity = 3, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ], + [ + @index 3 + @title "Dynamically Padded" + @variant "dynamic_padding" + ] + )] #[inline] fn deref(&self) -> &T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = self.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSlice`. let b = unsafe { self.as_byte_slice() }; @@ -824,11 +1265,12 @@ where return ptr.as_ref(); } - // PANICS: By postcondition on `as_byte_slice`, `b`'s size and alignment - // are valid for `T`, and by invariant on `ByteSlice`, these are - // preserved through `.deref()`, so this `unwrap` will not panic. + // PANICS: By postcondition on `as_byte_slice`, `b` is aligned for `T`, + // and its size is exactly the size encoded by `metadata`. By invariant + // on `ByteSlice`, these are preserved through `.deref()`, so this + // `unwrap` will not panic. let ptr = Ptr::from_ref(b) - .try_cast_into_no_leftover::<T, BecauseImmutable>(None) + .try_cast_into_no_leftover::<T, BecauseImmutable>(Some(metadata)) .expect("zerocopy internal error: Deref::deref should be infallible"); let ptr = ptr.recall_validity(); ptr.as_ref() @@ -843,11 +1285,32 @@ where // add a separate inherent method for this? T: FromBytes + IntoBytes + KnownLayout + Immutable + ?Sized, { + /// Mutably dereferences the value. + /// + #[doc = codegen_section!( + header = "h5", + bench = "ref_type_deref_mut", + format = "coco", + arity = 2, + [ + open + @index 1 + @title "Sized" + @variant "static_size" + ], + [ + @index 2 + @title "Unsized" + @variant "dynamic_size" + ] + )] #[inline] fn deref_mut(&mut self) -> &mut T { // Presumably unreachable, since we've guarded each constructor of `Ref`. static_assert_dst_is_not_zst!(T); + let metadata = self.pointer_metadata(); + // SAFETY: We don't call any methods on `b` other than those provided by // `ByteSliceMut`. let b = unsafe { self.as_byte_slice_mut() }; @@ -862,7 +1325,7 @@ where T, _, (BecauseRead, BecauseExclusive), - (BecauseMutationCompatible, BecauseInvariantsEq), + (BecauseMutationCompatible, BecauseSharedCompatible), >(ptr) }; @@ -873,12 +1336,12 @@ where return ptr.as_mut(); } - // PANICS: By postcondition on `as_byte_slice_mut`, `b`'s size and - // alignment are valid for `T`, and by invariant on `ByteSlice`, these - // are preserved through `.deref_mut()`, so this `unwrap` will not - // panic. + // PANICS: By postcondition on `as_byte_slice_mut`, `b` is aligned for + // `T`, and its size is exactly the size encoded by `metadata`. By + // invariant on `ByteSlice`, these are preserved through `.deref_mut()`, + // so this `unwrap` will not panic. let ptr = Ptr::from_mut(b) - .try_cast_into_no_leftover::<T, BecauseExclusive>(None) + .try_cast_into_no_leftover::<T, BecauseExclusive>(Some(metadata)) .expect("zerocopy internal error: DerefMut::deref_mut should be infallible"); let ptr = ptr.recall_validity::<_, (_, (_, BecauseExclusive))>(); ptr.as_mut() @@ -958,13 +1421,13 @@ where /// `T: Sized` and `ptr`'s referent must have size `size_of::<T>()`. #[inline(always)] unsafe fn cast_for_sized<'a, T, A, R, S>( - ptr: Ptr<'a, [u8], (A, Aligned, Valid)>, -) -> Ptr<'a, T, (A, Unaligned, Valid)> + ptr: Ptr<'a, [u8], (A, Aligned, Safe)>, +) -> Ptr<'a, T, (A, Unaligned, Safe)> where T: FromBytes + KnownLayout + ?Sized, A: crate::invariant::Aliasing, [u8]: MutationCompatible<T, A, Initialized, Initialized, R>, - T: TransmuteFromPtr<T, A, Initialized, Valid, crate::pointer::cast::IdCast, S>, + T: TransmuteFromPtr<T, A, Initialized, Safe, crate::pointer::cast::IdCast, S>, { use crate::pointer::cast::{Cast, Project}; @@ -990,7 +1453,7 @@ where ptr.recall_validity::<Initialized, (_, (_, _))>() .cast::<_, CastForSized, _>() - .recall_validity::<Valid, _>() + .recall_validity::<Safe, _>() } #[cfg(test)] @@ -1028,6 +1491,66 @@ mod tests { assert_eq!(buf_ptr, deref_ptr); } + #[test] + #[allow(clippy::clone_on_copy)] // Exercise `Ref`'s manual `Clone` impl. + fn test_with_elems_preserves_ambiguous_metadata() { + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, align(8))] + struct Frame { + header: [u8; 8], + body: [u8], + } + + #[derive(FromBytes, Immutable, KnownLayout)] + #[repr(C, align(8))] + struct OddHeader { + header: [u8; 7], + body: [u8], + } + + // Dynamic trailing padding makes these pairs of distinct element + // counts produce the same total byte size. Thus, the byte length alone + // cannot recover the count supplied to a `*_with_elems` constructor. + assert_eq!(Frame::size_for_metadata(1), Some(16)); + assert_eq!(Frame::size_for_metadata(8), Some(16)); + assert_eq!(OddHeader::size_for_metadata(0), Some(8)); + assert_eq!(OddHeader::size_for_metadata(1), Some(8)); + + let bytes = Align::<[u8; 32], AU64>::default(); + + let whole = Ref::<_, Frame>::from_bytes_with_elems(&bytes.t[..16], 1).unwrap(); + assert_eq!(whole.body.len(), 1); + assert_eq!(Ref::bytes(&whole).len(), 16); + assert_eq!(whole.clone().body.len(), 1); + let copied = whole; + assert_eq!(copied.body.len(), 1); + assert_eq!(Ref::into_ref(whole).body.len(), 1); + + let (prefix, rest) = Ref::<_, Frame>::from_prefix_with_elems(&bytes.t[..], 1).unwrap(); + assert_eq!(prefix.body.len(), 1); + assert_eq!(rest.len(), 16); + + let (before, suffix) = Ref::<_, Frame>::from_suffix_with_elems(&bytes.t[..24], 1).unwrap(); + assert_eq!(before.len(), 8); + assert_eq!(suffix.body.len(), 1); + + // Zero is real metadata, not a sentinel for "infer from the bytes". + let empty = Ref::<_, OddHeader>::from_bytes_with_elems(&bytes.t[..8], 0).unwrap(); + assert!(empty.body.is_empty()); + assert!(Ref::into_ref(empty).body.is_empty()); + + // The implicit constructors still choose the largest fitting count, + // and retain the metadata selected during their validating cast. + let inferred = Ref::<_, Frame>::from_bytes(&bytes.t[..16]).unwrap(); + assert_eq!(inferred.body.len(), 8); + let (inferred, rest) = Ref::<_, Frame>::from_prefix(&bytes.t[..16]).unwrap(); + assert_eq!(inferred.body.len(), 8); + assert!(rest.is_empty()); + let (before, inferred) = Ref::<_, Frame>::from_suffix(&bytes.t[..16]).unwrap(); + assert!(before.is_empty()); + assert_eq!(inferred.body.len(), 8); + } + // Verify that values written to a `Ref` are properly shared between the // typed and untyped representations, that reads via `deref` and `read` // behave the same, and that writes via `deref_mut` and `write` behave the @@ -1251,6 +1774,13 @@ mod tests { *rf = u64::MAX; assert_eq!(buf.t, [0xFF; 8]); + + // Exercise the unsized `into_mut` route, which must use the metadata + // retained by the constructor rather than infer fresh metadata. + let mut buf = Align::<[u8; 8], u16>::default(); + let r = Ref::<_, [u16]>::from_bytes_with_elems(&mut buf.t[..], 4).unwrap(); + let rf = Ref::into_mut(r); + assert_eq!(rf.len(), 4); } #[test] @@ -1325,8 +1855,13 @@ mod benches { let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); b.iter(move || { // SAFETY: The preceding `from_bytes` succeeded, and so we know that - // `buf` is validly-aligned and has the correct length. - let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + // `buf` is validly aligned and has the `size_of::<AU64>()` byte + // length required by `new_sized_unchecked` [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::<&mut [u8], AU64>::new_sized_unchecked(&mut *buf) }; test::black_box(Ref::into_mut(test::black_box(r))); }); } @@ -1349,8 +1884,13 @@ mod benches { let _ = Ref::<_, AU64>::from_bytes(&mut *buf).unwrap(); b.iter(|| { // SAFETY: The preceding `from_bytes` succeeded, and so we know that - // `buf` is validly-aligned and has the correct length. - let r = unsafe { Ref::<&mut [u8], AU64>::new_unchecked(&mut *buf) }; + // `buf` is validly aligned and has the `size_of::<AU64>()` byte + // length required by `new_sized_unchecked` [1]. + // + // [1] Per https://doc.rust-lang.org/1.56.0/core/mem/fn.size_of.html: + // + // Returns the size of a type in bytes. + let r = unsafe { Ref::<&mut [u8], AU64>::new_sized_unchecked(&mut *buf) }; let mut temp = test::black_box(r); test::black_box(temp.deref_mut()); }); |