| 1 |
// Copyright 2023 The Fuchsia Authors |
| 2 |
// |
| 3 |
// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0 |
| 4 |
// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT |
| 5 |
// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option. |
| 6 |
// This file may not be copied, modified, or distributed except according to |
| 7 |
// those terms. |
| 8 |
|
| 9 |
#[macro_use] |
| 10 |
pub(crate) mod macros; |
| 11 |
|
| 12 |
#[doc(hidden)] |
| 13 |
pub mod macro_util; |
| 14 |
|
| 15 |
use core::{ |
| 16 |
marker::PhantomData, |
| 17 |
mem::{self, ManuallyDrop}, |
| 18 |
num::NonZeroUsize, |
| 19 |
ptr::NonNull, |
| 20 |
}; |
| 21 |
|
| 22 |
use super::*; |
| 23 |
|
| 24 |
/// Like [`PhantomData`], but [`Send`] and [`Sync`] regardless of whether the |
| 25 |
/// wrapped `T` is. |
| 26 |
pub(crate) struct SendSyncPhantomData<T: ?Sized>(PhantomData<T>); |
| 27 |
|
| 28 |
// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound |
| 29 |
// to be called from multiple threads. |
| 30 |
unsafe impl<T: ?Sized> Send for SendSyncPhantomData<T> {} |
| 31 |
// SAFETY: `SendSyncPhantomData` does not enable any behavior which isn't sound |
| 32 |
// to be called from multiple threads. |
| 33 |
unsafe impl<T: ?Sized> Sync for SendSyncPhantomData<T> {} |
| 34 |
|
| 35 |
impl<T: ?Sized> Default for SendSyncPhantomData<T> { |
| 36 |
fn default() -> SendSyncPhantomData<T> { |
| 37 |
SendSyncPhantomData(PhantomData) |
| 38 |
} |
| 39 |
} |
| 40 |
|
| 41 |
impl<T: ?Sized> PartialEq for SendSyncPhantomData<T> { |
| 42 |
fn eq(&self, other: &Self) -> bool { |
| 43 |
self.0.eq(&other.0) |
| 44 |
} |
| 45 |
} |
| 46 |
|
| 47 |
impl<T: ?Sized> Eq for SendSyncPhantomData<T> {} |
| 48 |
|
| 49 |
pub(crate) trait AsAddress { |
| 50 |
fn addr(self) -> usize; |
| 51 |
} |
| 52 |
|
| 53 |
impl<T: ?Sized> AsAddress for &T { |
| 54 |
#[inline(always)] |
| 55 |
fn addr(self) -> usize { |
| 56 |
let ptr: *const T = self; |
| 57 |
AsAddress::addr(ptr) |
| 58 |
} |
| 59 |
} |
| 60 |
|
| 61 |
impl<T: ?Sized> AsAddress for &mut T { |
| 62 |
#[inline(always)] |
| 63 |
fn addr(self) -> usize { |
| 64 |
let ptr: *const T = self; |
| 65 |
AsAddress::addr(ptr) |
| 66 |
} |
| 67 |
} |
| 68 |
|
| 69 |
impl<T: ?Sized> AsAddress for NonNull<T> { |
| 70 |
#[inline(always)] |
| 71 |
fn addr(self) -> usize { |
| 72 |
AsAddress::addr(self.as_ptr()) |
| 73 |
} |
| 74 |
} |
| 75 |
|
| 76 |
impl<T: ?Sized> AsAddress for *const T { |
| 77 |
#[inline(always)] |
| 78 |
fn addr(self) -> usize { |
| 79 |
// FIXME(#181), FIXME(https://github.com/rust-lang/rust/issues/95228): |
| 80 |
// Use `.addr()` instead of `as usize` once it's stable, and get rid of |
| 81 |
// this `allow`. Currently, `as usize` is the only way to accomplish |
| 82 |
// this. |
| 83 |
#[allow(clippy::as_conversions)] |
| 84 |
#[cfg_attr( |
| 85 |
__ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS, |
| 86 |
allow(lossy_provenance_casts) |
| 87 |
)] |
| 88 |
return self.cast::<()>() as usize; |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
impl<T: ?Sized> AsAddress for *mut T { |
| 93 |
#[inline(always)] |
| 94 |
fn addr(self) -> usize { |
| 95 |
let ptr: *const T = self; |
| 96 |
AsAddress::addr(ptr) |
| 97 |
} |
| 98 |
} |
| 99 |
|
| 100 |
/// Validates that `t` is aligned to `align_of::<U>()`. |
| 101 |
#[inline(always)] |
| 102 |
pub(crate) fn validate_aligned_to<T: AsAddress, U>(t: T) -> Result<(), AlignmentError<(), U>> { |
| 103 |
// `mem::align_of::<U>()` is guaranteed to return a non-zero value, which in |
| 104 |
// turn guarantees that this mod operation will not panic. |
| 105 |
#[allow(clippy::arithmetic_side_effects)] |
| 106 |
let remainder = t.addr() % mem::align_of::<U>(); |
| 107 |
if remainder == 0 { |
| 108 |
Ok(()) |
| 109 |
} else { |
| 110 |
// SAFETY: We just confirmed that `t.addr() % align_of::<U>() != 0`. |
| 111 |
// That's only possible if `align_of::<U>() > 1`. |
| 112 |
Err(unsafe { AlignmentError::new_unchecked(()) }) |
| 113 |
} |
| 114 |
} |
| 115 |
|
| 116 |
/// Returns the bytes needed to pad `len` to the next multiple of `align`. |
| 117 |
/// |
| 118 |
/// This function assumes that align is a power of two; there are no guarantees |
| 119 |
/// on the answer it gives if this is not the case. |
| 120 |
#[cfg_attr( |
| 121 |
kani, |
| 122 |
kani::requires(len <= isize::MAX as usize), |
| 123 |
kani::requires(align.is_power_of_two()), |
| 124 |
kani::ensures(|&p| (len + p) % align.get() == 0), |
| 125 |
// Ensures that we add the minimum required padding. |
| 126 |
kani::ensures(|&p| p < align.get()), |
| 127 |
)] |
| 128 |
pub(crate) const fn padding_needed_for(len: usize, align: NonZeroUsize) -> usize { |
| 129 |
#[cfg(kani)] |
| 130 |
#[kani::proof_for_contract(padding_needed_for)] |
| 131 |
fn proof() { |
| 132 |
padding_needed_for(kani::any(), kani::any()); |
| 133 |
} |
| 134 |
|
| 135 |
// Abstractly, we want to compute: |
| 136 |
// align - (len % align). |
| 137 |
// Handling the case where len%align is 0. |
| 138 |
// Because align is a power of two, len % align = len & (align-1). |
| 139 |
// Guaranteed not to underflow as align is nonzero. |
| 140 |
#[allow(clippy::arithmetic_side_effects)] |
| 141 |
let mask = align.get() - 1; |
| 142 |
|
| 143 |
// To efficiently subtract this value from align, we can use the bitwise complement. |
| 144 |
// Note that ((!len) & (align-1)) gives us a number that with (len & |
| 145 |
// (align-1)) sums to align-1. So subtracting 1 from x before taking the |
| 146 |
// complement subtracts `len` from `align`. Some quick inspection of |
| 147 |
// cases shows that this also handles the case where `len % align = 0` |
| 148 |
// correctly too: len-1 % align then equals align-1, so the complement mod |
| 149 |
// align will be 0, as desired. |
| 150 |
// |
| 151 |
// The following reasoning can be verified quickly by an SMT solver |
| 152 |
// supporting the theory of bitvectors: |
| 153 |
// ```smtlib |
| 154 |
// ; Naive implementation of padding |
| 155 |
// (define-fun padding1 ( |
| 156 |
// (len (_ BitVec 32)) |
| 157 |
// (align (_ BitVec 32))) (_ BitVec 32) |
| 158 |
// (ite |
| 159 |
// (= (_ bv0 32) (bvand len (bvsub align (_ bv1 32)))) |
| 160 |
// (_ bv0 32) |
| 161 |
// (bvsub align (bvand len (bvsub align (_ bv1 32)))))) |
| 162 |
// |
| 163 |
// ; The implementation below |
| 164 |
// (define-fun padding2 ( |
| 165 |
// (len (_ BitVec 32)) |
| 166 |
// (align (_ BitVec 32))) (_ BitVec 32) |
| 167 |
// (bvand (bvnot (bvsub len (_ bv1 32))) (bvsub align (_ bv1 32)))) |
| 168 |
// |
| 169 |
// (define-fun is-power-of-two ((x (_ BitVec 32))) Bool |
| 170 |
// (= (_ bv0 32) (bvand x (bvsub x (_ bv1 32))))) |
| 171 |
// |
| 172 |
// (declare-const len (_ BitVec 32)) |
| 173 |
// (declare-const align (_ BitVec 32)) |
| 174 |
// ; Search for a case where align is a power of two and padding2 disagrees with padding1 |
| 175 |
// (assert (and (is-power-of-two align) |
| 176 |
// (not (= (padding1 len align) (padding2 len align))))) |
| 177 |
// (simplify (padding1 (_ bv300 32) (_ bv32 32))) ; 20 |
| 178 |
// (simplify (padding2 (_ bv300 32) (_ bv32 32))) ; 20 |
| 179 |
// (simplify (padding1 (_ bv322 32) (_ bv32 32))) ; 30 |
| 180 |
// (simplify (padding2 (_ bv322 32) (_ bv32 32))) ; 30 |
| 181 |
// (simplify (padding1 (_ bv8 32) (_ bv8 32))) ; 0 |
| 182 |
// (simplify (padding2 (_ bv8 32) (_ bv8 32))) ; 0 |
| 183 |
// (check-sat) ; unsat, also works for 64-bit bitvectors |
| 184 |
// ``` |
| 185 |
!(len.wrapping_sub(1)) & mask |
| 186 |
} |
| 187 |
|
| 188 |
/// Rounds `n` down to the largest value `m` such that `m <= n` and `m % align |
| 189 |
/// == 0`. |
| 190 |
/// |
| 191 |
/// # Panics |
| 192 |
/// |
| 193 |
/// May panic if `align` is not a power of two. Even if it doesn't panic in this |
| 194 |
/// case, it will produce nonsense results. |
| 195 |
#[inline(always)] |
| 196 |
#[cfg_attr( |
| 197 |
kani, |
| 198 |
kani::requires(align.is_power_of_two()), |
| 199 |
kani::ensures(|&m| m <= n && m % align.get() == 0), |
| 200 |
// Guarantees that `m` is the *largest* value such that `m % align == 0`. |
| 201 |
kani::ensures(|&m| { |
| 202 |
// If this `checked_add` fails, then the next multiple would wrap |
| 203 |
// around, which trivially satisfies the "largest value" requirement. |
| 204 |
m.checked_add(align.get()).map(|next_mul| next_mul > n).unwrap_or(true) |
| 205 |
}) |
| 206 |
)] |
| 207 |
pub(crate) const fn round_down_to_next_multiple_of_alignment( |
| 208 |
n: usize, |
| 209 |
align: NonZeroUsize, |
| 210 |
) -> usize { |
| 211 |
#[cfg(kani)] |
| 212 |
#[kani::proof_for_contract(round_down_to_next_multiple_of_alignment)] |
| 213 |
fn proof() { |
| 214 |
round_down_to_next_multiple_of_alignment(kani::any(), kani::any()); |
| 215 |
} |
| 216 |
|
| 217 |
let align = align.get(); |
| 218 |
#[cfg(zerocopy_panic_in_const_and_vec_try_reserve_1_57_0)] |
| 219 |
debug_assert!(align.is_power_of_two()); |
| 220 |
|
| 221 |
// Subtraction can't underflow because `align.get() >= 1`. |
| 222 |
#[allow(clippy::arithmetic_side_effects)] |
| 223 |
let mask = !(align - 1); |
| 224 |
n & mask |
| 225 |
} |
| 226 |
|
| 227 |
pub(crate) const fn max(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { |
| 228 |
if a.get() < b.get() { |
| 229 |
b |
| 230 |
} else { |
| 231 |
a |
| 232 |
} |
| 233 |
} |
| 234 |
|
| 235 |
pub(crate) const fn min(a: NonZeroUsize, b: NonZeroUsize) -> NonZeroUsize { |
| 236 |
if a.get() > b.get() { |
| 237 |
b |
| 238 |
} else { |
| 239 |
a |
| 240 |
} |
| 241 |
} |
| 242 |
|
| 243 |
/// Copies `src` into the prefix of `dst`. |
| 244 |
/// |
| 245 |
/// # Safety |
| 246 |
/// |
| 247 |
/// The caller guarantees that `src.len() <= dst.len()`. |
| 248 |
#[inline(always)] |
| 249 |
pub(crate) unsafe fn copy_unchecked(src: &[u8], dst: &mut [u8]) { |
| 250 |
debug_assert!(src.len() <= dst.len()); |
| 251 |
// SAFETY: This invocation satisfies the safety contract of |
| 252 |
// copy_nonoverlapping [1]: |
| 253 |
// - `src.as_ptr()` is trivially valid for reads of `src.len()` bytes |
| 254 |
// - `dst.as_ptr()` is valid for writes of `src.len()` bytes, because the |
| 255 |
// caller has promised that `src.len() <= dst.len()` |
| 256 |
// - `src` and `dst` are, trivially, properly aligned |
| 257 |
// - the region of memory beginning at `src` with a size of `src.len()` |
| 258 |
// bytes does not overlap with the region of memory beginning at `dst` |
| 259 |
// with the same size, because `dst` is derived from an exclusive |
| 260 |
// reference. |
| 261 |
unsafe { |
| 262 |
core::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr(), src.len()); |
| 263 |
}; |
| 264 |
} |
| 265 |
|
| 266 |
/// Unsafely transmutes the given `src` into a type `Dst`. |
| 267 |
/// |
| 268 |
/// # Safety |
| 269 |
/// |
| 270 |
/// The value `src` must be a valid instance of `Dst`. |
| 271 |
#[inline(always)] |
| 272 |
pub(crate) const unsafe fn transmute_unchecked<Src, Dst>(src: Src) -> Dst { |
| 273 |
static_assert!(Src, Dst => core::mem::size_of::<Src>() == core::mem::size_of::<Dst>()); |
| 274 |
|
| 275 |
#[repr(C)] |
| 276 |
union Transmute<Src, Dst> { |
| 277 |
src: ManuallyDrop<Src>, |
| 278 |
dst: ManuallyDrop<Dst>, |
| 279 |
} |
| 280 |
|
| 281 |
// SAFETY: Since `Transmute<Src, Dst>` is `#[repr(C)]`, its `src` and `dst` |
| 282 |
// fields both start at the same offset and the types of those fields are |
| 283 |
// transparent wrappers around `Src` and `Dst` [1]. Consequently, |
| 284 |
// initializng `Transmute` with with `src` and then reading out `dst` is |
| 285 |
// equivalent to transmuting from `Src` to `Dst` [2]. Transmuting from `src` |
| 286 |
// to `Dst` is valid because — by contract on the caller — `src` is a valid |
| 287 |
// instance of `Dst`. |
| 288 |
// |
| 289 |
// [1] Per https://doc.rust-lang.org/1.82.0/std/mem/struct.ManuallyDrop.html: |
| 290 |
// |
| 291 |
// `ManuallyDrop<T>` is guaranteed to have the same layout and bit |
| 292 |
// validity as `T`, and is subject to the same layout optimizations as |
| 293 |
// `T`. |
| 294 |
// |
| 295 |
// [2] Per https://doc.rust-lang.org/1.82.0/reference/items/unions.html#reading-and-writing-union-fields: |
| 296 |
// |
| 297 |
// Effectively, writing to and then reading from a union with the C |
| 298 |
// representation is analogous to a transmute from the type used for |
| 299 |
// writing to the type used for reading. |
| 300 |
unsafe { ManuallyDrop::into_inner(Transmute { src: ManuallyDrop::new(src) }.dst) } |
| 301 |
} |
| 302 |
|
| 303 |
/// Uses `allocate` to create a `Box<T>`. |
| 304 |
/// |
| 305 |
/// # Errors |
| 306 |
/// |
| 307 |
/// Returns an error on allocation failure. Allocation failure is guaranteed |
| 308 |
/// never to cause a panic or an abort. |
| 309 |
/// |
| 310 |
/// # Safety |
| 311 |
/// |
| 312 |
/// `allocate` must be either `alloc::alloc::alloc` or |
| 313 |
/// `alloc::alloc::alloc_zeroed`. The referent of the box returned by `new_box` |
| 314 |
/// has the same bit-validity as the referent of the pointer returned by the |
| 315 |
/// given `allocate` and sufficient size to store `T` with `meta`. |
| 316 |
#[must_use = "has no side effects (other than allocation)"] |
| 317 |
#[cfg(feature = "alloc")] |
| 318 |
#[inline] |
| 319 |
pub(crate) unsafe fn new_box<T>( |
| 320 |
meta: T::PointerMetadata, |
| 321 |
allocate: unsafe fn(core::alloc::Layout) -> *mut u8, |
| 322 |
) -> Result<alloc::boxed::Box<T>, AllocError> |
| 323 |
where |
| 324 |
T: ?Sized + crate::KnownLayout, |
| 325 |
{ |
| 326 |
let size = match T::size_for_metadata(meta) { |
| 327 |
Some(size) => size, |
| 328 |
None => return Err(AllocError), |
| 329 |
}; |
| 330 |
|
| 331 |
let align = T::LAYOUT.align.get(); |
| 332 |
// On stable Rust versions <= 1.64.0, `Layout::from_size_align` has a bug in |
| 333 |
// which sufficiently-large allocations (those which, when rounded up to the |
| 334 |
// alignment, overflow `isize`) are not rejected, which can cause undefined |
| 335 |
// behavior. See #64 for details. |
| 336 |
// |
| 337 |
// FIXME(#67): Once our MSRV is > 1.64.0, remove this assertion. |
| 338 |
#[allow(clippy::as_conversions)] |
| 339 |
let max_alloc = (isize::MAX as usize).saturating_sub(align); |
| 340 |
if size > max_alloc { |
| 341 |
return Err(AllocError); |
| 342 |
} |
| 343 |
|
| 344 |
// FIXME(https://github.com/rust-lang/rust/issues/55724): Use |
| 345 |
// `Layout::repeat` once it's stabilized. |
| 346 |
let layout = Layout::from_size_align(size, align).or(Err(AllocError))?; |
| 347 |
|
| 348 |
let ptr = if layout.size() != 0 { |
| 349 |
// SAFETY: By contract on the caller, `allocate` is either |
| 350 |
// `alloc::alloc::alloc` or `alloc::alloc::alloc_zeroed`. The above |
| 351 |
// check ensures their shared safety precondition: that the supplied |
| 352 |
// layout is not zero-sized type [1]. |
| 353 |
// |
| 354 |
// [1] Per https://doc.rust-lang.org/stable/std/alloc/trait.GlobalAlloc.html#tymethod.alloc: |
| 355 |
// |
| 356 |
// This function is unsafe because undefined behavior can result if |
| 357 |
// the caller does not ensure that layout has non-zero size. |
| 358 |
let ptr = unsafe { allocate(layout) }; |
| 359 |
match NonNull::new(ptr) { |
| 360 |
Some(ptr) => ptr, |
| 361 |
None => return Err(AllocError), |
| 362 |
} |
| 363 |
} else { |
| 364 |
let align = T::LAYOUT.align.get(); |
| 365 |
// We use `transmute` instead of an `as` cast since Miri (with strict |
| 366 |
// provenance enabled) notices and complains that an `as` cast creates a |
| 367 |
// pointer with no provenance. Miri isn't smart enough to realize that |
| 368 |
// we're only executing this branch when we're constructing a zero-sized |
| 369 |
// `Box`, which doesn't require provenance. |
| 370 |
// |
| 371 |
// SAFETY: any initialized bit sequence is a bit-valid `*mut u8`. All |
| 372 |
// bits of a `usize` are initialized. |
| 373 |
#[allow(unknown_lints)] // For `integer_to_ptr_transmutes` |
| 374 |
#[allow(clippy::useless_transmute, integer_to_ptr_transmutes)] |
| 375 |
let dangling = unsafe { mem::transmute::<usize, *mut u8>(align) }; |
| 376 |
// SAFETY: `dangling` is constructed from `T::LAYOUT.align`, which is a |
| 377 |
// `NonZeroUsize`, which is guaranteed to be non-zero. |
| 378 |
// |
| 379 |
// `Box<[T]>` does not allocate when `T` is zero-sized or when `len` is |
| 380 |
// zero, but it does require a non-null dangling pointer for its |
| 381 |
// allocation. |
| 382 |
// |
| 383 |
// FIXME(https://github.com/rust-lang/rust/issues/95228): Use |
| 384 |
// `std::ptr::without_provenance` once it's stable. That may optimize |
| 385 |
// better. As written, Rust may assume that this consumes "exposed" |
| 386 |
// provenance, and thus Rust may have to assume that this may consume |
| 387 |
// provenance from any pointer whose provenance has been exposed. |
| 388 |
unsafe { NonNull::new_unchecked(dangling) } |
| 389 |
}; |
| 390 |
|
| 391 |
let ptr = T::raw_from_ptr_len(ptr, meta); |
| 392 |
|
| 393 |
// FIXME(#429): Add a "SAFETY" comment and remove this `allow`. Make sure to |
| 394 |
// include a justification that `ptr.as_ptr()` is validly-aligned in the ZST |
| 395 |
// case (in which we manually construct a dangling pointer) and to justify |
| 396 |
// why `Box` is safe to drop (it's because `allocate` uses the system |
| 397 |
// allocator). |
| 398 |
#[allow(clippy::undocumented_unsafe_blocks)] |
| 399 |
Ok(unsafe { alloc::boxed::Box::from_raw(ptr.as_ptr()) }) |
| 400 |
} |
| 401 |
|
| 402 |
mod len_of { |
| 403 |
use super::*; |
| 404 |
|
| 405 |
/// A witness type for metadata of a valid instance of `&T`. |
| 406 |
pub(crate) struct MetadataOf<T: ?Sized + KnownLayout> { |
| 407 |
/// # Safety |
| 408 |
/// |
| 409 |
/// The size of an instance of `&T` with the given metadata is not |
| 410 |
/// larger than `isize::MAX`. |
| 411 |
meta: T::PointerMetadata, |
| 412 |
_p: PhantomData<T>, |
| 413 |
} |
| 414 |
|
| 415 |
impl<T: ?Sized + KnownLayout> Copy for MetadataOf<T> {} |
| 416 |
impl<T: ?Sized + KnownLayout> Clone for MetadataOf<T> { |
| 417 |
fn clone(&self) -> Self { |
| 418 |
*self |
| 419 |
} |
| 420 |
} |
| 421 |
|
| 422 |
impl<T: ?Sized> MetadataOf<T> |
| 423 |
where |
| 424 |
T: KnownLayout, |
| 425 |
{ |
| 426 |
/// Returns `None` if `meta` is greater than `t`'s metadata. |
| 427 |
#[inline(always)] |
| 428 |
pub(crate) fn new_in_bounds(t: &T, meta: usize) -> Option<Self> |
| 429 |
where |
| 430 |
T: KnownLayout<PointerMetadata = usize>, |
| 431 |
{ |
| 432 |
if meta <= Ptr::from_ref(t).len() { |
| 433 |
// SAFETY: We have checked that `meta` is not greater than `t`'s |
| 434 |
// metadata, which, by invariant on `&T`, addresses no more than |
| 435 |
// `isize::MAX` bytes [1][2]. |
| 436 |
// |
| 437 |
// [1] Per https://doc.rust-lang.org/1.85.0/std/primitive.reference.html#safety: |
| 438 |
// |
| 439 |
// For all types, `T: ?Sized`, and for all `t: &T` or `t: |
| 440 |
// &mut T`, when such values cross an API boundary, the |
| 441 |
// following invariants must generally be upheld: |
| 442 |
// |
| 443 |
// * `t` is non-null |
| 444 |
// * `t` is aligned to `align_of_val(t)` |
| 445 |
// * if `size_of_val(t) > 0`, then `t` is dereferenceable for |
| 446 |
// `size_of_val(t)` many bytes |
| 447 |
// |
| 448 |
// If `t` points at address `a`, being "dereferenceable" for |
| 449 |
// N bytes means that the memory range `[a, a + N)` is all |
| 450 |
// contained within a single allocated object. |
| 451 |
// |
| 452 |
// [2] Per https://doc.rust-lang.org/1.85.0/std/ptr/index.html#allocated-object: |
| 453 |
// |
| 454 |
// For any allocated object with `base` address, `size`, and |
| 455 |
// a set of `addresses`, the following are guaranteed: |
| 456 |
// - For all addresses `a` in `addresses`, `a` is in the |
| 457 |
// range `base .. (base + size)` (note that this requires |
| 458 |
// `a < base + size`, not `a <= base + size`) |
| 459 |
// - `base` is not equal to [`null()`] (i.e., the address |
| 460 |
// with the numerical value 0) |
| 461 |
// - `base + size <= usize::MAX` |
| 462 |
// - `size <= isize::MAX` |
| 463 |
Some(unsafe { Self::new_unchecked(meta) }) |
| 464 |
} else { |
| 465 |
None |
| 466 |
} |
| 467 |
} |
| 468 |
|
| 469 |
/// # Safety |
| 470 |
/// |
| 471 |
/// The size of an instance of `&T` with the given metadata is not |
| 472 |
/// larger than `isize::MAX`. |
| 473 |
pub(crate) unsafe fn new_unchecked(meta: T::PointerMetadata) -> Self { |
| 474 |
// SAFETY: The caller has promised that the size of an instance of |
| 475 |
// `&T` with the given metadata is not larger than `isize::MAX`. |
| 476 |
Self { meta, _p: PhantomData } |
| 477 |
} |
| 478 |
|
| 479 |
pub(crate) fn get(&self) -> T::PointerMetadata |
| 480 |
where |
| 481 |
T::PointerMetadata: Copy, |
| 482 |
{ |
| 483 |
self.meta |
| 484 |
} |
| 485 |
|
| 486 |
#[inline] |
| 487 |
pub(crate) fn padding_needed_for(&self) -> usize |
| 488 |
where |
| 489 |
T: KnownLayout<PointerMetadata = usize>, |
| 490 |
{ |
| 491 |
let trailing_slice_layout = crate::trailing_slice_layout::<T>(); |
| 492 |
// SAFETY: By invariant on `self`, a `&T` with metadata `self.meta` |
| 493 |
// describes an object of size `<= isize::MAX`. This computes the |
| 494 |
// size of such a `&T` without any trailing padding, and so neither |
| 495 |
// the multiplication nor the addition will overflow. |
| 496 |
// |
| 497 |
// FIXME(#67): Remove this allow. See NumExt for more details. |
| 498 |
#[allow(unstable_name_collisions, clippy::incompatible_msrv)] |
| 499 |
let unpadded_size = unsafe { |
| 500 |
let trailing_size = self.meta.unchecked_mul(trailing_slice_layout.elem_size); |
| 501 |
trailing_size.unchecked_add(trailing_slice_layout.offset) |
| 502 |
}; |
| 503 |
|
| 504 |
util::padding_needed_for(unpadded_size, T::LAYOUT.align) |
| 505 |
} |
| 506 |
|
| 507 |
#[inline(always)] |
| 508 |
pub(crate) fn validate_cast_and_convert_metadata( |
| 509 |
addr: usize, |
| 510 |
bytes_len: MetadataOf<[u8]>, |
| 511 |
cast_type: CastType, |
| 512 |
meta: Option<T::PointerMetadata>, |
| 513 |
) -> Result<(MetadataOf<T>, MetadataOf<[u8]>), MetadataCastError> { |
| 514 |
let layout = match meta { |
| 515 |
None => T::LAYOUT, |
| 516 |
// This can return `None` if the metadata describes an object |
| 517 |
// which can't fit in an `isize`. |
| 518 |
Some(meta) => { |
| 519 |
let size = match T::size_for_metadata(meta) { |
| 520 |
Some(size) => size, |
| 521 |
None => return Err(MetadataCastError::Size), |
| 522 |
}; |
| 523 |
DstLayout { |
| 524 |
align: T::LAYOUT.align, |
| 525 |
size_info: crate::SizeInfo::Sized { size }, |
| 526 |
statically_shallow_unpadded: false, |
| 527 |
} |
| 528 |
} |
| 529 |
}; |
| 530 |
// Lemma 0: By contract on `validate_cast_and_convert_metadata`, if |
| 531 |
// the result is `Ok(..)`, then a `&T` with `elems` trailing slice |
| 532 |
// elements is no larger in size than `bytes_len.get()`. |
| 533 |
let (elems, split_at) = |
| 534 |
layout.validate_cast_and_convert_metadata(addr, bytes_len.get(), cast_type)?; |
| 535 |
let elems = T::PointerMetadata::from_elem_count(elems); |
| 536 |
|
| 537 |
// For a slice DST type, if `meta` is `Some(elems)`, then we |
| 538 |
// synthesize `layout` to describe a sized type whose size is equal |
| 539 |
// to the size of the instance that we are asked to cast. For sized |
| 540 |
// types, `validate_cast_and_convert_metadata` returns `elems == 0`. |
| 541 |
// Thus, in this case, we need to use the `elems` passed by the |
| 542 |
// caller, not the one returned by |
| 543 |
// `validate_cast_and_convert_metadata`. |
| 544 |
// |
| 545 |
// Lemma 1: A `&T` with `elems` trailing slice elements is no larger |
| 546 |
// in size than `bytes_len.get()`. Proof: |
| 547 |
// - If `meta` is `None`, then `elems` satisfies this condition by |
| 548 |
// Lemma 0. |
| 549 |
// - If `meta` is `Some(meta)`, then `layout` describes an object |
| 550 |
// whose size is equal to the size of an `&T` with `meta` |
| 551 |
// metadata. By Lemma 0, that size is not larger than |
| 552 |
// `bytes_len.get()`. |
| 553 |
// |
| 554 |
// Lemma 2: A `&T` with `elems` trailing slice elements is no larger |
| 555 |
// than `isize::MAX` bytes. Proof: By Lemma 1, a `&T` with metadata |
| 556 |
// `elems` is not larger in size than `bytes_len.get()`. By |
| 557 |
// invariant on `MetadataOf<[u8]>`, a `&[u8]` with metadata |
| 558 |
// `bytes_len` is not larger than `isize::MAX`. Because |
| 559 |
// `size_of::<u8>()` is `1`, a `&[u8]` with metadata `bytes_len` has |
| 560 |
// size `bytes_len.get()` bytes. Therefore, a `&T` with metadata |
| 561 |
// `elems` has size not larger than `isize::MAX`. |
| 562 |
let elems = meta.unwrap_or(elems); |
| 563 |
|
| 564 |
// SAFETY: See Lemma 2. |
| 565 |
let elems = unsafe { MetadataOf::new_unchecked(elems) }; |
| 566 |
|
| 567 |
// SAFETY: Let `size` be the size of a `&T` with metadata `elems`. |
| 568 |
// By post-condition on `validate_cast_and_convert_metadata`, one of |
| 569 |
// the following conditions holds: |
| 570 |
// - `split_at == size`, in which case, by Lemma 2, `split_at <= |
| 571 |
// isize::MAX`. Since `size_of::<u8>() == 1`, a `[u8]` with |
| 572 |
// `split_at` elems has size not larger than `isize::MAX`. |
| 573 |
// - `split_at == bytes_len - size`. Since `bytes_len: |
| 574 |
// MetadataOf<u8>`, and since `size` is non-negative, `split_at` |
| 575 |
// addresses no more bytes than `bytes_len` does. Since |
| 576 |
// `bytes_len: MetadataOf<u8>`, `bytes_len` describes a `[u8]` |
| 577 |
// which has no more than `isize::MAX` bytes, and thus so does |
| 578 |
// `split_at`. |
| 579 |
let split_at = unsafe { MetadataOf::<[u8]>::new_unchecked(split_at) }; |
| 580 |
Ok((elems, split_at)) |
| 581 |
} |
| 582 |
} |
| 583 |
} |
| 584 |
|
| 585 |
pub(crate) use len_of::MetadataOf; |
| 586 |
|
| 587 |
/// Since we support multiple versions of Rust, there are often features which |
| 588 |
/// have been stabilized in the most recent stable release which do not yet |
| 589 |
/// exist (stably) on our MSRV. This module provides polyfills for those |
| 590 |
/// features so that we can write more "modern" code, and just remove the |
| 591 |
/// polyfill once our MSRV supports the corresponding feature. Without this, |
| 592 |
/// we'd have to write worse/more verbose code and leave FIXME comments sprinkled |
| 593 |
/// throughout the codebase to update to the new pattern once it's stabilized. |
| 594 |
/// |
| 595 |
/// Each trait is imported as `_` at the crate root; each polyfill should "just |
| 596 |
/// work" at usage sites. |
| 597 |
pub(crate) mod polyfills { |
| 598 |
use core::ptr::{self, NonNull}; |
| 599 |
|
| 600 |
// A polyfill for `NonNull::slice_from_raw_parts` that we can use before our |
| 601 |
// MSRV is 1.70, when that function was stabilized. |
| 602 |
// |
| 603 |
// The `#[allow(unused)]` is necessary because, on sufficiently recent |
| 604 |
// toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent |
| 605 |
// method rather than to this trait, and so this trait is considered unused. |
| 606 |
// |
| 607 |
// FIXME(#67): Once our MSRV is 1.70, remove this. |
| 608 |
#[allow(unused)] |
| 609 |
pub(crate) trait NonNullExt<T> { |
| 610 |
fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]>; |
| 611 |
} |
| 612 |
|
| 613 |
impl<T> NonNullExt<T> for NonNull<T> { |
| 614 |
// NOTE on coverage: this will never be tested in nightly since it's a |
| 615 |
// polyfill for a feature which has been stabilized on our nightly |
| 616 |
// toolchain. |
| 617 |
#[cfg_attr( |
| 618 |
all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), |
| 619 |
coverage(off) |
| 620 |
)] |
| 621 |
#[inline(always)] |
| 622 |
fn slice_from_raw_parts(data: Self, len: usize) -> NonNull<[T]> { |
| 623 |
let ptr = ptr::slice_from_raw_parts_mut(data.as_ptr(), len); |
| 624 |
// SAFETY: `ptr` is converted from `data`, which is non-null. |
| 625 |
unsafe { NonNull::new_unchecked(ptr) } |
| 626 |
} |
| 627 |
} |
| 628 |
|
| 629 |
// A polyfill for `Self::unchecked_sub` that we can use until methods like |
| 630 |
// `usize::unchecked_sub` is stabilized. |
| 631 |
// |
| 632 |
// The `#[allow(unused)]` is necessary because, on sufficiently recent |
| 633 |
// toolchain versions, `ptr.slice_from_raw_parts()` resolves to the inherent |
| 634 |
// method rather than to this trait, and so this trait is considered unused. |
| 635 |
// |
| 636 |
// FIXME(#67): Once our MSRV is high enough, remove this. |
| 637 |
#[allow(unused)] |
| 638 |
pub(crate) trait NumExt { |
| 639 |
/// Add without checking for overflow. |
| 640 |
/// |
| 641 |
/// # Safety |
| 642 |
/// |
| 643 |
/// The caller promises that the addition will not overflow. |
| 644 |
unsafe fn unchecked_add(self, rhs: Self) -> Self; |
| 645 |
|
| 646 |
/// Subtract without checking for underflow. |
| 647 |
/// |
| 648 |
/// # Safety |
| 649 |
/// |
| 650 |
/// The caller promises that the subtraction will not underflow. |
| 651 |
unsafe fn unchecked_sub(self, rhs: Self) -> Self; |
| 652 |
|
| 653 |
/// Multiply without checking for overflow. |
| 654 |
/// |
| 655 |
/// # Safety |
| 656 |
/// |
| 657 |
/// The caller promises that the multiplication will not overflow. |
| 658 |
unsafe fn unchecked_mul(self, rhs: Self) -> Self; |
| 659 |
} |
| 660 |
|
| 661 |
// NOTE on coverage: these will never be tested in nightly since they're |
| 662 |
// polyfills for a feature which has been stabilized on our nightly |
| 663 |
// toolchain. |
| 664 |
impl NumExt for usize { |
| 665 |
#[cfg_attr( |
| 666 |
all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), |
| 667 |
coverage(off) |
| 668 |
)] |
| 669 |
#[inline(always)] |
| 670 |
unsafe fn unchecked_add(self, rhs: usize) -> usize { |
| 671 |
match self.checked_add(rhs) { |
| 672 |
Some(x) => x, |
| 673 |
None => { |
| 674 |
// SAFETY: The caller promises that the addition will not |
| 675 |
// underflow. |
| 676 |
unsafe { core::hint::unreachable_unchecked() } |
| 677 |
} |
| 678 |
} |
| 679 |
} |
| 680 |
|
| 681 |
#[cfg_attr( |
| 682 |
all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), |
| 683 |
coverage(off) |
| 684 |
)] |
| 685 |
#[inline(always)] |
| 686 |
unsafe fn unchecked_sub(self, rhs: usize) -> usize { |
| 687 |
match self.checked_sub(rhs) { |
| 688 |
Some(x) => x, |
| 689 |
None => { |
| 690 |
// SAFETY: The caller promises that the subtraction will not |
| 691 |
// underflow. |
| 692 |
unsafe { core::hint::unreachable_unchecked() } |
| 693 |
} |
| 694 |
} |
| 695 |
} |
| 696 |
|
| 697 |
#[cfg_attr( |
| 698 |
all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), |
| 699 |
coverage(off) |
| 700 |
)] |
| 701 |
#[inline(always)] |
| 702 |
unsafe fn unchecked_mul(self, rhs: usize) -> usize { |
| 703 |
match self.checked_mul(rhs) { |
| 704 |
Some(x) => x, |
| 705 |
None => { |
| 706 |
// SAFETY: The caller promises that the multiplication will |
| 707 |
// not overflow. |
| 708 |
unsafe { core::hint::unreachable_unchecked() } |
| 709 |
} |
| 710 |
} |
| 711 |
} |
| 712 |
} |
| 713 |
} |
| 714 |
|
| 715 |
#[cfg(test)] |
| 716 |
pub(crate) mod testutil { |
| 717 |
use crate::*; |
| 718 |
|
| 719 |
/// A `T` which is aligned to at least `align_of::<A>()`. |
| 720 |
#[derive(Default)] |
| 721 |
pub(crate) struct Align<T, A> { |
| 722 |
pub(crate) t: T, |
| 723 |
_a: [A; 0], |
| 724 |
} |
| 725 |
|
| 726 |
impl<T: Default, A> Align<T, A> { |
| 727 |
pub(crate) fn set_default(&mut self) { |
| 728 |
self.t = T::default(); |
| 729 |
} |
| 730 |
} |
| 731 |
|
| 732 |
impl<T, A> Align<T, A> { |
| 733 |
pub(crate) const fn new(t: T) -> Align<T, A> { |
| 734 |
Align { t, _a: [] } |
| 735 |
} |
| 736 |
} |
| 737 |
|
| 738 |
/// A `T` which is guaranteed not to satisfy `align_of::<A>()`. |
| 739 |
/// |
| 740 |
/// It must be the case that `align_of::<T>() < align_of::<A>()` in order |
| 741 |
/// for this type to work properly. |
| 742 |
#[repr(C)] |
| 743 |
pub(crate) struct ForceUnalign<T: Unaligned, A> { |
| 744 |
// The outer struct is aligned to `A`, and, thanks to `repr(C)`, `t` is |
| 745 |
// placed at the minimum offset that guarantees its alignment. If |
| 746 |
// `align_of::<T>() < align_of::<A>()`, then that offset will be |
| 747 |
// guaranteed *not* to satisfy `align_of::<A>()`. |
| 748 |
// |
| 749 |
// Note that we need `T: Unaligned` in order to guarantee that there is |
| 750 |
// no padding between `_u` and `t`. |
| 751 |
_u: u8, |
| 752 |
pub(crate) t: T, |
| 753 |
_a: [A; 0], |
| 754 |
} |
| 755 |
|
| 756 |
impl<T: Unaligned, A> ForceUnalign<T, A> { |
| 757 |
pub(crate) fn new(t: T) -> ForceUnalign<T, A> { |
| 758 |
ForceUnalign { _u: 0, t, _a: [] } |
| 759 |
} |
| 760 |
} |
| 761 |
// A `u64` with alignment 8. |
| 762 |
// |
| 763 |
// Though `u64` has alignment 8 on some platforms, it's not guaranteed. By |
| 764 |
// contrast, `AU64` is guaranteed to have alignment 8 on all platforms. |
| 765 |
#[derive( |
| 766 |
KnownLayout, |
| 767 |
Immutable, |
| 768 |
FromBytes, |
| 769 |
IntoBytes, |
| 770 |
Eq, |
| 771 |
PartialEq, |
| 772 |
Ord, |
| 773 |
PartialOrd, |
| 774 |
Default, |
| 775 |
Debug, |
| 776 |
Copy, |
| 777 |
Clone, |
| 778 |
)] |
| 779 |
#[repr(C, align(8))] |
| 780 |
pub(crate) struct AU64(pub(crate) u64); |
| 781 |
|
| 782 |
impl AU64 { |
| 783 |
// Converts this `AU64` to bytes using this platform's endianness. |
| 784 |
pub(crate) fn to_bytes(self) -> [u8; 8] { |
| 785 |
crate::transmute!(self) |
| 786 |
} |
| 787 |
} |
| 788 |
|
| 789 |
impl Display for AU64 { |
| 790 |
#[cfg_attr( |
| 791 |
all(coverage_nightly, __ZEROCOPY_INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS), |
| 792 |
coverage(off) |
| 793 |
)] |
| 794 |
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { |
| 795 |
Display::fmt(&self.0, f) |
| 796 |
} |
| 797 |
} |
| 798 |
} |
| 799 |
|
| 800 |
#[cfg(test)] |
| 801 |
mod tests { |
| 802 |
use super::*; |
| 803 |
|
| 804 |
#[test] |
| 805 |
fn test_round_down_to_next_multiple_of_alignment() { |
| 806 |
fn alt_impl(n: usize, align: NonZeroUsize) -> usize { |
| 807 |
let mul = n / align.get(); |
| 808 |
mul * align.get() |
| 809 |
} |
| 810 |
|
| 811 |
for align in [1, 2, 4, 8, 16] { |
| 812 |
for n in 0..256 { |
| 813 |
let align = NonZeroUsize::new(align).unwrap(); |
| 814 |
let want = alt_impl(n, align); |
| 815 |
let got = round_down_to_next_multiple_of_alignment(n, align); |
| 816 |
assert_eq!(got, want, "round_down_to_next_multiple_of_alignment({}, {})", n, align); |
| 817 |
} |
| 818 |
} |
| 819 |
} |
| 820 |
|
| 821 |
#[rustversion::since(1.57.0)] |
| 822 |
#[test] |
| 823 |
#[should_panic] |
| 824 |
fn test_round_down_to_next_multiple_of_alignment_zerocopy_panic_in_const_and_vec_try_reserve() { |
| 825 |
round_down_to_next_multiple_of_alignment(0, NonZeroUsize::new(3).unwrap()); |
| 826 |
} |
| 827 |
} |