diff --git a/src/lib.rs b/src/lib.rs index 390cd9a..be183c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -181,7 +181,9 @@ use malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}; #[cfg(not(feature = "gecko-ffi"))] mod impl_details { pub type SizeType = usize; - pub const MAX_CAP: usize = !0; + // for ZSTs, store the length in the the NonNull as a NonZero, + // the length is thus off by one and can only reach usize::MAX - 1 + pub const MAX_CAP: usize = usize::MAX - 1; #[inline(always)] pub fn assert_size(x: usize) -> SizeType { @@ -476,6 +478,13 @@ fn header_with_capacity(cap: usize, is_auto: bool) -> NonNull
{ } } +/// Safety: len must be != 0 +unsafe fn len_to_ptr_unchecked(len: usize) -> NonNull { + use core::num::NonZeroUsize; + // NonNull::without_provenance polyfill + unsafe { mem::transmute(NonZeroUsize::new_unchecked(len)) } +} + /// See the crate's top level documentation for a description of this type. #[repr(C)] pub struct ThinVec { @@ -524,6 +533,12 @@ macro_rules! thin_vec { } impl ThinVec { + /// Return true if we can use ZST optimizations + #[inline(always)] + const fn is_zst() -> bool { + size_of::() == 0 && !cfg!(feature = "gecko-ffi") + } + /// Creates a new empty ThinVec. /// /// This will not allocate. @@ -597,7 +612,7 @@ impl ThinVec { /// let vec_units = ThinVec::<()>::with_capacity(10); /// /// // Only true **without** the gecko-ffi feature! - /// // assert_eq!(vec_units.capacity(), usize::MAX); + /// // assert_eq!(vec_units.capacity(), usize::MAX - 1); /// ``` pub fn with_capacity(cap: usize) -> ThinVec { // `padding` contains ~static assertions against types that are @@ -607,6 +622,16 @@ impl ThinVec { // `Drop` impl, trippng an assertion along that code path causes a // double panic. We duplicate the assertion here so that it is // testable, + + if Self::is_zst() { + unsafe { + return ThinVec { + ptr: len_to_ptr_unchecked(1), + boo: PhantomData, + }; + } + } + let _ = padding::(); if cap == 0 { @@ -626,13 +651,26 @@ impl ThinVec { // Accessor conveniences - fn ptr(&self) -> *mut Header { + /// # Safety + /// Self::is_zst() == false + unsafe fn ptr(&self) -> *mut Header { + debug_assert!(!Self::is_zst()); self.ptr.as_ptr() } - fn header(&self) -> &Header { + + /// # Safety + /// Self::is_zst() == false + unsafe fn header(&self) -> &Header { + debug_assert!(!Self::is_zst()); unsafe { self.ptr.as_ref() } } + fn data_raw(&self) -> *mut T { + if Self::is_zst() { + // Polyfill for ptr::dangling_mut(), stable from 1.84 + return NonNull::dangling().as_ptr(); + } + // `padding` contains ~static assertions against types that are // incompatible with the current feature flags. Even if we don't // care about its result, we should always call it before getting @@ -676,8 +714,10 @@ impl ThinVec { } } - // This is unsafe when the header is EMPTY_HEADER. + /// # Safety + /// This is unsafe when the header is EMPTY_HEADER or when T is a ZST. unsafe fn header_mut(&mut self) -> &mut Header { + debug_assert!(!Self::is_zst()); &mut *self.ptr() } @@ -693,7 +733,11 @@ impl ThinVec { /// assert_eq!(a.len(), 3); /// ``` pub fn len(&self) -> usize { - self.header().len() + if Self::is_zst() { + (self.ptr.as_ptr() as usize) - 1 + } else { + unsafe { self.header().len() } + } } /// Returns `true` if the vector contains no elements. @@ -725,7 +769,11 @@ impl ThinVec { /// assert_eq!(vec.capacity(), 10); /// ``` pub fn capacity(&self) -> usize { - self.header().cap() + if Self::is_zst() { + MAX_CAP + } else { + unsafe { self.header().cap() } + } } /// Returns `true` if the vector has the capacity to hold any element. @@ -815,7 +863,10 @@ impl ThinVec { /// Normally, here, one would use [`clear`] instead to correctly drop /// the contents and thus not leak memory. pub unsafe fn set_len(&mut self, len: usize) { - if self.is_singleton() { + if Self::is_zst() { + // since self.cap() return usize::MAX - 1 it's the caller reponsability to ensure len is < usize::MAX + self.set_len_zst(len); + } else if self.is_singleton() { // A prerequisite of `Vec::set_len` is that `new_len` must be // less than or equal to capacity(). The same applies here. debug_assert!(len == 0, "invalid set_len({}) on empty ThinVec", len); @@ -824,11 +875,37 @@ impl ThinVec { } } - // For internal use only, when setting the length and it's known to be the non-singleton. - unsafe fn set_len_non_singleton(&mut self, len: usize) { + /// For internal use only, when setting the length and it's known that T is a ZST. + /// # Safety + /// - This is unsafe when T is not a ZST. + /// - len must be < usize::MAX + unsafe fn set_len_zst(&mut self, len: usize) { + debug_assert!( + len <= MAX_CAP, + "invalid set_len(usize::MAX) on ZST ThinVec (max cap is usize::MAX - 1)" + ); + unsafe { self.ptr = len_to_ptr_unchecked(len + 1) } + } + + /// For internal use only, when setting the length and it's known that the header is owned. + /// # Safety + /// This is unsafe when the header is EMPTY_HEADER or when T is a ZST. + unsafe fn set_header_len(&mut self, len: usize) { self.header_mut().set_len(len) } + /// For internal use only, when setting the length and it's known to be the non-singleton. + /// # Safety + /// This is unsafe when the header is EMPTY_HEADER. + #[inline(always)] + unsafe fn set_len_non_singleton(&mut self, len: usize) { + if Self::is_zst() { + self.set_len_zst(len); + } else { + self.header_mut().set_len(len) + } + } + /// Appends an element to the back of a collection. /// /// # Panics @@ -846,7 +923,9 @@ impl ThinVec { /// ``` pub fn push(&mut self, val: T) { let old_len = self.len(); - if old_len == self.capacity() { + if Self::is_zst() { + assert!(old_len < MAX_CAP); + } else if old_len == self.capacity() { self.reserve(1); } unsafe { @@ -867,9 +946,12 @@ impl ThinVec { let old_len = self.len(); debug_assert!(old_len < self.capacity()); unsafe { - ptr::write(self.data_raw().add(old_len), val); - - // SAFETY: capacity > len >= 0, so capacity != 0, so this is not a singleton. + if Self::is_zst() { + mem::forget(val); + } else { + ptr::write(self.data_raw().add(old_len), val); + // SAFETY: capacity > len >= 0, so capacity != 0, so this is not a singleton. + } self.set_len_non_singleton(old_len + 1); } } @@ -894,7 +976,11 @@ impl ThinVec { unsafe { self.set_len_non_singleton(old_len - 1); - Some(ptr::read(self.data_raw().add(old_len - 1))) + if Self::is_zst() { + Some(mem::zeroed()) + } else { + Some(ptr::read(self.data_raw().add(old_len - 1))) + } } } @@ -920,6 +1006,14 @@ impl ThinVec { let old_len = self.len(); assert!(idx <= old_len, "Index out of bounds"); + if Self::is_zst() { + assert!(old_len < MAX_CAP); + mem::forget(elem); + unsafe { + self.set_len_zst(old_len + 1); + } + return; + } if old_len == self.capacity() { self.reserve(1); } @@ -927,7 +1021,7 @@ impl ThinVec { let ptr = self.data_raw(); ptr::copy(ptr.add(idx), ptr.add(idx + 1), old_len - idx); ptr::write(ptr.add(idx), elem); - self.set_len_non_singleton(old_len + 1); + self.set_header_len(old_len + 1); } } @@ -961,10 +1055,14 @@ impl ThinVec { unsafe { self.set_len_non_singleton(old_len - 1); - let ptr = self.data_raw(); - let val = ptr::read(self.data_raw().add(idx)); - ptr::copy(ptr.add(idx + 1), ptr.add(idx), old_len - idx - 1); - val + if Self::is_zst() { + mem::zeroed() + } else { + let ptr = self.data_raw(); + let val = ptr::read(self.data_raw().add(idx)); + ptr::copy(ptr.add(idx + 1), ptr.add(idx), old_len - idx - 1); + val + } } } @@ -1000,10 +1098,15 @@ impl ThinVec { assert!(idx < old_len, "Index out of bounds"); unsafe { - let ptr = self.data_raw(); - ptr::swap(ptr.add(idx), ptr.add(old_len - 1)); - self.set_len_non_singleton(old_len - 1); - ptr::read(ptr.add(old_len - 1)) + if Self::is_zst() { + self.set_len_zst(old_len - 1); + mem::zeroed() + } else { + let ptr = self.data_raw(); + ptr::swap(ptr.add(idx), ptr.add(old_len - 1)); + self.set_header_len(old_len - 1); + ptr::read(ptr.add(old_len - 1)) + } } } @@ -1063,7 +1166,12 @@ impl ThinVec { // doesn't re-drop the just-failed value. let new_len = self.len() - 1; self.set_len_non_singleton(new_len); - ptr::drop_in_place(self.data_raw().add(new_len)); + let ptr = if Self::is_zst() { + NonNull::dangling().as_ptr() + } else { + self.data_raw().add(new_len) + }; + ptr::drop_in_place(ptr); } } } @@ -1146,6 +1254,10 @@ impl ThinVec { if min_cap <= old_cap { return; } + // only way to get here is if min_cap == usize::MAX, which we can't handle. + if Self::is_zst() { + capacity_overflow(); + } // Ensure the new capacity is at least double, to guarantee exponential growth. let double_cap = if old_cap == 0 { // skip to 4 because tiny ThinVecs are dumb; but not if that would cause overflow @@ -1177,7 +1289,6 @@ impl ThinVec { if min_cap <= old_cap { return; } - // The growth logic can't handle zero-sized types, so we have to exit // early here. if elem_size == 0 { @@ -1230,6 +1341,9 @@ impl ThinVec { let new_cap = self.len().checked_add(additional).unwrap_cap_overflow(); let old_cap = self.capacity(); if new_cap > old_cap { + if Self::is_zst() { + capacity_overflow() + } unsafe { self.reallocate(new_cap); } @@ -1253,6 +1367,9 @@ impl ThinVec { /// assert!(vec.capacity() >= 3); /// ``` pub fn shrink_to_fit(&mut self) { + if Self::is_zst() { + return; + } let old_cap = self.capacity(); let new_cap = self.len(); if new_cap >= old_cap { @@ -1743,8 +1860,10 @@ impl ThinVec { /// Resize the buffer and update its capacity, without changing the length. /// Unsafe because it can cause length to be greater than capacity. + /// Must not be called if Self::is_zst() unsafe fn reallocate(&mut self, new_cap: usize) { debug_assert!(new_cap > 0); + debug_assert!(!Self::is_zst()); if self.has_allocation() { let old_cap = self.capacity(); let ptr = realloc( @@ -1779,7 +1898,7 @@ impl ThinVec { .add(1) .cast::() .copy_from_nonoverlapping(self.data_raw(), len); - self.set_len_non_singleton(0); + self.set_header_len(0); new_header.as_mut().set_len(len); } @@ -1790,7 +1909,13 @@ impl ThinVec { #[inline] #[allow(unused_unsafe)] fn is_singleton(&self) -> bool { - unsafe { self.ptr.as_ptr() as *const Header == &EMPTY_HEADER } + // could technicaly remove this branch + // but there is a 1/2^64 chance of the number of ZST being equal to &EMPTY_HEADER + if Self::is_zst() { + false + } else { + unsafe { self.ptr.as_ptr() as *const Header == &EMPTY_HEADER } + } } #[cfg(feature = "gecko-ffi")] @@ -1921,6 +2046,14 @@ impl ThinVec { } } +#[cold] +#[inline(never)] +fn drop_zsts(this: &mut ThinVec) { + unsafe { + ptr::drop_in_place(&mut this[..]); + } +} + #[cold] #[inline(never)] fn drop_non_singleton(this: &mut ThinVec) { @@ -1940,7 +2073,11 @@ impl Drop for ThinVec { #[inline] fn drop(&mut self) { if !self.is_singleton() { - drop_non_singleton(self); + if Self::is_zst() { + drop_zsts(self); + } else { + drop_non_singleton(self); + } } } } @@ -1950,7 +2087,11 @@ unsafe impl<#[may_dangle] T> Drop for ThinVec { #[inline] fn drop(&mut self) { if !self.is_singleton() { - drop_non_singleton(self); + if Self::is_zst() { + drop_zsts(self); + } else { + drop_non_singleton(self); + } } } } @@ -2146,7 +2287,7 @@ impl<'de, T: serde::Deserialize<'de>> serde::Deserialize<'de> for ThinVec { #[cfg(feature = "malloc_size_of")] impl MallocShallowSizeOf for ThinVec { fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.capacity() == 0 || self.uses_stack_allocated_buffer() { + if self.capacity() == 0 || self.uses_stack_allocated_buffer() || Self::is_zst() { // We're not a heap pointer. return 0; } @@ -4072,19 +4213,19 @@ mod std_tests { #[test] #[cfg(not(feature = "gecko-ffi"))] fn test_drain_max_vec_size() { - let mut v = ThinVec::<()>::with_capacity(usize::MAX); + let mut v = ThinVec::<()>::with_capacity(MAX_CAP); unsafe { - v.set_len(usize::MAX); + v.set_len(MAX_CAP); } - for _ in v.drain(usize::MAX - 1..) {} - assert_eq!(v.len(), usize::MAX - 1); + for _ in v.drain(MAX_CAP - 1..) {} + assert_eq!(v.len(), MAX_CAP - 1); - let mut v = ThinVec::<()>::with_capacity(usize::MAX); + let mut v = ThinVec::<()>::with_capacity(MAX_CAP); unsafe { - v.set_len(usize::MAX); + v.set_len(MAX_CAP); } - for _ in v.drain(usize::MAX - 1..=usize::MAX - 1) {} - assert_eq!(v.len(), usize::MAX - 1); + for _ in v.drain(MAX_CAP - 1..=MAX_CAP - 1) {} + assert_eq!(v.len(), MAX_CAP - 1); } #[test]