diff --git a/src/lib.rs b/src/lib.rs index 9ad1ea6..7d7d670 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -69,6 +69,7 @@ extern crate std; mod borsh; mod macros; mod rawsmallvec; +mod taggedlen; #[cfg(feature = "bytes")] use bytes::{ @@ -87,10 +88,6 @@ use malloc_size_of::{ MallocSizeOf, MallocSizeOfOps }; -#[cfg(feature = "internals")] -pub use rawsmallvec::RawSmallVec; -#[cfg(not(feature = "internals"))] -use rawsmallvec::RawSmallVec; #[cfg(feature = "serde")] use serde_core::{ de::{ @@ -138,6 +135,16 @@ use { } } }; +#[cfg(feature = "internals")] +pub use { + rawsmallvec::RawSmallVec, + taggedlen::TaggedLen +}; +#[cfg(not(feature = "internals"))] +use { + rawsmallvec::RawSmallVec, + taggedlen::TaggedLen +}; /// Error type for APIs with fallible heap allocation #[derive(Debug)] @@ -211,183 +218,6 @@ where R: core::ops::RangeBounds { } } -impl RawSmallVec { - const IS_ZST: bool = is_zst::(); - - #[inline] - const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) - } - - #[inline] - const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { - Self { - inline: ManuallyDrop::new(inline) - } - } - - #[inline] - const fn new_heap(ptr: NonNull, capacity: usize) -> Self { - Self { - heap: (ptr, capacity) - } - } - - #[inline] - const fn as_ptr_inline(&self) -> *const T { - // SAFETY: it is safe because we aren't reading the value, just getting - // a reference to it. reading it would be UB potentially, but - // for that downstream unsafe is required - #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw const self.inline }).cast() - } - - #[inline] - const fn as_mut_ptr_inline(&mut self) -> *mut T { - // SAFETY: same as above - #[allow(unused_unsafe, reason = "Unsafe in MSRV")] - (unsafe { &raw mut self.inline }).cast() - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_ptr_heap(&self) -> *const T { - return unsafe { self.heap.0.as_ptr() }; - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - return unsafe { self.heap.0.as_ptr() }; - } - - /// # Safety - /// - /// `new_capacity` must be non zero, and greater or equal to the length. - /// T must not be a ZST. - unsafe fn try_grow_raw( - &mut self, - len: TaggedLen, - new_capacity: usize - ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{ - alloc, - realloc - }; - debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); - - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - unsafe { self.as_mut_ptr_heap() } - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(); - - let new_layout = - Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); - } - - let new_ptr = if !was_on_heap { - // get a fresh allocation - let new_ptr = unsafe { alloc(new_layout) } as *mut T; // `new_layout` has nonzero size. - let new_ptr = NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { - layout: new_layout - })?; - unsafe { copy_nonoverlapping(ptr, new_ptr.as_ptr(), len) }; - new_ptr - } else { - // use realloc - - // this can't overflow since we already constructed an equivalent - // layout during the previous allocation - let old_layout = unsafe { - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()) - }; - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the - // previous memory block new_layout.size() is greater - // than zero does not overflow when rounded up to - // alignment. since it was constructed - // with Layout::array - let new_ptr = - unsafe { realloc(ptr as *mut u8, old_layout, new_layout.size()) } as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { - layout: new_layout - })? - }; - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } -} - -/// Vec guarantees that its length is always less than [`isize::MAX`] in -/// *bytes*. -/// -/// For a non ZST, this means that the length is less than `isize::MAX` objects, -/// which implies we have at least one free bit we can use. We use the least -/// significant bit for the tag. And store the length in the `usize::BITS - 1` -/// most significant bits. -/// -/// For a ZST, we never use the heap, so we just store the length directly. -#[repr(transparent)] -struct TaggedLen(usize, PhantomData); - -// Clone and Copy must be manually implemented because the generic interferes -// with the derive attribute implementations. -impl Clone for TaggedLen { - #[inline] - fn clone(&self) -> Self { - Self(self.0, PhantomData) - } - - #[inline] - fn clone_from(&mut self, source: &Self) { - self.0 = source.0; - } -} - -impl Copy for TaggedLen {} - -impl TaggedLen { - const IS_ZST: bool = is_zst::(); - - #[inline] - pub const fn new(len: usize, on_heap: bool) -> Self { - if Self::IS_ZST { - debug_assert!(!on_heap); - Self(len, PhantomData) - } else { - debug_assert!(len < isize::MAX as usize); - Self((len << 1) | on_heap as usize, PhantomData) - } - } - - #[inline] - #[must_use] - pub const fn on_heap(self) -> bool { - if Self::IS_ZST { - false - } else { - (self.0 & 1_usize) == 1 - } - } - - #[inline] - pub const fn value(self) -> usize { - if Self::IS_ZST { self.0 } else { self.0 >> 1 } - } -} - #[repr(C)] pub struct SmallVec { len: TaggedLen, diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index d0192ef..50ba5aa 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,9 +1,19 @@ -use core::{ - mem::{ - ManuallyDrop, - MaybeUninit +use { + super::{ + CollectionAllocErr, + taggedlen::TaggedLen }, - ptr::NonNull + core::{ + alloc::Layout, + mem::{ + ManuallyDrop, + MaybeUninit + }, + ptr::{ + NonNull, + copy_nonoverlapping + } + } }; /// Either a stack array with `length <= N` or a heap array @@ -16,3 +26,120 @@ pub union RawSmallVec { pub inline: ManuallyDrop>, pub heap: (NonNull, usize) } + +impl RawSmallVec { + const IS_ZST: bool = size_of::() == 0; + + #[inline] + pub const fn new() -> Self { + Self::new_inline(MaybeUninit::uninit()) + } + + #[inline] + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline) + } + } + + #[inline] + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self { + heap: (ptr, capacity) + } + } + + #[inline] + pub const fn as_ptr_inline(&self) -> *const T { + // SAFETY: it is safe because we aren't reading the value, just getting + // a reference to it. reading it would be UB potentially, but + // for that downstream unsafe is required + #[allow(unused_unsafe, reason = "Unsafe in MSRV")] + (unsafe { &raw const self.inline }).cast() + } + + #[inline] + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: same as above + #[allow(unused_unsafe, reason = "Unsafe in MSRV")] + (unsafe { &raw mut self.inline }).cast() + } + + /// # Safety + /// + /// The vector must be on the heap + #[inline] + pub const unsafe fn as_ptr_heap(&self) -> *const T { + self.heap.0.as_ptr() + } + + /// # Safety + /// + /// The vector must be on the heap + #[inline] + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { + self.heap.0.as_ptr() + } + + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + pub unsafe fn try_grow_raw( + &mut self, + len: TaggedLen, + new_capacity: usize + ) -> Result<(), CollectionAllocErr> { + use alloc::alloc::{ + alloc, + realloc + }; + debug_assert!(!Self::IS_ZST); + debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value()); + + let was_on_heap = len.on_heap(); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(); + + let new_layout = + Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(CollectionAllocErr::CapacityOverflow); + } + + let new_ptr = if !was_on_heap { + // get a fresh allocation + let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. + let new_ptr = NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { + layout: new_layout + })?; + copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); + new_ptr + } else { + // use realloc + + // this can't overflow since we already constructed an equivalent + // layout during the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the + // previous memory block new_layout.size() is greater + // than zero does not overflow when rounded up to + // alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { + layout: new_layout + })? + }; + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } +} diff --git a/src/taggedlen.rs b/src/taggedlen.rs new file mode 100644 index 0000000..d677419 --- /dev/null +++ b/src/taggedlen.rs @@ -0,0 +1,57 @@ +use core::marker::PhantomData; + +/// Vec guarantees that its length is always less than [`isize::MAX`] in +/// *bytes*. +/// +/// For a non ZST, this means that the length is less than `isize::MAX` objects, +/// which implies we have at least one free bit we can use. We use the least +/// significant bit for the tag. And store the length in the `usize::BITS - 1` +/// most significant bits. +/// +/// For a ZST, we never use the heap, so we just store the length directly. +#[repr(transparent)] +pub struct TaggedLen(usize, PhantomData); + +impl Clone for TaggedLen { + #[inline] + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } + + #[inline] + fn clone_from(&mut self, source: &Self) { + self.0 = source.0; + } +} + +impl Copy for TaggedLen {} + +impl TaggedLen { + const IS_ZST: bool = size_of::() == 0; + + #[inline] + pub const fn new(len: usize, on_heap: bool) -> Self { + if Self::IS_ZST { + debug_assert!(!on_heap); + Self(len, PhantomData) + } else { + debug_assert!(len < isize::MAX as usize); + Self((len << 1) | on_heap as usize, PhantomData) + } + } + + #[inline] + #[must_use] + pub const fn on_heap(self) -> bool { + if Self::IS_ZST { + false + } else { + (self.0 & 1_usize) == 1 + } + } + + #[inline] + pub const fn value(self) -> usize { + if Self::IS_ZST { self.0 } else { self.0 >> 1 } + } +}