diff --git a/vortex-buffer/src/allocation.rs b/vortex-buffer/src/allocation.rs index 69d9d4c0021..3206d52356b 100644 --- a/vortex-buffer/src/allocation.rs +++ b/vortex-buffer/src/allocation.rs @@ -2,6 +2,9 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! Allocator-backed storage for Vortex buffers. +//! +//! Each allocation retains its base pointer and original layout through slicing and ownership +//! transfers. Buffer data pointers and logical alignments do not change the layout used to free it. use std::alloc::Layout; use std::any::Any; @@ -22,7 +25,7 @@ use crate::BufferMut; /// An allocator that can back a Vortex buffer. /// -/// Vortex over-allocates raw storage and aligns the buffer within it. +/// Buffer allocations pass their byte capacity and effective alignment through [`Layout`]. pub trait BufferAllocator: Allocator + Debug + Send + Sync + 'static {} impl BufferAllocator for A where A: Allocator + Debug + Send + Sync + 'static {} @@ -235,7 +238,9 @@ unsafe impl Allocator for StaticBufferAllocator { static STATIC_ALLOCATOR: BufferAllocatorRef = BufferAllocatorRef(None); pub(crate) struct Allocation { + /// Allocation base, or an aligned dangling pointer when the layout has zero size. ptr: NonNull, + /// Layout used to allocate this block. Allocator excess is not exposed as buffer capacity. layout: Layout, allocator: BufferAllocatorRef, } @@ -318,6 +323,7 @@ impl Allocation { } pub(crate) fn grow(&mut self, new_layout: Layout) { + debug_assert!(new_layout.size() >= self.layout.size()); let allocation = if self.layout.size() == 0 { self.allocator.allocate(new_layout) } else { @@ -363,175 +369,3 @@ impl BufferBacking { } } } - -#[cfg(test)] -mod tests { - use std::alloc::Layout; - use std::ptr::NonNull; - use std::sync::Arc; - use std::sync::atomic::AtomicUsize; - use std::sync::atomic::Ordering; - - use allocator_api2::alloc::AllocError; - use allocator_api2::alloc::Allocator; - use allocator_api2::alloc::Global; - use rstest::rstest; - use vortex_error::VortexResult; - use vortex_error::vortex_err; - - use crate::Alignment; - use crate::BufferAllocatorRef; - use crate::BufferMut; - - #[derive(Clone, Debug, Default)] - struct TrackingAllocator { - state: Arc, - } - - #[derive(Debug, Default)] - struct TrackingState { - allocations: AtomicUsize, - deallocations: AtomicUsize, - grows: AtomicUsize, - alignment: AtomicUsize, - } - - // SAFETY: this forwards all memory operations to Global and only records call metadata. - unsafe impl Allocator for TrackingAllocator { - fn allocate(&self, layout: Layout) -> Result, AllocError> { - self.state.allocations.fetch_add(1, Ordering::Relaxed); - self.state - .alignment - .store(layout.align(), Ordering::Relaxed); - Global.allocate(layout) - } - - unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - self.state.deallocations.fetch_add(1, Ordering::Relaxed); - // SAFETY: the caller passes the pointer and layout returned by Global. - unsafe { Global.deallocate(ptr, layout) } - } - - unsafe fn grow( - &self, - ptr: NonNull, - old_layout: Layout, - new_layout: Layout, - ) -> Result, AllocError> { - self.state.grows.fetch_add(1, Ordering::Relaxed); - // SAFETY: the caller upholds the Allocator contract. - unsafe { Global.grow(ptr, old_layout, new_layout) } - } - } - - #[test] - fn allocator_identity() { - let static_allocator = BufferAllocatorRef::statically_allocated(); - assert!(static_allocator.ptr_eq(&BufferAllocatorRef::statically_allocated())); - - let custom_allocator = BufferAllocatorRef::new(TrackingAllocator::default()); - assert!(custom_allocator.ptr_eq(&custom_allocator.clone())); - assert!(!custom_allocator.ptr_eq(&static_allocator)); - assert!(!custom_allocator.ptr_eq(&BufferAllocatorRef::new(TrackingAllocator::default()))); - } - - #[test] - fn allocation_lives_until_last_view() { - let allocator = TrackingAllocator::default(); - let state = Arc::clone(&allocator.state); - let buffer = BufferAllocatorRef::new(allocator) - .copy_from([1u32, 2, 3, 4]) - .freeze(); - let view = buffer.slice(0..2); - - assert_eq!(state.allocations.load(Ordering::Relaxed), 1); - assert_eq!( - state.alignment.load(Ordering::Relaxed), - Alignment::of::().as_usize() - ); - drop(buffer); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 0); - drop(view); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 1); - } - - #[rstest] - fn buffer_growth_uses_allocator_grow(#[values(4, 64, 4096)] alignment: usize) { - let allocator = TrackingAllocator::default(); - let state = Arc::clone(&allocator.state); - let alignment = Alignment::new(alignment); - let mut buffer = - BufferAllocatorRef::new(allocator).with_capacity_aligned::(1, alignment); - let initial_capacity = buffer.capacity(); - buffer.extend(std::iter::repeat_n(7, initial_capacity)); - - buffer.push(u32::MAX); - assert!(alignment.is_ptr_aligned(buffer.as_ptr())); - - assert_eq!(&buffer[..initial_capacity], vec![7; initial_capacity]); - assert_eq!(buffer[initial_capacity], u32::MAX); - assert_eq!(state.allocations.load(Ordering::Relaxed), 1); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 0); - assert_eq!(state.grows.load(Ordering::Relaxed), 1); - - drop(buffer); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 1); - } - - #[test] - fn zero_capacity_does_not_allocate() { - let allocator = TrackingAllocator::default(); - let state = Arc::clone(&allocator.state); - let mut buffer = BufferAllocatorRef::new(allocator).with_capacity::(0); - - assert_eq!(buffer.capacity(), 0); - assert!(Alignment::DEFAULT_ALIGNMENT.is_offset_aligned(buffer.as_ptr().addr())); - assert_eq!(state.allocations.load(Ordering::Relaxed), 0); - - buffer.push(42); - - assert_eq!(buffer.as_slice(), [42]); - assert_eq!(state.allocations.load(Ordering::Relaxed), 1); - assert_eq!(state.grows.load(Ordering::Relaxed), 0); - } - - #[test] - fn empty_buffers_preserve_allocator_without_allocating() -> VortexResult<()> { - let allocator = TrackingAllocator::default(); - let state = Arc::clone(&allocator.state); - let allocator = BufferAllocatorRef::new(allocator); - let buffer = BufferMut::::zeroed_in(0, allocator.clone()); - let buffer = buffer.freeze(); - let copy = buffer.clone().into_mut(); - assert!(copy.allocator().ptr_eq(&allocator)); - let mut buffer = buffer - .try_into_mut() - .map_err(|_| vortex_err!("unique buffer"))?; - buffer.reserve(0); - assert!(buffer.is_empty()); - assert!(buffer.allocator().ptr_eq(&allocator)); - drop((copy, buffer)); - assert_eq!(state.allocations.load(Ordering::Relaxed), 0); - assert_eq!(state.grows.load(Ordering::Relaxed), 0); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 0); - Ok(()) - } - - #[test] - fn shared_into_mut_preserves_allocator() { - let allocator = TrackingAllocator::default(); - let state = Arc::clone(&allocator.state); - let allocator = BufferAllocatorRef::new(allocator); - let original = allocator.copy_from([1u32, 2, 3]).freeze(); - let mut copy = original.clone().into_mut(); - assert!(copy.allocator().ptr_eq(&allocator)); - copy[0] = 42; - assert_eq!(original.as_slice(), [1, 2, 3]); - assert_eq!(copy.as_slice(), [42, 2, 3]); - assert_eq!(state.allocations.load(Ordering::Relaxed), 2); - drop(copy); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 1); - drop(original); - assert_eq!(state.deallocations.load(Ordering::Relaxed), 2); - } -} diff --git a/vortex-buffer/src/buffer_mut.rs b/vortex-buffer/src/buffer_mut.rs index 71e27c4f8ce..9b25d25da71 100644 --- a/vortex-buffer/src/buffer_mut.rs +++ b/vortex-buffer/src/buffer_mut.rs @@ -36,15 +36,15 @@ use crate::trusted_len::TrustedLen; /// let _ = BufferMut::<()>::zeroed(3); /// ``` pub struct BufferMut { - /// The owned allocation, including any bytes before `ptr` used for alignment. + /// Owns the allocation base and the layout used to allocate it. pub(crate) allocation: Allocation, - /// The first element, aligned to `alignment`; it may dangle for an empty buffer. + /// Aligned data pointer, possibly dangling for empty buffers or interior to a sliced allocation. pub(crate) ptr: std::ptr::NonNull, - /// The number of initialized `T` values starting at `ptr`. + /// Number of initialized elements, at most `capacity`. pub(crate) length: usize, - /// The number of `T` values that fit from `ptr`. + /// Number of usable elements starting at `ptr`, excluding any allocation prefix. pub(crate) capacity: usize, - /// The minimum alignment maintained for `ptr` across reallocations. + /// Requested alignment. The backing allocation can have a stronger preferred alignment. pub(crate) alignment: Alignment, /// Marks the buffer as logically owning values of `T` despite storing an erased allocation. pub(crate) _marker: std::marker::PhantomData, @@ -135,22 +135,11 @@ impl BufferMut { let size = capacity .checked_mul(size_of::()) .vortex_expect("buffer capacity overflow"); - let layout = if size == 0 { - Layout::from_size_align(0, actual.as_usize()) - .unwrap_or_else(|_| vortex_panic!("invalid empty buffer alignment")) - } else { - let allocation_size = size - .checked_add(actual.as_usize()) - .vortex_expect("buffer capacity overflow"); - Layout::from_size_align(allocation_size, 1).unwrap_or_else(|_| { - vortex_panic!("buffer capacity exceeds maximum allocation size") - }) - }; + let layout = Layout::from_size_align(size, actual.as_usize()) + .unwrap_or_else(|_| vortex_panic!("buffer capacity exceeds maximum allocation size")); let allocation = Allocation::allocate(layout, allocator); - let offset = allocation.ptr().as_ptr().align_offset(actual.as_usize()); - // SAFETY: the allocation includes enough padding to reach this aligned pointer. - let ptr = unsafe { allocation.ptr().add(offset).cast() }; - let capacity = (allocation.size() - offset) / size_of::(); + let ptr = allocation.ptr().cast(); + Self { allocation, ptr, @@ -220,34 +209,30 @@ impl BufferMut { allocator: BufferAllocatorRef, ) -> Self { const { assert!(size_of::() != 0, "ZSTs are not supported") }; + + if !alignment.is_aligned_to(Alignment::of::()) { + vortex_panic!( + "Alignment {} must align to the scalar type's alignment {}", + alignment, + align_of::() + ); + } + let preferred_alignment = preferred_alignment.unwrap_or(Alignment::of::()); let actual_alignment = max(preferred_alignment, alignment); let size = len .checked_mul(size_of::()) .vortex_expect("buffer length overflow"); - let layout = if size == 0 { - Layout::from_size_align(0, actual_alignment.as_usize()) - .unwrap_or_else(|_| vortex_panic!("invalid empty buffer alignment")) - } else { - let allocation_size = size - .checked_add(actual_alignment.as_usize()) - .vortex_expect("buffer length overflow"); - Layout::from_size_align(allocation_size, 1) - .unwrap_or_else(|_| vortex_panic!("buffer length exceeds maximum allocation size")) - }; + let layout = Layout::from_size_align(size, actual_alignment.as_usize()) + .unwrap_or_else(|_| vortex_panic!("buffer length exceeds maximum allocation size")); let allocation = Allocation::allocate_zeroed(layout, allocator); - let offset = allocation - .ptr() - .as_ptr() - .align_offset(actual_alignment.as_usize()); - // SAFETY: the allocation includes enough padding to reach this aligned pointer. - let ptr = unsafe { allocation.ptr().add(offset).cast() }; - let capacity = (allocation.size() - offset) / size_of::(); + let ptr = allocation.ptr().cast(); + Self { allocation, ptr, length: len, - capacity, + capacity: len, alignment, _marker: Default::default(), } @@ -477,7 +462,6 @@ impl BufferMut { return; } - // Otherwise, reserve additional + alignment bytes in case we need to realign the buffer. self.reserve_allocate(additional); } @@ -490,7 +474,6 @@ impl BufferMut { let required_size = required .checked_mul(size_of::()) .vortex_expect("buffer capacity overflow"); - let alignment = self.alignment; let current_size = self .capacity .checked_mul(size_of::()) @@ -498,55 +481,43 @@ impl BufferMut { let logical_size = required_size .max(current_size.saturating_mul(2)) .max(Alignment::DEFAULT_ALIGNMENT.as_usize()); - let allocation_size = logical_size - .checked_add(alignment.as_usize()) - .vortex_expect("buffer capacity overflow"); - let allocation_alignment = if self.allocation.size() == 0 { - 1 - } else { - self.allocation.alignment() - }; - let layout = Layout::from_size_align(allocation_size, allocation_alignment) + let alignment = self.alignment.as_usize().max(self.allocation.alignment()); + let layout = Layout::from_size_align(logical_size, alignment) .unwrap_or_else(|_| vortex_panic!("buffer capacity exceeds maximum allocation size")); let old_offset = self.ptr.cast::().addr().get() - self.allocation.ptr().addr().get(); - let new_offset = if self.allocation.allocator().is_statically_allocated() { - let allocation = - Allocation::allocate(layout, BufferAllocatorRef::statically_allocated()); - let new_offset = allocation.ptr().as_ptr().align_offset(alignment.as_usize()); + // A short slice can need more capacity while still needing fewer bytes than its backing + // allocation. Allocator::grow cannot shrink that backing layout. The static path also uses + // a fresh allocation so it copies only initialized data. + if self.allocation.allocator().is_statically_allocated() + || layout.size() < self.allocation.size() + { + let allocation = Allocation::allocate(layout, self.allocation.allocator().clone()); // SAFETY: both allocations have room for the initialized elements and do not overlap. unsafe { std::ptr::copy_nonoverlapping( self.ptr.cast::().as_ptr(), - allocation.ptr().as_ptr().add(new_offset), + allocation.ptr().as_ptr(), self.length * size_of::(), ); } self.allocation = allocation; - new_offset } else { self.allocation.grow(layout); - let new_offset = self - .allocation - .ptr() - .as_ptr() - .align_offset(alignment.as_usize()); - if new_offset != old_offset { - // SAFETY: grow preserved the initialized elements at old_offset. The new allocation - // has room for the requested elements plus alignment padding, and copy permits - // overlap. + if old_offset != 0 { + // SAFETY: grow preserves the old layout's bytes, including the initialized range + // at old_offset. The new base is aligned, has enough capacity, and copy permits + // overlap when moving the slice to the base. unsafe { std::ptr::copy( self.allocation.ptr().as_ptr().add(old_offset), - self.allocation.ptr().as_ptr().add(new_offset), + self.allocation.ptr().as_ptr(), self.length * size_of::(), ); } } - new_offset - }; - // SAFETY: new_offset was computed within the allocation for alignment. - self.ptr = unsafe { self.allocation.ptr().add(new_offset).cast() }; + } + self.ptr = self.allocation.ptr().cast(); self.capacity = logical_size / size_of::(); } @@ -557,9 +528,7 @@ impl BufferMut { /// reading from a file) before marking the data as initialized using the /// [`set_len`] method. /// - /// Note that the returned slice may be larger than the capacity requested at - /// construction, since the underlying allocation can be rounded up (e.g. to - /// satisfy alignment requirements). + /// Growth and ownership transfers can provide more capacity than originally requested. /// /// [`set_len`]: BufferMut::set_len /// [`Vec::spare_capacity_mut`]: Vec::spare_capacity_mut @@ -994,7 +963,7 @@ impl FromIterator for BufferMut { } #[cfg(test)] -mod test { +mod tests { use crate::Alignment; use crate::BufferMut; use crate::buffer_mut; diff --git a/vortex-buffer/tests/allocation.rs b/vortex-buffer/tests/allocation.rs new file mode 100644 index 00000000000..377bef5de60 --- /dev/null +++ b/vortex-buffer/tests/allocation.rs @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Check the allocator boundary through public buffer operations. +//! +//! The allocator records requested layouts, checks ownership, and always moves on growth. Its +//! backing alignment is stronger than requested so metadata-only alignment changes are repeatable. + +#![cfg(test)] +#![allow(clippy::expect_used)] +#![allow( + clippy::disallowed_types, + reason = "the test recorder does not need parking_lot" +)] + +use std::alloc::Layout; +use std::ptr::NonNull; +use std::sync::Arc; +use std::sync::Mutex; + +use allocator_api2::alloc::AllocError; +use allocator_api2::alloc::Allocator; +use allocator_api2::alloc::Global; +use rstest::rstest; +use vortex_buffer::Alignment; +use vortex_buffer::BufferAllocatorRef; +use vortex_buffer::BufferMut; + +#[derive(Clone, Debug, Default)] +struct TrackingAllocator { + state: Arc>, +} + +#[derive(Debug, Default)] +struct TrackingState { + requests: Vec, + live: Vec<(usize, Layout, Layout)>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Request { + Allocate { layout: Layout, zeroed: bool }, + Grow { old: Layout, new: Layout }, + Deallocate(Layout), +} + +impl TrackingAllocator { + fn as_ref(&self) -> BufferAllocatorRef { + BufferAllocatorRef::new(self.clone()) + } + + fn allocate_impl(&self, layout: Layout, zeroed: bool) -> Result, AllocError> { + let backing = Layout::from_size_align(layout.size(), layout.align().max(4096)) + .map_err(|_| AllocError)?; + let allocation = if zeroed { + Global.allocate_zeroed(backing)? + } else { + Global.allocate(backing)? + }; + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + state.requests.push(Request::Allocate { layout, zeroed }); + state + .live + .push((allocation.cast::().addr().get(), layout, backing)); + + Ok(allocation) + } + + fn requests(&self) -> Vec { + self.state + .lock() + .expect("tracking state is not poisoned") + .requests + .clone() + } + + #[track_caller] + fn assert_all_freed(&self) { + assert!( + self.state + .lock() + .expect("tracking state is not poisoned") + .live + .is_empty() + ); + } +} + +// SAFETY: The shared state keeps each requested layout paired with its Global backing layout. +// Growth allocates before freeing, and every free checks the pointer and the original layout. +unsafe impl Allocator for TrackingAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + self.allocate_impl(layout, false) + } + + fn allocate_zeroed(&self, layout: Layout) -> Result, AllocError> { + self.allocate_impl(layout, true) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + let mut state = self.state.lock().expect("tracking state is not poisoned"); + let index = state + .live + .iter() + .position(|(address, ..)| *address == ptr.addr().get()) + .expect("the buffer must free a live allocation base"); + let (_, requested, backing) = state.live.remove(index); + assert_eq!(layout, requested); + state.requests.push(Request::Deallocate(layout)); + + // SAFETY: The live entry identifies the original pointer and Global layout. + unsafe { Global.deallocate(ptr, backing) } + } + + unsafe fn grow( + &self, + ptr: NonNull, + old: Layout, + new: Layout, + ) -> Result, AllocError> { + { + let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); + assert!(state.live.iter().any(|(address, requested, _)| { + *address == ptr.addr().get() && *requested == old + })); + assert!(new.size() >= old.size()); + state.requests.push(Request::Grow { old, new }); + } + let allocation = self.allocate(new)?; + + // SAFETY: The checked old layout fits the live block. The new block is large enough and + // cannot overlap it because allocation precedes deallocation. + unsafe { + std::ptr::copy_nonoverlapping(ptr.as_ptr(), allocation.cast().as_ptr(), old.size()); + self.deallocate(ptr, old); + } + + Ok(allocation) + } +} + +#[test] +fn allocator_identity() { + let static_allocator = BufferAllocatorRef::statically_allocated(); + assert!(static_allocator.ptr_eq(&BufferAllocatorRef::statically_allocated())); + + let custom_allocator = TrackingAllocator::default().as_ref(); + assert!(custom_allocator.ptr_eq(&custom_allocator.clone())); + assert!(!custom_allocator.ptr_eq(&static_allocator)); + assert!(!custom_allocator.ptr_eq(&TrackingAllocator::default().as_ref())); +} + +#[rstest] +#[case::page(4096, None, 4096)] +#[case::preferred_page(4, Some(4096), 4096)] +#[case::requested_wins(4096, Some(256), 4096)] +#[case::default(4, Some(256), 256)] +#[case::natural(4, None, 4)] +fn requested_layout_reaches_allocator( + #[case] requested: usize, + #[case] preferred: Option, + #[case] effective: usize, + #[values(false, true)] zeroed: bool, +) { + let allocator = TrackingAllocator::default(); + let alignment = Alignment::new(requested); + let preferred = preferred.map(Alignment::new); + let buffer = if zeroed { + BufferMut::::zeroed_preferred_aligned_in( + 1024, + alignment, + preferred, + allocator.as_ref(), + ) + } else { + BufferMut::::with_capacity_preferred_aligned_in( + 1024, + alignment, + preferred, + allocator.as_ref(), + ) + }; + let layout = Layout::from_size_align(4096, effective).expect("valid test layout"); + + assert_eq!(allocator.requests(), [Request::Allocate { layout, zeroed }]); + assert_eq!(buffer.alignment(), alignment); + assert!(Alignment::new(effective).is_ptr_aligned(buffer.as_ptr())); + assert!(buffer.capacity() >= 1024); + if zeroed { + assert_eq!(buffer.as_slice(), [0; 1024]); + } else { + assert!(buffer.is_empty()); + } + + drop(buffer); + assert_eq!( + allocator.requests().last(), + Some(&Request::Deallocate(layout)) + ); + allocator.assert_all_freed(); +} + +#[test] +fn allocation_lives_until_last_view() { + let allocator = TrackingAllocator::default(); + let buffer = allocator.as_ref().copy_from([1u32, 2, 3, 4]).freeze(); + let view = buffer.slice(1..3).into_byte_buffer().into_bytes(); + drop(buffer); + + assert_eq!(allocator.requests().len(), 1); + assert_eq!(view.len(), 8); + drop(view); + allocator.assert_all_freed(); +} + +#[rstest] +fn growth_moves_initialized_and_spare_contents(#[values(4, 64, 4096)] alignment: usize) { + let allocator = TrackingAllocator::default(); + let alignment = Alignment::new(alignment); + let mut buffer = allocator + .as_ref() + .with_capacity_aligned::(17, alignment); + let capacity = buffer.capacity(); + buffer.push(7); + buffer + .spare_capacity_mut() + .fill(std::mem::MaybeUninit::new(11)); + // SAFETY: The first element and every element of the spare capacity are initialized. + unsafe { buffer.set_len(capacity) }; + let expected = buffer.as_slice().to_vec(); + let old_ptr = buffer.as_ptr(); + + buffer.push(u32::MAX); + + assert_ne!(buffer.as_ptr(), old_ptr); + assert_eq!(&buffer[..capacity], expected); + assert_eq!(buffer[capacity], u32::MAX); + assert!(Alignment::DEFAULT_ALIGNMENT.is_ptr_aligned(buffer.as_ptr())); + assert!(alignment.is_ptr_aligned(buffer.as_ptr())); + assert!(matches!(allocator.requests()[1], Request::Grow { .. })); + drop(buffer); + allocator.assert_all_freed(); +} + +#[rstest] +#[case::large_tail(32, true)] +#[case::small_tail(1000, false)] +fn sliced_growth_preserves_contents(#[case] begin: usize, #[case] grows: bool) { + let allocator = TrackingAllocator::default(); + let mut original = allocator.as_ref().with_capacity::(1024); + original.extend(0..1024); + let original = original.freeze(); + let sliced = original.slice(begin..begin + 8); + drop(original); + let mut sliced = sliced.try_into_mut().expect("the slice is uniquely owned"); + let capacity = sliced.capacity(); + sliced.push_n(777, capacity - sliced.len()); + let expected = sliced.as_slice().to_vec(); + + sliced.push(u32::MAX); + + assert_eq!(&sliced[..capacity], expected); + assert_eq!(sliced[capacity], u32::MAX); + assert!(Alignment::DEFAULT_ALIGNMENT.is_ptr_aligned(sliced.as_ptr())); + assert_eq!( + allocator + .requests() + .iter() + .any(|request| matches!(request, Request::Grow { .. })), + grows + ); + drop(sliced.freeze()); + allocator.assert_all_freed(); +} + +#[test] +fn realigning_a_slice_copies_with_its_allocator() { + let allocator = TrackingAllocator::default(); + let original = allocator.as_ref().copy_from([1u32, 2, 3, 4]).freeze(); + let sliced = original.slice(1..3); + drop(original); + let buffer = sliced.try_into_mut().expect("the slice is uniquely owned"); + let capacity = buffer.capacity(); + let alignment = Alignment::new(4096); + assert!(!alignment.is_ptr_aligned(buffer.as_ptr())); + + let mut buffer = buffer.aligned(alignment); + + assert!(alignment.is_ptr_aligned(buffer.as_ptr())); + assert_eq!(buffer.capacity(), capacity); + buffer.push(7); + buffer.reserve(buffer.capacity()); + assert_eq!(buffer.as_slice(), [2, 3, 7]); + drop(buffer); + allocator.assert_all_freed(); +} + +#[rstest] +#[case::raise(4, 4096)] +#[case::lower(4096, 4)] +fn changing_alignment_uses_original_layout_for_growth( + #[case] original_alignment: usize, + #[case] new_alignment: usize, +) { + let allocator = TrackingAllocator::default(); + let mut buffer = BufferMut::::with_capacity_preferred_aligned_in( + 4, + Alignment::new(original_alignment), + None, + allocator.as_ref(), + ); + buffer.extend([1, 2, 3, 4]); + let ptr = buffer.as_ptr(); + let mut buffer = buffer.aligned(Alignment::new(new_alignment)); + assert_eq!(buffer.as_ptr(), ptr); + + buffer.reserve(buffer.capacity()); + + assert_eq!(buffer.as_slice(), [1, 2, 3, 4]); + assert_eq!(buffer.alignment(), Alignment::new(new_alignment)); + assert!(Alignment::new(4096).is_ptr_aligned(buffer.as_ptr())); + let Request::Grow { old, new } = allocator.requests()[1] else { + panic!("the allocator must grow the original block"); + }; + assert_eq!(old.align(), original_alignment); + assert_eq!(new.align(), 4096); + drop(buffer); + allocator.assert_all_freed(); +} + +#[rstest] +fn zero_capacity_does_not_allocate(#[values(false, true)] zeroed: bool) { + let allocator = TrackingAllocator::default(); + let alignment = Alignment::new(4096); + let mut buffer = if zeroed { + BufferMut::::zeroed_aligned_in(0, alignment, allocator.as_ref()) + } else { + BufferMut::::with_capacity_aligned_in(0, alignment, allocator.as_ref()) + }; + assert_eq!(buffer.capacity(), 0); + assert!(alignment.is_ptr_aligned(buffer.as_ptr())); + assert!(allocator.requests().is_empty()); + + buffer.push(42); + + assert_eq!(buffer.as_slice(), [42]); + assert!(alignment.is_ptr_aligned(buffer.as_ptr())); + assert_eq!(allocator.requests().len(), 1); + drop(buffer); + allocator.assert_all_freed(); +} + +#[test] +fn empty_buffers_preserve_allocator_without_allocating() { + let allocator = TrackingAllocator::default(); + let handle = allocator.as_ref(); + let buffer = BufferMut::::zeroed_in(0, handle.clone()).freeze(); + let copy = buffer.clone().into_mut(); + assert!(copy.allocator().ptr_eq(&handle)); + let mut buffer = buffer.try_into_mut().expect("the buffer is uniquely owned"); + + buffer.reserve(0); + + assert!(buffer.is_empty()); + assert!(buffer.allocator().ptr_eq(&handle)); + drop((copy, buffer)); + assert!(allocator.requests().is_empty()); +} + +#[test] +fn shared_into_mut_preserves_allocator() { + let allocator = TrackingAllocator::default(); + let handle = allocator.as_ref(); + let original = handle.copy_from([1u32, 2, 3]).freeze(); + let mut copy = original.clone().into_mut(); + assert!(copy.allocator().ptr_eq(&handle)); + + copy[0] = 42; + + assert_eq!(original.as_slice(), [1, 2, 3]); + assert_eq!(copy.as_slice(), [42, 2, 3]); + assert_eq!(allocator.requests().len(), 2); + drop(copy); + assert!(matches!(allocator.requests()[2], Request::Deallocate(_))); + drop(original); + assert!(matches!(allocator.requests()[3], Request::Deallocate(_))); + allocator.assert_all_freed(); +} + +#[test] +fn empty_max_alignment() { + let buffer = BufferMut::::zeroed_aligned(0, Alignment::MAX); + assert!(Alignment::MAX.is_ptr_aligned(buffer.as_ptr())); + assert_eq!(buffer.capacity(), 0); + drop(buffer.freeze().into_mut()); +} + +#[test] +#[should_panic(expected = "must align to the scalar type")] +fn zeroed_rejects_alignment_below_element_alignment() { + drop(BufferMut::::zeroed_preferred_aligned( + 1, + Alignment::none(), + None, + )); +}