diff --git a/AGENTS.md b/AGENTS.md index d91f533..880228f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,7 +156,8 @@ unavoidable external C library dependency. ## Technology Stack - **Language**: C++20 (`std::span`, `std::popcount`, and `` are required). -- **Build**: CMake 3.18 or newer. +- **Build**: CMake 3.18 or newer. The optional Pasta comparison backend + requires CMake 3.25 or newer. - **Testing**: Google Test 1.17.0, registered with CTest. - **Benchmarking**: Google Benchmark 1.9.4. - **SIMD**: AVX-512 when available, AVX2 fallback, and scalar fallback. @@ -172,6 +173,7 @@ dependencies only as needed. Direct CMake defaults and their effects are: | `PIXIE_TESTS` | `ON` standalone, `OFF` downstream | Builds tests and fetches Google Test. | | `PIXIE_BENCHMARKS` | `OFF` | Builds native Google Benchmark targets. | | `PIXIE_THIRD_PARTY_BACKENDS` | `OFF` | Enables optional SDSL adapters and their comparison targets. | +| `PIXIE_3STAR_SOURCE_DIR` | empty | Permitted local 3star-rankselect checkout at Pixie's pinned revision for its benchmark-only adapter; Pixie never fetches or redistributes it. | | `PIXIE_DIAGNOSTICS` | `OFF` | Enables diagnostic logging for profiling experiments. | | `PIXIE_DOCS` | `OFF` | Enables the Doxygen `docs` target. | | `PIXIE_COVERAGE` | `OFF` | Adds GCC coverage instrumentation. | @@ -179,9 +181,12 @@ dependencies only as needed. Direct CMake defaults and their effects are: `MappedFile` uses native POSIX memory mapping on Linux/Unix. A default FetchContent consumer therefore receives no third-party dependency, while a standalone default build fetches Google Test. Enabling third-party backends also -fetches SDSL and pasta-toolbox dependencies, but only SDSL currently has a -registered Pixie adapter/comparison benchmark. Do not describe pasta-toolbox as -an available backend until Pixie adds and registers one. +fetches SDSL and pasta-toolbox dependencies. SDSL has registered RmM comparison +targets; Pasta provides an owning rank/select comparison backend and is not a +default library dependency. Pasta is GPLv3-or-later; keep its adapter strictly +behind the optional comparison-backend build option. The 3-star research +artifact has no published license; its benchmark-only adapter requires a +user-provided, permitted local checkout and must not fetch or redistribute it. ## Build and Test Presets @@ -275,6 +280,9 @@ when debugging a focused Google Test filter. cases over more undirected random input. Cover border correction, same-leaf paths, prefix/suffix selectors, sparse-overlay hit/miss paths, partial final blocks, and first-minimum ties. +- **Rank/select boundaries**: test a target at the final position of a full + 2^16-bit superblock; its local one or zero rank is 65,536 and cannot be + passed through a 16-bit basic-block search key. - **Storage tests**: test owners and views through the same specification, including nested byte subranges, alignment constraints, serialization, and owner/view lifetime rules. diff --git a/CMakeLists.txt b/CMakeLists.txt index f6aa200..2dc215c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,9 +50,40 @@ option(PIXIE_BENCHMARKS "Build benchmarks" OFF) option(PIXIE_THIRD_PARTY_BACKENDS "Build optional third-party backend integrations" OFF) option(PIXIE_DIAGNOSTICS "Include diagnostic logs" OFF) option(PIXIE_DOCS "Build Doxygen documentation" OFF) +set(PIXIE_3STAR_SOURCE_DIR "" CACHE PATH + "Path to a permitted local 3star-rankselect checkout for benchmark-only use") + +if (PIXIE_3STAR_SOURCE_DIR) + if (NOT PIXIE_THIRD_PARTY_BACKENDS) + message(FATAL_ERROR + "PIXIE_3STAR_SOURCE_DIR requires PIXIE_THIRD_PARTY_BACKENDS") + endif () + foreach (three_star_file m3.hpp Tree3.hpp bv.hpp alignedallocator.hpp) + if (NOT EXISTS "${PIXIE_3STAR_SOURCE_DIR}/${three_star_file}") + message(FATAL_ERROR + "PIXIE_3STAR_SOURCE_DIR is missing ${three_star_file}") + endif () + endforeach () + execute_process( + COMMAND git -C "${PIXIE_3STAR_SOURCE_DIR}" rev-parse HEAD + OUTPUT_VARIABLE PIXIE_3STAR_REVISION + OUTPUT_STRIP_TRAILING_WHITESPACE + RESULT_VARIABLE PIXIE_3STAR_REVISION_RESULT) + if (NOT PIXIE_3STAR_REVISION_RESULT EQUAL 0 OR + NOT PIXIE_3STAR_REVISION STREQUAL "8b66e5b24944b483f8a886e215e147a11457cb30") + message(FATAL_ERROR + "PIXIE_3STAR_SOURCE_DIR must be 3star-rankselect revision " + "8b66e5b24944b483f8a886e215e147a11457cb30") + endif () + set(PIXIE_3STAR_SUPPORT ON) +endif () if (PIXIE_THIRD_PARTY_BACKENDS) - add_compile_definitions(SDSL_SUPPORT) + if (CMAKE_VERSION VERSION_LESS 3.25) + message(FATAL_ERROR + "PIXIE_THIRD_PARTY_BACKENDS requires CMake 3.25 for pasta::bit_vector") + endif () + add_compile_definitions(SDSL_SUPPORT PIXIE_PASTA_SUPPORT) endif () if (PIXIE_DIAGNOSTICS) @@ -97,7 +128,7 @@ if (PIXIE_THIRD_PARTY_BACKENDS) FetchContent_Declare( pasta_bit_vector GIT_REPOSITORY https://github.com/pasta-toolbox/bit_vector.git - GIT_TAG origin/main + GIT_TAG 3ffb6e5a2e58c76425de8197bfe554eb1bf9fd94 ) FetchContent_MakeAvailable(pasta_bit_vector) @@ -148,15 +179,38 @@ function (pixie_enable_project_warnings target) endif () endfunction () +if (PIXIE_3STAR_SUPPORT) + function (pixie_enable_three_star target) + target_include_directories(${target} + PRIVATE src/benchmarks) + target_include_directories(${target} + SYSTEM PRIVATE ${PIXIE_3STAR_SOURCE_DIR}) + target_compile_definitions(${target} PRIVATE + PIXIE_3STAR_SUPPORT + ALIGNMENT=2097152 + CHECK_ALL_DENSE_SPARSE=0 + COUNT_HIT_TYPES=0 + CUSTOM_L0=2048 + POP=2 + PREFETCH_NT=1 + PREFETCH_T0=1 + REVERSE=2 + TREE_DIVISION_EXP=3 + TREE_EXP_LVL0=19 + TREE_TYPE=3) + endfunction () +endif () + if (PIXIE_DIAGNOSTICS) target_compile_definitions(pixie INTERFACE PIXIE_DIAGNOSTICS) target_link_libraries(pixie INTERFACE spdlog::spdlog_header_only) endif () if (PIXIE_THIRD_PARTY_BACKENDS) - target_compile_definitions(pixie INTERFACE SDSL_SUPPORT) + target_compile_definitions(pixie INTERFACE SDSL_SUPPORT PIXIE_PASTA_SUPPORT) target_include_directories(pixie INTERFACE $) + target_link_libraries(pixie INTERFACE pasta_bit_vector) endif () # --------------------------------------------------------------------------- @@ -191,6 +245,27 @@ if (PIXIE_TESTS) gtest gtest_main ${PIXIE_DIAGNOSTICS_LIBS}) + if (PIXIE_THIRD_PARTY_BACKENDS) + target_link_libraries(rank_select_tests pasta_bit_vector) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + # Pasta's debug-assertion helper is a static function that becomes + # unused in optimized builds. + target_compile_options(rank_select_tests PRIVATE -Wno-unused-function) + endif () + endif () + + if (PIXIE_3STAR_SUPPORT) + add_executable(three_star_benchmark_tests + src/tests/three_star_benchmark_tests.cpp + src/benchmarks/three_star_adapter.cpp) + target_include_directories(three_star_benchmark_tests + PUBLIC include) + target_link_libraries(three_star_benchmark_tests + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + pixie_enable_three_star(three_star_benchmark_tests) + endif () add_executable(benchmark_tests src/tests/benchmark_tests.cpp) @@ -325,6 +400,9 @@ if (PIXIE_TESTS) select512_experimental_tests excess_record_lows_tests rmq_tests) + if (PIXIE_3STAR_SUPPORT) + list(APPEND PIXIE_TEST_TARGETS three_star_benchmark_tests) + endif () foreach (test_target IN LISTS PIXIE_TEST_TARGETS) pixie_enable_project_warnings(${test_target}) gtest_discover_tests(${test_target} @@ -345,6 +423,18 @@ if (PIXIE_BENCHMARKS) target_link_libraries(rank_select_benchmarks benchmark ${PIXIE_DIAGNOSTICS_LIBS}) + if (PIXIE_THIRD_PARTY_BACKENDS) + target_link_libraries(rank_select_benchmarks pasta_bit_vector) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + # See the matching rank_select_tests exception above. + target_compile_options(rank_select_benchmarks PRIVATE -Wno-unused-function) + endif () + endif () + if (PIXIE_3STAR_SUPPORT) + target_sources(rank_select_benchmarks PRIVATE + src/benchmarks/three_star_adapter.cpp) + pixie_enable_three_star(rank_select_benchmarks) + endif () add_executable(rmm_benchmarks src/benchmarks/rmm_benchmarks.cpp) @@ -431,6 +521,15 @@ if (PIXIE_BENCHMARKS) benchmark benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + if (PIXIE_THIRD_PARTY_BACKENDS) + target_include_directories(serialization_benchmarks + PRIVATE ${sdsl_lite_SOURCE_DIR}/include) + target_link_libraries(serialization_benchmarks pasta_bit_vector) + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + target_compile_options(serialization_benchmarks + PRIVATE -Wno-unused-function) + endif () + endif () add_executable(integer_vector_benchmarks src/benchmarks/integer_vector_benchmarks.cpp) diff --git a/include/pixie/rank_select/implementations.h b/include/pixie/rank_select/implementations.h index 3787d43..bf7cda8 100644 --- a/include/pixie/rank_select/implementations.h +++ b/include/pixie/rank_select/implementations.h @@ -7,6 +7,8 @@ * - `RankSelectSupport`: non-owning source bits with * storage-backed rank/select metadata. * - `RankSelectSupportView`: non-owning source bits with read-only metadata. + * - `PastaRankSelectSupport`: optional Pasta backend owning a converted source + * copy and its rank/select metadata. */ // clang-format off @@ -76,8 +78,43 @@ * | FNBP | 2^22 | 0.139 | 0.020 | 0.041 | * | FNBP | 2^26 | 2.289 | 0.291 | 0.036 | * | FNBP | 2^30 | 36.093 | 4.627 | 0.036 | + * + * Current native 2^30 snapshot, 2026-09-13. + * + * One CPU-pinned Release pass per row with the standard 0.2 s warmup and 1.0 s + * minimum time. Query CPU times are ns; build CPU time is ms. The index uses + * SelectSupport::kBoth and is non-owning; its source remains external. + * + * | source | build ms | rank1 | rank0 | select1 | select0 | aux MiB | + * | :----------- | -------: | ------: | ------: | ------: | ------: | ------: | + * | random 12.5% | 39.700 | 40.400 | 66.500 | 180.000 | 152.000 | 4.625 | + * | random 50% | 39.800 | 38.100 | 37.200 | 140.000 | 153.000 | 4.625 | + * | random 87.5% | 45.400 | 37.800 | 51.200 | 134.000 | 169.000 | 4.625 | + * + * Pasta FlatRankSelect<> snapshot, 2026-09-13. + * + * One CPU-pinned Release pass per row with a deterministic random 50% source + * and the standard 0.2 s warmup and 1.0 s minimum time. Both query pools and + * index construction are outside timed query regions. Pasta uses its default + * `FlatRankSelect<>` configuration (`DONT_CARE` and linear L2 search). Build + * time is CPU ms; query times are CPU ns. `aux MiB` is logical index storage; + * `source MiB` is the copied source that native RankSelectSupport does not own. + * + * | N | build ms | rank1 | rank0 | select1 | select0 | aux MiB | source MiB | + * | ---: | -------: | ------: | ------: | ------: | ------: | ------: | ---------: | + * | 2^10 | 0.000 | 10.900 | 11.400 | 17.200 | 16.600 | 0.000 | 0.000 | + * | 2^14 | 0.000 | 10.300 | 11.100 | 30.600 | 33.400 | 0.000 | 0.002 | + * | 2^18 | 0.003 | 11.300 | 10.100 | 30.100 | 36.300 | 0.001 | 0.031 | + * | 2^22 | 0.057 | 12.600 | 11.500 | 35.300 | 47.000 | 0.018 | 0.500 | + * | 2^26 | 1.540 | 38.300 | 29.400 | 71.600 | 146.000 | 0.281 | 8.000 | + * | 2^30 | 143.000 | 47.100 | 49.100 | 140.000 | 325.000 | 4.500 | 128.000 | + * | 2^34 | 2889.000 | 109.000 | 88.700 | 291.000 | 552.000 | 72.000 | 2048.000 | */ // clang-format on #include #include + +#ifdef PIXIE_PASTA_SUPPORT +#include +#endif diff --git a/include/pixie/rank_select/pasta.h b/include/pixie/rank_select/pasta.h new file mode 100644 index 0000000..ba14166 --- /dev/null +++ b/include/pixie/rank_select/pasta.h @@ -0,0 +1,132 @@ +#pragma once + +#ifndef PIXIE_PASTA_SUPPORT +#error "PastaRankSelectSupport requires PIXIE_THIRD_PARTY_BACKENDS" +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Pasta flat rank/select adapter over a copied packed bit sequence. + * + * @details The constructor owns a normalized copy of the logical source words; + * source mutations do not affect this index. Both select directions are always + * available. Public positions are zero-based, prefix ranks use `[0, end)`, and + * selects use one-based ranks with the Pixie rank-zero and out-of-range + * sentinels. + */ +class PastaRankSelectSupport : public RankSelectBase { + private: + using PastaSupport = pasta::FlatRankSelect<>; + + static uint64_t logical_word(std::span source, + std::size_t word_index, + std::size_t num_bits) { + uint64_t word = source[word_index]; + const std::size_t remaining = num_bits - word_index * 64; + if (remaining < 64) { + word &= (uint64_t{1} << remaining) - 1; + } + return word; + } + + pasta::BitVector bits_; + PastaSupport support_; + std::size_t num_bits_ = 0; + std::size_t max_rank_ = 0; + + public: + /** + * @brief Construct an owning Pasta index from caller-provided packed words. + * @param source_words Packed least-significant-bit-first source words. + * @param num_bits Number of valid input bits, clamped to @p source_words. + */ + explicit PastaRankSelectSupport(std::span source_words, + std::size_t num_bits) + : bits_(std::min(num_bits, source_words.size() * 64)), + num_bits_(std::min(num_bits, source_words.size() * 64)) { + std::fill(bits_.data().begin(), bits_.data().end(), 0); + const std::size_t word_count = (num_bits_ + 63) / 64; + for (std::size_t word_index = 0; word_index < word_count; ++word_index) { + bits_.data()[word_index] = + logical_word(source_words, word_index, num_bits_); + } + support_ = PastaSupport(bits_); + max_rank_ = num_bits_ == 0 ? 0 : support_.rank1(num_bits_); + } + + /** @brief Return the number of valid source bits. */ + std::size_t size_impl() const { return num_bits_; } + + /** @brief Return the bit at a zero-based valid position. */ + int bit_impl(std::size_t position) const { + return static_cast(bits_[position]); + } + + /** @brief Return the one rank in `[0, end_position)`. */ + uint64_t rank_impl(std::size_t end_position) const { + return end_position >= num_bits_ ? max_rank_ : support_.rank1(end_position); + } + + /** @brief Return the zero-based position of a one-based one rank. */ + uint64_t select_impl(std::size_t rank) const { + if (rank == 0) { + return 0; + } + return rank > max_rank_ ? num_bits_ : support_.select1(rank); + } + + /** @brief Return the zero-based position of a one-based zero rank. */ + uint64_t select0_impl(std::size_t rank) const { + if (rank == 0) { + return 0; + } + return rank > num_bits_ - max_rank_ ? num_bits_ : support_.select0(rank); + } + + /** @brief Pasta constructs support for one selects unconditionally. */ + bool supports_select1_impl() const { return true; } + + /** @brief Pasta constructs support for zero selects unconditionally. */ + bool supports_select0_impl() const { return true; } + + /** + * @brief Return bytes owned by the converted source and Pasta metadata. + * @details Pasta's `FlatRankSelect::space_usage()` omits its inherited L1/L2 + * table, so account for that table explicitly from its documented layout. + */ + std::size_t memory_usage_bytes_impl() const { + return source_copy_bytes() + index_logical_bytes() + + (sizeof(*this) - sizeof(bits_) - sizeof(support_)); + } + + /** @brief Return bytes in Pasta's owned normalized source copy. */ + std::size_t source_copy_bytes() const { + return bits_.space_usage() - sizeof(pasta::BitVector); + } + + /** + * @brief Return Pasta's logical index bytes, excluding its owned source. + * @details Pasta reports logical vector sizes rather than allocator capacity. + */ + std::size_t index_logical_bytes() const { + const std::size_t l12_entries = + bits_.data().size() / pasta::FlatRankSelectConfig::L1_WORD_SIZE + 1; + return (support_.space_usage() - sizeof(PastaSupport)) + + l12_entries * sizeof(pasta::BigL12Type); + } +}; + +} // namespace pixie diff --git a/include/pixie/rank_select/support.h b/include/pixie/rank_select/support.h index 5b6d878..4462940 100644 --- a/include/pixie/rank_select/support.h +++ b/include/pixie/rank_select/support.h @@ -713,7 +713,6 @@ class RankSelectSupport const size_t block_begin = kBlocksPerSuperBlock * s_block; const size_t block_count = std::min(kBlocksPerSuperBlock, basic_block_rank.size() - block_begin); - for (size_t pos = 0; pos < block_count; pos += 32) { auto count = lower_bound_32x16(&basic_block_rank[block_begin + pos], local_rank); @@ -758,15 +757,10 @@ class RankSelectSupport * @param s_block Super block * index. * @details - * Similar to find_basicblock but initial guess is - * based on linear - * interpolation, for random data it should make initial - * guess correct - * most of the times, we start from the 32 wide block with - * interpolation - * guess at the center, if we see that select result lie in - * lower blocks - * we backoff to find_basicblock + * @details Interpolation predicts a basic block from the superblock's one + * ranks. The predicted block is validated directly from the existing + * one-prefix metadata; a miss uses the centered SIMD search and its scalar + * fallback. */ uint64_t find_basicblock_is(uint16_t local_rank, uint64_t s_block) const { auto super_block_rank = super_block_rank_.as_words64(); @@ -779,7 +773,19 @@ class RankSelectSupport auto lower = super_block_rank[s_block]; auto upper = super_block_rank[s_block + 1]; - uint64_t pos = block_count * local_rank / (upper - lower); + uint64_t interpolation = block_count * local_rank / (upper - lower); + const uint64_t block_offset = + std::min(interpolation, block_count - 1); + const uint64_t block = block_begin + block_offset; + const uint64_t ones_before = basic_block_rank[block]; + const uint64_t ones_after = block_offset + 1 == block_count + ? upper - lower + : basic_block_rank[block + 1]; + if (ones_before < local_rank && local_rank <= ones_after) { + return block; + } + + uint64_t pos = interpolation; pos = pos + 16 < 32 ? 0 : (pos - 16); pos = std::min(pos, last_group); while (pos < last_group) { @@ -1066,7 +1072,10 @@ class RankSelectSupport uint64_t s_block = find_superblock(rank); rank -= super_block_rank[s_block]; - auto pos = find_basicblock_is(rank, s_block); + const uint64_t pos = + rank == kSuperBlockSize + ? s_block * kBlocksPerSuperBlock + (kBlocksPerSuperBlock - 1) + : find_basicblock_is(static_cast(rank), s_block); rank -= basic_block_rank[pos]; return select_in_words(pos * kWordsPerBlock, rank, true); } @@ -1093,7 +1102,10 @@ class RankSelectSupport uint64_t s_block = find_superblock_zeros(rank0); rank0 -= kSuperBlockSize * s_block - super_block_rank[s_block]; - auto pos = find_basicblock_is_zeros(rank0, s_block); + const uint64_t pos = + rank0 == kSuperBlockSize + ? s_block * kBlocksPerSuperBlock + (kBlocksPerSuperBlock - 1) + : find_basicblock_is_zeros(static_cast(rank0), s_block); auto pos_in_super_block = pos & (kBlocksPerSuperBlock - 1); rank0 -= kBasicBlockSize * pos_in_super_block - basic_block_rank[pos]; return select_in_words(pos * kWordsPerBlock, rank0, false); diff --git a/src/benchmarks/rank_select_benchmarks.cpp b/src/benchmarks/rank_select_benchmarks.cpp index 122f893..10752b2 100644 --- a/src/benchmarks/rank_select_benchmarks.cpp +++ b/src/benchmarks/rank_select_benchmarks.cpp @@ -3,6 +3,10 @@ #include #include +#ifdef PIXIE_3STAR_SUPPORT +#include "three_star_adapter.h" +#endif + #include #include #include @@ -29,9 +33,7 @@ constexpr std::array kSizes = { // expose large-source behavior while retaining the existing scaling points. constexpr std::array kFnbpSizes = { 1ull << 10, 1ull << 14, 1ull << 18, 1ull << 22, 1ull << 26, 1ull << 30}; -using RankSelect = pixie::RankSelectSupport<>; -constexpr RankSelect::SelectSupport kMeasuredSelectSupport = - RankSelect::SelectSupport::kBoth; +using PixieRankSelect = pixie::RankSelectSupport<>; static_assert(kQueryCount > 0 && (kQueryCount & (kQueryCount - 1)) == 0); @@ -237,6 +239,42 @@ class FnbpDataset { std::size_t size_ = 0; }; +template +Support make_support(const Dataset& dataset) { + return Support(dataset.source().padded_view().as_words64(), dataset.size()); +} + +template +constexpr bool backend_owns_source() { + return requires(const Support& support) { support.source_copy_bytes(); }; +} + +template +std::size_t index_logical_bytes(const Support& support) { + if constexpr (requires { support.index_logical_bytes(); }) { + return support.index_logical_bytes(); + } + return support.memory_usage_bytes(); +} + +template +std::size_t source_copy_bytes(const Support& support) { + if constexpr (requires { support.source_copy_bytes(); }) { + return support.source_copy_bytes(); + } + return 0; +} + +template +bool supports_select1(const Support& support) { + return support.supports_select1(); +} + +template +bool supports_select0(const Support& support) { + return support.supports_select0(); +} + std::vector make_query_pool(std::size_t first, std::size_t last, std::uint64_t seed) { @@ -252,60 +290,81 @@ std::vector make_query_pool(std::size_t first, void set_common_counters(benchmark::State& state, std::size_t size, Fill fill, - std::size_t auxiliary_bytes, + std::size_t index_bytes, + std::size_t owned_bytes, + std::size_t source_copy_bytes, + bool owns_source, + bool select1_enabled, + bool select0_enabled, std::uint64_t repetition_index) { const double input_bytes = static_cast((size + 7) / 8); - const double auxiliary_bytes_as_double = static_cast(auxiliary_bytes); + const double index_bytes_as_double = static_cast(index_bytes); state.counters["N"] = static_cast(size); state.counters["one_fill_percent"] = fill_spec(fill).expected_one_fill_percent; state.counters["input_bytes"] = input_bytes; - state.counters["aux_bytes"] = auxiliary_bytes_as_double; - state.counters["aux_mib"] = auxiliary_bytes_as_double / (1024.0 * 1024.0); + state.counters["aux_bytes"] = index_bytes_as_double; + state.counters["aux_mib"] = index_bytes_as_double / (1024.0 * 1024.0); state.counters["aux_bits_per_input_bit"] = - size == 0 ? 0.0 : 8.0 * auxiliary_bytes_as_double / size; - state.counters["select1_enabled"] = 1.0; - state.counters["select0_enabled"] = 1.0; + size == 0 ? 0.0 : 8.0 * index_bytes_as_double / size; + state.counters["backend_owned_bytes"] = static_cast(owned_bytes); + state.counters["source_copy_bytes"] = static_cast(source_copy_bytes); + state.counters["select1_enabled"] = select1_enabled ? 1.0 : 0.0; + state.counters["select0_enabled"] = select0_enabled ? 1.0 : 0.0; state.counters["seed_repetition"] = static_cast(repetition_index); + state.counters["backend_owns_source"] = owns_source ? 1.0 : 0.0; } void set_fnbp_counters(benchmark::State& state, std::size_t size, - std::size_t auxiliary_bytes, + std::size_t index_bytes, + std::size_t owned_bytes, + std::size_t source_copy_bytes, + bool owns_source, + bool select1_enabled, + bool select0_enabled, std::uint64_t repetition_index) { - set_common_counters(state, size, Fill::k50, auxiliary_bytes, - repetition_index); + set_common_counters(state, size, Fill::k50, index_bytes, owned_bytes, + source_copy_bytes, owns_source, select1_enabled, + select0_enabled, repetition_index); state.counters["fnbp_source"] = 1.0; } -template +template void run_build(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const SeedContext seeds = make_seed_context(state, size, fill); const BitDataset dataset(size, fill, seeds.source_seed); - std::size_t auxiliary_bytes = 0; + std::size_t index_bytes = 0; + std::size_t owned_bytes = 0; + std::size_t copied_source_bytes = 0; + bool select1_enabled = false; + bool select0_enabled = false; for (auto _ : state) { - RankSelect support(dataset.source(), dataset.size(), - kMeasuredSelectSupport); - auxiliary_bytes = support.memory_usage_bytes(); - benchmark::DoNotOptimize(auxiliary_bytes); + Support support = make_support(dataset); + index_bytes = index_logical_bytes(support); + owned_bytes = support.memory_usage_bytes(); + copied_source_bytes = source_copy_bytes(support); + select1_enabled = supports_select1(support); + select0_enabled = supports_select0(support); + benchmark::DoNotOptimize(owned_bytes); benchmark::ClobberMemory(); } - set_common_counters(state, size, fill, auxiliary_bytes, - seeds.repetition_index); + set_common_counters(state, size, fill, index_bytes, owned_bytes, + copied_source_bytes, backend_owns_source(), + select1_enabled, select0_enabled, seeds.repetition_index); state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(size)); } -template +template void run_query(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const SeedContext seeds = make_seed_context(state, size, fill); const BitDataset dataset(size, fill, seeds.source_seed); - const RankSelect support(dataset.source(), dataset.size(), - kMeasuredSelectSupport); + const Support support = make_support(dataset); const std::size_t one_count = support.rank(support.size()); const std::size_t zero_count = support.rank0(support.size()); @@ -341,37 +400,48 @@ void run_query(benchmark::State& state) { } } - set_common_counters(state, size, fill, support.memory_usage_bytes(), - seeds.repetition_index); + set_common_counters(state, size, fill, index_logical_bytes(support), + support.memory_usage_bytes(), source_copy_bytes(support), + backend_owns_source(), supports_select1(support), + supports_select0(support), seeds.repetition_index); state.SetItemsProcessed(static_cast(state.iterations())); } +template void run_fnbp_build(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const SeedContext seeds = make_seed_context(state, size, Fill::k50); const FnbpDataset dataset(size, seeds.source_seed); - std::size_t auxiliary_bytes = 0; + std::size_t index_bytes = 0; + std::size_t owned_bytes = 0; + std::size_t copied_source_bytes = 0; + bool select1_enabled = false; + bool select0_enabled = false; for (auto _ : state) { - RankSelect support(dataset.source(), dataset.size(), - kMeasuredSelectSupport); - auxiliary_bytes = support.memory_usage_bytes(); - benchmark::DoNotOptimize(auxiliary_bytes); + Support support = make_support(dataset); + index_bytes = index_logical_bytes(support); + owned_bytes = support.memory_usage_bytes(); + copied_source_bytes = source_copy_bytes(support); + select1_enabled = supports_select1(support); + select0_enabled = supports_select0(support); + benchmark::DoNotOptimize(owned_bytes); benchmark::ClobberMemory(); } - set_fnbp_counters(state, size, auxiliary_bytes, seeds.repetition_index); + set_fnbp_counters(state, size, index_bytes, owned_bytes, copied_source_bytes, + backend_owns_source(), select1_enabled, + select0_enabled, seeds.repetition_index); state.SetItemsProcessed(static_cast(state.iterations()) * static_cast(size)); } -template +template void run_fnbp_query(benchmark::State& state) { const std::size_t size = static_cast(state.range(0)); const SeedContext seeds = make_seed_context(state, size, Fill::k50); const FnbpDataset dataset(size, seeds.source_seed); - const RankSelect support(dataset.source(), dataset.size(), - kMeasuredSelectSupport); + const Support support = make_support(dataset); const std::size_t one_count = support.rank(support.size()); const std::size_t zero_count = support.rank0(support.size()); if (one_count != size / 2 || zero_count != size / 2) { @@ -403,16 +473,19 @@ void run_fnbp_query(benchmark::State& state) { } } - set_fnbp_counters(state, size, support.memory_usage_bytes(), - seeds.repetition_index); + set_fnbp_counters(state, size, index_logical_bytes(support), + support.memory_usage_bytes(), source_copy_bytes(support), + backend_owns_source(), supports_select1(support), + supports_select0(support), seeds.repetition_index); state.SetItemsProcessed(static_cast(state.iterations())); } -template -void register_build_row() { - const std::string name = - "rank_select_build_both_" + std::string(fill_spec(fill).name); - auto* row = benchmark::RegisterBenchmark(name.c_str(), &run_build); +template +void register_build_row(std::string_view backend_name) { + const std::string name = std::string(backend_name) + "_build_both_" + + std::string(fill_spec(fill).name); + auto* row = + benchmark::RegisterBenchmark(name.c_str(), &run_build); for (const std::size_t size : kSizes) { row->Arg(static_cast(size)); } @@ -422,12 +495,14 @@ void register_build_row() { ->MinTime(kBenchmarkMinSeconds); } -template -void register_query_row(std::string_view operation_name) { - const std::string name = "rank_select_" + std::string(operation_name) + "_" + +template +void register_query_row(std::string_view backend_name, + std::string_view operation_name) { + const std::string name = std::string(backend_name) + "_" + + std::string(operation_name) + "_" + std::string(fill_spec(fill).name); - auto* row = - benchmark::RegisterBenchmark(name.c_str(), &run_query); + auto* row = benchmark::RegisterBenchmark( + name.c_str(), &run_query); for (const std::size_t size : kSizes) { row->Arg(static_cast(size)); } @@ -437,18 +512,24 @@ void register_query_row(std::string_view operation_name) { ->MinTime(kBenchmarkMinSeconds); } -template -void register_fill_rows() { - register_build_row(); - register_query_row("rank1"); - register_query_row("rank0"); - register_query_row("select1"); - register_query_row("select0"); +template +void register_fill_rows(std::string_view backend_name) { + constexpr auto kRank1 = QueryOperation::kRank1; + constexpr auto kRank0 = QueryOperation::kRank0; + constexpr auto kSelect1 = QueryOperation::kSelect1; + constexpr auto kSelect0 = QueryOperation::kSelect0; + register_build_row(backend_name); + register_query_row(backend_name, "rank1"); + register_query_row(backend_name, "rank0"); + register_query_row(backend_name, "select1"); + register_query_row(backend_name, "select0"); } -void register_fnbp_build_row() { - auto* row = benchmark::RegisterBenchmark("rank_select_fnbp_build_both", - &run_fnbp_build); +template +void register_fnbp_build_row(std::string_view backend_name) { + const std::string name = std::string(backend_name) + "_fnbp_build_both"; + auto* row = + benchmark::RegisterBenchmark(name.c_str(), &run_fnbp_build); for (const std::size_t size : kFnbpSizes) { row->Arg(static_cast(size)); } @@ -458,11 +539,13 @@ void register_fnbp_build_row() { ->MinTime(kBenchmarkMinSeconds); } -template -void register_fnbp_query_row(std::string_view operation_name) { - const std::string name = "rank_select_fnbp_" + std::string(operation_name); - auto* row = - benchmark::RegisterBenchmark(name.c_str(), &run_fnbp_query); +template +void register_fnbp_query_row(std::string_view backend_name, + std::string_view operation_name) { + const std::string name = + std::string(backend_name) + "_fnbp_" + std::string(operation_name); + const auto callback = &run_fnbp_query; + auto* row = benchmark::RegisterBenchmark(name.c_str(), callback); for (const std::size_t size : kFnbpSizes) { row->Arg(static_cast(size)); } @@ -472,19 +555,56 @@ void register_fnbp_query_row(std::string_view operation_name) { ->MinTime(kBenchmarkMinSeconds); } -void register_fnbp_rows() { - register_fnbp_build_row(); - register_fnbp_query_row("rank1"); - register_fnbp_query_row("rank0"); - register_fnbp_query_row("select1"); - register_fnbp_query_row("select0"); +template +void register_fnbp_rows(std::string_view backend_name) { + constexpr auto kRank1 = QueryOperation::kRank1; + constexpr auto kRank0 = QueryOperation::kRank0; + constexpr auto kSelect1 = QueryOperation::kSelect1; + constexpr auto kSelect0 = QueryOperation::kSelect0; + register_fnbp_build_row(backend_name); + register_fnbp_query_row(backend_name, "rank1"); + register_fnbp_query_row(backend_name, "rank0"); + register_fnbp_query_row(backend_name, "select1"); + if constexpr (select0_enabled) { + register_fnbp_query_row(backend_name, "select0"); + } +} + +template +void register_select1_only_fill_rows(std::string_view backend_name) { + constexpr auto kRank1 = QueryOperation::kRank1; + constexpr auto kRank0 = QueryOperation::kRank0; + constexpr auto kSelect1 = QueryOperation::kSelect1; + register_build_row(backend_name); + register_query_row(backend_name, "rank1"); + register_query_row(backend_name, "rank0"); + register_query_row(backend_name, "select1"); } void register_benchmarks() { - register_fill_rows(); - register_fill_rows(); - register_fill_rows(); - register_fnbp_rows(); + register_fill_rows("rank_select"); + register_fill_rows("rank_select"); + register_fill_rows("rank_select"); + register_fnbp_rows("rank_select"); +#ifdef PIXIE_PASTA_SUPPORT + register_fill_rows( + "rank_select_pasta"); + register_fill_rows( + "rank_select_pasta"); + register_fill_rows( + "rank_select_pasta"); + register_fnbp_rows("rank_select_pasta"); +#endif +#ifdef PIXIE_3STAR_SUPPORT + using ThreeStarRankSelect = pixie::benchmarks::ThreeStarRankSelectSupport; + register_select1_only_fill_rows( + "rank_select_3star"); + register_select1_only_fill_rows( + "rank_select_3star"); + register_select1_only_fill_rows( + "rank_select_3star"); + register_fnbp_rows("rank_select_3star"); +#endif } } // namespace diff --git a/src/benchmarks/three_star_adapter.cpp b/src/benchmarks/three_star_adapter.cpp new file mode 100644 index 0000000..9b84225 --- /dev/null +++ b/src/benchmarks/three_star_adapter.cpp @@ -0,0 +1,128 @@ +#include "three_star_adapter.h" + +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t kWordBits = 64; +constexpr std::size_t kWordsPerBasicBlock = 32; +constexpr std::size_t kWordsPerSuperblock = 1024; +constexpr std::size_t kSentinelSuperblocks = 5; + +std::uint64_t logical_word(std::span source, + std::size_t word_index, + std::size_t num_bits) { + std::uint64_t word = source[word_index]; + const std::size_t remaining = num_bits - word_index * kWordBits; + if (remaining < kWordBits) { + word &= (std::uint64_t{1} << remaining) - 1; + } + return word; +} + +} // namespace + +uint64_t SAMPLEDIST_EXP = 0; +uint64_t SAMPLEDIST = 0; + +bool show_overhead = false; +std::string BITGEN; +std::string BVNAME; +bool doBench = false; +bool doValid = false; +bool doValidw = false; +bool doMultibench = false; +uint64_t synth_bits = 0; +bool useSynth = false; +bool do_manual_bits = false; +bool do_fractals = false; +bool do_chunky = false; +bool do_smooth = false; +bool do_quicksmooth = false; +bool do_gidneysmooth = false; +bool do_alternate = false; +bool do_every_kth = false; +bool do_bimodal = false; +bool do_patches = false; +bool do_sinus = false; +int bimodal_factor = 0; +int every_kth_spacing = 0; +int synth_seqsize = 0; +int synth_01ratio = 0; +bool invert_all_bits = false; +bool do_random_queries = false; +uint64_t synth_accesses = 0; +uint64_t synth_ranks = 0; +uint64_t synth_selects = 0; +uint64_t synth_select1s = 0; +uint64_t synth_select0s = 0; +std::vector query_type_counter; +uint64_t N_queries = 0; +uint64_t N_bits = 0; +uint64_t N_ones = 0; +uint64_t N_zeros = 0; +uint64_t N_words = 0; +uint64_t seed = 0; +double eff_01ratio = 0.0; +int parameter_cycles = 1; +int curr_parameter_cycle = 0; +int curr_instance = 0; +double sin_threshhold = 0.0; +std::vector> bits; +std::vector> queries; +std::vector> + bits_64bit_reversed; +uint64_t total_used_space_in_bits = 0; +tp tCopyFinished; +double dRead = 0.0; +double dCopy = 0.0; +double dBuild = 0.0; +double dQueries = 0.0; +double dPerQuery = 0.0; + +std::string get_unixtimestamp() { + return {}; +} + +namespace pixie::benchmarks { + +ThreeStarRankSelectSupport::ThreeStarRankSelectSupport( + std::span source_words, + std::size_t num_bits) + : num_bits_(std::min(num_bits, source_words.size() * kWordBits)) { + const std::size_t source_word_count = (num_bits_ + kWordBits - 1) / kWordBits; + N_bits = num_bits_; + N_words = source_word_count; + bits.assign(source_word_count, 0); + one_count_ = 0; + for (std::size_t word_index = 0; word_index < source_word_count; + ++word_index) { + const std::uint64_t word = + logical_word(source_words, word_index, num_bits_); + bits[word_index] = word; + one_count_ += std::popcount(word); + } + N_ones = one_count_; + N_zeros = num_bits_ - one_count_; + + // Match the paper's uncompressed robust configuration: L0=2048, a*, + // alpha=16, and its two-level L0/L1 summary tree. + ALPHA = 16; + SUMMARY_LEVELS = 1; + TREE3_STRATEGY = GET_LIFTED_CUBIC_THEORY_PARAMS; + BV_COMPRESSION = 0; + + support_.build_auxiliaries(); + + const std::size_t stored_words = + ((source_word_count + kSentinelSuperblocks * kWordsPerSuperblock) / + kWordsPerBasicBlock) * + kWordsPerBasicBlock; + source_copy_bytes_ = stored_words * sizeof(std::uint64_t); +} + +} // namespace pixie::benchmarks diff --git a/src/benchmarks/three_star_adapter.h b/src/benchmarks/three_star_adapter.h new file mode 100644 index 0000000..5e108e8 --- /dev/null +++ b/src/benchmarks/three_star_adapter.h @@ -0,0 +1,95 @@ +#pragma once + +#ifndef PIXIE_3STAR_SUPPORT +#error "ThreeStarRankSelectSupport requires PIXIE_3STAR_SOURCE_DIR" +#endif + +#include +#include +#include +#include + +// The upstream header emits unconditional construction diagnostics. Suppress +// only those diagnostics so benchmark construction measures the index rather +// than terminal I/O. +#define printf(...) static_cast(0) +#include +#undef printf + +static void print_overhead_line(const std::string&, int64_t) {} + +static void print_separator() {} + +namespace pixie::benchmarks { + +/** + * @brief Benchmark-only adapter for a permitted local 3-star checkout. + * + * @details This adapter exposes only the upstream operations that are complete + * in the pinned local source: one-rank and one-select. Its backing input and + * metadata are owned by the upstream process-global implementation, so only + * one instance may be live at a time. It is intentionally not a public Pixie + * rank/select implementation. Upstream construction diagnostics are suppressed + * to keep terminal I/O out of the timed build operation. + */ +class ThreeStarRankSelectSupport { + public: + /** + * @brief Build the upstream uncompressed 3-star configuration. + * @param source_words Packed least-significant-bit-first source words. + * @param num_bits Logical source length, clamped to @p source_words. + */ + explicit ThreeStarRankSelectSupport( + std::span source_words, + std::size_t num_bits); + + /** @brief Return the logical source length. */ + std::size_t size() const { return num_bits_; } + + /** @brief Return the one rank in `[0, end_position)`. */ + std::uint64_t rank(std::size_t end_position) const { + return end_position >= num_bits_ ? one_count_ + : support_.rank_1(end_position); + } + + /** @brief Return the derived zero rank in `[0, end_position)`. */ + std::uint64_t rank0(std::size_t end_position) const { + return end_position >= num_bits_ ? num_bits_ - one_count_ + : end_position - rank(end_position); + } + + /** @brief Return the zero-based position of a one-based one rank. */ + std::uint64_t select(std::size_t rank) const { + if (rank == 0) { + return 0; + } + return rank > one_count_ ? num_bits_ : support_.select_1(rank); + } + + /** @brief 3-star implements one-select, but not zero-select. */ + bool supports_select1() const { return true; } + + /** @brief 3-star's upstream zero-select implementation is incomplete. */ + bool supports_select0() const { return false; } + + /** @brief Return upstream-reported logical bytes for source and metadata. */ + std::size_t memory_usage_bytes() const { + return static_cast(support_.space_in_bits() / 8); + } + + /** @brief Return logical bytes in 3-star's uncompressed source copy. */ + std::size_t source_copy_bytes() const { return source_copy_bytes_; } + + /** @brief Return upstream-reported logical metadata bytes. */ + std::size_t index_logical_bytes() const { + return memory_usage_bytes() - source_copy_bytes_; + } + + private: + m3 support_; + std::size_t num_bits_ = 0; + std::size_t one_count_ = 0; + std::size_t source_copy_bytes_ = 0; +}; + +} // namespace pixie::benchmarks diff --git a/src/tests/rank_select_tests.cpp b/src/tests/rank_select_tests.cpp index ac8a374..6385e8e 100644 --- a/src/tests/rank_select_tests.cpp +++ b/src/tests/rank_select_tests.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -68,7 +68,12 @@ RankSelectSampleLayout locate_rank_select_samples( template class RankSelectSpecificationTest : public testing::Test {}; +#ifdef PIXIE_PASTA_SUPPORT +using RankSelectImplementations = + testing::Types, pixie::PastaRankSelectSupport>; +#else using RankSelectImplementations = testing::Types>; +#endif TYPED_TEST_SUITE(RankSelectSpecificationTest, RankSelectImplementations); TYPED_TEST(RankSelectSpecificationTest, EmptySequence) { diff --git a/src/tests/rank_select_unittests.cc b/src/tests/rank_select_unittests.cc index 4547144..069156d 100644 --- a/src/tests/rank_select_unittests.cc +++ b/src/tests/rank_select_unittests.cc @@ -347,6 +347,17 @@ TEST(RankSelectSupportTest, SelectZeroSkewedDistributionFallback) { EXPECT_EQ(bv.select0(kZeroBasicBlocks * 512u + 1), bv.size()); } +TEST(RankSelectSupportTest, SelectHandlesFullSuperblockRank) { + constexpr size_t kWordsPerSuperblock = 65536 / 64; + std::vector ones(kWordsPerSuperblock, ~uint64_t{0}); + const RankSelectSupport one_support(ones, 65536); + EXPECT_EQ(one_support.select(65536), 65535u); + + std::vector zeros(kWordsPerSuperblock, 0); + const RankSelectSupport zero_support(zeros, 65536); + EXPECT_EQ(zero_support.select0(65536), 65535u); +} + TEST(RankSelectSupportTest, MainRankZeroTest) { std::mt19937_64 rng(42); std::vector bits(65536 * 32); diff --git a/src/tests/three_star_benchmark_tests.cpp b/src/tests/three_star_benchmark_tests.cpp new file mode 100644 index 0000000..4475529 --- /dev/null +++ b/src/tests/three_star_benchmark_tests.cpp @@ -0,0 +1,57 @@ +#include + +#include +#include +#include +#include +#include + +#include "three_star_adapter.h" + +namespace { + +std::uint64_t splitmix64(std::uint64_t value) { + value += 0x9E3779B97F4A7C15ull; + value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ull; + value = (value ^ (value >> 27)) * 0x94D049BB133111EBull; + return value ^ (value >> 31); +} + +TEST(ThreeStarBenchmarkAdapterTest, MatchesOneRankAndSelect) { + constexpr std::size_t kBits = 1 << 16; + std::vector words(kBits / 64); + for (std::size_t word_index = 0; word_index < words.size(); ++word_index) { + words[word_index] = splitmix64(word_index); + } + + pixie::benchmarks::ThreeStarRankSelectSupport support(words, kBits); + std::uint64_t rank = 0; + for (std::size_t position = 0; position < kBits; ++position) { + const bool bit = (words[position >> 6] >> (position & 63)) & 1; + EXPECT_EQ(support.rank(position), rank) << position; + if (bit) { + ++rank; + EXPECT_EQ(support.select(rank), position) << rank; + } + } + EXPECT_EQ(support.rank(kBits), rank); +} + +TEST(ThreeStarBenchmarkAdapterTest, HandlesSmallSources) { + constexpr std::array kWords{ + 0x8000000000000001ull, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x8000000000000000ull, + }; + pixie::benchmarks::ThreeStarRankSelectSupport support(kWords, 1024); + + EXPECT_EQ(support.rank(0), 0); + EXPECT_EQ(support.rank(1), 1); + EXPECT_EQ(support.rank(64), 2); + EXPECT_EQ(support.rank(1024), 3); + EXPECT_EQ(support.select(1), 0); + EXPECT_EQ(support.select(2), 63); + EXPECT_EQ(support.select(3), 1023); + EXPECT_EQ(support.select(4), 1024); +} + +} // namespace