From f62f94b8eaea139299706700d4ac91ee0a8eec2a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 19 Aug 2026 21:21:29 +0000 Subject: [PATCH 1/8] Add SlateDB SlateDB is an embedded LSM-tree key-value store on object storage (the local filesystem here). One key-value pair per row; the 43 SQL queries run through embedded Apache DataFusion with a custom TableProvider doing parallel SlateDB range scans. Co-Authored-By: Claude Fable 5 --- slatedb/.gitignore | 3 + slatedb/README.md | 56 + slatedb/benchmark.sh | 8 + slatedb/check | 5 + slatedb/create.sql | 4 + slatedb/data-size | 5 + slatedb/hits-slatedb/Cargo.lock | 4219 ++++++++++++++++++++++++++++++ slatedb/hits-slatedb/Cargo.toml | 22 + slatedb/hits-slatedb/src/main.rs | 823 ++++++ slatedb/install | 32 + slatedb/load | 10 + slatedb/queries.sql | 43 + slatedb/query | 10 + slatedb/start | 3 + slatedb/stop | 2 + slatedb/template.json | 12 + 16 files changed, 5257 insertions(+) create mode 100644 slatedb/.gitignore create mode 100644 slatedb/README.md create mode 100755 slatedb/benchmark.sh create mode 100755 slatedb/check create mode 100644 slatedb/create.sql create mode 100755 slatedb/data-size create mode 100644 slatedb/hits-slatedb/Cargo.lock create mode 100644 slatedb/hits-slatedb/Cargo.toml create mode 100644 slatedb/hits-slatedb/src/main.rs create mode 100755 slatedb/install create mode 100755 slatedb/load create mode 100644 slatedb/queries.sql create mode 100755 slatedb/query create mode 100755 slatedb/start create mode 100755 slatedb/stop create mode 100644 slatedb/template.json diff --git a/slatedb/.gitignore b/slatedb/.gitignore new file mode 100644 index 0000000000..eb3a846aeb --- /dev/null +++ b/slatedb/.gitignore @@ -0,0 +1,3 @@ +hits-slatedb/target/ +db/ +rust-init.sh diff --git a/slatedb/README.md b/slatedb/README.md new file mode 100644 index 0000000000..446ea30ec0 --- /dev/null +++ b/slatedb/README.md @@ -0,0 +1,56 @@ +# SlateDB + +[SlateDB](https://slatedb.io/) is an embedded LSM-tree key-value store, written +in Rust, that keeps all of its state (SSTs, WAL, manifests) in an object store. +In this benchmark the object store is the local filesystem +(`object_store::LocalFileSystem`), which is SlateDB's standard single-node setup +and keeps the entry comparable to the other locally-run systems. + +SlateDB has no query language — its API is `put`/`get`/`scan` over byte keys — +so this entry follows the usual approach for key-value stores: the storage +layout and query execution are implemented by the client, in +[`hits-slatedb`](hits-slatedb/): + +- **Storage**: one key-value pair per row. Key = 8-byte big-endian row index; + value = the row in a compact positional encoding (little-endian fixed-width + integers, varint-length-prefixed strings). The schema and row count are kept + under two metadata keys. +- **Queries**: the 43 SQL queries run unmodified through [Apache + DataFusion](https://datafusion.apache.org/) embedded in the same binary, with + a custom `TableProvider` whose partitions are parallel SlateDB range scans + (one contiguous key range per CPU). Projection pushdown skips the decode of + unused columns, but every query still pays for scanning the full rows out of + SlateDB — there is no columnar shortcut past the KV store. + +So the numbers here measure SlateDB's scan path (SST decode, block fetch from +the object store, merge across sorted runs) plus row decode, with DataFusion +providing SQL on top. The comparison against `datafusion` (same SQL engine +reading Parquet directly) isolates what the KV storage layer costs. + +Non-default settings, per the fine-tuning rules: + +- `compression_codec = zstd` and 64 KiB SST blocks: without compression the + row-oriented database is ~90 GB; SST block compression is the standard + production option for scan-heavy data. +- `wal_enabled = false` during the bulk load (with an explicit flush at the + end): the WAL would double the load I/O for no durability benefit since the + load ends with a flush + clean close. Queries open the database with a + read-only `DbReader`. + +The concurrent-QPS test is skipped: each query forks a fresh full-machine +process with no shared server (see issue #946). + +## Manual run + +``` +wget --continue https://datasets.clickhouse.com/hits_compatible/hits.parquet +./install +./load +echo 'SELECT COUNT(*) FROM hits;' | ./query +``` + +Or the full benchmark on a fresh VM: `bash benchmark.sh`. + +For validating results, the binary also has a `queryp` mode that runs the same +SQL directly against the Parquet file with the same DataFusion version: +`echo '' | hits-slatedb/target/release/hits-slatedb queryp hits.parquet create.sql`. diff --git a/slatedb/benchmark.sh b/slatedb/benchmark.sh new file mode 100755 index 0000000000..27ed18dae5 --- /dev/null +++ b/slatedb/benchmark.sh @@ -0,0 +1,8 @@ +#!/bin/bash +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +# Embedded engine: no daemon to restart, one process per query. +export BENCH_RESTARTABLE=no +# Single-process engine: each query forks a fresh full-machine process, so +# the concurrent-QPS test would only oversubscribe RAM (see issue #946). +export BENCH_CONCURRENT_DURATION="${BENCH_CONCURRENT_DURATION:-0}" +exec ../lib/benchmark-common.sh diff --git a/slatedb/check b/slatedb/check new file mode 100755 index 0000000000..12f22aab97 --- /dev/null +++ b/slatedb/check @@ -0,0 +1,5 @@ +#!/bin/bash +# Embedded engine: nothing to probe beyond the binary being built. +set -e + +test -x hits-slatedb/target/release/hits-slatedb diff --git a/slatedb/create.sql b/slatedb/create.sql new file mode 100644 index 0000000000..a4629f217f --- /dev/null +++ b/slatedb/create.sql @@ -0,0 +1,4 @@ +CREATE VIEW hits AS +SELECT * EXCEPT ("EventDate"), + CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate" +FROM hits_raw; diff --git a/slatedb/data-size b/slatedb/data-size new file mode 100755 index 0000000000..c98f7a672c --- /dev/null +++ b/slatedb/data-size @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +# Size of the SlateDB database: SSTs, manifests, and checkpoint state. +du -sb db | awk '{print $1}' diff --git a/slatedb/hits-slatedb/Cargo.lock b/slatedb/hits-slatedb/Cargo.lock new file mode 100644 index 0000000000..1703bcf4b4 --- /dev/null +++ b/slatedb/hits-slatedb/Cargo.lock @@ -0,0 +1,4219 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" +dependencies = [ + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +dependencies = [ + "serde", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link 0.2.1", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_affinity" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap", + "itertools 0.15.0", + "liblzma", + "log", + "object_store 0.13.2", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store 0.13.2", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.15.0", + "log", + "object_store 0.13.2", + "percent-encoding", +] + +[[package]] +name = "datafusion-common" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "libc", + "log", + "num-traits", + "object_store 0.13.2", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.15.0", + "liblzma", + "log", + "object_store 0.13.2", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.15.0", + "object_store 0.13.2", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store 0.13.2", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store 0.13.2", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store 0.13.2", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" + +[[package]] +name = "datafusion-execution" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "bytes", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store 0.13.2", + "parking_lot", + "pin-project-lite", + "rand 0.9.5", + "tempfile", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "itertools 0.15.0", + "recursive", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-functions" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.15.0", + "log", + "md-5", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.15.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" +dependencies = [ + "datafusion-doc", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "datafusion-optimizer" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap", + "itertools 0.15.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "itertools 0.15.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap", + "itertools 0.15.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser", + "stacker", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "duration-str" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" +dependencies = [ + "rust_decimal", + "serde", + "thiserror 2.0.20", + "time", + "winnow 0.6.26", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fail-parallel" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" +dependencies = [ + "log", + "once_cell", + "rand 0.9.5", + "tokio", +] + +[[package]] +name = "fastant" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e825441bfb2d831c47c97d05821552db8832479f44c571b97fededbf0099c07" +dependencies = [ + "small_ctor", + "web-time", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "serde_json", + "serde_yaml", + "toml", + "uncased", + "version_check", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "foyer" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" +dependencies = [ + "anyhow", + "equivalent", + "foyer-common", + "foyer-memory", + "foyer-storage", + "foyer-tokio", + "futures-util", + "mea", + "mixtrics", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-common" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" +dependencies = [ + "anyhow", + "bincode", + "bytes", + "cfg-if", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "serde", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" +dependencies = [ + "anyhow", + "bitflags", + "cmsketch", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-storage" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" +dependencies = [ + "allocator-api2", + "anyhow", + "bytes", + "core_affinity", + "equivalent", + "fastant", + "foyer-common", + "foyer-memory", + "foyer-tokio", + "fs4", + "futures-core", + "futures-util", + "hashbrown 0.16.1", + "io-uring", + "itertools 0.14.0", + "libc", + "lz4", + "mea", + "parking_lot", + "pin-project", + "rand 0.9.5", + "serde", + "tracing", + "twox-hash", + "zstd", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" +dependencies = [ + "tokio", +] + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hits-slatedb" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "datafusion", + "futures", + "slatedb", + "tokio", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core 0.62.2", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + +[[package]] +name = "io-uring" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64d8ca234d152948ceaede1f419b6a83983a5ecccaac05fb337a809c96d3aa6" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "liblzma" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fe0a34ca854fd4f20c07f696fc8675aec78f87d88d29f5e10257a7490a1b2e1" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dad045e4b1b7b170be4b60b54b780cafb4490165461bac7d1cf7b703f61d5f" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "mea" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c709842c4ce65cb91e2666ad5319dfc1efc3af0d34f02075eddca9000d9f8afb" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.20", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "object_store" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures-channel", + "futures-core", + "futures-util", + "http", + "humantime", + "itertools 0.15.0", + "nix", + "parking_lot", + "percent-encoding", + "thiserror 2.0.20", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link 0.2.1", +] + +[[package]] +name = "parquet" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "object_store 0.13.2", + "seq-macro", + "simdutf8", + "snap", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "num-traits", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "slatedb" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" +dependencies = [ + "async-channel", + "async-trait", + "atomic", + "backon", + "bitflags", + "bytes", + "chrono", + "crc32fast", + "crossbeam-skiplist", + "dotenvy", + "duration-str", + "fail-parallel", + "figment", + "flatbuffers", + "foyer", + "futures", + "log", + "lru", + "object_store 0.14.1", + "ouroboros", + "parking_lot", + "rand 0.9.5", + "serde", + "serde_json", + "siphasher", + "slatedb-common", + "slatedb-txn-obj", + "smallvec", + "sysinfo", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "ulid", + "url", + "uuid", + "walkdir", + "zstd", +] + +[[package]] +name = "slatedb-common" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" +dependencies = [ + "chrono", + "log", + "object_store 0.14.1", + "rand 0.9.5", + "rand_xoshiro", + "serde", + "thread_local", + "tokio", +] + +[[package]] +name = "slatedb-txn-obj" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "log", + "object_store 0.14.1", + "parking_lot", + "slatedb-common", + "thiserror 1.0.69", +] + +[[package]] +name = "small_ctor" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sysinfo" +version = "0.35.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "futures-util", + "hashbrown 0.15.5", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow 0.7.15", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +dependencies = [ + "rand 0.10.2", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "ulid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" +dependencies = [ + "rand 0.9.5", + "serde", + "web-time", +] + +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link 0.2.1", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.6.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" +dependencies = [ + "memchr", +] + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b5c6b5976d66c1d703c4fd17d3f5e43c8cedaacf604961b171adc7130896d8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/slatedb/hits-slatedb/Cargo.toml b/slatedb/hits-slatedb/Cargo.toml new file mode 100644 index 0000000000..fc168942be --- /dev/null +++ b/slatedb/hits-slatedb/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "hits-slatedb" +version = "0.1.0" +edition = "2021" + +[dependencies] +# SlateDB is the system under test: an embedded LSM-tree key-value store on +# top of an object store (the local filesystem here). `zstd` enables SST +# block compression, `wal_disable` allows turning the WAL off for the bulk +# load, `foyer` provides the default in-memory block cache. +slatedb = { version = "0.15.0", default-features = false, features = ["zstd", "wal_disable", "foyer"] } +# DataFusion provides the SQL frontend; all table data is scanned out of +# SlateDB through a custom TableProvider. +datafusion = "55" +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +futures = "0.3" +async-trait = "0.1" +anyhow = "1" + +[profile.release] +lto = "thin" +codegen-units = 1 diff --git a/slatedb/hits-slatedb/src/main.rs b/slatedb/hits-slatedb/src/main.rs new file mode 100644 index 0000000000..135820d3c9 --- /dev/null +++ b/slatedb/hits-slatedb/src/main.rs @@ -0,0 +1,823 @@ +// ClickBench harness for SlateDB (https://slatedb.io). +// +// SlateDB is an embedded LSM-tree key-value store that keeps all of its +// state in an object store; here the object store is the local filesystem. +// It has no query language, so this harness stores each row of the hits +// dataset as one key-value pair (key = 8-byte big-endian row index, value = +// a compact positional row encoding) and runs the ClickBench SQL through +// Apache DataFusion with a custom TableProvider whose partitions are +// parallel SlateDB range scans. +// +// Usage: +// hits-slatedb load +// hits-slatedb query [create.sql] # SQL statement on stdin + +use std::io::Read; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{anyhow, bail, Context, Result}; +use async_trait::async_trait; +use futures::stream::{self, StreamExt, TryStreamExt}; + +use datafusion::arrow::array::{ + ArrayRef, AsArray, Int16Array, Int32Array, Int64Array, RecordBatch, RecordBatchOptions, + StringBuilder, UInt16Array, UInt32Array, UInt64Array, +}; +use datafusion::arrow::datatypes::{ + DataType, Field, Int16Type, Int32Type, Int64Type, Schema, SchemaRef, UInt16Type, UInt32Type, + UInt64Type, +}; +use datafusion::arrow::util::pretty::pretty_format_batches; +use datafusion::catalog::Session; +use datafusion::common::stats::{Precision, Statistics}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::datasource::{TableProvider, TableType}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::Expr; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + SendableRecordBatchStream, +}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; + +use slatedb::bytes::Bytes; +use slatedb::config::{CompressionCodec, ScanOptions, Settings, WriteOptions}; +use slatedb::object_store::local::LocalFileSystem; +use slatedb::object_store::ObjectStore; +use slatedb::{Db, DbReader, SstBlockSize, WriteBatch}; + +// Row keys are 8-byte big-endian indexes, so they sort in row order and any +// key starting with 0xff sorts after all of them (the dataset has far fewer +// than 2^56 rows). +const META_SCHEMA: &[u8] = b"\xffschema"; +const META_COUNT: &[u8] = b"\xffcount"; +const BATCH_ROWS: usize = 8192; +const DB_PATH: &str = "hits"; + +fn row_key(i: u64) -> [u8; 8] { + i.to_be_bytes() +} + +fn object_store(db_dir: &str) -> Result> { + Ok(Arc::new(LocalFileSystem::new_with_prefix(db_dir)?)) +} + +fn db_settings() -> Settings { + Settings { + // The load is a bulk import with an explicit flush at the end; + // writing a WAL on top of the SSTs would only double the I/O. + wal_enabled: false, + compression_codec: Some(CompressionCodec::Zstd), + ..Default::default() + } +} + +// --------------------------------------------------------------------------- +// Row encoding: fixed-width integers little-endian, strings varint-length +// prefixed, all columns concatenated in schema order. The dataset has no +// NULLs (all parquet fields are REQUIRED), which the loader verifies. +// --------------------------------------------------------------------------- + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum ColType { + I16, + I32, + I64, + U16, + U32, + U64, + Str, +} + +impl ColType { + fn tag(self) -> &'static str { + match self { + ColType::I16 => "i16", + ColType::I32 => "i32", + ColType::I64 => "i64", + ColType::U16 => "u16", + ColType::U32 => "u32", + ColType::U64 => "u64", + ColType::Str => "str", + } + } + + fn from_tag(s: &str) -> Result { + Ok(match s { + "i16" => ColType::I16, + "i32" => ColType::I32, + "i64" => ColType::I64, + "u16" => ColType::U16, + "u32" => ColType::U32, + "u64" => ColType::U64, + "str" => ColType::Str, + other => bail!("unknown column tag {other:?}"), + }) + } + + fn from_arrow(dt: &DataType) -> Result { + Ok(match dt { + DataType::Int16 => ColType::I16, + DataType::Int32 => ColType::I32, + DataType::Int64 => ColType::I64, + DataType::UInt16 => ColType::U16, + DataType::UInt32 => ColType::U32, + DataType::UInt64 => ColType::U64, + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView => ColType::Str, + other => bail!("unsupported column type {other:?}"), + }) + } + + fn arrow_type(self) -> DataType { + match self { + ColType::I16 => DataType::Int16, + ColType::I32 => DataType::Int32, + ColType::I64 => DataType::Int64, + ColType::U16 => DataType::UInt16, + ColType::U32 => DataType::UInt32, + ColType::U64 => DataType::UInt64, + ColType::Str => DataType::Utf8, + } + } +} + +fn write_varint(buf: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + buf.push(b); + break; + } + buf.push(b | 0x80); + } +} + +fn read_varint(data: &[u8], pos: &mut usize) -> u64 { + let mut v = 0u64; + let mut shift = 0; + loop { + let b = data[*pos]; + *pos += 1; + v |= ((b & 0x7f) as u64) << shift; + if b & 0x80 == 0 { + return v; + } + shift += 7; + } +} + +fn put_bytes(row: &mut Vec, b: &[u8]) { + write_varint(row, b.len() as u64); + row.extend_from_slice(b); +} + +fn encode_str_col(arr: &ArrayRef, rows: &mut [Vec]) -> Result<()> { + match arr.data_type() { + DataType::Utf8 => { + let a = arr.as_string::(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i).as_bytes()); + } + } + DataType::LargeUtf8 => { + let a = arr.as_string::(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i).as_bytes()); + } + } + DataType::Utf8View => { + let a = arr.as_string_view(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i).as_bytes()); + } + } + DataType::Binary => { + let a = arr.as_binary::(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i)); + } + } + DataType::LargeBinary => { + let a = arr.as_binary::(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i)); + } + } + DataType::BinaryView => { + let a = arr.as_binary_view(); + for (i, row) in rows.iter_mut().enumerate() { + put_bytes(row, a.value(i)); + } + } + other => bail!("unsupported string column type {other:?}"), + } + Ok(()) +} + +fn encode_rows(batch: &RecordBatch, tags: &[ColType]) -> Result>> { + let n = batch.num_rows(); + let mut rows: Vec> = (0..n).map(|_| Vec::with_capacity(1024)).collect(); + for (ci, tag) in tags.iter().enumerate() { + let arr = batch.column(ci); + if arr.null_count() > 0 { + bail!("NULL values are not supported (column {ci})"); + } + match tag { + ColType::I16 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::I32 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::I64 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::U16 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::U32 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::U64 => { + let a = arr.as_primitive::(); + for (i, row) in rows.iter_mut().enumerate() { + row.extend_from_slice(&a.value(i).to_le_bytes()); + } + } + ColType::Str => encode_str_col(arr, &mut rows)?, + } + } + Ok(rows) +} + +// --------------------------------------------------------------------------- +// Row decoding into Arrow arrays, skipping columns outside the projection. +// --------------------------------------------------------------------------- + +const SKIP: usize = usize::MAX; + +enum ColBuf { + I16(Vec), + I32(Vec), + I64(Vec), + U16(Vec), + U32(Vec), + U64(Vec), + Str(StringBuilder), +} + +impl ColBuf { + fn new(tag: ColType) -> Self { + match tag { + ColType::I16 => ColBuf::I16(Vec::with_capacity(BATCH_ROWS)), + ColType::I32 => ColBuf::I32(Vec::with_capacity(BATCH_ROWS)), + ColType::I64 => ColBuf::I64(Vec::with_capacity(BATCH_ROWS)), + ColType::U16 => ColBuf::U16(Vec::with_capacity(BATCH_ROWS)), + ColType::U32 => ColBuf::U32(Vec::with_capacity(BATCH_ROWS)), + ColType::U64 => ColBuf::U64(Vec::with_capacity(BATCH_ROWS)), + ColType::Str => ColBuf::Str(StringBuilder::new()), + } + } + + fn finish(&mut self) -> ArrayRef { + match self { + ColBuf::I16(v) => Arc::new(Int16Array::from(std::mem::take(v))), + ColBuf::I32(v) => Arc::new(Int32Array::from(std::mem::take(v))), + ColBuf::I64(v) => Arc::new(Int64Array::from(std::mem::take(v))), + ColBuf::U16(v) => Arc::new(UInt16Array::from(std::mem::take(v))), + ColBuf::U32(v) => Arc::new(UInt32Array::from(std::mem::take(v))), + ColBuf::U64(v) => Arc::new(UInt64Array::from(std::mem::take(v))), + ColBuf::Str(b) => Arc::new(b.finish()), + } + } +} + +struct Decoder { + tags: Arc>, + // Output slot per source column; SKIP when the column is not projected. + slots: Vec, + bufs: Vec, + out_schema: SchemaRef, + rows: usize, +} + +impl Decoder { + fn new(tags: Arc>, projection: &[usize], out_schema: SchemaRef) -> Self { + let mut slots = vec![SKIP; tags.len()]; + for (slot, &ci) in projection.iter().enumerate() { + slots[ci] = slot; + } + let bufs = projection.iter().map(|&ci| ColBuf::new(tags[ci])).collect(); + Decoder { tags, slots, bufs, out_schema, rows: 0 } + } + + fn push_row(&mut self, data: &[u8]) { + let mut pos = 0usize; + for (ci, tag) in self.tags.iter().enumerate() { + let slot = self.slots[ci]; + match tag { + ColType::I16 => { + if slot != SKIP { + let v = i16::from_le_bytes(data[pos..pos + 2].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::I16(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 2; + } + ColType::I32 => { + if slot != SKIP { + let v = i32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::I32(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 4; + } + ColType::I64 => { + if slot != SKIP { + let v = i64::from_le_bytes(data[pos..pos + 8].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::I64(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 8; + } + ColType::U16 => { + if slot != SKIP { + let v = u16::from_le_bytes(data[pos..pos + 2].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::U16(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 2; + } + ColType::U32 => { + if slot != SKIP { + let v = u32::from_le_bytes(data[pos..pos + 4].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::U32(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 4; + } + ColType::U64 => { + if slot != SKIP { + let v = u64::from_le_bytes(data[pos..pos + 8].try_into().unwrap()); + match &mut self.bufs[slot] { + ColBuf::U64(b) => b.push(v), + _ => unreachable!(), + } + } + pos += 8; + } + ColType::Str => { + let len = read_varint(data, &mut pos) as usize; + if slot != SKIP { + // The dataset comes from ClickHouse where strings are + // raw bytes; reinterpret without validation, exactly + // like DataFusion's own `binary_as_string` option. + let s = unsafe { std::str::from_utf8_unchecked(&data[pos..pos + len]) }; + match &mut self.bufs[slot] { + ColBuf::Str(b) => b.append_value(s), + _ => unreachable!(), + } + } + pos += len; + } + } + } + self.rows += 1; + } + + fn finish(&mut self) -> DFResult { + let arrays: Vec = self.bufs.iter_mut().map(|b| b.finish()).collect(); + let opts = RecordBatchOptions::new().with_row_count(Some(self.rows)); + self.rows = 0; + RecordBatch::try_new_with_options(self.out_schema.clone(), arrays, &opts) + .map_err(DataFusionError::from) + } +} + +// --------------------------------------------------------------------------- +// DataFusion integration: TableProvider + ExecutionPlan over SlateDB scans. +// --------------------------------------------------------------------------- + +fn ext_err(e: slatedb::Error) -> DataFusionError { + DataFusionError::External(Box::new(e)) +} + +struct HitsTable { + db: Arc, + schema: SchemaRef, + tags: Arc>, + row_count: u64, +} + +impl std::fmt::Debug for HitsTable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "HitsTable(rows={})", self.row_count) + } +} + +#[async_trait] +impl TableProvider for HitsTable { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + limit: Option, + ) -> DFResult> { + let projection: Vec = match projection { + Some(p) => p.clone(), + None => (0..self.schema.fields().len()).collect(), + }; + let out_schema = Arc::new(self.schema.project(&projection)?); + // One partition per target partition, but keep at least 64k rows in + // each so tiny scans don't fan out for nothing. + let max_parts = self.row_count.div_ceil(65536).max(1); + let partitions = (state.config().target_partitions() as u64).min(max_parts) as usize; + Ok(Arc::new(SlateScanExec::new( + self.db.clone(), + self.tags.clone(), + projection, + out_schema, + self.row_count, + partitions, + limit, + ))) + } + + fn statistics(&self) -> Option { + Some( + Statistics::new_unknown(&self.schema) + .with_num_rows(Precision::Exact(self.row_count as usize)), + ) + } +} + +struct SlateScanExec { + db: Arc, + tags: Arc>, + projection: Vec, + out_schema: SchemaRef, + row_count: u64, + partitions: usize, + limit: Option, + props: Arc, +} + +impl SlateScanExec { + #[allow(clippy::too_many_arguments)] + fn new( + db: Arc, + tags: Arc>, + projection: Vec, + out_schema: SchemaRef, + row_count: u64, + partitions: usize, + limit: Option, + ) -> Self { + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(out_schema.clone()), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + )); + SlateScanExec { db, tags, projection, out_schema, row_count, partitions, limit, props } + } +} + +impl std::fmt::Debug for SlateScanExec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "SlateScanExec") + } +} + +impl DisplayAs for SlateScanExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "SlateScanExec: rows={}, partitions={}, projection={:?}", + self.row_count, self.partitions, self.projection + ) + } +} + +impl ExecutionPlan for SlateScanExec { + fn name(&self) -> &str { + "SlateScanExec" + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> DFResult> { + Ok(self) + } + + fn execute( + &self, + partition: usize, + _context: Arc, + ) -> DFResult { + let chunk = self.row_count.div_ceil(self.partitions as u64); + let lo = (partition as u64) * chunk; + let hi = ((partition as u64 + 1) * chunk).min(self.row_count); + let db = self.db.clone(); + let mut remaining = self.limit.unwrap_or(usize::MAX); + let decoder = Decoder::new(self.tags.clone(), &self.projection, self.out_schema.clone()); + + let batches = stream::once(async move { + let opts = ScanOptions { + read_ahead_bytes: 4 << 20, + max_fetch_tasks: 2, + cache_blocks: false, + ..Default::default() + }; + let iter = if lo < hi { + let range = Bytes::copy_from_slice(&row_key(lo))..Bytes::copy_from_slice(&row_key(hi)); + Some(db.scan_with_options(range, &opts).await.map_err(ext_err)?) + } else { + None + }; + Ok::<_, DataFusionError>(stream::try_unfold( + (iter, decoder), + move |(mut iter, mut decoder)| async move { + let Some(it) = iter.as_mut() else { + return Ok(None); + }; + while decoder.rows < BATCH_ROWS && remaining > 0 { + match it.next().await.map_err(ext_err)? { + Some(kv) => { + decoder.push_row(&kv.value); + remaining -= 1; + } + None => break, + } + } + if decoder.rows == 0 { + Ok(None) + } else { + let batch = decoder.finish()?; + Ok(Some((batch, (iter, decoder)))) + } + }, + )) + }) + .try_flatten(); + + Ok(Box::pin(RecordBatchStreamAdapter::new(self.out_schema.clone(), batches))) + } + + fn partition_statistics(&self, partition: Option) -> DFResult> { + let mut stats = Statistics::new_unknown(&self.out_schema); + if self.limit.is_none() { + let rows = match partition { + None => self.row_count, + Some(p) => { + let chunk = self.row_count.div_ceil(self.partitions as u64); + let lo = (p as u64) * chunk; + let hi = ((p as u64 + 1) * chunk).min(self.row_count); + hi.saturating_sub(lo) + } + }; + stats = stats.with_num_rows(Precision::Exact(rows as usize)); + } + Ok(Arc::new(stats)) + } +} + +// --------------------------------------------------------------------------- +// Load: hits.parquet -> SlateDB. +// --------------------------------------------------------------------------- + +async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { + std::fs::create_dir_all(db_dir)?; + let db = Arc::new( + Db::builder(DB_PATH, object_store(db_dir)?) + .with_settings(db_settings()) + .with_sst_block_size(SstBlockSize::Block64Kib) + .build() + .await?, + ); + + let ctx = SessionContext::new(); + let df = ctx.read_parquet(parquet_path, ParquetReadOptions::default()).await?; + let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone()); + let tags: Vec = schema + .fields() + .iter() + .map(|f| ColType::from_arrow(f.data_type())) + .collect::>()?; + let tags = Arc::new(tags); + let schema_txt: String = schema + .fields() + .iter() + .zip(tags.iter()) + .map(|(f, t)| format!("{}\t{}\n", f.name(), t.tag())) + .collect(); + + let streams = df.execute_stream_partitioned().await?; + let counter = Arc::new(AtomicU64::new(0)); + let mut handles = Vec::new(); + for mut s in streams { + let db = db.clone(); + let tags = tags.clone(); + let counter = counter.clone(); + handles.push(tokio::spawn(async move { + let wopts = WriteOptions { await_durable: false, ..Default::default() }; + while let Some(batch) = s.next().await { + let batch = batch?; + let rows = encode_rows(&batch, &tags)?; + let base = counter.fetch_add(rows.len() as u64, Ordering::Relaxed); + let mut wb = WriteBatch::new(); + for (i, row) in rows.iter().enumerate() { + wb.put(row_key(base + i as u64), row); + } + db.write_with_options(wb, &wopts).await?; + } + Ok::<_, anyhow::Error>(()) + })); + } + for h in handles { + h.await??; + } + + let total = counter.load(Ordering::Relaxed); + let mut wb = WriteBatch::new(); + wb.put(META_COUNT, total.to_le_bytes()); + wb.put(META_SCHEMA, schema_txt.as_bytes()); + db.write_with_options(wb, &WriteOptions { await_durable: false, ..Default::default() }) + .await?; + db.flush().await?; + db.close().await?; + println!("Loaded {total} rows"); + Ok(()) +} + +// --------------------------------------------------------------------------- +// Query: run one SQL statement from stdin via DataFusion. +// --------------------------------------------------------------------------- + +async fn query(db_dir: &str, create_sql: Option<&str>) -> Result<()> { + let reader = Arc::new(DbReader::builder(DB_PATH, object_store(db_dir)?).build().await?); + + let schema_txt = reader + .get(META_SCHEMA) + .await? + .context("missing schema metadata; run `load` first")?; + let count_raw = reader.get(META_COUNT).await?.context("missing row count metadata")?; + let row_count = u64::from_le_bytes(count_raw.as_ref().try_into()?); + + let mut fields = Vec::new(); + let mut tags = Vec::new(); + for line in std::str::from_utf8(&schema_txt)?.lines() { + let (name, tag) = line + .split_once('\t') + .ok_or_else(|| anyhow!("malformed schema line {line:?}"))?; + let tag = ColType::from_tag(tag)?; + fields.push(Field::new(name, tag.arrow_type(), false)); + tags.push(tag); + } + let schema = Arc::new(Schema::new(fields)); + + let ctx = SessionContext::new(); + ctx.register_table( + "hits_raw", + Arc::new(HitsTable { db: reader, schema, tags: Arc::new(tags), row_count }), + )?; + + if let Some(path) = create_sql { + let ddl = std::fs::read_to_string(path)?; + for stmt in ddl.split(';') { + let stmt = stmt.trim(); + if !stmt.is_empty() { + ctx.sql(stmt).await?.collect().await?; + } + } + } + + let mut sql = String::new(); + std::io::stdin().read_to_string(&mut sql)?; + let sql = sql.trim(); + if sql.is_empty() { + bail!("no query on stdin"); + } + + let start = Instant::now(); + let results = ctx.sql(sql).await?.collect().await?; + let elapsed = start.elapsed().as_secs_f64(); + + if !results.is_empty() { + println!("{}", pretty_format_batches(&results)?); + } + eprintln!("{elapsed:.6}"); + Ok(()) +} + +// Validation helper (not used by the benchmark): run the same SQL directly +// against the parquet file so results can be diffed against the SlateDB path. +async fn query_parquet(parquet_path: &str, create_sql: Option<&str>) -> Result<()> { + let ctx = SessionContext::new(); + { + let state = ctx.state_ref(); + let mut state = state.write(); + state.config_mut().options_mut().execution.parquet.binary_as_string = true; + } + ctx.register_parquet("hits_raw", parquet_path, ParquetReadOptions::default()).await?; + + if let Some(path) = create_sql { + let ddl = std::fs::read_to_string(path)?; + for stmt in ddl.split(';') { + let stmt = stmt.trim(); + if !stmt.is_empty() { + ctx.sql(stmt).await?.collect().await?; + } + } + } + + let mut sql = String::new(); + std::io::stdin().read_to_string(&mut sql)?; + let start = Instant::now(); + let results = ctx.sql(sql.trim()).await?.collect().await?; + let elapsed = start.elapsed().as_secs_f64(); + if !results.is_empty() { + println!("{}", pretty_format_batches(&results)?); + } + eprintln!("{elapsed:.6}"); + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + match args.get(1).map(String::as_str) { + Some("load") if args.len() == 4 => load(&args[2], &args[3]).await, + Some("query") if args.len() >= 3 => query(&args[2], args.get(3).map(String::as_str)).await, + Some("queryp") if args.len() >= 3 => { + query_parquet(&args[2], args.get(3).map(String::as_str)).await + } + _ => { + eprintln!("usage: hits-slatedb load "); + eprintln!(" hits-slatedb query [create.sql] (SQL on stdin)"); + std::process::exit(2); + } + } +} diff --git a/slatedb/install b/slatedb/install new file mode 100755 index 0000000000..30f997aff1 --- /dev/null +++ b/slatedb/install @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +# Build the benchmark harness (SlateDB + DataFusion embedded in one binary). +# Idempotent: skip the build if the binary is already present. + +if [ -x hits-slatedb/target/release/hits-slatedb ]; then + exit 0 +fi + +if ! command -v cargo >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust-init.sh + bash rust-init.sh -y +fi +export HOME=${HOME:=~} +# shellcheck disable=SC1091 +source "$HOME/.cargo/env" + +# Low-memory hosts need swap to compile DataFusion. +if [ "$(free -g | awk '/^Mem:/{print $2}')" -lt 12 ]; then + if [ "$(swapon --noheadings --show | wc -l)" -eq 0 ]; then + sudo fallocate -l 8G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + fi +fi + +sudo apt-get update -y +sudo apt-get install -y gcc + +cargo build --release --locked --manifest-path hits-slatedb/Cargo.toml diff --git a/slatedb/load b/slatedb/load new file mode 100755 index 0000000000..ba3e0c2355 --- /dev/null +++ b/slatedb/load @@ -0,0 +1,10 @@ +#!/bin/bash +# Load hits.parquet into a SlateDB database at ./db: one key-value pair per +# row (key = 8-byte big-endian row index, value = compact row encoding). +set -e + +rm -rf db +hits-slatedb/target/release/hits-slatedb load hits.parquet db + +rm -f hits.parquet +sync diff --git a/slatedb/queries.sql b/slatedb/queries.sql new file mode 100644 index 0000000000..5be85b3f87 --- /dev/null +++ b/slatedb/queries.sql @@ -0,0 +1,43 @@ +SELECT COUNT(*) FROM hits; +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; +SELECT AVG("UserID") FROM hits; +SELECT COUNT(DISTINCT "UserID") FROM hits; +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; +SELECT MIN("EventDate"), MAX("EventDate") FROM hits; +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; +SELECT "CounterID", AVG(octet_length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(octet_length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +SELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +SELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/slatedb/query b/slatedb/query new file mode 100755 index 0000000000..1b961408d9 --- /dev/null +++ b/slatedb/query @@ -0,0 +1,10 @@ +#!/bin/bash +# Reads a SQL query from stdin and runs it with the hits-slatedb binary +# (DataFusion SQL over parallel SlateDB range scans). +# Stdout: query result. +# Stderr: query runtime in fractional seconds on the last line (printed by +# the binary itself; it measures SQL execution, not process startup). +# Exit non-zero on error. +set -e + +hits-slatedb/target/release/hits-slatedb query db create.sql diff --git a/slatedb/start b/slatedb/start new file mode 100755 index 0000000000..60f00961ef --- /dev/null +++ b/slatedb/start @@ -0,0 +1,3 @@ +#!/bin/bash +# Embedded engine — no daemon to start. +exit 0 diff --git a/slatedb/stop b/slatedb/stop new file mode 100755 index 0000000000..06bd986563 --- /dev/null +++ b/slatedb/stop @@ -0,0 +1,2 @@ +#!/bin/bash +exit 0 diff --git a/slatedb/template.json b/slatedb/template.json new file mode 100644 index 0000000000..506d90ffa9 --- /dev/null +++ b/slatedb/template.json @@ -0,0 +1,12 @@ +{ + "system": "SlateDB", + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": [ + "Rust", + "key-value", + "row-oriented", + "embedded" + ] +} From 24c6225f4a7d8769ef394bd41d783cd9c35df6b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 19 Aug 2026 22:56:16 +0000 Subject: [PATCH 2/8] slatedb: jemalloc + pin compactor-panic fix - glibc malloc fragmentation grew load RSS unboundedly (9.4 GB at 4 cores); jemalloc keeps it flat. - v0.15.0's embedded compactor intermittently panics on large bulk loads ("compaction source view not found in L0"); pin the upstream fix (slatedb/slatedb#2002) until it lands in a release. - Adapt to the post-0.15.0 write API (WriteHandle instead of WriteOptions::await_durable). Co-Authored-By: Claude Fable 5 --- slatedb/README.md | 10 ++++++++++ slatedb/hits-slatedb/Cargo.lock | 30 ++++++++++++++++++++++++------ slatedb/hits-slatedb/Cargo.toml | 10 +++++++++- slatedb/hits-slatedb/src/main.rs | 21 ++++++++++++--------- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/slatedb/README.md b/slatedb/README.md index 446ea30ec0..a618725308 100644 --- a/slatedb/README.md +++ b/slatedb/README.md @@ -36,6 +36,16 @@ Non-default settings, per the fine-tuning rules: end): the WAL would double the load I/O for no durability benefit since the load ends with a flush + clean close. Queries open the database with a read-only `DbReader`. +- The harness binary uses jemalloc: with glibc malloc, the allocation churn of + the load path fragments the heap and RSS grows several GB beyond live data, + OOMing the smaller machines. + +SlateDB is pinned to v0.15.0 plus 17 commits (rev `d0c3d63`) because v0.15.0's +embedded compactor intermittently panics during large bulk loads +("compaction source view not found in L0") and takes the database down; +the fix ([slatedb#2002]) is merged upstream but not yet in a crates.io release. + +[slatedb#2002]: https://github.com/slatedb/slatedb/pull/2002 The concurrent-QPS test is skipped: each query forks a fresh full-machine process with no shared server (see issue #946). diff --git a/slatedb/hits-slatedb/Cargo.lock b/slatedb/hits-slatedb/Cargo.lock index 1703bcf4b4..9ac20f702d 100644 --- a/slatedb/hits-slatedb/Cargo.lock +++ b/slatedb/hits-slatedb/Cargo.lock @@ -1989,6 +1989,7 @@ dependencies = [ "datafusion", "futures", "slatedb", + "tikv-jemallocator", "tokio", ] @@ -3180,8 +3181,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" +source = "git+https://github.com/slatedb/slatedb.git?rev=d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed#d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed" dependencies = [ "async-channel", "async-trait", @@ -3226,8 +3226,7 @@ dependencies = [ [[package]] name = "slatedb-common" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" +source = "git+https://github.com/slatedb/slatedb.git?rev=d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed#d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed" dependencies = [ "chrono", "log", @@ -3242,8 +3241,7 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" +source = "git+https://github.com/slatedb/slatedb.git?rev=d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed#d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed" dependencies = [ "async-trait", "bytes", @@ -3436,6 +3434,26 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.55" diff --git a/slatedb/hits-slatedb/Cargo.toml b/slatedb/hits-slatedb/Cargo.toml index fc168942be..f33a2f9cff 100644 --- a/slatedb/hits-slatedb/Cargo.toml +++ b/slatedb/hits-slatedb/Cargo.toml @@ -8,7 +8,11 @@ edition = "2021" # top of an object store (the local filesystem here). `zstd` enables SST # block compression, `wal_disable` allows turning the WAL off for the bulk # load, `foyer` provides the default in-memory block cache. -slatedb = { version = "0.15.0", default-features = false, features = ["zstd", "wal_disable", "foyer"] } +# Pinned to v0.15.0 + the fix for a compactor panic that kills long bulk +# loads ("compaction source view not found in L0", fixed upstream in +# slatedb/slatedb#2002); switch back to a crates.io release once one +# containing that fix is published. +slatedb = { git = "https://github.com/slatedb/slatedb.git", rev = "d0c3d635b6d5d69053f52fe15bdd3ee4487e28ed", default-features = false, features = ["zstd", "wal_disable", "foyer"] } # DataFusion provides the SQL frontend; all table data is scanned out of # SlateDB through a custom TableProvider. datafusion = "55" @@ -16,6 +20,10 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros"] } futures = "0.3" async-trait = "0.1" anyhow = "1" +# glibc malloc fragments badly under the allocation churn of the load path +# (row buffers + SST builds across many threads), growing RSS without bound; +# jemalloc keeps it flat. +tikv-jemallocator = "0.6" [profile.release] lto = "thin" diff --git a/slatedb/hits-slatedb/src/main.rs b/slatedb/hits-slatedb/src/main.rs index 135820d3c9..c071d03a21 100644 --- a/slatedb/hits-slatedb/src/main.rs +++ b/slatedb/hits-slatedb/src/main.rs @@ -47,11 +47,14 @@ use datafusion::physical_plan::{ use datafusion::prelude::{ParquetReadOptions, SessionContext}; use slatedb::bytes::Bytes; -use slatedb::config::{CompressionCodec, ScanOptions, Settings, WriteOptions}; +use slatedb::config::{CompressionCodec, ScanOptions, Settings}; use slatedb::object_store::local::LocalFileSystem; use slatedb::object_store::ObjectStore; use slatedb::{Db, DbReader, SstBlockSize, WriteBatch}; +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + // Row keys are 8-byte big-endian indexes, so they sort in row order and any // key starting with 0xff sorts after all of them (the dataset has far fewer // than 2^56 rows). @@ -581,7 +584,7 @@ impl ExecutionPlan for SlateScanExec { let lo = (partition as u64) * chunk; let hi = ((partition as u64 + 1) * chunk).min(self.row_count); let db = self.db.clone(); - let mut remaining = self.limit.unwrap_or(usize::MAX); + let limit = self.limit.unwrap_or(usize::MAX); let decoder = Decoder::new(self.tags.clone(), &self.projection, self.out_schema.clone()); let batches = stream::once(async move { @@ -598,8 +601,8 @@ impl ExecutionPlan for SlateScanExec { None }; Ok::<_, DataFusionError>(stream::try_unfold( - (iter, decoder), - move |(mut iter, mut decoder)| async move { + (iter, decoder, limit), + move |(mut iter, mut decoder, mut remaining)| async move { let Some(it) = iter.as_mut() else { return Ok(None); }; @@ -616,7 +619,7 @@ impl ExecutionPlan for SlateScanExec { Ok(None) } else { let batch = decoder.finish()?; - Ok(Some((batch, (iter, decoder)))) + Ok(Some((batch, (iter, decoder, remaining)))) } }, )) @@ -682,7 +685,6 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { let tags = tags.clone(); let counter = counter.clone(); handles.push(tokio::spawn(async move { - let wopts = WriteOptions { await_durable: false, ..Default::default() }; while let Some(batch) = s.next().await { let batch = batch?; let rows = encode_rows(&batch, &tags)?; @@ -691,7 +693,9 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { for (i, row) in rows.iter().enumerate() { wb.put(row_key(base + i as u64), row); } - db.write_with_options(wb, &wopts).await?; + // Fire-and-forget (the returned WriteHandle is not awaited); + // durability comes from the final flush + close. + db.write(wb).await?; } Ok::<_, anyhow::Error>(()) })); @@ -704,8 +708,7 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { let mut wb = WriteBatch::new(); wb.put(META_COUNT, total.to_le_bytes()); wb.put(META_SCHEMA, schema_txt.as_bytes()); - db.write_with_options(wb, &WriteOptions { await_durable: false, ..Default::default() }) - .await?; + db.write(wb).await?; db.flush().await?; db.close().await?; println!("Loaded {total} rows"); From 191a3562ca151e393fe3a309bba3fdae8b1f0c36 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 19 Aug 2026 23:59:06 +0000 Subject: [PATCH 3/8] slatedb: install make for tikv-jemalloc-sys Co-Authored-By: Claude Fable 5 --- slatedb/install | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/slatedb/install b/slatedb/install index 30f997aff1..9d7abcae51 100755 --- a/slatedb/install +++ b/slatedb/install @@ -26,7 +26,8 @@ if [ "$(free -g | awk '/^Mem:/{print $2}')" -lt 12 ]; then fi fi +# gcc + make: tikv-jemalloc-sys compiles jemalloc's C sources at build time. sudo apt-get update -y -sudo apt-get install -y gcc +sudo apt-get install -y gcc make cargo build --release --locked --manifest-path hits-slatedb/Cargo.toml From fad4c37d10d7d26997359f3b587155c1c35ad848 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:25:58 +0000 Subject: [PATCH 4/8] Add benchmark results for slatedb (c6a.4xlarge) --- slatedb/results/20260820/c6a.4xlarge.json | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 slatedb/results/20260820/c6a.4xlarge.json diff --git a/slatedb/results/20260820/c6a.4xlarge.json b/slatedb/results/20260820/c6a.4xlarge.json new file mode 100644 index 0000000000..0c4eee6ba7 --- /dev/null +++ b/slatedb/results/20260820/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "SlateDB", + "date": "2026-08-20", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["Rust","key-value","row-oriented","embedded"], + "load_time": 874, + "data_size": 71918094006, + "concurrent_qps": null, + "concurrent_error_ratio": null, + "result": [ + [0.074, 0.001, 0.001], + [57.985, 20.458, 20.724], + [58.102, 20.483, 21.144], + [58.018, 20.609, 20.464], + [58.247, 21.354, 21.242], + [58.228, 21.39, 21.588], + [58.028, 20.363, 20.753], + [58.105, 21.003, 21.27], + [58.197, 21.265, 21.698], + [58.378, 21.685, 21.725], + [58.098, 20.941, 21.172], + [58.046, 20.813, 21.008], + [58.214, 21.348, 20.963], + [58.441, 21.534, 21.815], + [58.157, 21.733, 21.279], + [58.245, 21.107, 21.132], + [58.437, 22.291, 22.265], + [58.405, 22.044, 22.451], + [58.857, 23.463, 23.247], + [58.077, 20.539, 20.739], + [58.061, 21.332, 20.932], + [58.143, 20.973, 21.115], + [58.184, 21.454, 21.868], + [58.169, 27.354, 27.788], + [58.068, 21.251, 20.934], + [58.105, 20.591, 21.361], + [58.049, 20.661, 20.751], + [58.105, 21.049, 21.309], + [58.411, 24.012, 23.486], + [58.117, 20.767, 20.503], + [58.233, 21.437, 21.701], + [58.119, 21.517, 21.702], + [58.513, 23.874, 24.192], + [58.935, 23.399, 23.647], + [58.973, 23.669, 23.898], + [58.198, 20.993, 21.513], + [58.112, 20.995, 21.014], + [58.128, 20.861, 21.039], + [58.152, 21.13, 21.316], + [58.138, 21.265, 21.436], + [58.134, 21.124, 21.159], + [58.136, 21.033, 21.574], + [58.087, 21.113, 20.712] +] + } + \ No newline at end of file From 22762b624d796068126a818af8adedf9eaf494ce Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 20 Aug 2026 03:44:45 +0000 Subject: [PATCH 5/8] slatedb: run one-shot GC at the end of load The background GC never deletes objects younger than 5 minutes, so a fast load reported ~72 GB in data-size while only ~24 GB were live SSTs. Run an immediate GC pass after close (6 s on the full dataset). Co-Authored-By: Claude Fable 5 --- slatedb/hits-slatedb/src/main.rs | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/slatedb/hits-slatedb/src/main.rs b/slatedb/hits-slatedb/src/main.rs index c071d03a21..fa166f7932 100644 --- a/slatedb/hits-slatedb/src/main.rs +++ b/slatedb/hits-slatedb/src/main.rs @@ -46,8 +46,12 @@ use datafusion::physical_plan::{ }; use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use slatedb::admin::Admin; use slatedb::bytes::Bytes; -use slatedb::config::{CompressionCodec, ScanOptions, Settings}; +use slatedb::config::{ + CompressionCodec, GarbageCollectorDirectoryOptions, GarbageCollectorOptions, ScanOptions, + Settings, +}; use slatedb::object_store::local::LocalFileSystem; use slatedb::object_store::ObjectStore; use slatedb::{Db, DbReader, SstBlockSize, WriteBatch}; @@ -711,10 +715,37 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { db.write(wb).await?; db.flush().await?; db.close().await?; + + // The background GC never deletes objects younger than 5 minutes, so a + // fast load would report several GB of compaction garbage in data-size. + // Run one immediate GC pass now that the database is closed. + gc(db_dir).await?; + println!("Loaded {total} rows"); Ok(()) } +// Delete unreferenced SSTs/manifests left behind by compaction. Safe to run +// with zero min-age because no other process has the database open; the WAL +// fence directory keeps its default min-age since deleting fresh fence files +// could un-fence a concurrent writer. +async fn gc(db_dir: &str) -> Result<()> { + let admin = Admin::builder(DB_PATH, object_store(db_dir)?).build(); + let eager = GarbageCollectorDirectoryOptions { + min_age: std::time::Duration::ZERO, + ..Default::default() + }; + let gc_opts = GarbageCollectorOptions { + manifest_options: Some(eager), + wal_options: Some(eager), + compacted_options: Some(eager), + compactions_options: Some(eager), + ..Default::default() + }; + admin.run_gc_once(gc_opts).await?; + Ok(()) +} + // --------------------------------------------------------------------------- // Query: run one SQL statement from stdin via DataFusion. // --------------------------------------------------------------------------- @@ -817,6 +848,7 @@ async fn main() -> Result<()> { Some("queryp") if args.len() >= 3 => { query_parquet(&args[2], args.get(3).map(String::as_str)).await } + Some("gc") if args.len() == 3 => gc(&args[2]).await, _ => { eprintln!("usage: hits-slatedb load "); eprintln!(" hits-slatedb query [create.sql] (SQL on stdin)"); From 355373378a3c3972f9f5162c4c81c13082b87f2f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:18:15 +0000 Subject: [PATCH 6/8] Add benchmark results for slatedb (c6a.4xlarge) --- slatedb/results/20260820/c6a.4xlarge.json | 90 +++++++++++------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/slatedb/results/20260820/c6a.4xlarge.json b/slatedb/results/20260820/c6a.4xlarge.json index 0c4eee6ba7..f13524c073 100644 --- a/slatedb/results/20260820/c6a.4xlarge.json +++ b/slatedb/results/20260820/c6a.4xlarge.json @@ -7,54 +7,54 @@ "hardware": "cpu", "tuned": "no", "tags": ["Rust","key-value","row-oriented","embedded"], - "load_time": 874, - "data_size": 71918094006, + "load_time": 906, + "data_size": 71088729577, "concurrent_qps": null, "concurrent_error_ratio": null, "result": [ - [0.074, 0.001, 0.001], - [57.985, 20.458, 20.724], - [58.102, 20.483, 21.144], - [58.018, 20.609, 20.464], - [58.247, 21.354, 21.242], - [58.228, 21.39, 21.588], - [58.028, 20.363, 20.753], - [58.105, 21.003, 21.27], - [58.197, 21.265, 21.698], - [58.378, 21.685, 21.725], - [58.098, 20.941, 21.172], - [58.046, 20.813, 21.008], - [58.214, 21.348, 20.963], - [58.441, 21.534, 21.815], - [58.157, 21.733, 21.279], - [58.245, 21.107, 21.132], - [58.437, 22.291, 22.265], - [58.405, 22.044, 22.451], - [58.857, 23.463, 23.247], - [58.077, 20.539, 20.739], - [58.061, 21.332, 20.932], - [58.143, 20.973, 21.115], - [58.184, 21.454, 21.868], - [58.169, 27.354, 27.788], - [58.068, 21.251, 20.934], - [58.105, 20.591, 21.361], - [58.049, 20.661, 20.751], - [58.105, 21.049, 21.309], - [58.411, 24.012, 23.486], - [58.117, 20.767, 20.503], - [58.233, 21.437, 21.701], - [58.119, 21.517, 21.702], - [58.513, 23.874, 24.192], - [58.935, 23.399, 23.647], - [58.973, 23.669, 23.898], - [58.198, 20.993, 21.513], - [58.112, 20.995, 21.014], - [58.128, 20.861, 21.039], - [58.152, 21.13, 21.316], - [58.138, 21.265, 21.436], - [58.134, 21.124, 21.159], - [58.136, 21.033, 21.574], - [58.087, 21.113, 20.712] + [0.098, 0.001, 0.001], + [58.123, 20.918, 21.024], + [58.111, 21.337, 20.742], + [57.991, 20.901, 21.956], + [58.257, 21.491, 21.878], + [58.235, 21.589, 21.728], + [58.065, 20.768, 20.972], + [58.028, 20.917, 21.435], + [58.163, 21.618, 22.02], + [58.343, 22.232, 22.487], + [58.114, 21.681, 21.363], + [58.068, 21.763, 21.498], + [58.126, 21.252, 21.462], + [58.486, 22.406, 21.954], + [58.154, 21.73, 21.583], + [58.258, 21.642, 21.644], + [58.493, 22.84, 22.516], + [58.322, 22.572, 22.34], + [58.759, 23.429, 23.873], + [58.031, 20.984, 21.821], + [58.109, 21.739, 22.071], + [58.015, 21.466, 21.286], + [58.209, 22.59, 21.745], + [58.309, 27.669, 28.253], + [58.111, 20.949, 21.217], + [58.086, 21.276, 21.168], + [58.034, 21.691, 21.512], + [58.23, 21.56, 21.49], + [58.404, 24.128, 24.205], + [58.134, 21.419, 21.344], + [58.159, 21.777, 21.702], + [58.093, 21.71, 21.824], + [58.41, 24.546, 24.217], + [58.893, 24.491, 24.335], + [58.879, 24.796, 24.143], + [58.328, 21.399, 21.491], + [58.124, 21.215, 21.658], + [58.087, 21.748, 21.105], + [58.007, 21.699, 21.777], + [58.21, 22.001, 21.769], + [58.08, 21.532, 21.407], + [58.167, 22.073, 22.46], + [58.113, 22.055, 22.477] ] } \ No newline at end of file From 2ca262a6ffd7dc7fd981a3a4020e653c682ea0a4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 20 Aug 2026 07:12:29 +0000 Subject: [PATCH 7/8] slatedb: quiesce compaction and drop leftover checkpoints before GC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first end-of-load GC pass freed nothing: closing mid-compaction pins the collector's low watermark at the oldest pending compaction, and the 74 checkpoints left behind by compaction workers pin old manifests and every SST they reference. Wait for the compactor to drain, delete the checkpoints, then collect — data-size now matches live SSTs (15.8 GB instead of 71 GB), and queries scan 3 merged runs instead of dozens. Co-Authored-By: Claude Fable 5 --- slatedb/README.md | 6 +++ slatedb/hits-slatedb/Cargo.toml | 2 +- slatedb/hits-slatedb/src/main.rs | 68 ++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) diff --git a/slatedb/README.md b/slatedb/README.md index a618725308..25028c9790 100644 --- a/slatedb/README.md +++ b/slatedb/README.md @@ -39,6 +39,12 @@ Non-default settings, per the fine-tuning rules: - The harness binary uses jemalloc: with glibc malloc, the allocation churn of the load path fragments the heap and RSS grows several GB beyond live data, OOMing the smaller machines. +- The load waits for the embedded compactor to drain its queue, deletes the + checkpoints left behind by compaction workers, and runs one GC pass before + exiting. Without this the database reports ~4x its live size: closing + mid-compaction pins the GC's low watermark, leftover checkpoints pin old + manifests and every SST they reference, and the GC never deletes objects + younger than 5 minutes. SlateDB is pinned to v0.15.0 plus 17 commits (rev `d0c3d63`) because v0.15.0's embedded compactor intermittently panics during large bulk loads diff --git a/slatedb/hits-slatedb/Cargo.toml b/slatedb/hits-slatedb/Cargo.toml index f33a2f9cff..beaaa60c77 100644 --- a/slatedb/hits-slatedb/Cargo.toml +++ b/slatedb/hits-slatedb/Cargo.toml @@ -16,7 +16,7 @@ slatedb = { git = "https://github.com/slatedb/slatedb.git", rev = "d0c3d635b6d5d # DataFusion provides the SQL frontend; all table data is scanned out of # SlateDB through a custom TableProvider. datafusion = "55" -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } futures = "0.3" async-trait = "0.1" anyhow = "1" diff --git a/slatedb/hits-slatedb/src/main.rs b/slatedb/hits-slatedb/src/main.rs index fa166f7932..0a01f6a1ff 100644 --- a/slatedb/hits-slatedb/src/main.rs +++ b/slatedb/hits-slatedb/src/main.rs @@ -714,6 +714,14 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { wb.put(META_SCHEMA, schema_txt.as_bytes()); db.write(wb).await?; db.flush().await?; + + // Keep the database open until the embedded compactor drains its queue. + // Closing with compactions still pending leaves the LSM un-merged (every + // scan pays for the extra sorted runs) and pins the garbage collector's + // low watermark at the oldest pending compaction, so the input SSTs of + // already-committed compactions could never be collected. + wait_for_compaction_quiesce(db_dir).await?; + db.close().await?; // The background GC never deletes objects younger than 5 minutes, so a @@ -725,12 +733,51 @@ async fn load(parquet_path: &str, db_dir: &str) -> Result<()> { Ok(()) } +// Poll the compactions state until nothing is submitted, scheduled, running, +// or awaiting commit. Capped at 2 hours; on timeout the load proceeds with +// whatever state the compactor reached. +async fn wait_for_compaction_quiesce(db_dir: &str) -> Result<()> { + let admin = Admin::builder(DB_PATH, object_store(db_dir)?).build(); + let mut quiet = 0; + for _ in 0..1440 { + // Compaction and CompactionStatus are not exported from the crate, so + // classify via the status Debug representation. + let active = match admin.read_compactions(None).await? { + Some(vc) => vc.recent_compactions().any(|c| { + matches!( + format!("{:?}", c.status()).as_str(), + "Submitted" | "Scheduled" | "Running" | "Compacted" + ) + }), + None => false, + }; + quiet = if active { 0 } else { quiet + 1 }; + if quiet >= 2 { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + eprintln!("warning: compaction did not quiesce within 2 hours"); + Ok(()) +} + // Delete unreferenced SSTs/manifests left behind by compaction. Safe to run // with zero min-age because no other process has the database open; the WAL // fence directory keeps its default min-age since deleting fresh fence files // could un-fence a concurrent writer. async fn gc(db_dir: &str) -> Result<()> { let admin = Admin::builder(DB_PATH, object_store(db_dir)?).build(); + + // Compaction workers leave checkpoints behind (74 after a full load), + // and checkpoints pin old manifests and every SST those reference, so + // the GC would keep ~2x the live bytes. The database is closed and this + // is the only process, so none of them are needed. + if let Some(m) = admin.read_manifest(None).await? { + for cp in m.checkpoints() { + admin.delete_checkpoint(cp.id).await?; + } + } + let eager = GarbageCollectorDirectoryOptions { min_age: std::time::Duration::ZERO, ..Default::default() @@ -849,6 +896,27 @@ async fn main() -> Result<()> { query_parquet(&args[2], args.get(3).map(String::as_str)).await } Some("gc") if args.len() == 3 => gc(&args[2]).await, + Some("state") if args.len() == 3 => { + let admin = Admin::builder(DB_PATH, object_store(&args[2])?).build(); + match admin.read_compactions(None).await? { + Some(vc) => { + for c in vc.recent_compactions() { + println!("{:?} start={}", c.status(), c.id().datetime().duration_since(std::time::UNIX_EPOCH)?.as_secs()); + } + } + None => println!("no compactions file"), + } + if let Some(m) = admin.read_manifest(None).await? { + let l0_bytes: u64 = m.l0().iter().map(|v| v.estimate_size()).sum(); + let sr_bytes: u64 = m.compacted().iter().map(|sr| sr.estimate_size()).sum(); + println!("manifest id={} l0={} ({} bytes)", m.id(), m.l0().len(), l0_bytes); + for sr in m.compacted() { + println!(" run {}: {} ssts, {} bytes", sr.id, sr.sst_views().len(), sr.estimate_size()); + } + println!("live total: {} bytes, checkpoints: {}", l0_bytes + sr_bytes, m.checkpoints().len()); + } + Ok(()) + } _ => { eprintln!("usage: hits-slatedb load "); eprintln!(" hits-slatedb query [create.sql] (SQL on stdin)"); From 060b56aed94350d2ee89b6e777031e8334f90b81 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:18:43 +0000 Subject: [PATCH 8/8] Add benchmark results for slatedb (c6a.4xlarge) --- slatedb/results/20260820/c6a.4xlarge.json | 90 +++++++++++------------ 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/slatedb/results/20260820/c6a.4xlarge.json b/slatedb/results/20260820/c6a.4xlarge.json index f13524c073..c6650bc135 100644 --- a/slatedb/results/20260820/c6a.4xlarge.json +++ b/slatedb/results/20260820/c6a.4xlarge.json @@ -7,54 +7,54 @@ "hardware": "cpu", "tuned": "no", "tags": ["Rust","key-value","row-oriented","embedded"], - "load_time": 906, - "data_size": 71088729577, + "load_time": 916, + "data_size": 15493367866, "concurrent_qps": null, "concurrent_error_ratio": null, "result": [ - [0.098, 0.001, 0.001], - [58.123, 20.918, 21.024], - [58.111, 21.337, 20.742], - [57.991, 20.901, 21.956], - [58.257, 21.491, 21.878], - [58.235, 21.589, 21.728], - [58.065, 20.768, 20.972], - [58.028, 20.917, 21.435], - [58.163, 21.618, 22.02], - [58.343, 22.232, 22.487], - [58.114, 21.681, 21.363], - [58.068, 21.763, 21.498], - [58.126, 21.252, 21.462], - [58.486, 22.406, 21.954], - [58.154, 21.73, 21.583], - [58.258, 21.642, 21.644], - [58.493, 22.84, 22.516], - [58.322, 22.572, 22.34], - [58.759, 23.429, 23.873], - [58.031, 20.984, 21.821], - [58.109, 21.739, 22.071], - [58.015, 21.466, 21.286], - [58.209, 22.59, 21.745], - [58.309, 27.669, 28.253], - [58.111, 20.949, 21.217], - [58.086, 21.276, 21.168], - [58.034, 21.691, 21.512], - [58.23, 21.56, 21.49], - [58.404, 24.128, 24.205], - [58.134, 21.419, 21.344], - [58.159, 21.777, 21.702], - [58.093, 21.71, 21.824], - [58.41, 24.546, 24.217], - [58.893, 24.491, 24.335], - [58.879, 24.796, 24.143], - [58.328, 21.399, 21.491], - [58.124, 21.215, 21.658], - [58.087, 21.748, 21.105], - [58.007, 21.699, 21.777], - [58.21, 22.001, 21.769], - [58.08, 21.532, 21.407], - [58.167, 22.073, 22.46], - [58.113, 22.055, 22.477] + [0.101, 0.001, 0.001], + [58.023, 20.615, 20.782], + [58.009, 20.455, 20.487], + [58.06, 20.616, 20.716], + [58.067, 21.205, 21.348], + [58.081, 21.581, 21.332], + [58.017, 20.776, 20.738], + [57.998, 20.994, 20.912], + [58.275, 21.606, 21.419], + [58.588, 21.681, 21.515], + [58.175, 21.263, 20.826], + [58.107, 20.748, 21.295], + [58.09, 21.605, 21.251], + [58.46, 21.531, 21.868], + [58.214, 21.499, 21.033], + [58.283, 21.353, 21.27], + [58.683, 22.253, 21.9], + [58.521, 21.836, 22.159], + [58.703, 23.322, 23.119], + [58.111, 20.655, 20.53], + [58.064, 20.9, 20.828], + [58.225, 21.267, 21.159], + [58.205, 21.521, 21.084], + [58.218, 27.359, 27.326], + [58.173, 20.94, 20.891], + [58.048, 20.572, 20.856], + [58.037, 20.666, 20.945], + [58.187, 21.056, 21.765], + [58.545, 23.946, 23.967], + [58.064, 20.423, 20.424], + [58.147, 21.746, 21.053], + [58.208, 21.266, 21.432], + [58.214, 24.462, 24.178], + [59.216, 23.61, 23.846], + [58.839, 23.978, 24.257], + [58.368, 21.176, 21.341], + [58.022, 21.742, 21.322], + [58.274, 20.758, 21.206], + [58.272, 21.101, 21.651], + [58.241, 21.218, 21.581], + [58.017, 21.002, 20.933], + [58.052, 20.878, 20.749], + [58.101, 20.914, 20.568] ] } \ No newline at end of file