From 98736325ccd29c5ea58a9ab9c79eb9aaef79b765 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 10 Sep 2026 16:05:13 +0000 Subject: [PATCH 1/4] bench: enable Lance compression across the compression benchmarks `LanceCompressor` wrote storage version 2.0, the pre-structural-encoding layout that applies almost no compression. The other two Lance writers in `lance-bench` (`convert.rs`, `random_access.rs`) already use 2.1, which is also Lance's own default and the first version with the compressive encodings (bitpacking, FSST, general compression). The compression suite was therefore charting a near-uncompressed Lance file against compressed Vortex and Parquet. Lance was also missing from pull-request compression runs: `pr-bench-compress.yml` left `with_lance` at its default of false, so the binary was built without the feature and `compress-split.py` defaulted to `arrow-ipc,parquet,vortex`. Only the `develop` run covered Lance. - Write Lance at storage version 2.1 in `LanceCompressor`. - Put `lance` in `compress-split.py`'s default `--formats`, matching `random-access-split.py`, and drop the now-redundant explicit list from `develop-bench.yml`. - Set `with_lance: true` for the PR compression benchmark. - Document the feature, the format list and the storage version in the compress-bench README. Lance rows in the PR `Compression` comment will read as a large change against the S3 baseline, which was recorded at 2.0. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TkDpMmCYNaj43ENFUyDULe --- .github/workflows/develop-bench.yml | 2 +- .github/workflows/pr-bench-compress.yml | 1 + .github/workflows/pr-bench-runner.yml | 2 ++ benchmarks/compress-bench/README.md | 16 +++++++++++++++- benchmarks/lance-bench/src/compress.rs | 6 +++++- scripts/compress-split.py | 5 +++-- 6 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 80106b43af8..5a7b5cb149e 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -156,7 +156,7 @@ jobs: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | - python3 scripts/compress-split.py --formats arrow-ipc,parquet,lance,vortex --emit-ingest-records + python3 scripts/compress-split.py --emit-ingest-records - name: Run ${{ matrix.benchmark.name }} benchmark (per-column-encoder) if: matrix.benchmark.id == 'string-bench' diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml index 6d7abd7e86f..2b358b98ed2 100644 --- a/.github/workflows/pr-bench-compress.yml +++ b/.github/workflows/pr-bench-compress.yml @@ -18,3 +18,4 @@ jobs: with: benchmark_id: compress-bench benchmark_name: Compression + with_lance: true diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index bc68ee2ef73..15b0b8ce066 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -135,6 +135,8 @@ jobs: run: | python3 scripts/random-access-split.py + # `compress-split.py` includes Lance in its default `--formats`, so compress-bench must be + # called with `with_lance: true` or the run fails on an unimplemented Lance compressor. - name: Run ${{ inputs.benchmark_name }} benchmark (per-dataset) if: inputs.benchmark_id == 'compress-bench' shell: bash diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index c84f46f5a1b..8190ec85eca 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -1,7 +1,7 @@ # Compression benchmark Measures compression and decompression throughput, plus resulting file sizes, for Vortex, -Parquet, uncompressed Arrow IPC, and optionally Lance. +Parquet, uncompressed Arrow IPC, and Lance. [Arrow IPC](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) is Apache Arrow's built-in file format, formerly called Feather V2. This suite writes it without @@ -23,6 +23,20 @@ See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--format cargo run -p compress-bench --profile release_debug ``` +Lance is behind the `lance` feature, and `--formats` accepts it only in a binary built with +it. CI always builds with the feature, so both the PR `Compression` comment and the `develop` +run cover Lance: + +```bash +cargo run -p compress-bench --profile release_debug --features lance \ + -- --formats arrow-ipc,parquet,lance,vortex +``` + +Lance files are written at storage version 2.1, the first version with structural encoding and +therefore the first that applies Lance's compressive encodings. Version 2.0 writes an +essentially uncompressed file, which is not a meaningful comparison against compressed Vortex +and Parquet. + ## GPU decompression `--gpu-decompress` is opt-in, requires the `cuda` feature, and restricts the suite to the diff --git a/benchmarks/lance-bench/src/compress.rs b/benchmarks/lance-bench/src/compress.rs index d21e0462feb..ba72db1f874 100644 --- a/benchmarks/lance-bench/src/compress.rs +++ b/benchmarks/lance-bench/src/compress.rs @@ -148,7 +148,11 @@ impl Compressor for LanceCompressor { .to_str() .ok_or_else(|| anyhow!("Failed to convert path to str"))?; let reader_iter = RecordBatchIterator::new(batches.into_iter().map(Ok), Arc::clone(schema)); - let write_params = WriteParams::with_storage_version(LanceFileVersion::V2_0); + // 2.1 is Lance's default and the first version with structural encoding, which is what + // turns on its compressive encodings (bitpacking, FSST, general compression). Writing 2.0 + // here measured a near-uncompressed Lance file against compressed Vortex and Parquet. + // The other Lance writers in this crate already use 2.1. + let write_params = WriteParams::with_storage_version(LanceFileVersion::V2_1); Dataset::write(reader_iter, path_str, Some(write_params)).await?; let elapsed = start.elapsed(); diff --git a/scripts/compress-split.py b/scripts/compress-split.py index e216ff52bcf..e4d972d82f2 100755 --- a/scripts/compress-split.py +++ b/scripts/compress-split.py @@ -83,8 +83,9 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--formats", - default="arrow-ipc,parquet,vortex", - help="comma-separated formats to forward to compress-bench", + default="arrow-ipc,parquet,lance,vortex", + help="comma-separated formats to forward to compress-bench " + "(lance requires a binary built with --features lance)", ) parser.add_argument( "--emit-ingest-records", From 81d974e474d3cedeb5c0e516aa9dd59e107f6822 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Thu, 10 Sep 2026 16:26:30 +0000 Subject: [PATCH 2/4] bench: keep Lance out of PR compression runs Revert the pull-request side of the previous commit. Lance stays on the post-merge `develop` benchmark only, which already passes `--formats arrow-ipc,parquet,lance,vortex`; the PR `Compression` matrix is left as it was so it does not grow. `pr-bench-compress.yml`, `pr-bench-runner.yml` and `develop-bench.yml` are back to their original contents, and `compress-split.py` keeps its original default format list. What remains is the storage-version fix plus docs. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TkDpMmCYNaj43ENFUyDULe --- .github/workflows/develop-bench.yml | 2 +- .github/workflows/pr-bench-compress.yml | 1 - .github/workflows/pr-bench-runner.yml | 2 -- benchmarks/compress-bench/README.md | 10 +++++----- scripts/compress-split.py | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/develop-bench.yml b/.github/workflows/develop-bench.yml index 5a7b5cb149e..80106b43af8 100644 --- a/.github/workflows/develop-bench.yml +++ b/.github/workflows/develop-bench.yml @@ -156,7 +156,7 @@ jobs: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | - python3 scripts/compress-split.py --emit-ingest-records + python3 scripts/compress-split.py --formats arrow-ipc,parquet,lance,vortex --emit-ingest-records - name: Run ${{ matrix.benchmark.name }} benchmark (per-column-encoder) if: matrix.benchmark.id == 'string-bench' diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml index 2b358b98ed2..6d7abd7e86f 100644 --- a/.github/workflows/pr-bench-compress.yml +++ b/.github/workflows/pr-bench-compress.yml @@ -18,4 +18,3 @@ jobs: with: benchmark_id: compress-bench benchmark_name: Compression - with_lance: true diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml index 15b0b8ce066..bc68ee2ef73 100644 --- a/.github/workflows/pr-bench-runner.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -135,8 +135,6 @@ jobs: run: | python3 scripts/random-access-split.py - # `compress-split.py` includes Lance in its default `--formats`, so compress-bench must be - # called with `with_lance: true` or the run fails on an unimplemented Lance compressor. - name: Run ${{ inputs.benchmark_name }} benchmark (per-dataset) if: inputs.benchmark_id == 'compress-bench' shell: bash diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 8190ec85eca..90a12e55f0b 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -1,7 +1,7 @@ # Compression benchmark Measures compression and decompression throughput, plus resulting file sizes, for Vortex, -Parquet, uncompressed Arrow IPC, and Lance. +Parquet, uncompressed Arrow IPC, and optionally Lance. [Arrow IPC](https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format) is Apache Arrow's built-in file format, formerly called Feather V2. This suite writes it without @@ -24,8 +24,8 @@ cargo run -p compress-bench --profile release_debug ``` Lance is behind the `lance` feature, and `--formats` accepts it only in a binary built with -it. CI always builds with the feature, so both the PR `Compression` comment and the `develop` -run cover Lance: +it. Only the post-merge `develop` benchmark covers Lance; the pull-request `Compression` +comment does not, to keep the PR matrix cheap: ```bash cargo run -p compress-bench --profile release_debug --features lance \ @@ -33,8 +33,8 @@ cargo run -p compress-bench --profile release_debug --features lance \ ``` Lance files are written at storage version 2.1, the first version with structural encoding and -therefore the first that applies Lance's compressive encodings. Version 2.0 writes an -essentially uncompressed file, which is not a meaningful comparison against compressed Vortex +therefore the first that applies Lance's compressive encodings. Version 2.0 wrote an +essentially uncompressed file, which was not a meaningful comparison against compressed Vortex and Parquet. ## GPU decompression diff --git a/scripts/compress-split.py b/scripts/compress-split.py index e4d972d82f2..0bba221f83b 100755 --- a/scripts/compress-split.py +++ b/scripts/compress-split.py @@ -83,7 +83,7 @@ def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--formats", - default="arrow-ipc,parquet,lance,vortex", + default="arrow-ipc,parquet,vortex", help="comma-separated formats to forward to compress-bench " "(lance requires a binary built with --features lance)", ) From c40a4b0d4b2c821e9d016bf5cf875030ee72347b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 11 Sep 2026 09:10:13 +0000 Subject: [PATCH 3/4] bench: write Lance at storage version 2.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2.2 is the newest stable Lance storage version. `is_unstable` is `self >= Next`, and the variant order is `Legacy < V2_0 < V2_1 < Stable < V2_2 < Next < V2_3`, so 2.2 sits below the unstable boundary. lance-file maps it to `ConcreteFileVersion::V2_2` for writing and resolves `(2, 2)` back to it when reading, and `is_structural = version >= V2_1` still holds, so the compressive encodings stay on. Named explicitly rather than through `LanceFileVersion::Stable`, which resolves to `default()` — that is 2.1, the default version for new datasets, not the newest stable version. `convert.rs` and `random_access.rs` are left at 2.1 so the SQL and random-access suites keep writing the data they already write. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TkDpMmCYNaj43ENFUyDULe --- benchmarks/compress-bench/README.md | 9 +++++---- benchmarks/lance-bench/src/compress.rs | 13 ++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 0c8d673bf1a..5ab8cf4a32d 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -32,10 +32,11 @@ cargo run -p compress-bench --profile release_debug --features lance \ -- --formats arrow-ipc,parquet,lance,vortex ``` -Lance files are written at storage version 2.1, the first version with structural encoding and -therefore the first that applies Lance's compressive encodings. Version 2.0 wrote an -essentially uncompressed file, which was not a meaningful comparison against compressed Vortex -and Parquet. +Lance files are written at storage version 2.2, the newest stable version. Structural encoding +arrives in 2.1, and that is what applies Lance's compressive encodings, so version 2.0 wrote an +essentially uncompressed file — not a meaningful comparison against compressed Vortex and +Parquet. Note that `LanceFileVersion::Stable` resolves to 2.1, the default for new datasets, +rather than to the newest stable version, so 2.2 has to be named explicitly. ## GPU decompression diff --git a/benchmarks/lance-bench/src/compress.rs b/benchmarks/lance-bench/src/compress.rs index ba72db1f874..972d81e6ccc 100644 --- a/benchmarks/lance-bench/src/compress.rs +++ b/benchmarks/lance-bench/src/compress.rs @@ -148,11 +148,14 @@ impl Compressor for LanceCompressor { .to_str() .ok_or_else(|| anyhow!("Failed to convert path to str"))?; let reader_iter = RecordBatchIterator::new(batches.into_iter().map(Ok), Arc::clone(schema)); - // 2.1 is Lance's default and the first version with structural encoding, which is what - // turns on its compressive encodings (bitpacking, FSST, general compression). Writing 2.0 - // here measured a near-uncompressed Lance file against compressed Vortex and Parquet. - // The other Lance writers in this crate already use 2.1. - let write_params = WriteParams::with_storage_version(LanceFileVersion::V2_1); + // 2.2 is the newest stable storage version: `is_unstable` is `self >= Next`, and 2.2 + // orders below `Next`. Anything at or above 2.1 has structural encoding, which is what + // turns on Lance's compressive encodings (bitpacking, FSST, general compression); writing + // 2.0 here measured a near-uncompressed Lance file against compressed Vortex and Parquet. + // + // Named explicitly rather than via `LanceFileVersion::Stable`, which resolves to the + // default version for new datasets (2.1) rather than to the newest stable one. + let write_params = WriteParams::with_storage_version(LanceFileVersion::V2_2); Dataset::write(reader_iter, path_str, Some(write_params)).await?; let elapsed = start.elapsed(); From b7e0285c8621e7fa2093522c21465eebcc479979 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 09:38:34 +0000 Subject: [PATCH 4/4] bench: trim the Lance storage-version comments The code comment carried the `is_unstable` variant-order proof and the 2.0 history, which belong in the commit message and the pull request, not next to the line. What a reader of `compress.rs` needs is the two non-obvious facts: 2.1 and above is where structural encoding applies the compressive encodings, and `LanceFileVersion::Stable` is not a way to ask for the newest stable version. The README loses the same 2.0 history, which describes a bug no longer present in the code. Signed-off-by: Joe Isaacs Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TkDpMmCYNaj43ENFUyDULe --- benchmarks/compress-bench/README.md | 9 ++++----- benchmarks/lance-bench/src/compress.rs | 10 +++------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md index 5ab8cf4a32d..48375b2397a 100644 --- a/benchmarks/compress-bench/README.md +++ b/benchmarks/compress-bench/README.md @@ -32,11 +32,10 @@ cargo run -p compress-bench --profile release_debug --features lance \ -- --formats arrow-ipc,parquet,lance,vortex ``` -Lance files are written at storage version 2.2, the newest stable version. Structural encoding -arrives in 2.1, and that is what applies Lance's compressive encodings, so version 2.0 wrote an -essentially uncompressed file — not a meaningful comparison against compressed Vortex and -Parquet. Note that `LanceFileVersion::Stable` resolves to 2.1, the default for new datasets, -rather than to the newest stable version, so 2.2 has to be named explicitly. +Lance files are written at storage version 2.2, the newest stable version. Storage versions at or +above 2.1 use structural encoding, which is what applies Lance's compressive encodings — bitpacking, +FSST, general compression. Note that `LanceFileVersion::Stable` resolves to 2.1, the default for new +datasets, rather than to the newest stable version, so 2.2 has to be named explicitly. ## GPU decompression diff --git a/benchmarks/lance-bench/src/compress.rs b/benchmarks/lance-bench/src/compress.rs index 972d81e6ccc..d47d7162b14 100644 --- a/benchmarks/lance-bench/src/compress.rs +++ b/benchmarks/lance-bench/src/compress.rs @@ -148,13 +148,9 @@ impl Compressor for LanceCompressor { .to_str() .ok_or_else(|| anyhow!("Failed to convert path to str"))?; let reader_iter = RecordBatchIterator::new(batches.into_iter().map(Ok), Arc::clone(schema)); - // 2.2 is the newest stable storage version: `is_unstable` is `self >= Next`, and 2.2 - // orders below `Next`. Anything at or above 2.1 has structural encoding, which is what - // turns on Lance's compressive encodings (bitpacking, FSST, general compression); writing - // 2.0 here measured a near-uncompressed Lance file against compressed Vortex and Parquet. - // - // Named explicitly rather than via `LanceFileVersion::Stable`, which resolves to the - // default version for new datasets (2.1) rather than to the newest stable one. + // Storage versions at or above 2.1 use structural encoding, which is what applies + // Lance's compressive encodings. Not `Stable`: that resolves to the default version for + // new datasets, 2.1, rather than to the newest stable one, which is 2.2. let write_params = WriteParams::with_storage_version(LanceFileVersion::V2_2); Dataset::write(reader_iter, path_str, Some(write_params)).await?;