feat(uring): add benchmark and diagnostic foundations

.github/workflows/ci.yml

@@ -41,24 +41,35 @@ jobs:
         run: cargo fmt --check
       - name: clippy
         run: cargo clippy --all-targets --all-features -- -D warnings
+      - name: ABBA evidence gate tests
+        run: python3 -m unittest discover -s scripts -p 'test_bench_abba.py' -v
 
   test:
     name: test (real io_uring on runner)
     runs-on: ubuntu-latest
+    env:
+      URING_REQUIRE_DIRECT: "1"
     steps:
       - uses: actions/checkout@v7
       - name: Install toolchain
         run: rustup toolchain install stable --profile minimal && rustup default stable
       - name: Build tests
-        run: cargo build --tests --all-features
+        run: cargo build --locked --tests --all-features
       - name: Test and assert io_uring actually ran
         run: |
           set -o pipefail
-          cargo test --all-features -- --nocapture --test-threads=1 2>&1 | tee test.out
+          cargo test --locked --all-features -- --nocapture --test-threads=1 2>&1 | tee test.out
           if grep -q "SKIP " test.out; then
             echo "::error::a test skipped — io_uring did not run on this runner (vacuous pass)"
             exit 1
           fi
+          grep -q "DIRECT_OK direct_read_returns_exact_unaligned_ranges" test.out
+      - name: Benchmark schema and correctness smoke
+        run: bash scripts/test-benchmark-cli.sh
+      - name: Instrumented benchmark schema and correctness smoke
+        env:
+          BENCH_DIAGNOSTICS: "1"
+        run: bash scripts/test-benchmark-cli.sh
 
   docker-two-legs:
     name: docker (degradation + real io_uring)

CHANGELOG.md

@@ -23,6 +23,24 @@ aims to follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Opt-in `diagnostics` feature with per-shard sampled driver-stage histograms,
+  aggregate snapshots, and measurement-interval deltas. Default builds compile
+  out the timing fields and sampling work.
+- Native O_DIRECT execution gate and byte-exact benchmark CLI smoke coverage.
+- Warm-cache ABBA evidence runner with per-leg provenance/resource records,
+  drift and environment gates, and bounded benchmark process-group cleanup.
+
+### Changed
+
+- Benchmark CSV schema v2 obtains headers from the executable and reports
+  independent setup, workload, and teardown timings, configurable ring depth,
+  workers, warmup, and instrumentation status. Positional CSV consumers must
+  migrate to the new header.
+- Clarified that driver submission counters count accepted logical reads and
+  delivery counters count successful channel sends, not caller consumption.
+
 ## [0.2.2] - 2026-09-07
 
 This patch release documents and hardens the public read API while preserving

Cargo.toml

@@ -33,6 +33,9 @@ categories = ["asynchronous", "filesystem"]
 # probe-drain leak paths deterministically. All gated code is `#[cfg(feature =
 # "fault-injection")]`, so a default build compiles none of it.
 fault-injection = []
+# Opt-in sampled timing. No timestamps, histogram storage, or tracing allocations
+# are compiled into the default driver.
+diagnostics = []
 
 [dependencies]
 # tracing for driver diagnostics: unifies with the RustFS tracing pipeline for

README.md

@@ -78,8 +78,16 @@ The public API is intentionally small and read-only:
 `submitted == delivered + orphan_reclaimed` holds after all completions have
 been reaped; `in_flight == 0` indicates a clean shutdown.
 
+The optional `diagnostics` feature exposes sampled stage histograms through
+`UringDriver::diagnostics()` and `shard_diagnostics()`. It is off by default.
+See [the measurement guide](docs/benchmarking.md#sampled-diagnostics) for sampling,
+stage overlap, cancellation, and instrumentation-overhead boundaries.
+
 ## Testing
 
+Benchmark configuration, CSV schema, timing boundaries, and performance gates
+are documented in [the benchmarking guide](docs/benchmarking.md).
+
 Linux only; on other hosts `cargo check` builds the empty stub.
 
 ```bash

bench-concurrent-pread.sh

@@ -19,7 +19,7 @@
 # Sweeps strategy × read_size × concurrency. Caches are dropped before every
 # timed run so reads hit the device (a large file + random offsets means the
 # page cache would otherwise skew results run-to-run). Needs root for
-# drop_caches; the bench host azure-20780104 runs as root.
+# drop_caches; run cold sweeps only on an isolated test host.
 set -euo pipefail
 cd "$(dirname "$0")"
 
@@ -28,13 +28,14 @@ REPEAT="${REPEAT:-3}"
 BIN="${CARGO_TARGET_DIR:-target}/release/examples/concurrent_pread_bench"
 
 FILE_SIZE="${FILE_SIZE:-4294967296}" # 4 GiB, >> page cache reuse for random reads
-READ_SIZES=(${READ_SIZES:-65536 1048576})
-CONCURRENCIES=(${CONCURRENCIES:-1 8 32 128})
+read -r -a READ_SIZES <<<"${READ_SIZES:-65536 1048576}"
+read -r -a CONCURRENCIES <<<"${CONCURRENCIES:-1 8 32 128}"
+read -r -a SHARD_COUNTS <<<"${SHARD_COUNTS:-1}"
 # Bound each cold run's transfer instead of fixing the op count: a cold 1 MiB
 # read costs ~16x a 64 KiB one, so a fixed op count would make the large-read
 # legs dominate wall-clock for no extra signal.
 TOTAL_BYTES="${TOTAL_BYTES:-268435456}" # 256 MiB per run
-MIN_OPS="${MIN_OPS:-512}"               # enough samples for a p999
+MIN_OPS="${MIN_OPS:-512}"               # smoke-sized; not a reliable p999 population
 MAX_OPS="${MAX_OPS:-4096}"
 STRATS=(std_open_pread std_cached_pread uring_open_read uring_cached_read)
 
@@ -46,19 +47,19 @@ ops_for() { # read_size -> op count, clamped
 }
 
 mkdir -p "$DIR"
-cargo build --release --example concurrent_pread_bench >&2
+build_features=()
+if [[ "${BENCH_DIAGNOSTICS:-0}" == 1 ]]; then build_features=(--features diagnostics); fi
+cargo build --locked --release --example concurrent_pread_bench "${build_features[@]}" >&2
 
 # Correctness preflight (untimed; output discarded). IOPS cannot distinguish a
 # strategy that reads the right *number* of bytes from one that reads the wrong
 # offsets, so every strategy first replays a small workload under BENCH_VERIFY=1,
 # which checks each delivered byte against the file's offset-addressable pattern.
 # A mismatch aborts before any measurement is taken.
 preflight_verify() {
-    local vdir="$DIR/verify.$$" vfile strat
-    rm -rf "$vdir"
-    mkdir -p "$vdir"
-    # shellcheck disable=SC2064
-    trap "rm -rf '$vdir'" RETURN
+    local vdir vfile strat
+    vdir=$(mktemp -d "$DIR/verify.XXXXXX")
+    trap 'rm -rf -- "$vdir"; trap - RETURN' RETURN
     vfile="$vdir/verify.bin"
     for strat in "${STRATS[@]}"; do
         # Unaligned read size on purpose: exercises the offset bookkeeping.
@@ -72,24 +73,38 @@ FILE="$DIR/pread_${FILE_SIZE}.bin"
 # Create once, untimed, so every cold run below is genuinely cold.
 "$BIN" std_cached_pread "$FILE" "$FILE_SIZE" 65536 1 1 >/dev/null
 
-echo "cache,strategy,file_size,read_size,concurrency,ops,secs,IOPS,MBps,p50_us,p99_us,p999_us"
+HEADER=$("$BIN" --header)
+FIELDS=$(awk -F, '{print NF}' <<<"$HEADER")
+printf 'cache,%s\n' "$HEADER"
 for cache in ${CACHES:-cold warm}; do
+    if [[ "$cache" == cold && "${BENCH_WARMUP_OPS:-0}" != 0 ]]; then
+        echo "cold sweeps require BENCH_WARMUP_OPS=0" >&2
+        exit 1
+    fi
     for read_size in "${READ_SIZES[@]}"; do
         ops=$(ops_for "$read_size")
         for conc in "${CONCURRENCIES[@]}"; do
             for strat in "${STRATS[@]}"; do
-                for _ in $(seq 1 "$REPEAT"); do
-                    if [ "$cache" = cold ]; then
-                        sync
-                        echo 3 >/proc/sys/vm/drop_caches
-                    else
-                        # Warm takes the device out of the picture, isolating the
-                        # software cost (open, blocking-pool hop, submission).
-                        # On a throughput-throttled disk the cold leg saturates
-                        # and hides exactly the overhead we are pricing.
-                        cat "$FILE" >/dev/null
-                    fi
-                    echo "$cache,$("$BIN" "$strat" "$FILE" "$FILE_SIZE" "$read_size" "$conc" "$ops")"
+                shards_for_strategy=(1)
+                if [[ "$strat" == uring_* ]]; then
+                    shards_for_strategy=("${SHARD_COUNTS[@]}")
+                fi
+                for shards in "${shards_for_strategy[@]}"; do
+                    for _ in $(seq 1 "$REPEAT"); do
+                        if [ "$cache" = cold ]; then
+                            sync
+                            echo 3 >/proc/sys/vm/drop_caches
+                        else
+                            # Warm takes the device out of the picture, isolating the
+                            # software cost (open, blocking-pool hop, submission).
+                            # On a throughput-throttled disk the cold leg saturates
+                            # and hides exactly the overhead we are pricing.
+                            cat "$FILE" >/dev/null
+                        fi
+                        row=$("$BIN" "$strat" "$FILE" "$FILE_SIZE" "$read_size" "$conc" "$ops" "$shards")
+                        awk -F, -v expected="$FIELDS" 'NF != expected || $1 != 2 || $2 != "measure" {exit 1}' <<<"$row"
+                        printf '%s,%s\n' "$cache" "$row"
+                    done
                 done
             done
         done

bench-streaming.sh

@@ -22,7 +22,7 @@
 #   cold  — `drop_caches` before each timed run, so every read hits the device.
 #
 # Emits one CSV to stdout. Cold mode needs root (drop_caches); the bench host
-# azure-20780104 runs as root. REPEAT>1 runs each config N times so the caller
+# must be isolated from other workloads. REPEAT>1 repeats each config so the caller
 # can take the median.
 #
 # BENCH_DIR only ever holds files this script created: the example refuses to
@@ -39,12 +39,14 @@ BIN="${CARGO_TARGET_DIR:-target}/release/examples/streaming_bench"
 STRATEGIES=(std_buffered std_odirect uring_read_at uring_read_at_direct)
 
 # Sizes: metadata-ish, mid object, large object.
-SIZES=(${SIZES:-65536 16777216 268435456})
-CHUNKS=(${CHUNKS:-131072 1048576})
-QDS=(${QDS:-1 4 16})
+read -r -a SIZES <<<"${SIZES:-65536 16777216 268435456}"
+read -r -a CHUNKS <<<"${CHUNKS:-131072 1048576}"
+read -r -a QDS <<<"${QDS:-1 4 16}"
 
 mkdir -p "$DIR"
-cargo build --release --example streaming_bench >&2
+build_features=()
+if [[ "${BENCH_DIAGNOSTICS:-0}" == 1 ]]; then build_features=(--features diagnostics); fi
+cargo build --locked --release --example streaming_bench "${build_features[@]}" >&2
 
 # Correctness preflight (untimed; its output is discarded). Throughput cannot
 # distinguish a strategy that reads the right *number* of bytes from one that
@@ -56,11 +58,9 @@ cargo build --release --example streaming_bench >&2
 #   - queue depths 1 and 4, covering the pipelined path's offset bookkeeping.
 # A mismatch aborts the script before any measurement is taken.
 preflight_verify() {
-    local vdir="$DIR/verify.$$" chunk=131072 size strat qd
-    rm -rf "$vdir"
-    mkdir -p "$vdir"
-    # shellcheck disable=SC2064
-    trap "rm -rf '$vdir'" RETURN
+    local vdir chunk=131072 size strat qd
+    vdir=$(mktemp -d "$DIR/verify.XXXXXX")
+    trap 'rm -rf -- "$vdir"; trap - RETURN' RETURN
     for size in $((1048576 + 4096)) $((1048576 + 4097)); do
         for strat in "${STRATEGIES[@]}"; do
             for qd in 1 4; do
@@ -94,12 +94,21 @@ run() { # strategy size chunk qd cache
     local file="$DIR/bench_${size}.bin"
     for _ in $(seq 1 "$REPEAT"); do
         drop_or_warm "$cache" "$size" "$chunk"
-        echo "$cache,$("$BIN" "$strat" "$file" "$size" "$chunk" "$qd" "$ALIGN")"
+        local row
+        row=$("$BIN" "$strat" "$file" "$size" "$chunk" "$qd" "$ALIGN")
+        awk -F, -v expected="$FIELDS" 'NF != expected || $1 != 2 || $2 != "measure" {exit 1}' <<<"$row"
+        printf '%s,%s\n' "$cache" "$row"
     done
 }
 
-echo "cache,strategy,size,chunk,qd,align,bytes,secs,MBps,ops"
+HEADER=$("$BIN" --header)
+FIELDS=$(awk -F, '{print NF}' <<<"$HEADER")
+printf 'cache,%s\n' "$HEADER"
 for cache in warm cold; do
+    if [[ "$cache" == cold && "${BENCH_WARMUP_RUNS:-0}" != 0 ]]; then
+        echo "cold sweeps require BENCH_WARMUP_RUNS=0" >&2
+        exit 1
+    fi
     for size in "${SIZES[@]}"; do
         for chunk in "${CHUNKS[@]}"; do
             run std_buffered "$size" "$chunk" 1 "$cache"