diff --git a/firebolt/benchmark.sh b/firebolt/benchmark.sh index 617422ddc2..db4c7b3ecb 100755 --- a/firebolt/benchmark.sh +++ b/firebolt/benchmark.sh @@ -1,6 +1,7 @@ #!/bin/bash # Thin shim — actual flow is in lib/benchmark-common.sh. -export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned" export BENCH_DURABLE=yes export BENCH_RESTARTABLE=no +./cleanup # a previous run may have left its container and data behind exec ../lib/benchmark-common.sh diff --git a/firebolt/cleanup b/firebolt/cleanup new file mode 100755 index 0000000000..8ca9374ad4 --- /dev/null +++ b/firebolt/cleanup @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu + +# Remove everything a run leaves behind: the container, the engine's data +# directory and the staged parquet source. +sudo docker rm -f firebolt-core >/dev/null 2>&1 || true +# The engine owns its data directory as uid 3473, hence sudo. +sudo rm -rf fb-volume +rm -rf data +rm -f hits_*.parquet diff --git a/firebolt/create.sql b/firebolt/create.sql index 2e4bb93669..02f9fe9e5b 100644 --- a/firebolt/create.sql +++ b/firebolt/create.sql @@ -111,7 +111,7 @@ CREATE EXTERNAL TABLE hits_external "CLID" INTEGER NOT NULL ) URL = 'file:///firebolt-core/clickbench' -OBJECT_PATTERN = 'hits.parquet' +OBJECT_PATTERN = 'hits_*.parquet' TYPE = PARQUET; CREATE TABLE hits diff --git a/firebolt/data-size b/firebolt/data-size index b5fe999ff8..a7043437eb 100755 --- a/firebolt/data-size +++ b/firebolt/data-size @@ -1,6 +1,7 @@ #!/bin/bash set -e -# Firebolt-core writes its database state under /firebolt-core/volume -# inside the container, which we bind-mount to ./fb-volume on the host. -du -bcs fb-volume 2>/dev/null | awk '/total$/ { print $1 }' +# The engine writes its database state under /var/lib/firebolt inside the +# container, which we bind-mount to ./fb-volume on the host. It owns that +# directory as uid 3473 mode 2770, so du without sudo reports 0. +sudo du -bcs fb-volume | awk '/total$/ { print $1 }' diff --git a/firebolt/install b/firebolt/install index 38799727d9..9f61da7b5c 100755 --- a/firebolt/install +++ b/firebolt/install @@ -1,6 +1,9 @@ #!/bin/bash set -eu -sudo apt-get update -y -sudo apt-get install -y docker.io jq -sudo docker pull ghcr.io/firebolt-db/firebolt-core:preview-rc +# Installing docker.io over a docker that came from elsewhere breaks it. +if ! command -v docker >/dev/null || ! command -v jq >/dev/null; then + sudo apt-get update -y + sudo apt-get install -y docker.io jq +fi +sudo docker pull ghcr.io/firebolt-db/engine:dev diff --git a/firebolt/load b/firebolt/load index 8b03982e1e..432611cfc5 100755 --- a/firebolt/load +++ b/firebolt/load @@ -1,12 +1,13 @@ #!/bin/bash set -eu -# Stage hits.parquet where the container can see it (./data is +# Stage the parquet source where the container can see it (./data is # bind-mounted as /firebolt-core/clickbench). mkdir -p data -if [ -f hits.parquet ]; then - mv -f hits.parquet data/hits.parquet -fi +for f in hits_*.parquet; do + [ -e "$f" ] || break + mv -f "$f" data/ +done run_sql() { local url="$1" data="$2" body_file http_code @@ -31,7 +32,7 @@ run_sql() { fi } -# create.sql CREATEs hits_external pointing at the parquet file, then +# create.sql CREATEs hits_external pointing at the parquet files, then # INSERTs into the managed `hits` table — the ingested-to-Firebolt # variant of the benchmark. run_sql 'http://localhost:3473/?enable_multi_query_requests=true' \ @@ -39,5 +40,5 @@ run_sql 'http://localhost:3473/?enable_multi_query_requests=true' \ run_sql 'http://localhost:3473/?database=clickbench&enable_multi_query_requests=true' \ @create.sql -rm -f data/hits.parquet +rm -f data/hits_*.parquet sync diff --git a/firebolt/results/20260819/c8g.metal-48xl.json b/firebolt/results/20260819/c8g.metal-48xl.json new file mode 100644 index 0000000000..79c7aa9ccc --- /dev/null +++ b/firebolt/results/20260819/c8g.metal-48xl.json @@ -0,0 +1,59 @@ +{ + "system": "Firebolt", + "date": "2026-08-19", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "yes", + "hardware": "cpu", + "tuned": "no", + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "load_time": 170, + "data_size": 15829362967, + "concurrent_qps": 42.822, + "concurrent_error_ratio": 0, + "result": [ + [0.008, 0.002, 0.002], + [0.4, 0.168, 0.114], + [0.175, 0.033, 0.025], + [0.163, 0.015, 0.015], + [0.052, 0.046, 0.049], + [1.258, 0.066, 0.063], + [0.071, 0.013, 0.013], + [0.051, 0.106, 0.112], + [0.562, 0.088, 0.103], + [0.109, 0.08, 0.09], + [0.069, 0.041, 0.039], + [0.192, 0.065, 0.021], + [0.048, 0.043, 0.041], + [0.079, 0.1, 0.066], + [0.081, 0.049, 0.049], + [0.046, 0.046, 0.045], + [0.095, 0.085, 0.085], + [0.071, 0.072, 0.07], + [4.894, 0.146, 0.103], + [0.019, 0.007, 0.007], + [11.688, 0.026, 0.013], + [0.016, 0.011, 0.011], + [10.167, 0.034, 0.021], + [0.217, 0.066, 0.066], + [0.019, 0.018, 0.017], + [0.016, 0.015, 0.016], + [0.018, 0.018, 0.017], + [0.072, 0.063, 0.058], + [9.543, 0.181, 0.121], + [0.029, 0.026, 0.026], + [0.654, 0.038, 0.04], + [4.92, 0.078, 0.048], + [0.205, 0.184, 0.157], + [0.176, 0.16, 0.159], + [0.157, 0.148, 0.154], + [0.052, 0.056, 0.05], + [0.049, 0.025, 0.02], + [0.026, 0.018, 0.016], + [0.031, 0.011, 0.01], + [0.099, 0.046, 0.043], + [0.046, 0.012, 0.011], + [0.042, 0.014, 0.01], + [0.016, 0.012, 0.014] +] +} diff --git a/firebolt/start b/firebolt/start index 652146394e..6d9ade00a2 100755 --- a/firebolt/start +++ b/firebolt/start @@ -8,12 +8,10 @@ if curl -sS --max-time 5 'http://localhost:3473/' \ fi mkdir -p data fb-volume -# firebolt-core runs as UID/GID 1111 inside the container and refuses -# to start if its data dir is not writeable by that uid (the engine -# self-checks and aborts with "directory ... is not readable or -# writeable by the Firebolt Core process"). Set the host-side -# ownership accordingly so the bind-mounted dir is usable. -sudo chown 1111:1111 fb-volume +# The engine runs as UID/GID 3473 inside the container and refuses to +# start if its data dir is not writeable by that uid. World-writeable is +# what the official installer at get.firebolt.io does. +sudo chmod 0777 fb-volume # If the container exists (stopped from a prior agent pre-snapshot # cycle), just start it back — the data lives on the bind-mounted @@ -22,10 +20,10 @@ sudo chown 1111:1111 fb-volume if sudo docker ps -a --format '{{.Names}}' | grep -qx firebolt-core; then sudo docker start firebolt-core >/dev/null else - # `firebolt-core` is the public self-hosted image. Container needs + # `engine` is the public self-hosted image. Container needs # memlock 8 GiB and seccomp unconfined per upstream's run docs. # /firebolt-core/clickbench: parquet source (read at load time). - # /firebolt-core/volume: engine data directory (must persist + # /var/lib/firebolt: engine data directory (must persist # across the agent's pre-snapshot # stop+start cycle or the snapshot # ships an empty DB). @@ -33,9 +31,9 @@ else --ulimit memlock=8589934592:8589934592 \ --security-opt seccomp=unconfined \ -p 127.0.0.1:3473:3473 \ - -v "$(pwd)/fb-volume:/firebolt-core/volume" \ + -v "$(pwd)/fb-volume:/var/lib/firebolt" \ -v "$(pwd)/data:/firebolt-core/clickbench" \ - ghcr.io/firebolt-db/firebolt-core:preview-rc >/dev/null + "${FB_IMAGE:-ghcr.io/firebolt-db/engine:dev}" >/dev/null fi # Wait for the cluster to be "actually" ready. firebolt-core's HTTP