From 0e080327396c6c46e94cb72e87bc9439a1e96358 Mon Sep 17 00:00:00 2001
From: Tobias Schmidt <tobias.schmidt@in.tum.de>
Date: Tue, 18 Aug 2026 21:18:17 +0000
Subject: [PATCH 1/2] datafusion(-vortex): fail the query on any stderr output

datafusion-cli and vortex-datafusion-cli exit 0 even when a query errors
mid-batch: they print the failure to stderr and carry on, leaving only
create.sql's own ~0.01s Elapsed line on stdout. The query scripts merged
stderr into stdout (2>&1) and reported the last Elapsed line, so a failed
query was silently recorded as a ~0.011s runtime.

On c8g.metal-48xl this masked real failures: wide scans over the 100
partitioned .vortex files exhaust the open-file limit ('Too many open
files (os error 24)') and were published as bogus 40-200x speedups.

Capture stderr separately and treat any stderr output (or a nonzero exit)
as a failed run, so the driver records null and surfaces the error instead
of a fake near-zero time. A successful query writes nothing to stderr.
---
 datafusion-partitioned/query        | 13 +++++++++----
 datafusion-vortex-partitioned/query | 13 +++++++++----
 datafusion-vortex/query             | 13 +++++++++----
 datafusion/query                    | 13 +++++++++----
 4 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/datafusion-partitioned/query b/datafusion-partitioned/query
index c3625f4b1b..8d9ca6162a 100755
--- a/datafusion-partitioned/query
+++ b/datafusion-partitioned/query
@@ -9,14 +9,19 @@ DF=arrow-datafusion/target/release/datafusion-cli
 
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion.XXXXXX.sql)
-trap 'rm -f "$tmp"' EXIT
+err=$(mktemp /tmp/datafusion.XXXXXX.err)
+trap 'rm -f "$tmp" "$err"' EXIT
 printf '%s\n' "$query" > "$tmp"
 
-out=$("$DF" -f create.sql "$tmp" 2>&1) && status=0 || status=$?
+# datafusion-cli exits 0 even when a query errors: it prints to stderr and leaves
+# only create.sql's "Elapsed" on stdout, which the driver would mis-record as the
+# query time. Treat any stderr output (or a nonzero exit) as failure.
+out=$("$DF" -f create.sql "$tmp" 2>"$err") && status=0 || status=$?
 
-if [ "$status" -ne 0 ]; then
+if [ "$status" -ne 0 ] || [ -s "$err" ]; then
+    cat "$err" >&2
     printf '%s\n' "$out" >&2
-    exit "$status"
+    exit 1
 fi
 
 printf '%s\n' "$out" | grep -v 'Elapsed' || true
diff --git a/datafusion-vortex-partitioned/query b/datafusion-vortex-partitioned/query
index 2e4df9d6a9..64015bcd49 100755
--- a/datafusion-vortex-partitioned/query
+++ b/datafusion-vortex-partitioned/query
@@ -7,14 +7,19 @@ DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
 
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion-vortex.XXXXXX.sql)
-trap 'rm -f "$tmp"' EXIT
+err=$(mktemp /tmp/datafusion-vortex.XXXXXX.err)
+trap 'rm -f "$tmp" "$err"' EXIT
 printf '%s\n' "$query" > "$tmp"
 
-out=$("$DF" -f create.sql -f "$tmp" 2>&1) && status=0 || status=$?
+# The CLI exits 0 even when a query errors: it prints to stderr and leaves only
+# create.sql's tiny "Elapsed" on stdout, which the driver would mis-record as the
+# query time. Treat any stderr output (or a nonzero exit) as failure.
+out=$("$DF" -f create.sql -f "$tmp" 2>"$err") && status=0 || status=$?
 
-if [ "$status" -ne 0 ]; then
+if [ "$status" -ne 0 ] || [ -s "$err" ]; then
+    cat "$err" >&2
     printf '%s\n' "$out" >&2
-    exit "$status"
+    exit 1
 fi
 
 printf '%s\n' "$out" | grep -v 'Elapsed' || true
diff --git a/datafusion-vortex/query b/datafusion-vortex/query
index 2e4df9d6a9..64015bcd49 100755
--- a/datafusion-vortex/query
+++ b/datafusion-vortex/query
@@ -7,14 +7,19 @@ DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
 
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion-vortex.XXXXXX.sql)
-trap 'rm -f "$tmp"' EXIT
+err=$(mktemp /tmp/datafusion-vortex.XXXXXX.err)
+trap 'rm -f "$tmp" "$err"' EXIT
 printf '%s\n' "$query" > "$tmp"
 
-out=$("$DF" -f create.sql -f "$tmp" 2>&1) && status=0 || status=$?
+# The CLI exits 0 even when a query errors: it prints to stderr and leaves only
+# create.sql's tiny "Elapsed" on stdout, which the driver would mis-record as the
+# query time. Treat any stderr output (or a nonzero exit) as failure.
+out=$("$DF" -f create.sql -f "$tmp" 2>"$err") && status=0 || status=$?
 
-if [ "$status" -ne 0 ]; then
+if [ "$status" -ne 0 ] || [ -s "$err" ]; then
+    cat "$err" >&2
     printf '%s\n' "$out" >&2
-    exit "$status"
+    exit 1
 fi
 
 printf '%s\n' "$out" | grep -v 'Elapsed' || true
diff --git a/datafusion/query b/datafusion/query
index 65cc944ea0..ff2957fb3c 100755
--- a/datafusion/query
+++ b/datafusion/query
@@ -10,14 +10,19 @@ DF=arrow-datafusion/target/release/datafusion-cli
 
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion.XXXXXX.sql)
-trap 'rm -f "$tmp"' EXIT
+err=$(mktemp /tmp/datafusion.XXXXXX.err)
+trap 'rm -f "$tmp" "$err"' EXIT
 printf '%s\n' "$query" > "$tmp"
 
-out=$("$DF" -f create.sql "$tmp" 2>&1) && status=0 || status=$?
+# datafusion-cli exits 0 even when a query errors: it prints to stderr and leaves
+# only create.sql's "Elapsed" on stdout, which the driver would mis-record as the
+# query time. Treat any stderr output (or a nonzero exit) as failure.
+out=$("$DF" -f create.sql "$tmp" 2>"$err") && status=0 || status=$?
 
-if [ "$status" -ne 0 ]; then
+if [ "$status" -ne 0 ] || [ -s "$err" ]; then
+    cat "$err" >&2
     printf '%s\n' "$out" >&2
-    exit "$status"
+    exit 1
 fi
 
 # Print everything that's not an "Elapsed" timing line as the result.

From 0a2766aa11303ee6238b5edc19e514f3a140151f Mon Sep 17 00:00:00 2001
From: Tobias Schmidt <tobias.schmidt@in.tum.de>
Date: Tue, 18 Aug 2026 21:18:44 +0000
Subject: [PATCH 2/2] datafusion-vortex: raise open-file limit so wide scans
 complete
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

vortex-io opens all partition files concurrently, so on many-core hosts
(e.g. c8g.metal-48xl, 192 vCPU) a wide scan over the 100 .vortex files
exceeds the default open-file soft limit of 1024 and fails with 'Too many
open files (os error 24)'. The failure is a flaky race — a different subset
of queries fails each run.

Raise the soft limit to the hard cap (no privilege required) at the top of
the query script; the spawned CLI inherits it.

Also add the c8g.metal-48xl results (partitioned and single) produced with
the fix in place: the full 43-query sweep completes with zero failures and
real times instead of the earlier bogus ~0.011s.
---
 datafusion-vortex-partitioned/query           |   4 +
 .../results/20260818/c8g.metal-48xl.json      | 236 ++++++++++++++++++
 datafusion-vortex/query                       |   4 +
 .../results/20260818/c8g.metal-48xl.json      | 236 ++++++++++++++++++
 4 files changed, 480 insertions(+)
 create mode 100644 datafusion-vortex-partitioned/results/20260818/c8g.metal-48xl.json
 create mode 100644 datafusion-vortex/results/20260818/c8g.metal-48xl.json

diff --git a/datafusion-vortex-partitioned/query b/datafusion-vortex-partitioned/query
index 64015bcd49..4fbadaa925 100755
--- a/datafusion-vortex-partitioned/query
+++ b/datafusion-vortex-partitioned/query
@@ -5,6 +5,10 @@ set -e
 
 DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
 
+# vortex-io opens all partitions at once; on many-core hosts a wide scan can
+# exceed the default 1024 open-file limit. Raise the soft limit to the hard cap.
+ulimit -n "$(ulimit -Hn)" 2>/dev/null || true
+
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion-vortex.XXXXXX.sql)
 err=$(mktemp /tmp/datafusion-vortex.XXXXXX.err)
diff --git a/datafusion-vortex-partitioned/results/20260818/c8g.metal-48xl.json b/datafusion-vortex-partitioned/results/20260818/c8g.metal-48xl.json
new file mode 100644
index 0000000000..1ce00b6825
--- /dev/null
+++ b/datafusion-vortex-partitioned/results/20260818/c8g.metal-48xl.json
@@ -0,0 +1,236 @@
+{
+      "system": "DataFusion (Vortex, partitioned)",
+      "date": "2026-08-18",
+      "machine": "c8g.metal-48xl",
+      "cluster_size": 1,
+      "proprietary": "no",
+      "hardware": "cpu",
+      "tuned": "no",
+      "tags": [
+            "Rust",
+            "column-oriented",
+            "embedded",
+            "stateless"
+      ],
+      "load_time": 62,
+      "data_size": 15329147920,
+      "concurrent_qps": null,
+      "concurrent_error_ratio": null,
+      "result": [
+            [
+                  0.036,
+                  0.002,
+                  0.002
+            ],
+            [
+                  0.1,
+                  0.041,
+                  0.042
+            ],
+            [
+                  0.117,
+                  0.055,
+                  0.059
+            ],
+            [
+                  0.479,
+                  0.06,
+                  0.065
+            ],
+            [
+                  0.962,
+                  0.175,
+                  0.158
+            ],
+            [
+                  1.012,
+                  0.15,
+                  0.15
+            ],
+            [
+                  0.033,
+                  0.002,
+                  0.002
+            ],
+            [
+                  0.104,
+                  0.052,
+                  0.052
+            ],
+            [
+                  0.856,
+                  0.191,
+                  0.191
+            ],
+            [
+                  1.428,
+                  0.261,
+                  0.242
+            ],
+            [
+                  0.572,
+                  0.089,
+                  0.093
+            ],
+            [
+                  1.008,
+                  0.109,
+                  0.098
+            ],
+            [
+                  1.153,
+                  0.137,
+                  0.134
+            ],
+            [
+                  2.543,
+                  0.231,
+                  0.242
+            ],
+            [
+                  0.939,
+                  0.136,
+                  0.141
+            ],
+            [
+                  0.768,
+                  0.152,
+                  0.143
+            ],
+            [
+                  2.475,
+                  0.252,
+                  0.262
+            ],
+            [
+                  2.227,
+                  0.265,
+                  0.279
+            ],
+            [
+                  3.413,
+                  0.507,
+                  0.522
+            ],
+            [
+                  0.243,
+                  0.064,
+                  0.046
+            ],
+            [
+                  14.054,
+                  0.185,
+                  0.193
+            ],
+            [
+                  16.44,
+                  0.186,
+                  0.197
+            ],
+            [
+                  22.192,
+                  0.546,
+                  0.224
+            ],
+            [
+                  53.545,
+                  1.727,
+                  1.403
+            ],
+            [
+                  1.449,
+                  0.061,
+                  0.054
+            ],
+            [
+                  1.266,
+                  0.08,
+                  0.082
+            ],
+            [
+                  1.926,
+                  0.062,
+                  0.071
+            ],
+            [
+                  14.394,
+                  0.189,
+                  0.202
+            ],
+            [
+                  11.641,
+                  1.596,
+                  1.926
+            ],
+            [
+                  0.219,
+                  0.116,
+                  0.124
+            ],
+            [
+                  2.267,
+                  0.141,
+                  0.14
+            ],
+            [
+                  5.469,
+                  0.191,
+                  0.193
+            ],
+            [
+                  3.792,
+                  0.742,
+                  0.747
+            ],
+            [
+                  14.043,
+                  0.467,
+                  0.466
+            ],
+            [
+                  14.038,
+                  0.444,
+                  0.457
+            ],
+            [
+                  0.395,
+                  0.191,
+                  0.197
+            ],
+            [
+                  0.161,
+                  0.066,
+                  0.067
+            ],
+            [
+                  0.135,
+                  0.043,
+                  0.044
+            ],
+            [
+                  0.187,
+                  0.031,
+                  0.03
+            ],
+            [
+                  0.31,
+                  0.126,
+                  0.124
+            ],
+            [
+                  0.127,
+                  0.029,
+                  0.03
+            ],
+            [
+                  0.122,
+                  0.03,
+                  0.03
+            ],
+            [
+                  0.112,
+                  0.028,
+                  0.029
+            ]
+      ]
+}
diff --git a/datafusion-vortex/query b/datafusion-vortex/query
index 64015bcd49..142eae7919 100755
--- a/datafusion-vortex/query
+++ b/datafusion-vortex/query
@@ -5,6 +5,10 @@ set -e
 
 DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
 
+# vortex-io opens files concurrently; on many-core hosts a wide scan can exceed
+# the default 1024 open-file limit. Raise the soft limit to the hard cap.
+ulimit -n "$(ulimit -Hn)" 2>/dev/null || true
+
 query=$(cat)
 tmp=$(mktemp /tmp/datafusion-vortex.XXXXXX.sql)
 err=$(mktemp /tmp/datafusion-vortex.XXXXXX.err)
diff --git a/datafusion-vortex/results/20260818/c8g.metal-48xl.json b/datafusion-vortex/results/20260818/c8g.metal-48xl.json
new file mode 100644
index 0000000000..feea6c7923
--- /dev/null
+++ b/datafusion-vortex/results/20260818/c8g.metal-48xl.json
@@ -0,0 +1,236 @@
+{
+      "system": "DataFusion (Vortex, single)",
+      "date": "2026-08-18",
+      "machine": "c8g.metal-48xl",
+      "cluster_size": 1,
+      "proprietary": "no",
+      "hardware": "cpu",
+      "tuned": "no",
+      "tags": [
+            "Rust",
+            "column-oriented",
+            "embedded",
+            "stateless"
+      ],
+      "load_time": 104,
+      "data_size": 15269901360,
+      "concurrent_qps": null,
+      "concurrent_error_ratio": null,
+      "result": [
+            [
+                  0.036,
+                  0.001,
+                  0.001
+            ],
+            [
+                  0.623,
+                  0.489,
+                  0.487
+            ],
+            [
+                  0.527,
+                  0.485,
+                  0.488
+            ],
+            [
+                  0.716,
+                  0.56,
+                  0.49
+            ],
+            [
+                  0.806,
+                  0.547,
+                  0.559
+            ],
+            [
+                  1.154,
+                  0.584,
+                  0.583
+            ],
+            [
+                  0.033,
+                  0.001,
+                  0.001
+            ],
+            [
+                  0.56,
+                  0.496,
+                  0.587
+            ],
+            [
+                  1.156,
+                  0.593,
+                  0.611
+            ],
+            [
+                  1.5,
+                  0.661,
+                  0.698
+            ],
+            [
+                  0.807,
+                  0.535,
+                  0.535
+            ],
+            [
+                  0.849,
+                  0.536,
+                  0.536
+            ],
+            [
+                  1.28,
+                  0.579,
+                  0.603
+            ],
+            [
+                  2.675,
+                  0.705,
+                  0.699
+            ],
+            [
+                  1.436,
+                  0.588,
+                  0.599
+            ],
+            [
+                  0.822,
+                  0.559,
+                  0.558
+            ],
+            [
+                  2.521,
+                  0.83,
+                  0.802
+            ],
+            [
+                  2.505,
+                  0.667,
+                  0.708
+            ],
+            [
+                  3.526,
+                  0.855,
+                  0.849
+            ],
+            [
+                  0.736,
+                  0.59,
+                  0.497
+            ],
+            [
+                  14.45,
+                  0.763,
+                  0.732
+            ],
+            [
+                  16.518,
+                  0.934,
+                  1.264
+            ],
+            [
+                  21.147,
+                  1.149,
+                  1.157
+            ],
+            [
+                  57.583,
+                  3.471,
+                  3.829
+            ],
+            [
+                  2.218,
+                  0.556,
+                  0.556
+            ],
+            [
+                  1.237,
+                  0.516,
+                  0.523
+            ],
+            [
+                  2.27,
+                  0.549,
+                  0.56
+            ],
+            [
+                  14.581,
+                  0.83,
+                  0.808
+            ],
+            [
+                  11.855,
+                  1.81,
+                  1.837
+            ],
+            [
+                  0.557,
+                  0.575,
+                  0.534
+            ],
+            [
+                  2.537,
+                  0.609,
+                  0.582
+            ],
+            [
+                  5.571,
+                  0.636,
+                  0.63
+            ],
+            [
+                  3.897,
+                  0.977,
+                  0.978
+            ],
+            [
+                  14.491,
+                  0.828,
+                  0.854
+            ],
+            [
+                  14.48,
+                  0.873,
+                  0.909
+            ],
+            [
+                  0.705,
+                  0.644,
+                  0.587
+            ],
+            [
+                  0.657,
+                  0.582,
+                  0.592
+            ],
+            [
+                  0.615,
+                  0.569,
+                  0.544
+            ],
+            [
+                  0.637,
+                  0.575,
+                  0.592
+            ],
+            [
+                  0.732,
+                  0.69,
+                  0.754
+            ],
+            [
+                  0.601,
+                  0.526,
+                  0.526
+            ],
+            [
+                  0.667,
+                  0.528,
+                  0.532
+            ],
+            [
+                  0.575,
+                  0.512,
+                  0.511
+            ]
+      ]
+}