DO NOT MERGE: end-to-end test of bare metal job callbacks

.github/e2e/image/Dockerfile

@@ -0,0 +1,2 @@
+FROM busybox:1.37
+COPY --chmod=755 e2e-bench /usr/local/bin/e2e-bench

.github/e2e/image/e2e-bench

@@ -0,0 +1,21 @@
+#!/bin/sh
+# A stand-in benchmark: `run` prints one Bencher Metric Format result,
+# `fail` exits with an error, and `sleep <seconds>` runs long enough to cancel.
+set -eu
+
+case "${1:-}" in
+run) ;;
+fail)
+	echo "e2e-bench: failing on purpose" >&2
+	exit 1
+	;;
+sleep)
+	sleep "${2:?sleep needs a number of seconds}"
+	;;
+*)
+	echo "usage: e2e-bench run | fail | sleep <seconds>" >&2
+	exit 2
+	;;
+esac
+
+echo '{"e2e::callback": {"latency": {"value": 1000.0}}}'

.github/e2e/install-bencher/action.yml

@@ -0,0 +1,63 @@
+name: Install Bencher from the job callbacks branch
+description: Install the Bencher CLI or runner built from the tip of the job callbacks branch, cached by its commit.
+
+inputs:
+  component:
+    description: "`cli` for the `bencher` CLI, or `runner` for the runner daemon"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve the Bencher ${{ inputs.component }} commit
+      id: commit
+      shell: bash
+      env:
+        COMPONENT: ${{ inputs.component }}
+      run: |
+        case "$COMPONENT" in
+        cli | runner) ;;
+        *)
+          echo "::error::Unknown component: $COMPONENT"
+          exit 1
+          ;;
+        esac
+        sha="$(git ls-remote https://github.com/bencherdev/bencher refs/heads/u/ep/callback/smoke | cut -f1)"
+        test -n "$sha"
+        echo "Bencher $COMPONENT from bencherdev/bencher@$sha"
+        echo "sha=$sha" >> "$GITHUB_OUTPUT"
+        echo "key=bencher-$COMPONENT-${ImageOS:?}-$RUNNER_ARCH-$sha" >> "$GITHUB_OUTPUT"
+    - name: Restore the Bencher ${{ inputs.component }}
+      id: restore
+      uses: actions/cache/restore@v5
+      with:
+        path: ~/.bencher-${{ inputs.component }}
+        key: ${{ steps.commit.outputs.key }}
+    - name: Install the Bencher CLI
+      if: inputs.component == 'cli' && steps.restore.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        BENCHER_SHA: ${{ steps.commit.outputs.sha }}
+      run: cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --root ~/.bencher-cli bencher_cli
+    - name: Install the Bencher runner
+      if: inputs.component == 'runner' && steps.restore.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        BENCHER_SHA: ${{ steps.commit.outputs.sha }}
+      run: |
+        sudo apt-get update && sudo apt-get install -y musl-tools
+        rustup target add x86_64-unknown-linux-musl
+        # The release runner embeds a static `bencher-init` for its VMs.
+        cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --target x86_64-unknown-linux-musl --root "$RUNNER_TEMP/bencher-init" bencher_init
+        BENCHER_INIT_PATH="$RUNNER_TEMP/bencher-init/bin/bencher-init" cargo install --git https://github.com/bencherdev/bencher --rev "$BENCHER_SHA" --locked --root ~/.bencher-runner bencher_runner_cli
+    - name: Save the Bencher ${{ inputs.component }}
+      if: steps.restore.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v5
+      with:
+        path: ~/.bencher-${{ inputs.component }}
+        key: ${{ steps.commit.outputs.key }}
+    - name: Add the Bencher ${{ inputs.component }} to the path
+      shell: bash
+      env:
+        COMPONENT: ${{ inputs.component }}
+      run: echo "$HOME/.bencher-$COMPONENT/bin" >> "$GITHUB_PATH"

.github/e2e/runner.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Host a dev runner for the scenarios' Jobs, after every submit job has finished:
+#
+#   runner.sh serve    serve `test-spec` as `test-runner` for 10 minutes
+#   runner.sh cancel   serve `no-sandbox-spec` as `test-runner-no-sandbox`, and kill the runner
+#                      once the cancel scenario's Job is running, so the server cancels that Job
+#                      when its timeout and grace period pass
+set -euo pipefail
+
+now() {
+	date -u +%Y-%m-%dT%H:%M:%SZ
+}
+
+rotate_key() {
+	local key
+	key="$(bencher runner key --token "$BENCHER_ADMIN_API_TOKEN" "$1" | jq -r '.key // empty')"
+	if [ -z "$key" ]; then
+		echo "::error::Rotating the key of $1 returned no key"
+		return 1
+	fi
+	echo "::add-mask::$key"
+	export BENCHER_RUNNER_KEY="$key"
+}
+
+serve() {
+	rotate_key test-runner
+	echo "Runner up at $(now)"
+	local status=0
+	timeout --kill-after 60s 10m runner up --runner test-runner --no-auto-update || status=$?
+	# `timeout` exits 124 after it stops the runner, or 137 if the runner needed a SIGKILL.
+	if [ "$status" -eq 124 ] || [ "$status" -eq 137 ]; then
+		echo "Runner down at $(now)"
+		return 0
+	fi
+	return "$status"
+}
+
+cancel() {
+	rotate_key test-runner-no-sandbox
+	local jobs count job
+	jobs="$(bencher job list "$PAID_PROJECT" --status pending --sort created --direction desc --per-page 255)"
+	count="$(jq '[.[] | select(.spec.slug == "no-sandbox-spec")] | length' <<< "$jobs")"
+	job="$(jq -r '[.[] | select(.spec.slug == "no-sandbox-spec")][0].uuid // empty' <<< "$jobs")"
+	if [ -z "$job" ]; then
+		echo "::error::There is no pending no-sandbox-spec Job in $PAID_PROJECT to cancel"
+		return 1
+	fi
+	if [ "$count" -gt 1 ]; then
+		echo "::warning::$count no-sandbox-spec Jobs are pending in $PAID_PROJECT, and the runner may run older ones before $job"
+	fi
+
+	local log="$RUNNER_TEMP/runner.log"
+	echo "Runner up at $(now), waiting for Job $job to start"
+	runner up --runner test-runner-no-sandbox --danger-allow-no-sandbox --no-auto-update > "$log" 2>&1 &
+	local pid=$!
+	local deadline=$((SECONDS + 900))
+	until grep -qF "Starting iteration 1/1 for job $job" "$log"; do
+		if ! kill -0 "$pid" 2> /dev/null; then
+			cat "$log"
+			echo "::error::The runner exited before Job $job started"
+			return 1
+		fi
+		if [ "$SECONDS" -ge "$deadline" ]; then
+			kill -KILL "$pid"
+			cat "$log"
+			echo "::error::Job $job did not start within 15 minutes"
+			return 1
+		fi
+		sleep 1
+	done
+	# A SIGTERM would let the runner finish the Job first, so the runner gets no chance to.
+	kill -KILL "$pid"
+	wait "$pid" || true
+	cat "$log"
+	echo "Killed the runner at $(now) while Job $job was running"
+}
+
+case "${1:-}" in
+serve) serve ;;
+cancel) cancel ;;
+*)
+	echo "usage: runner.sh serve | cancel" >&2
+	exit 2
+	;;
+esac

.github/e2e/setup.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Make sure dev has what the scenarios need. Every step is idempotent,
+# so it runs as is after every deploy wipes the dev database.
+set -euo pipefail
+
+ensure_project() {
+	local organization="$1" slug="$2" name="$3"
+	if bencher project view "$slug" > /dev/null 2>&1; then
+		echo "Project $slug exists"
+	else
+		bencher project create "$organization" --name "$name" --slug "$slug" > /dev/null
+		echo "Created project $slug in $organization"
+	fi
+}
+
+# The seed recreates the paid organization on every deploy, and only a plan makes it paid.
+bencher organization view "$PAID_ORGANIZATION" > /dev/null
+if bencher plan view --attempts 3 "$PAID_ORGANIZATION" > /dev/null 2>&1; then
+	echo "Organization $PAID_ORGANIZATION has a plan"
+elif [ -z "${BENCHER_DEV_SUBSCRIPTION:-}" ]; then
+	echo "::warning::The BENCHER_DEV_SUBSCRIPTION repository variable is not set, so $PAID_ORGANIZATION has no plan and every callback is skipped"
+else
+	# The level only matters to a licensed plan: a metered plan reads its level from the subscription.
+	bencher plan create "$PAID_ORGANIZATION" \
+		--checkout "$BENCHER_DEV_SUBSCRIPTION" \
+		--level "${BENCHER_DEV_PLAN_LEVEL:-team}" \
+		--skip-remote > /dev/null
+	echo "Attached the subscription to $PAID_ORGANIZATION"
+fi
+ensure_project "$PAID_ORGANIZATION" "$PAID_PROJECT" "Callback E2E"
+
+if bencher organization view "$FREE_ORGANIZATION" > /dev/null 2>&1; then
+	echo "Organization $FREE_ORGANIZATION exists"
+else
+	bencher organization create --name "Callback E2E Free" --slug "$FREE_ORGANIZATION" > /dev/null
+	echo "Created organization $FREE_ORGANIZATION"
+fi
+if bencher plan view --attempts 3 "$FREE_ORGANIZATION" > /dev/null 2>&1; then
+	echo "::error::Organization $FREE_ORGANIZATION has a plan, but the no plan scenario needs one without"
+	exit 1
+fi
+ensure_project "$FREE_ORGANIZATION" "$FREE_PROJECT" "Callback E2E Free"
+
+docker build --tag e2e-bench "$(dirname "$0")/image"
+printf '%s' "$BENCHER_API_TOKEN" | docker login "$BENCHER_REGISTRY" --username "$DEV_USER_EMAIL" --password-stdin
+for project in "$PAID_PROJECT" "$FREE_PROJECT"; do
+	docker tag e2e-bench "$BENCHER_REGISTRY/$project:$IMAGE_TAG"
+	docker push "$BENCHER_REGISTRY/$project:$IMAGE_TAG"
+done
+docker logout "$BENCHER_REGISTRY"

.github/e2e/submit.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Run a detached `bencher run` and check its raw output, before GitHub masks the log:
+#
+#   submit.sh <scenario> bencher run ...
+#
+# REDACTED lists the environment variables whose values must never be printed,
+# EXPECT_SKIPPED says whether the server should skip the callback for want of a plan,
+# and CALLBACK_PATH is the callback URL's path, which the CLI must never print (default `/dispatches`).
+set -euo pipefail
+
+scenario="$1"
+shift
+
+out="$RUNNER_TEMP/bencher-run.stdout"
+err="$RUNNER_TEMP/bencher-run.stderr"
+status=0
+"$@" > "$out" 2> "$err" || status=$?
+cat "$out"
+cat "$err" >&2
+
+failed=0
+fail() {
+	echo "::error title=$scenario::$1"
+	failed=1
+}
+
+for name in $REDACTED; do
+	value="${!name:-}"
+	if [ -z "$value" ]; then
+		fail "$name is empty, so there is nothing to check its redaction against"
+	elif grep -qF -- "$value" "$out" "$err"; then
+		fail "the CLI printed the raw value of $name"
+	fi
+done
+if ! grep -qF '"authorization": "************"' "$out"; then
+	fail "the Bencher New Report echo does not show the CLI's mask for the authorization header"
+fi
+if grep -qF -- "${CALLBACK_PATH:-/dispatches}" "$out" "$err"; then
+	fail "the CLI printed the callback URL past its origin"
+fi
+
+skipped=false
+if grep -qxF 'callback skipped: requires a Bencher Plus plan' "$err"; then
+	skipped=true
+fi
+if [ "$skipped" != "$EXPECT_SKIPPED" ]; then
+	fail "expected a skipped callback to be $EXPECT_SKIPPED, but it was $skipped"
+fi
+
+job="$(sed -n 's/^Remote job submitted successfully: //p' "$err" | head -n 1)"
+check="$(grep -oE '"check": [0-9]+' "$out" | head -n 1 | grep -oE '[0-9]+' || true)"
+if [ -z "$job" ]; then
+	fail "the CLI did not print the submitted Job"
+fi
+echo "::notice title=$scenario::Job ${job:-none}, check run ${check:-none}, commit $HEAD_SHA"
+{
+	echo "| Scenario | Job | Check run | Commit |"
+	echo "| --- | --- | --- | --- |"
+	echo "| $scenario | ${job:-none} | ${check:-none} | $HEAD_SHA |"
+} >> "$GITHUB_STEP_SUMMARY"
+
+if [ "$status" -ne 0 ]; then
+	exit "$status"
+fi
+exit "$failed"