From 818ff7ebcedd6cc24524685d71594aa1e9eadf4b Mon Sep 17 00:00:00 2001
From: Colin Walters <walters@verbum.org>
Date: Thu, 10 Sep 2026 09:06:40 -0400
Subject: [PATCH] bootc-ubuntu-setup: Use fixed QEMU from Resolute

The Plucky QEMU used on Noble lacks the synchronous vhost-user notifier update needed to avoid lost virtiofs interrupts. Select Resolute's fixed emulator for virtualization jobs, including its matching iPXE ROM package so guests can actually boot.

Keep this limited to Ubuntu 24.04 disposable runners: the package transaction also upgrades runtime dependencies. Exercise guest boot on the amd64 KVM runner. Since bcvk requires KVM and has no arm64 release asset, validate the installed arm64 emulator with a bounded TCG startup probe instead.

Generated-by: AI

Signed-off-by: Colin Walters <walters@verbum.org>
---
 .github/workflows/test-virtualization.yml | 152 ++++++++++++++++++++++
 bootc-ubuntu-setup/action.yml             | 114 ++++++++++++++--
 2 files changed, 257 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/test-virtualization.yml

diff --git a/.github/workflows/test-virtualization.yml b/.github/workflows/test-virtualization.yml
new file mode 100644
index 0000000..b674491
--- /dev/null
+++ b/.github/workflows/test-virtualization.yml
@@ -0,0 +1,152 @@
+name: Test virtualization setup
+
+on:
+  pull_request:
+    paths:
+      - bootc-ubuntu-setup/action.yml
+      - .github/workflows/test-virtualization.yml
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  qemu-smoke:
+    name: 'QEMU smoke: ${{ matrix.name }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            architecture: amd64
+            name: amd64 virtiofs (KVM required)
+            full_vm_smoke: true
+          - runner: ubuntu-24.04-arm
+            architecture: arm64
+            name: arm64 QEMU startup (TCG)
+            full_vm_smoke: false
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 25
+    steps:
+      - name: Checkout actions
+        uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+
+      - name: Set up Ubuntu host
+        uses: ./bootc-ubuntu-setup
+        with:
+          libvirt: true
+
+      - name: Verify host QEMU and select acceleration
+        id: qemu
+        shell: bash
+        env:
+          FULL_VM_SMOKE: ${{ matrix.full_vm_smoke }}
+        run: |
+          set -euo pipefail
+          qemu="qemu-system-$(uname -m)"
+          "$qemu" --version
+          if [ "$(uname -m)" = x86_64 ]; then
+            machine=q35
+          else
+            machine=virt
+          fi
+          "$qemu" -machine help | grep -Eq "(^|[[:space:]])${machine}([[:space:]]|$)"
+          if [ "$FULL_VM_SMOKE" = true ]; then
+            if ! [ -c /dev/kvm ] || ! [ -r /dev/kvm ] || ! [ -w /dev/kvm ]; then
+              printf 'KVM is required for the full VM smoke, but /dev/kvm is unavailable.\n' >&2
+              exit 1
+            fi
+            echo 'full_vm_smoke=true' >> "$GITHUB_OUTPUT"
+          else
+            # bcvk requires KVM and is not available for arm64.  Always use TCG
+            # here, even if a future arm64 runner exposes /dev/kvm.
+            printf 'Smoke testing QEMU startup with TCG.\n'
+            tcg_status=0
+            timeout --foreground --signal=TERM --kill-after=5s 10s \
+              "$qemu" -machine "${machine},accel=tcg" -display none -nodefaults -S || \
+              tcg_status=$?
+            if [ "$tcg_status" -ne 124 ]; then
+              printf 'QEMU TCG smoke expected timeout status 124, got %s.\n' "$tcg_status" >&2
+              exit 1
+            fi
+            echo 'full_vm_smoke=false' >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Pull smoke image
+        if: ${{ steps.qemu.outputs.full_vm_smoke == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          image=quay.io/centos-bootc/centos-bootc:stream9
+          timeout --foreground --signal=TERM --kill-after=30s 5m podman pull -q "$image"
+          podman image inspect "$image" --format '{{.RepoDigests}}' >"$RUNNER_TEMP/image-digest.txt"
+
+      - name: Run ephemeral VM
+        if: ${{ steps.qemu.outputs.full_vm_smoke == 'true' }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          artifact_dir="$RUNNER_TEMP/qemu-smoke"
+          log_dir="$artifact_dir/vm-logs"
+          name="qemu-smoke-${GITHUB_RUN_ID}"
+          mkdir -p "$log_dir"
+          limit_log() {
+            local log=$1
+            if [ -f "$log" ] && [ "$(stat -c %s "$log")" -gt 5242880 ]; then
+              truncate --size=5242880 "$log"
+            fi
+          }
+          # shellcheck disable=SC2317 # invoked by the EXIT trap below
+          cleanup() {
+            timeout --foreground --signal=TERM --kill-after=5s 20s podman stop --time 20 "$name" >/dev/null 2>&1 || :
+            timeout --foreground --signal=TERM --kill-after=5s 10s podman rm -f "$name" >/dev/null 2>&1 || :
+          }
+          trap cleanup EXIT
+
+          set +e
+          timeout --foreground --signal=TERM --kill-after=10s 90s \
+            bcvk ephemeral run -d -K --name "$name" --console \
+              --log-dir "journal,console=$log_dir" \
+              quay.io/centos-bootc/centos-bootc:stream9 \
+              >"$artifact_dir/launcher.log" 2>&1
+          launcher_status=$?
+          main_status=$launcher_status
+          if [ "$launcher_status" -eq 0 ]; then
+            deadline=$((SECONDS + 90))
+            main_status=1
+            while [ "$SECONDS" -lt "$deadline" ]; do
+              if timeout --foreground --signal=TERM --kill-after=2s 10s \
+                bcvk ephemeral ssh "$name" 'uname -a && true' \
+                >"$artifact_dir/guest-command.log" 2>&1; then
+                ssh_status=0
+              else
+                ssh_status=$?
+              fi
+              if [ "$ssh_status" -eq 0 ] && grep -Eq '^Linux ' "$artifact_dir/guest-command.log"; then
+                main_status=0
+                break
+              fi
+              sleep 2
+            done
+          fi
+          printf '%s\n' "$launcher_status" >"$artifact_dir/launcher-exit-status.txt"
+          printf '%s\n' "$main_status" >"$artifact_dir/main-exit-status.txt"
+          timeout --foreground --signal=TERM --kill-after=5s 10s podman logs "$name" >"$artifact_dir/container.log" 2>&1 || :
+          limit_log "$artifact_dir/launcher.log"
+          limit_log "$artifact_dir/container.log"
+          for log in "$log_dir"/journal.json "$log_dir"/journal-initrd.json "$log_dir"/console.txt; do
+            limit_log "$log"
+          done
+          exit "$main_status"
+
+      - name: Upload smoke diagnostics
+        if: ${{ always() && steps.qemu.outputs.full_vm_smoke == 'true' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: qemu-virtiofs-smoke-${{ matrix.architecture }}
+          path: |
+            ${{ runner.temp }}/qemu-smoke
+            ${{ runner.temp }}/image-digest.txt
+          retention-days: 7
diff --git a/bootc-ubuntu-setup/action.yml b/bootc-ubuntu-setup/action.yml
index 30298f2..f78f5f0 100644
--- a/bootc-ubuntu-setup/action.yml
+++ b/bootc-ubuntu-setup/action.yml
@@ -90,10 +90,14 @@ runs:
       shell: bash
       run: |
         set -xeuo pipefail
-        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-        sudo udevadm control --reload-rules
-        sudo udevadm trigger --name-match=kvm
-        ls -l /dev/kvm
+        if [ -c /dev/kvm ]; then
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+          ls -l /dev/kvm
+        else
+          printf 'No /dev/kvm character device; skipping KVM permission setup.\n'
+        fi
     # Used by a few workflows, but generally useful
     - name: Set architecture variable
       id: set_arch
@@ -105,24 +109,116 @@ runs:
       shell: bash
       run: |
         set -xeuo pipefail
-        # renovate: datasource=github-releases depName=bootc-dev/bcvk
-        export BCVK_VERSION=0.19.0
         # see https://github.com/bootc-dev/bcvk/issues/176
         /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system python3-virt-firmware
+
+    # bcvk requires KVM and v0.19.0 has no arm64 release asset.
+    - name: Install bcvk
+      if: ${{ inputs.libvirt == 'true' && runner.arch == 'X64' }}
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        # renovate: datasource=github-releases depName=bootc-dev/bcvk
+        export BCVK_VERSION=0.19.0
         # Something in the stack is overriding this, but we want session right now for bcvk
         echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV
         td=$(mktemp -d)
         cd $td
         # Install bcvk
         target=bcvk-$(arch)-unknown-linux-gnu
-        /bin/time -f '%E %C' curl --retry 35 --retry-delay 30 --retry-max-time 1200 -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz
-        tar xzf ${target}.tar.gz
-        sudo install -T ${target} /usr/bin/bcvk
+        /bin/time -f '%E %C' curl --fail --show-error --location --retry 35 --retry-delay 30 --retry-max-time 1200 --output "${target}.tar.gz" "https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz"
+        tar xzf "${target}.tar.gz"
+        sudo install -T "${target}" /usr/bin/bcvk
         cd -
         rm -rf "$td"
 
         # Also bump the default fd limit as a workaround for https://github.com/bootc-dev/bcvk/issues/65
         sudo sed -i -e 's,^\* hard nofile 65536,* hard nofile 524288,' /etc/security/limits.conf
+    # Ubuntu 24.04's QEMU loses virtiofs IRQ notifications.  Use the fixed
+    # package from Resolute on Ubuntu 24.04 for the disposable CI host; the
+    # Ubuntu 26.04 setup may provide its own native stack.  This is the same
+    # race investigated in https://github.com/bootc-dev/bootc/pull/2290:
+    # https://github.com/qemu/qemu/commit/1ba9a5220325dd5260a0c37b6299ce38364a5120
+    - name: Install fixed QEMU from Ubuntu Resolute
+      if: ${{ inputs.libvirt == 'true' }}
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        idv=$(. /usr/lib/os-release && printf '%s-%s' "$ID" "$VERSION_ID")
+        if [ "$idv" != ubuntu-24.04 ]; then
+          printf 'Skipping Resolute QEMU setup on %s\n' "$idv"
+          exit 0
+        fi
+        case "$(dpkg --print-architecture)" in
+          amd64)
+            qemu_package=qemu-system-x86
+            ;;
+          arm64)
+            qemu_package=qemu-system-arm
+            ;;
+          *)
+            printf 'Unsupported architecture for Resolute QEMU: %s\n' \
+              "$(dpkg --print-architecture)" >&2
+            exit 1
+            ;;
+        esac
+        qemu_binary="qemu-system-$(uname -m)"
+
+        # Keep Resolute below the normal candidate for every package.  The
+        # explicitly requested QEMU packages still bring their dependency
+        # closure (including glibc 2.43); that is acceptable on this
+        # disposable CI runner; copying just the executable would leave its
+        # runtime libraries behind.
+        printf '%s\n' \
+          'Package: *' \
+          'Pin: release n=resolute' \
+          'Pin-Priority: 50' | sudo tee /etc/apt/preferences.d/resolute-qemu >/dev/null
+
+        if [ "$(dpkg --print-architecture)" = amd64 ]; then
+          if [ -f /etc/apt/apt-mirrors.txt ]; then
+            mirror='mirror+file:/etc/apt/apt-mirrors.txt'
+          else
+            mirror='http://archive.ubuntu.com/ubuntu'
+          fi
+        else
+          mirror='http://ports.ubuntu.com/ubuntu-ports'
+        fi
+        printf 'deb %s resolute main universe\n' "$mirror" | \
+          sudo tee /etc/apt/sources.list.d/resolute-qemu.list >/dev/null
+        timeout --foreground --signal=TERM --kill-after=30s 5m sudo apt-get update
+
+        # Keep ipxe-qemu in this explicit target transaction: its ROM must
+        # match the QEMU userspace rather than the older package from Plucky.
+        timeout --foreground --signal=TERM --kill-after=30s 5m \
+          sudo apt-get -t resolute install -y --no-install-recommends \
+            "$qemu_package" qemu-system-common qemu-system-data qemu-utils ipxe-qemu
+
+        printf 'Selected QEMU: %s\n' "$qemu_binary"
+        "$qemu_binary" --version
+        dpkg-query -W -f='${binary:Package} ${Version}\n' \
+          "$qemu_package" qemu-system-common qemu-system-data qemu-utils ipxe-qemu
+        rom=
+        if [ "$(dpkg --print-architecture)" = amd64 ]; then
+          rom_names=(efi-virtio.rom)
+        else
+          rom_names=(efi-aa64.rom efi-virtio.rom)
+        fi
+        for rom_name in "${rom_names[@]}"; do
+          for candidate in "/usr/share/qemu/$rom_name" "/usr/lib/ipxe/qemu/$rom_name"; do
+            if [ -f "$candidate" ]; then
+              rom=$candidate
+              break
+            fi
+          done
+          [ -n "$rom" ] && break
+        done
+        if [ -n "$rom" ]; then
+          printf 'EFI ROM: %s (%s)\n' "$rom" "$(dpkg-query -S "$rom")"
+          sha256sum "$rom"
+        else
+          printf 'No expected EFI ROM found after installing ipxe-qemu\n' >&2
+          exit 1
+        fi
     - name: Cleanup status
       shell: bash
       run: |