Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 152 additions & 0 deletions .github/workflows/test-virtualization.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
name: Test virtualization setup

on:
pull_request:
paths:
- bootc-ubuntu-setup/action.yml
- .github/workflows/test-virtualization.yml
workflow_dispatch:

permissions:
contents: read

jobs:
qemu-smoke:
name: 'QEMU smoke: ${{ matrix.name }}'
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-24.04
architecture: amd64
name: amd64 virtiofs (KVM required)
full_vm_smoke: true
- runner: ubuntu-24.04-arm
architecture: arm64
name: arm64 QEMU startup (TCG)
full_vm_smoke: false
runs-on: ${{ matrix.runner }}
timeout-minutes: 25
steps:
- name: Checkout actions
uses: actions/checkout@v7
with:
persist-credentials: false

- name: Set up Ubuntu host
uses: ./bootc-ubuntu-setup
with:
libvirt: true

- name: Verify host QEMU and select acceleration
id: qemu
shell: bash
env:
FULL_VM_SMOKE: ${{ matrix.full_vm_smoke }}
run: |
set -euo pipefail
qemu="qemu-system-$(uname -m)"
"$qemu" --version
if [ "$(uname -m)" = x86_64 ]; then
machine=q35
else
machine=virt
fi
"$qemu" -machine help | grep -Eq "(^|[[:space:]])${machine}([[:space:]]|$)"
if [ "$FULL_VM_SMOKE" = true ]; then
if ! [ -c /dev/kvm ] || ! [ -r /dev/kvm ] || ! [ -w /dev/kvm ]; then
printf 'KVM is required for the full VM smoke, but /dev/kvm is unavailable.\n' >&2
exit 1
fi
echo 'full_vm_smoke=true' >> "$GITHUB_OUTPUT"
else
# bcvk requires KVM and is not available for arm64. Always use TCG
# here, even if a future arm64 runner exposes /dev/kvm.
printf 'Smoke testing QEMU startup with TCG.\n'
tcg_status=0
timeout --foreground --signal=TERM --kill-after=5s 10s \
"$qemu" -machine "${machine},accel=tcg" -display none -nodefaults -S || \
tcg_status=$?
if [ "$tcg_status" -ne 124 ]; then
printf 'QEMU TCG smoke expected timeout status 124, got %s.\n' "$tcg_status" >&2
exit 1
fi
echo 'full_vm_smoke=false' >> "$GITHUB_OUTPUT"
fi

- name: Pull smoke image
if: ${{ steps.qemu.outputs.full_vm_smoke == 'true' }}
shell: bash
run: |
set -euo pipefail
image=quay.io/centos-bootc/centos-bootc:stream9
timeout --foreground --signal=TERM --kill-after=30s 5m podman pull -q "$image"
podman image inspect "$image" --format '{{.RepoDigests}}' >"$RUNNER_TEMP/image-digest.txt"

- name: Run ephemeral VM
if: ${{ steps.qemu.outputs.full_vm_smoke == 'true' }}
shell: bash
run: |
set -euo pipefail
artifact_dir="$RUNNER_TEMP/qemu-smoke"
log_dir="$artifact_dir/vm-logs"
name="qemu-smoke-${GITHUB_RUN_ID}"
mkdir -p "$log_dir"
limit_log() {
local log=$1
if [ -f "$log" ] && [ "$(stat -c %s "$log")" -gt 5242880 ]; then
truncate --size=5242880 "$log"
fi
}
# shellcheck disable=SC2317 # invoked by the EXIT trap below
cleanup() {
timeout --foreground --signal=TERM --kill-after=5s 20s podman stop --time 20 "$name" >/dev/null 2>&1 || :
timeout --foreground --signal=TERM --kill-after=5s 10s podman rm -f "$name" >/dev/null 2>&1 || :
}
trap cleanup EXIT

set +e
timeout --foreground --signal=TERM --kill-after=10s 90s \

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, this is all gross. I think what we should really do is have something like bcvk ephemeral selftest --image quay.io/centos-bootc/centos-bootc:stream9, like move some core smoke testing of functionality into the binary itself.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bcvk ephemeral run -d -K --name "$name" --console \
--log-dir "journal,console=$log_dir" \
quay.io/centos-bootc/centos-bootc:stream9 \
>"$artifact_dir/launcher.log" 2>&1
launcher_status=$?
main_status=$launcher_status
if [ "$launcher_status" -eq 0 ]; then
deadline=$((SECONDS + 90))
main_status=1
while [ "$SECONDS" -lt "$deadline" ]; do
if timeout --foreground --signal=TERM --kill-after=2s 10s \
bcvk ephemeral ssh "$name" 'uname -a && true' \
>"$artifact_dir/guest-command.log" 2>&1; then
ssh_status=0
else
ssh_status=$?
fi
if [ "$ssh_status" -eq 0 ] && grep -Eq '^Linux ' "$artifact_dir/guest-command.log"; then
main_status=0
break
fi
sleep 2
done
fi
printf '%s\n' "$launcher_status" >"$artifact_dir/launcher-exit-status.txt"
printf '%s\n' "$main_status" >"$artifact_dir/main-exit-status.txt"
timeout --foreground --signal=TERM --kill-after=5s 10s podman logs "$name" >"$artifact_dir/container.log" 2>&1 || :
limit_log "$artifact_dir/launcher.log"
limit_log "$artifact_dir/container.log"
for log in "$log_dir"/journal.json "$log_dir"/journal-initrd.json "$log_dir"/console.txt; do
limit_log "$log"
done
exit "$main_status"

- name: Upload smoke diagnostics
if: ${{ always() && steps.qemu.outputs.full_vm_smoke == 'true' }}
uses: actions/upload-artifact@v7
with:
name: qemu-virtiofs-smoke-${{ matrix.architecture }}
path: |
${{ runner.temp }}/qemu-smoke
${{ runner.temp }}/image-digest.txt
retention-days: 7
114 changes: 105 additions & 9 deletions bootc-ubuntu-setup/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,14 @@ runs:
shell: bash
run: |
set -xeuo pipefail
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
ls -l /dev/kvm
if [ -c /dev/kvm ]; then
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
ls -l /dev/kvm
else
printf 'No /dev/kvm character device; skipping KVM permission setup.\n'
fi
# Used by a few workflows, but generally useful
- name: Set architecture variable
id: set_arch
Expand All @@ -105,24 +109,116 @@ runs:
shell: bash
run: |
set -xeuo pipefail
# renovate: datasource=github-releases depName=bootc-dev/bcvk
export BCVK_VERSION=0.19.0
# see https://github.com/bootc-dev/bcvk/issues/176
/bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system python3-virt-firmware

# bcvk requires KVM and v0.19.0 has no arm64 release asset.
- name: Install bcvk
if: ${{ inputs.libvirt == 'true' && runner.arch == 'X64' }}
shell: bash
run: |
set -xeuo pipefail
# renovate: datasource=github-releases depName=bootc-dev/bcvk
export BCVK_VERSION=0.19.0
# Something in the stack is overriding this, but we want session right now for bcvk
echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV
td=$(mktemp -d)
cd $td
# Install bcvk
target=bcvk-$(arch)-unknown-linux-gnu
/bin/time -f '%E %C' curl --retry 35 --retry-delay 30 --retry-max-time 1200 -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz
tar xzf ${target}.tar.gz
sudo install -T ${target} /usr/bin/bcvk
/bin/time -f '%E %C' curl --fail --show-error --location --retry 35 --retry-delay 30 --retry-max-time 1200 --output "${target}.tar.gz" "https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz"
tar xzf "${target}.tar.gz"
sudo install -T "${target}" /usr/bin/bcvk
cd -
rm -rf "$td"

# Also bump the default fd limit as a workaround for https://github.com/bootc-dev/bcvk/issues/65
sudo sed -i -e 's,^\* hard nofile 65536,* hard nofile 524288,' /etc/security/limits.conf
# Ubuntu 24.04's QEMU loses virtiofs IRQ notifications. Use the fixed
# package from Resolute on Ubuntu 24.04 for the disposable CI host; the
# Ubuntu 26.04 setup may provide its own native stack. This is the same
# race investigated in https://github.com/bootc-dev/bootc/pull/2290:
# https://github.com/qemu/qemu/commit/1ba9a5220325dd5260a0c37b6299ce38364a5120
- name: Install fixed QEMU from Ubuntu Resolute
if: ${{ inputs.libvirt == 'true' }}
shell: bash
run: |
set -xeuo pipefail
idv=$(. /usr/lib/os-release && printf '%s-%s' "$ID" "$VERSION_ID")
if [ "$idv" != ubuntu-24.04 ]; then
printf 'Skipping Resolute QEMU setup on %s\n' "$idv"
exit 0
fi
case "$(dpkg --print-architecture)" in
amd64)
qemu_package=qemu-system-x86
;;
arm64)
qemu_package=qemu-system-arm
;;
*)
printf 'Unsupported architecture for Resolute QEMU: %s\n' \
"$(dpkg --print-architecture)" >&2
exit 1
;;
esac
qemu_binary="qemu-system-$(uname -m)"

# Keep Resolute below the normal candidate for every package. The
# explicitly requested QEMU packages still bring their dependency
# closure (including glibc 2.43); that is acceptable on this
# disposable CI runner; copying just the executable would leave its
# runtime libraries behind.
printf '%s\n' \
'Package: *' \
'Pin: release n=resolute' \
'Pin-Priority: 50' | sudo tee /etc/apt/preferences.d/resolute-qemu >/dev/null

if [ "$(dpkg --print-architecture)" = amd64 ]; then
if [ -f /etc/apt/apt-mirrors.txt ]; then
mirror='mirror+file:/etc/apt/apt-mirrors.txt'
else
mirror='http://archive.ubuntu.com/ubuntu'
fi
else
mirror='http://ports.ubuntu.com/ubuntu-ports'
fi
printf 'deb %s resolute main universe\n' "$mirror" | \
sudo tee /etc/apt/sources.list.d/resolute-qemu.list >/dev/null
timeout --foreground --signal=TERM --kill-after=30s 5m sudo apt-get update

# Keep ipxe-qemu in this explicit target transaction: its ROM must
# match the QEMU userspace rather than the older package from Plucky.
timeout --foreground --signal=TERM --kill-after=30s 5m \
sudo apt-get -t resolute install -y --no-install-recommends \
"$qemu_package" qemu-system-common qemu-system-data qemu-utils ipxe-qemu

printf 'Selected QEMU: %s\n' "$qemu_binary"
"$qemu_binary" --version
dpkg-query -W -f='${binary:Package} ${Version}\n' \
"$qemu_package" qemu-system-common qemu-system-data qemu-utils ipxe-qemu
rom=
if [ "$(dpkg --print-architecture)" = amd64 ]; then
rom_names=(efi-virtio.rom)
else
rom_names=(efi-aa64.rom efi-virtio.rom)
fi
for rom_name in "${rom_names[@]}"; do
for candidate in "/usr/share/qemu/$rom_name" "/usr/lib/ipxe/qemu/$rom_name"; do
if [ -f "$candidate" ]; then
rom=$candidate
break
fi
done
[ -n "$rom" ] && break
done
if [ -n "$rom" ]; then
printf 'EFI ROM: %s (%s)\n' "$rom" "$(dpkg-query -S "$rom")"
sha256sum "$rom"
else
printf 'No expected EFI ROM found after installing ipxe-qemu\n' >&2
exit 1
fi
- name: Cleanup status
shell: bash
run: |
Expand Down