From f785a0f2aa1c476f799dafb05cc4a754b6e0e84c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 1 Sep 2026 20:26:10 -0700 Subject: [PATCH] fix(os/image): enable MMCONFIG so Blackwell GPUs can be probed pci=nommconf confines PCI config space access to the legacy CF8h/CFCh port I/O path, which physically cannot reach past the first 256 bytes. pci_find_ext_capability() then short-circuits on dev->cfg_size and the whole PCIe extended config space (offset >= 0x100) becomes unreadable. Blackwell GPU drivers walk that space to find NVIDIA's vendor DVSEC. With MMCONFIG off the reads return 0xFFFFFFFF, the driver asserts on every probe, and every later cuInit() fails with CUDA_ERROR_SYSTEM_NOT_READY (802). No other mechanism reaches extended config space, so GPU passthrough cannot work without MMCONFIG. pci=noearly is deliberately kept. It gates early type 1 scanning, which runs before ACPI is parsed and dispatches into vendor fixups keyed on a vendor/device/class triple read straight out of host-controlled config space. GPU drivers bind during normal PCI enumeration and do not depend on it, and Intel's confidential-computing guest hardening names early PCI code as something to disable, so leaving it off costs nothing here. The command line is stated once per image backend, so the mkosi shared definition and the yocto UKI recipe move together. This changes the measured guest command line: the UKI PE section digests, metadata.json and the derived RTMRs all shift, so attestation baselines have to be refreshed with the image that ships it. --- os/image/kernel-cmdline.sh | 19 ++++++++++++++++++- .../recipes-core/images/dstack-uki.bb | 5 ++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/os/image/kernel-cmdline.sh b/os/image/kernel-cmdline.sh index 1847bc5b6..c900899be 100644 --- a/os/image/kernel-cmdline.sh +++ b/os/image/kernel-cmdline.sh @@ -13,6 +13,23 @@ # one would have produced measurements describing a command line the guest # never booted with -- and every existing check would still have passed. +# `pci=nommconf` is deliberately absent. It confines PCI config space access to +# the legacy CF8h/CFCh port I/O path, which physically cannot reach past the +# first 256 bytes, so `pci_find_ext_capability()` short-circuits on +# `dev->cfg_size <= PCI_CFG_SPACE_SIZE` and the whole PCIe extended config space +# (offset >= 0x100) becomes unreadable. Blackwell GPU drivers walk that space to +# find NVIDIA's vendor DVSEC; with MMCONFIG off the reads return 0xFFFFFFFF, the +# driver asserts on every probe, and every later cuInit() fails with +# CUDA_ERROR_SYSTEM_NOT_READY (802). There is no substitute mechanism, so GPU +# passthrough requires MMCONFIG. +# +# `pci=noearly` is kept. It gates a different mechanism -- early type 1 scanning, +# which runs before ACPI is parsed and dispatches into vendor fixups keyed on the +# vendor/device/class triple read straight out of a host-controlled config space. +# GPU drivers bind during normal PCI enumeration and do not depend on it, and +# Intel's confidential-computing guest hardening names early PCI code as +# something to disable, so it stays off. + # Emit the guest kernel command line. # $1 dm-verity root hash # $2 rootfs data size in bytes @@ -20,7 +37,7 @@ dstack_kernel_cmdline() { local root_hash=${1:?root hash required} local data_size=${2:?data size required} echo "console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0" \ - "mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y" \ + "mce=off oops=panic pci=noearly random.trust_cpu=y" \ "random.trust_bootloader=n tsc=reliable no-kvmclock" \ "dstack.rootfs_hash=$root_hash dstack.rootfs_size=$data_size" } diff --git a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb index 600240518..41c530bff 100644 --- a/os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb +++ b/os/yocto/layers/meta-dstack/recipes-core/images/dstack-uki.bb @@ -22,8 +22,11 @@ KERNEL_IMAGETYPE = "bzImage" # Do NOT embed per-app dstack.mr_config_id here: AWS app/config binding is # measured from the shared-disk MrConfigV3 into NitroTPM PCR8 at guest setup, # so the UKI/AMI stays app-independent (PCR4 is OS-only). +# Keep in sync with dstack_kernel_cmdline() in os/image/kernel-cmdline.sh, which +# documents why pci=nommconf is absent (Blackwell needs PCIe extended config +# space) and why pci=noearly stays. UKI_CMDLINE_BASE = "console=ttyS0 init=/init panic=1 net.ifnames=0 biosdevname=0 \ -mce=off oops=panic pci=noearly pci=nommconf random.trust_cpu=y random.trust_bootloader=n \ +mce=off oops=panic pci=noearly random.trust_cpu=y random.trust_bootloader=n \ tsc=reliable no-kvmclock" # Flavor settings (should match dstack-rootfs.bb, set via multiconfig)