From dc4b9c8f1333abcb4818ada4c2340f5fbcad2f2a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Wed, 2 Sep 2026 02:20:35 -0700 Subject: [PATCH] feat(vmm): default the 64-bit PCI hole to 8T The shipped default is 0, which leaves QEMU's own 32 GiB hole in place. That cannot host a GPU: an H200 SXM's VRAM aperture is 256 GiB and a B300's is 512 GiB, measured on hardware. Anyone starting from the file in this repository gets a host that cannot pass a GPU through, while every GPU host in production has been setting the value by hand. 8T rather than the 1P production runs. The hole is global -- one setting for every guest on the host -- and its size decides where the top of the guest address space lands: unset end 0x980000000 36 bits 8T end 0x80180000000 44 bits 1P end 0x4000180000000 51 bits 51 bits is past the 48-bit ceiling of 4-level paging, so a 1 PiB hole means 5-level EPT for every guest, and on TDX forces GPAW=52. GPU-less guests share this setting and would pay that for nothing. 8T keeps the top at 44 bits, under both that ceiling and the 46 physical address bits the fleet's hosts report, while still covering eight B300s twice over -- 8 x 512 GiB is 4 TiB. No performance delta was measured; the argument is structural, and 1P is demonstrably workable since the fleet runs it. But it is 250 times more than the largest supported topology needs, and crossing a paging boundary for that is a poor trade to make on behalf of every guest. Deployments needing more can still set it explicitly. This moves the measurement baseline. pci_hole64_size feeds qemu-acpi's MachineConfig, so the generated tables change and RTMR0 with them. Confirmed with the generator: hole=0, 8T and 1P each produce a distinct tables.bin, and 8T differs from 0 by four bytes -- the _CRS length field. The value is modelled correctly at all three sizes, so the baseline shifts rather than breaks. --- dstack/vmm/vmm.toml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 823884a8b..c8e8c8b94 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -64,11 +64,19 @@ use_mrconfigid = true # suffix: "512G", "2T", "1P" -- all binary multipliers, as are the "1PB" and # "1PiB" spellings. # -# 0 leaves the hole at QEMU's default, which is far below what GPU passthrough -# needs: a data center card's VRAM aperture alone is 256 GiB on an H200 SXM and -# 512 GiB on a B300, so eight of them want 2 to 4 TiB. Hosts running GPUs set -# this to "1P", which is what Phala's H200 fleet runs today. -qemu_pci_hole64_size = 0 +# QEMU's own default is far below what GPU passthrough needs: a data center +# card's VRAM aperture alone is 256 GiB on an H200 SXM and 512 GiB on a B300, +# so eight of them want 2 to 4 TiB. The hole costs guest address space rather +# than memory -- it is where firmware and the guest place 64-bit BARs -- so +# sizing it generously is cheaper than sizing it exactly. +# +# 8T covers eight B300s twice over while keeping the top of the guest address +# space at 44 bits. That matters because the hole is global: a larger one is +# not free for the GPU-less guests that share this setting. 1P, which the H200 +# fleet runs today, puts the top at 51 bits, past the 48-bit ceiling of 4-level +# paging -- so every guest, GPU or not, needs 5-level EPT and, on TDX, GPAW=52. +# Raise it explicitly if a deployment ever needs more than 8 TiB of BARs. +qemu_pci_hole64_size = "8T" qemu_hotplug_off = false # TDX attestation/hash scheme policy: # - "legacy": digest.txt + legacy verifier