diff --git a/os/mkosi/components/kernel/kernel.config b/os/mkosi/components/kernel/kernel.config index 706a33d36..822e539b9 100644 --- a/os/mkosi/components/kernel/kernel.config +++ b/os/mkosi/components/kernel/kernel.config @@ -37,6 +37,32 @@ CONFIG_VSOCKETS=y CONFIG_NET_9P=y CONFIG_NET_9P_VIRTIO=y CONFIG_9P_FS=y + +# Let the SWIOTLB bounce buffer grow at runtime. Every DMA in a CVM is bounced +# through SWIOTLB because the host cannot reach the guest's private memory, and +# without this the boot-time pool is all the guest ever gets -- 6% of RAM +# clamped to 1 GiB by arch/x86/mm/mem_encrypt.c. Bursty virtio traffic exhausts +# it ("virtio-pci 0000:00:02.0: swiotlb buffer is full (sz: 262144 bytes), +# total 524288 (slots), used 523564 (slots)") and the mapping then fails with +# -ENOMEM, which surfaces as dropped packets or block I/O errors rather than as +# anything naming SWIOTLB. +# +# Preferred over a larger fixed pool from a swiotlb= command line argument. That +# command line is measured into the RTMRs and has to stay in step across +# dstack-uki.bb and os/image/kernel-cmdline.sh, and raising it cannot buy much +# anyway: the boot pool comes from memblock_alloc_low() -- the x86 KVM path +# never sets SWIOTLB_ANY, only Xen does -- so it has to fit below 4 GiB, and a +# request that does not fit is silently halved rather than refused. Pools +# allocated at runtime are bounded by the device's DMA mask instead. +# +# The cost falls on the unmap and sync paths, where swiotlb_find_pool() becomes +# an RCU list walk rather than a range check against the single pool. The +# synchronous allocation path also needs CONFIG_DMA_COHERENT_POOL, which +# CONFIG_AMD_MEM_ENCRYPT already selects. +# +# Kept in step with meta-dstack's dstack.cfg. +CONFIG_SWIOTLB_DYNAMIC=y + CONFIG_BLK_DEV_NVME=y CONFIG_NVME_CORE=y CONFIG_NET_VENDOR_GOOGLE=y diff --git a/os/mkosi/parity.json b/os/mkosi/parity.json index dc42891b5..0b10d4802 100644 --- a/os/mkosi/parity.json +++ b/os/mkosi/parity.json @@ -112,6 +112,7 @@ }, "required_kernel_config": [ "CONFIG_ACPI_TABLE_UPGRADE=y", + "CONFIG_SWIOTLB_DYNAMIC=y", "CONFIG_INTEL_TDX_GUEST=y", "CONFIG_TDX_GUEST_DRIVER=y", "CONFIG_AMD_MEM_ENCRYPT=y", diff --git a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg index 9f49c406b..7c55f7502 100644 --- a/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg +++ b/os/yocto/layers/meta-dstack/recipes-kernel/linux/files/dstack.cfg @@ -9,6 +9,31 @@ CONFIG_PCI=y CONFIG_TUN=m CONFIG_VIRTIO_PCI=y +# Let the SWIOTLB bounce buffer grow at runtime. Every DMA in a CVM is bounced +# through SWIOTLB because the host cannot reach the guest's private memory, and +# without this the boot-time pool is all the guest ever gets -- 6% of RAM +# clamped to 1 GiB by arch/x86/mm/mem_encrypt.c. Bursty virtio traffic exhausts +# it ("virtio-pci 0000:00:02.0: swiotlb buffer is full (sz: 262144 bytes), +# total 524288 (slots), used 523564 (slots)") and the mapping then fails with +# -ENOMEM, which surfaces as dropped packets or block I/O errors rather than as +# anything naming SWIOTLB. +# +# Preferred over a larger fixed pool from a swiotlb= command line argument. That +# command line is measured into the RTMRs and has to stay in step across +# dstack-uki.bb and os/image/kernel-cmdline.sh, and raising it cannot buy much +# anyway: the boot pool comes from memblock_alloc_low() -- the x86 KVM path +# never sets SWIOTLB_ANY, only Xen does -- so it has to fit below 4 GiB, and a +# request that does not fit is silently halved rather than refused. Pools +# allocated at runtime are bounded by the device's DMA mask instead. +# +# The cost falls on the unmap and sync paths, where swiotlb_find_pool() becomes +# an RCU list walk rather than a range check against the single pool. The +# synchronous allocation path also needs CONFIG_DMA_COHERENT_POOL, which +# CONFIG_AMD_MEM_ENCRYPT already selects. +# +# Kept in step with os/mkosi/components/kernel/kernel.config. +CONFIG_SWIOTLB_DYNAMIC=y + # Network NIC drivers. GCP C3/modern instances use gVNIC (the Google gve # driver); older/virtio instances use virtio-net. Without gve, a C3 CVM gets # no NIC -> no DHCP -> no network. Build both in.