Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions Sources/Containerization/LinuxContainer.swift
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,16 @@ public final class LinuxContainer: Container, Sendable {
public var sockets: [UnixSocketConfiguration] = []
/// The mounts for the container.
public var mounts: [Mount] = LinuxContainer.defaultMounts()
/// Paths inside the container that vmexec hides from the workload.
/// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``),
/// matching the restricted capability baseline. Set to `[]` to opt out,
/// or append to extend it.
public var maskedPaths: [String] = LinuxContainer.defaultMaskedPaths()
/// Paths inside the container that vmexec marks read-only.
/// Defaults to the OCI standard set (``LinuxContainer/defaultReadonlyPaths()``),
/// matching the restricted capability baseline. Set to `[]` to opt out,
/// or append to extend it.
public var readonlyPaths: [String] = LinuxContainer.defaultReadonlyPaths()
/// The DNS configuration for the container.
public var dns: DNS?
/// The hosts to add to /etc/hosts for the container.
Expand Down Expand Up @@ -100,6 +110,8 @@ public final class LinuxContainer: Container, Sendable {
interfaces: [any Interface] = [],
sockets: [UnixSocketConfiguration] = [],
mounts: [Mount] = LinuxContainer.defaultMounts(),
maskedPaths: [String] = LinuxContainer.defaultMaskedPaths(),
readonlyPaths: [String] = LinuxContainer.defaultReadonlyPaths(),
dns: DNS? = nil,
hosts: Hosts? = nil,
virtualization: Bool = false,
Expand All @@ -117,6 +129,8 @@ public final class LinuxContainer: Container, Sendable {
self.interfaces = interfaces
self.sockets = sockets
self.mounts = mounts
self.maskedPaths = maskedPaths
self.readonlyPaths = readonlyPaths
self.dns = dns
self.hosts = hosts
self.virtualization = virtualization
Expand Down Expand Up @@ -394,6 +408,8 @@ public final class LinuxContainer: Container, Sendable {

// Linux toggles.
spec.linux?.sysctl = config.sysctl
spec.linux?.maskedPaths = config.maskedPaths
spec.linux?.readonlyPaths = config.readonlyPaths

// If the rootfs was requested as read-only, set it in the OCI spec.
// We let the OCI runtime remount as ro, instead of doing it originally.
Expand Down Expand Up @@ -438,6 +454,44 @@ public final class LinuxContainer: Container, Sendable {
]
}

/// The default set of paths to mask inside a container, matching the OCI
/// runtime spec defaults that runc and other production runtimes apply.
/// Each path is hidden from the workload (replaced by `/dev/null` for files
/// or an empty tmpfs for directories) by `vmexec` after `pivot_root`.
///
/// Applied by default (see ``Configuration/maskedPaths``); set
/// `config.maskedPaths = []` to opt out, or append to extend the set.
public static func defaultMaskedPaths() -> [String] {
[
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware",
"/sys/devices/virtual/powercap",
]
}

/// The default set of paths to mark read-only inside a container, matching
/// the OCI runtime spec defaults that runc and other production runtimes apply.
///
/// Applied by default (see ``Configuration/readonlyPaths``); set
/// `config.readonlyPaths = []` to opt out, or append to extend the set.
public static func defaultReadonlyPaths() -> [String] {
[
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger",
]
}

/// A more traditional default set of mounts that OCI runtimes typically employ.
public static func defaultOCIMounts() -> [Mount] {
let defaultOptions = ["nosuid", "noexec", "nodev"]
Expand Down
12 changes: 12 additions & 0 deletions Sources/Containerization/LinuxPod.swift
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ public final class LinuxPod: Sendable {
public var sysctl: [String: String] = [:]
/// The mounts for the container.
public var mounts: [Mount] = LinuxContainer.defaultMounts()
/// Paths inside the container that vmexec hides from the workload.
/// Defaults to the OCI standard set (``LinuxContainer/defaultMaskedPaths()``),
/// matching the restricted capability baseline. Set to `[]` to opt out,
/// or append to extend it.
public var maskedPaths: [String] = LinuxContainer.defaultMaskedPaths()
/// Paths inside the container that vmexec marks read-only.
/// Defaults to the OCI standard set (``LinuxContainer/defaultReadonlyPaths()``),
/// matching the restricted capability baseline. Set to `[]` to opt out,
/// or append to extend it.
public var readonlyPaths: [String] = LinuxContainer.defaultReadonlyPaths()
/// The Unix domain socket relays to setup for the container.
public var sockets: [UnixSocketConfiguration] = []
/// The DNS configuration for the container.
Expand Down Expand Up @@ -290,6 +300,8 @@ public final class LinuxPod: Sendable {

// Linux toggles
spec.linux?.sysctl = config.sysctl
spec.linux?.maskedPaths = config.maskedPaths
spec.linux?.readonlyPaths = config.readonlyPaths

// If the rootfs was requested as read-only, set it in the OCI spec.
// We let the OCI runtime remount as ro, instead of doing it originally.
Expand Down
64 changes: 64 additions & 0 deletions Sources/Integration/ContainerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1697,6 +1697,70 @@ extension IntegrationSuite {
}
}

func testDefaultMaskedAndReadonlyPaths() async throws {
let id = "test-masked-readonly-defaults"

// A default container (default capabilities + default masked/readonly
// paths) must have the OCI standard set enforced by vmexec without any
// opt-in. Probe from inside the guest:
// 1. readonlyPaths: writing under /proc/sys fails with EROFS. EROFS
// (not EPERM) proves the read-only remount rather than a mere
// capability denial — the default (restricted) caps already lack
// CAP_SYS_ADMIN, so a writable /proc/sys would fail with EPERM.
// The write is wrapped in a brace group so the shell's redirection
// failure ("can't create ...: Read-only file system") is captured:
// a trailing `2>&1` on the command misses it, because the failing
// `>` redirection aborts before `2>&1` is applied.
// 2. maskedPaths: at least one default-masked path is mounted over
// (visible in /proc/self/mountinfo). Checked via mountinfo rather
// than reading the target, since some masked paths (e.g.
// /proc/kcore) require CAP_SYS_RAWIO to read and would appear empty
// even if masking were broken, yielding a false pass.
let probe = """
set -u
rerr=$( { echo x > /proc/sys/kernel/hostname; } 2>&1 )
case "$rerr" in
*"Read-only file system"*) ;;
*) echo "RO-FAIL: writing /proc/sys expected EROFS, got: ${rerr:-<write succeeded>}"; exit 1 ;;
esac
masked=0
for p in /proc/kcore /proc/keys /proc/scsi /proc/sched_debug /sys/firmware /sys/devices/virtual/powercap; do
grep -q " $p " /proc/self/mountinfo && masked=$((masked + 1))
done
if [ "$masked" -eq 0 ]; then
echo "MASK-FAIL: no default masked path mounted"
cat /proc/self/mountinfo
exit 1
fi
echo "MASKED-RO-OK masked=$masked"
"""

let bs = try await bootstrap(id)
let buffer = BufferWriter()
let container = try LinuxContainer(id, rootfs: bs.rootfs, vmm: bs.vmm) { config in
// Intentionally leave config.maskedPaths / readonlyPaths /
// process.capabilities untouched — the point is that the defaults
// are secure out of the box.
config.process.arguments = ["/bin/sh", "-c", probe]
config.process.stdout = buffer
config.bootLog = bs.bootLog
}

try await container.create()
try await container.start()

let status = try await container.wait()
try await container.stop()

let output = String(data: buffer.data, encoding: .utf8) ?? "<non-utf8 output>"
guard status.exitCode == 0 else {
throw IntegrationError.assert(msg: "default masked/readonly enforcement failed (exit \(status.exitCode)): \(output)")
}
guard output.contains("MASKED-RO-OK") else {
throw IntegrationError.assert(msg: "expected MASKED-RO-OK sentinel, got: \(output)")
}
}

func testStat() async throws {
let id = "test-stat"

Expand Down
3 changes: 3 additions & 0 deletions Sources/Integration/Suite.swift
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,9 @@ struct IntegrationSuite: AsyncParsableCommand {
Test("container capabilities all capabilities", testCapabilitiesAllCapabilities),
Test("container capabilities file ownership", testCapabilitiesFileOwnership),

// Masked / read-only paths
Test("container default masked and read-only paths", testDefaultMaskedAndReadonlyPaths),

// Stat / Copy
Test("container stat", testStat),
Test("container copy in", testCopyIn),
Expand Down
27 changes: 27 additions & 0 deletions Tests/ContainerizationTests/LinuxContainerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -92,4 +92,31 @@ struct LinuxContainerTests {
#expect(viaProperty.capabilities.ambient == expected.ambient)
#expect(viaInit.capabilities.bounding == expected.bounding)
}

@Test func defaultMaskedAndReadonlyPathsAreOCISet() {
// Regression guard: masked/readonly paths must default to the OCI
// standard set now that capabilities default to the restricted baseline.
// Without CAP_SYS_ADMIN a workload can't unmount these, so the defaults
// are meaningful defense-in-depth — shipping empty defaults would leave
// /proc/kcore and friends exposed. Cover both construction paths and
// both configuration types.
let expectedMasked = LinuxContainer.defaultMaskedPaths()
let expectedReadonly = LinuxContainer.defaultReadonlyPaths()

// Sensitive kernel paths must actually be in the defaults.
#expect(expectedMasked.contains("/proc/kcore"))
#expect(expectedMasked.contains("/sys/firmware"))
#expect(expectedReadonly.contains("/proc/sys"))

let containerViaProperty = LinuxContainer.Configuration()
let containerViaInit = LinuxContainer.Configuration(process: LinuxProcessConfiguration(arguments: ["/bin/sh"]))
let pod = LinuxPod.ContainerConfiguration()

for config in [containerViaProperty, containerViaInit] {
#expect(config.maskedPaths == expectedMasked)
#expect(config.readonlyPaths == expectedReadonly)
}
#expect(pod.maskedPaths == expectedMasked)
#expect(pod.readonlyPaths == expectedReadonly)
}
}
78 changes: 77 additions & 1 deletion vminitd/Sources/vmexec/RunCommand.swift
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,10 @@ struct RunCommand: ParsableCommand {
}
}

private func childRootSetup(rootfs: ContainerizationOCI.Root, mounts: [ContainerizationOCI.Mount]) throws {
private func childRootSetup(
rootfs: ContainerizationOCI.Root,
mounts: [ContainerizationOCI.Mount]
) throws {
// setup rootfs
try prepareRoot(rootfs: rootfs.path)
try mountRootfs(rootfs: rootfs.path, mounts: mounts)
Expand All @@ -69,6 +72,71 @@ struct RunCommand: ParsableCommand {
try reOpenDevNull()
}

/// Mask paths per OCI `linux.maskedPaths`. Files (and any non-directory)
/// get `/dev/null` bind-mounted on top; directories get an empty read-only
/// tmpfs. Missing paths are skipped silently — matches runc's `maskPath`.
private func applyMaskedPaths(_ paths: [String]) throws {
for path in paths {
var st = stat()
if stat(path, &st) != 0 {
if errno == ENOENT {
continue
}
throw App.Errno(stage: "stat(\(path)) for mask")
}

if (st.st_mode & S_IFMT) == S_IFDIR {
// Match runc: mask directories with a read-only tmpfs. MS_RDONLY
// is what actually prevents writes into the masked dir; a
// `size=0k` option would be a no-op (the kernel treats tmpfs
// size=0 as "no limit", not an empty filesystem).
guard mount("tmpfs", path, "tmpfs", UInt(MS_RDONLY | MS_NOSUID | MS_NODEV | MS_NOEXEC), nil) == 0 else {
throw App.Errno(stage: "mount(tmpfs mask \(path))")
}
} else {
guard mount("/dev/null", path, "bind", UInt(MS_BIND), nil) == 0 else {
throw App.Errno(stage: "mount(bind /dev/null -> \(path))")
}
}
}
}

/// Make paths read-only per OCI `linux.readonlyPaths` by bind-mounting
/// each onto itself and remounting with `MS_RDONLY`. Missing paths are
/// skipped silently — matches runc's `readonlyPath`. The statfs fallback
/// mirrors `remountRootfsReadOnly()` for filesystems whose existing flags
/// (e.g. nosuid, nodev) must be preserved on the remount.
private func applyReadonlyPaths(_ paths: [String]) throws {
for path in paths {
var st = stat()
if stat(path, &st) != 0 {
if errno == ENOENT {
continue
}
throw App.Errno(stage: "stat(\(path)) for readonly")
}

guard mount(path, path, "", UInt(MS_BIND | MS_REC), nil) == 0 else {
throw App.Errno(stage: "mount(bind \(path))")
}

var flags = UInt(MS_BIND | MS_REMOUNT | MS_RDONLY)
if mount("", path, "", flags, "") == 0 {
continue
}

var s = statfs()
guard statfs(path, &s) == 0 else {
throw App.Errno(stage: "statfs(\(path))")
}
flags |= UInt(s.f_flags)

guard mount("", path, "", flags, "") == 0 else {
throw App.Errno(stage: "mount remount-ro \(path)")
}
}
}

private func remountRootfsReadOnly() throws {
var flags = UInt(MS_BIND | MS_REMOUNT | MS_RDONLY)

Expand Down Expand Up @@ -181,6 +249,14 @@ struct RunCommand: ParsableCommand {
}
}

// Apply OCI maskedPaths/readonlyPaths AFTER sysctls (writes to
// /proc/sys/* would otherwise fail once /proc/sys is remounted ro)
// and BEFORE the user/capability change (mount() requires
// CAP_SYS_ADMIN, which we still have here as root). Mask runs first
// so a path appearing in both lists is hidden, not just locked.
try self.applyMaskedPaths(spec.linux?.maskedPaths ?? [])
try self.applyReadonlyPaths(spec.linux?.readonlyPaths ?? [])

// Apply O_CLOEXEC to all file descriptors except stdio.
// This ensures that all unwanted fds we may have accidentally
// inherited are marked close-on-exec so they stay out of the
Expand Down
6 changes: 3 additions & 3 deletions vminitd/Sources/vmexec/vmexec.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@
//===----------------------------------------------------------------------===//

/// NOTE: This binary implements a very small subset of the OCI runtime spec, mostly just
/// the process configurations. Mounts are somewhat functional, but masked and read only paths
/// aren't checked today. Today the namespaces are also ignored, and we always spawn a new pid
/// and mount namespace.
/// the process configurations. Mounts, masked paths, and read-only paths are enforced.
/// The `network` namespace is currently ignored and we always spawn a new pid and mount
/// namespace.

import ArgumentParser
import ContainerizationError
Expand Down
Loading