From 3047690cc8b8e9097f3656488795a0f5e8592ca8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:00:27 -0700 Subject: [PATCH 01/22] feat(sandbox): split capability-free supervisor runtime Signed-off-by: Drew Newberry --- .github/workflows/branch-checks.yml | 10 +- AGENTS.md | 2 +- Cargo.lock | 86 +- Cargo.toml | 2 +- architecture/build.md | 18 +- architecture/compute-runtimes.md | 189 +- architecture/sandbox.md | 228 +- architecture/security-policy.md | 16 +- .../src/provider_credentials.rs | 86 +- crates/openshell-core/src/sandbox_env.rs | 18 + .../openshell-isolation-interface/Cargo.toml | 15 +- .../src/boundary_protocol.rs | 1183 +++ .../src/contract.rs | 124 +- .../src/contract/tests.rs | 43 +- .../openshell-isolation-interface/src/lib.rs | 9 +- .../src/linux/landlock.rs | 22 + .../src/linux/proc_fd.rs | 76 + .../src/linux/seccomp_notify.rs | 75 +- .../src/linux/socket_registry.rs | 53 +- .../src/linux/task_memory.rs | 209 +- .../src/remote.rs | 1806 +++++ crates/openshell-sandbox/Cargo.toml | 59 +- .../src/boundary_exec.rs | 122 +- .../src/boundary_io.rs | 84 +- .../openshell-sandbox/src/boundary_server.rs | 3300 +++++++++ .../src/child_env.rs | 38 - crates/openshell-sandbox/src/delegated.rs | 241 + .../src/google_cloud_metadata.rs | 536 -- .../src/identity.rs | 0 crates/openshell-sandbox/src/lib.rs | 6334 +---------------- crates/openshell-sandbox/src/main.rs | 2441 +++++-- crates/openshell-sandbox/src/main_session.rs | 1067 +++ .../src/managed_children.rs | 0 .../openshell-sandbox/src/metadata_server.rs | 231 - .../openshell-sandbox/src/network_broker.rs | 1797 +++++ .../src/process.rs | 684 +- crates/openshell-sandbox/src/pty.rs | 144 + .../src/sandbox/linux/landlock.rs | 73 +- .../src/sandbox/linux/mod.rs | 50 +- .../src/sandbox/linux/seccomp.rs | 7 +- .../src/sandbox/mod.rs | 0 .../openshell-sandbox/src/sidecar_control.rs | 1210 ---- .../openshell-sandbox/tests/stdout_logging.rs | 19 +- crates/openshell-server/src/grpc/sandbox.rs | 17 +- .../data/sandbox-policy.rego | 13 +- .../src/l7/rest.rs | 59 +- .../src/l7/tls.rs | 91 +- .../openshell-supervisor-network/src/lib.rs | 1 + .../openshell-supervisor-network/src/opa.rs | 47 +- .../src/policy_dns/mod.rs | 159 +- .../src/policy_dns/runtime.rs | 72 + .../src/policy_dns/store.rs | 11 + .../openshell-supervisor-network/src/proxy.rs | 923 ++- .../src/proxy/egress.rs | 2 + .../src/proxy/tests/compatibility.rs | 208 +- .../openshell-supervisor-network/src/run.rs | 53 +- .../src/spiffe_endpoint.rs | 18 + .../src/upstream_proxy.rs | 47 +- .../openshell-supervisor-process/Cargo.toml | 12 +- .../src/bypass_monitor/mod.rs | 651 -- .../src/bypass_monitor/procfs.rs | 318 - .../src/delegated.rs | 256 + .../openshell-supervisor-process/src/lib.rs | 24 +- .../src/main_session.rs | 267 +- .../src/netns/mod.rs | 1239 ---- .../src/netns/nft_ruleset.rs | 875 --- .../openshell-supervisor-process/src/run.rs | 830 --- .../openshell-supervisor-process/src/ssh.rs | 2437 ++----- .../src/supervisor_session.rs | 156 +- crates/openshell-supervisor/Cargo.toml | 58 + .../src/activity_aggregator.rs | 2 +- .../src/denial_aggregator.rs | 2 +- crates/openshell-supervisor/src/lib.rs | 5493 ++++++++++++++ crates/openshell-supervisor/src/main.rs | 327 + .../src/mechanistic_mapper.rs | 2 +- deploy/docker/Dockerfile.supervisor | 20 +- e2e/rust/tests/bypass_detection.rs | 33 +- e2e/rust/tests/credential_gating.rs | 26 +- e2e/rust/tests/forward_proxy_graphql_l7.rs | 92 +- e2e/rust/tests/forward_proxy_jsonrpc_l7.rs | 66 +- e2e/rust/tests/live_policy_update.rs | 178 - e2e/rust/tests/no_proxy.rs | 25 +- e2e/rust/tests/websocket_conformance.rs | 77 +- rfc/0012-isolation-backend/README.md | 164 +- rfc/0012-isolation-backend/topology-matrix.md | 25 +- tasks/rust.toml | 12 +- tasks/scripts/docker-build-image.sh | 2 +- tasks/scripts/stage-prebuilt-binaries.sh | 13 +- .../verify-defaults-without-telemetry.sh | 2 +- 89 files changed, 21599 insertions(+), 16513 deletions(-) create mode 100644 crates/openshell-isolation-interface/src/boundary_protocol.rs create mode 100644 crates/openshell-isolation-interface/src/remote.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/boundary_exec.rs (86%) rename crates/{openshell-supervisor-process => openshell-sandbox}/src/boundary_io.rs (78%) create mode 100644 crates/openshell-sandbox/src/boundary_server.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/child_env.rs (59%) create mode 100644 crates/openshell-sandbox/src/delegated.rs delete mode 100644 crates/openshell-sandbox/src/google_cloud_metadata.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/identity.rs (100%) create mode 100644 crates/openshell-sandbox/src/main_session.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/managed_children.rs (100%) delete mode 100644 crates/openshell-sandbox/src/metadata_server.rs create mode 100644 crates/openshell-sandbox/src/network_broker.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/process.rs (88%) create mode 100644 crates/openshell-sandbox/src/pty.rs rename crates/{openshell-supervisor-process => openshell-sandbox}/src/sandbox/linux/landlock.rs (91%) rename crates/{openshell-supervisor-process => openshell-sandbox}/src/sandbox/linux/mod.rs (78%) rename crates/{openshell-supervisor-process => openshell-sandbox}/src/sandbox/linux/seccomp.rs (99%) rename crates/{openshell-supervisor-process => openshell-sandbox}/src/sandbox/mod.rs (100%) delete mode 100644 crates/openshell-sandbox/src/sidecar_control.rs create mode 100644 crates/openshell-supervisor-network/src/spiffe_endpoint.rs delete mode 100644 crates/openshell-supervisor-process/src/bypass_monitor/mod.rs delete mode 100644 crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs create mode 100644 crates/openshell-supervisor-process/src/delegated.rs delete mode 100644 crates/openshell-supervisor-process/src/netns/mod.rs delete mode 100644 crates/openshell-supervisor-process/src/netns/nft_ruleset.rs delete mode 100644 crates/openshell-supervisor-process/src/run.rs create mode 100644 crates/openshell-supervisor/Cargo.toml rename crates/{openshell-sandbox => openshell-supervisor}/src/activity_aggregator.rs (99%) rename crates/{openshell-sandbox => openshell-supervisor}/src/denial_aggregator.rs (98%) create mode 100644 crates/openshell-supervisor/src/lib.rs create mode 100644 crates/openshell-supervisor/src/main.rs rename crates/{openshell-sandbox => openshell-supervisor}/src/mechanistic_mapper.rs (99%) diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 5d4a98c61a..089f1b9b4c 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -179,20 +179,20 @@ jobs: tasks/scripts/verify-telemetry-compiled-out.sh present target/debug/openshell-gateway cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway - cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry - tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox + cargo build -p openshell-supervisor --bin openshell-supervisor --no-default-features --features defaults-without-telemetry + tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-supervisor - name: Verify the defaults-without-telemetry feature alias tracks the default feature set run: tasks/scripts/verify-defaults-without-telemetry.sh - name: Verify system CA roots build mode compiles and excludes bundled Mozilla roots run: | - cargo check -p openshell-sandbox --all-targets --no-default-features --features system-ca-roots - if cargo tree -p openshell-sandbox -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then + cargo check -p openshell-supervisor --all-targets --no-default-features --features system-ca-roots + if cargo tree -p openshell-supervisor -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo "ERROR: webpki-roots found in system CA roots build" >&2 exit 1 fi - if cargo tree -p openshell-sandbox -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then + if cargo tree -p openshell-supervisor -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo "ERROR: webpki-root-certs found in system CA roots build" >&2 exit 1 fi diff --git a/AGENTS.md b/AGENTS.md index 0a4a9218a1..d0049c85a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,7 +68,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution | | `crates/openshell-supervisor-middleware-builtins/` | Built-in middleware | First-party in-process middleware implementations | | `crates/openshell-supervisor-network/` | Network supervisor | Proxying, L7 enforcement, policy evaluation, and inference routing | -| `crates/openshell-supervisor-process/` | Process supervisor | Process lifecycle, namespace, and bypass monitoring | +| `crates/openshell-supervisor-process/` | Supervisor process runtime | Gateway sessions, SSH access, and remote sandbox process control | | `crates/openshell-vfio/` | VFIO support | PCI and GPU passthrough preparation and lifecycle | | `python/openshell/` | Python SDK | Python bindings and CLI packaging | | `sdk/typescript/` | TypeScript SDK | Native Connect client, curated sandbox API, and generated protobuf types | diff --git a/Cargo.lock b/Cargo.lock index 0f5bbfd419..83bea726ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4178,8 +4178,19 @@ dependencies = [ "async-trait", "libc", "openshell-core", + "rcgen", "rustix 1.1.4", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "sha2 0.10.9", + "socket2", + "thiserror 2.0.18", "tokio", + "tokio-rustls", + "tracing", + "uuid", ] [[package]] @@ -4287,32 +4298,38 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "bytes", + "capctl", "clap", - "futures", + "hex", + "ipnet", + "landlock", + "libc", "miette", "nix 0.29.0", + "openshell-binary-identity", "openshell-core", - "openshell-extension-core", + "openshell-isolation-interface", "openshell-ocsf", "openshell-policy", - "openshell-supervisor-middleware", - "openshell-supervisor-middleware-builtins", - "openshell-supervisor-network", - "openshell-supervisor-process", - "prost", - "prost-types", + "rand 0.10.2", + "rcgen", + "rustix 1.1.4", "rustls", + "rustls-pemfile", + "seccompiler", "serde", "serde_json", - "temp-env", + "sha2 0.10.9", + "socket2", "tempfile", "tokio", - "tokio-tungstenite 0.26.2", - "tonic", + "tokio-rustls", "tracing", - "tracing-appender", "tracing-subscriber", - "uuid", ] [[package]] @@ -4439,6 +4456,40 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "openshell-supervisor" +version = "0.0.0" +dependencies = [ + "clap", + "futures", + "miette", + "nix 0.29.0", + "openshell-core", + "openshell-extension-core", + "openshell-isolation-interface", + "openshell-ocsf", + "openshell-policy", + "openshell-supervisor-middleware", + "openshell-supervisor-middleware-builtins", + "openshell-supervisor-network", + "openshell-supervisor-process", + "prost", + "prost-types", + "rustix 1.1.4", + "rustls", + "serde", + "serde_json", + "temp-env", + "tempfile", + "tokio", + "tokio-tungstenite 0.26.2", + "tonic", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", +] + [[package]] name = "openshell-supervisor-middleware" version = "0.0.0" @@ -4535,24 +4586,17 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "capctl", "hex", - "ipnet", - "landlock", "libc", "miette", "nix 0.29.0", "openshell-core", "openshell-isolation-interface", "openshell-ocsf", - "openshell-policy", "rand 0.10.2", "russh", - "rustix 1.1.4", - "seccompiler", "serde_json", "sha2 0.10.9", - "socket2", "tempfile", "tokio", "tokio-stream", @@ -5570,6 +5614,7 @@ dependencies = [ "ring", "rustls-pki-types", "time", + "x509-parser", "yasna", ] @@ -8656,6 +8701,7 @@ dependencies = [ "lazy_static", "nom", "oid-registry", + "ring", "rusticata-macros", "thiserror 1.0.69", "time", diff --git a/Cargo.toml b/Cargo.toml index 6936439a3b..16845307a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ http-body-util = "0.1" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "ring"] } rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12", "ring"] } rustls-pemfile = "2" -rcgen = { version = "0.13", features = ["crypto", "pem"] } +rcgen = { version = "0.13", features = ["crypto", "pem", "x509-parser"] } webpki-roots = "1" rustls-native-certs = "0.8" diff --git a/architecture/build.md b/architecture/build.md index 972f847bb9..69ddba420a 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -47,8 +47,7 @@ Cargo cannot subtract a single default feature, so each of the three binary crates also defines a `defaults-without-telemetry` alias listing every default except `telemetry`. Telemetry-free builds use `--no-default-features --features defaults-without-telemetry` and stay correct -as the default set grows, instead of dropping unrelated defaults the way a bare -`--no-default-features` does on `openshell-sandbox`. The alias is a keep-list, +as the default set grows. The alias is a keep-list, not a switch: enabling it on top of the defaults would otherwise yield a telemetry-on binary that reads as telemetry-free, so each crate root carries a `compile_error!` for the `telemetry` + `defaults-without-telemetry` combination. @@ -63,7 +62,7 @@ roots through `webpki-roots` plus locally-installed CAs from the system bundle. Building without `bundled-ca-roots` switches to the platform trust store via `rustls-native-certs` and excludes bundled Mozilla root crates such as `webpki-roots` and `webpki-root-certs` from the dependency graph. The -`system-ca-roots` feature alias on `openshell-sandbox` includes all other +`system-ca-roots` feature alias on `openshell-supervisor` includes all other defaults (currently `telemetry`) except `bundled-ca-roots`, so Linux distribution builds (e.g. RPM) can use `--no-default-features --features system-ca-roots` without manually re-adding @@ -196,13 +195,12 @@ Runtime layout: cache action runs. An explicitly configured VM runtime bundle is required to contain every non-empty embedding input; the driver build fails before packaging when an input is absent or empty. -- **Supervisor**: Alpine base with `nftables`, static binary at - `/openshell-sandbox` (musl by default; see `SUPERVISOR_LIBC` above). Static - linkage keeps the binary usable when the image is mounted/extracted into - sandbox environments (Docker extraction, Podman image volumes, Kubernetes - init-container copy-self), whose libc and glibc version are not known at build - time, while `nftables` supports Kubernetes supervisor sidecar egress - enforcement. The VM driver bundles its own supervisor build +- **Sandbox and supervisor**: Alpine base with separate static + `/openshell-sandbox` and `/openshell-supervisor` binaries (musl by default; + see `SUPERVISOR_LIBC` above). Static linkage keeps the sandbox executable + usable when a driver stages it into an arbitrary workload image. The image + entrypoint is the external supervisor; drivers copy only the sandbox binary + into the workload trust domain. The VM driver bundles both builds (`tasks/scripts/vm/build-supervisor-bundle.sh`) and does not read `SUPERVISOR_LIBC`. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index e1e731a0ce..0396f51bc0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -1,10 +1,9 @@ # Compute Runtimes Compute runtimes create, stop, start, delete, and watch sandbox workloads for the -gateway. Supervisor-controlled runtimes start a workload that runs the -`openshell-sandbox` supervisor, which enforces the sandbox contract locally. -Driver-controlled runtimes apply the canonical sandbox policy while -provisioning and report workload readiness directly. +gateway. A supported runtime provisions `openshell-sandbox` inside the workload, +`openshell-supervisor` outside it, a protected channel between them, and an +independent outer network fence. Drivers do not implement policy evaluation. ## Driver Contract @@ -12,11 +11,15 @@ Each runtime receives a sandbox spec and canonical policy from the gateway and is responsible for: - Selecting the sandbox image. -- For supervisor-controlled runtimes, injecting sandbox identity and gateway - callback configuration, supplying callback credentials, and providing the - supervisor binary or image. -- For runtimes without the standard supervisor, validating and applying the - canonical policy before launching the workload. +- Resolving an immutable non-root sandbox identity before workload creation. +- Supplying separate sandbox and supervisor bootstrap material. +- Delivering `openshell-sandbox` to the workload and `openshell-supervisor` only + to the external supervisor placement. +- Provisioning protected control and boundary configs plus a private Unix socket, + TLS-authenticated TCP, or vsock transport when the supervisor is separated. + Runtime-specific code supplies immutable resource claims and transport + coordinates; the shared boundary protocol supplies lifecycle, exec, signaling, + forwarding, and binary identity semantics. - Forwarding the exact canonical main-process argv and TTY mode without shell reconstruction. The sandbox-level environment and policy workspace apply to the main process. @@ -247,10 +250,10 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | -| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Docker | Local development with Docker available. | Capability-free workload container. | Uses `network_mode=none`; a separate capability-free supervisor container mediates egress and access over a private daemon-local Unix socket volume. | +| Podman | Existing rootless driver. | Container. | Not converted by this isolation stack. | +| Kubernetes | Cluster deployment through Helm. | Capability-free sandbox Pod. | Uses empty-egress NetworkPolicy, paired-only supervisor ingress, and a separate capability-free supervisor Deployment over mutually authenticated TLS. It requires an enforcing CNI and trusted sandbox namespace. | +| VM | Experimental microVM isolation. | Per-sandbox libkrun or QEMU VM. | The NIC-less guest runs `openshell-sandbox` as PID 1; host `openshell-supervisor` owns gateway networking and reaches the guest over vsock. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through @@ -278,25 +281,17 @@ operator override because they place gateway-host filesystem state inside the sandbox and can negate OpenShell workspace isolation and filesystem-policy controls. Driver-owned supervisor, token, and TLS bind mounts stay reserved. -Network features follow the existing driver/substrate split. Compute drivers -advertise only the runtime mechanics they can guarantee: namespace and -capability ownership, DNS/TCP capture installation, and coupled -restart ordering. The shared supervisor remains the sole owner of DNS -eligibility, synthetic mappings, process authorization, destination filtering, -pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise -`policy-dns-transparent-tcp`; other runtimes reject explicit TCP policy until -they implement and validate the same complete contract. The capability marker -is driver-owned supervisor input and is removed from workload environments. - -Kubernetes deployments may set an AppArmor profile on sandbox agent containers -through the driver configuration. The Helm chart defaults sandbox agents to -`Unconfined` so runtime/default AppArmor profiles do not block supervisor -network namespace setup on AppArmor-enabled nodes. +Network features follow the driver/substrate split. Drivers own only the outer +fence and protected channel. The sandbox owns seccomp notification, local DNS, +socket virtualization, process observation, and binary identity. The supervisor +owns DNS eligibility, policy authorization, destination filtering, upstream +dials, relay behavior, credential rewriting, and OCSF decisions. No supported +path requires nftables, a workload network namespace, proxy environment +variables, added capabilities, or an unconfined AppArmor profile. The Kubernetes deployment packaging has two ownership boundaries. The gateway chart owns the gateway workload, configuration, Services, PKI, and -cluster-scoped gateway resources. It can retain the legacy combined behavior, -or omit workspace resources. The workspace chart is installed into a +cluster-scoped gateway resources. The workspace chart is installed into a pre-provisioned sandbox namespace and owns only the sandbox ServiceAccount, namespaced RBAC, and sandbox ingress NetworkPolicy. Its RoleBinding names the gateway ServiceAccount and namespace explicitly, so the two releases have @@ -321,113 +316,57 @@ Runtime-specific implementation notes belong in the driver crate README: - `crates/openshell-driver-kubernetes/README.md` - `crates/openshell-driver-vm/README.md` -The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun -executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves -that identity when it execs the supervisor after mounting and network setup. +The VM guest bootstrap runs once as root to prepare mounts, loopback, and the +safe port-53 sysctl. It then drops to the resolved identity with empty +capability sets and executes `openshell-sandbox` as guest PID 1. ## Supervisor Delivery -The supervisor must be available inside each sandbox workload: +Drivers deliver the two binaries to separate trust domains: | Runtime | Delivery model | |---|---| -| Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | -| Podman | Read-only OCI image volume by default; host-cached bind mount when `userns` is configured. | -| Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | -| VM | Embedded in the guest rootfs bundle. | +| Docker | A digest-pinned daemon-local volume supplies `openshell-sandbox`; the companion image runs `openshell-supervisor`. | +| Podman | Existing driver behavior; not converted by this stack. | +| Kubernetes | A non-root init container stages `openshell-sandbox` into a memory volume; the separate Deployment image runs `openshell-supervisor`. | +| VM | `openshell-sandbox` is embedded in the guest rootfs; a separately digest-checked native `openshell-supervisor` runs on the host. | | Extension | Defined by the out-of-tree driver. | -Driver-controlled environment variables must override sandbox image or template -values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS -paths, and command metadata. +Driver-controlled sandbox bootstrap must override image or template values for +sandbox identity, command metadata, resolver configuration, and public trust +paths. Gateway endpoints, callback credentials, policy, and private TLS material +belong only to the supervisor placement. ## Process Identity -The gateway preserves whether each policy process field was omitted. The active -driver then supplies one authoritative identity input to the supervisor: - -- Docker and Podman inspect the final sandbox image, pin container creation to - its immutable image ID, and pass its raw OCI `Config.User`. Docker also - resolves the workspace from OCI `Config.WorkingDir` during that inspection. -- Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift - SCC-derived values. -- VM keeps its existing guest identity behavior. - -Explicit numeric workload identities may use any Linux UID/GID from `1` -through `u32::MAX - 1`. UID/GID `0` remains prohibited as root, and -`u32::MAX` remains prohibited because Linux APIs and POSIX ACLs use it as an -invalid identity sentinel. Infrastructure identities use separate validation: -the Kubernetes network proxy UID remains at least `1000` and must not match the -workload UID because its traffic bypasses the pod egress fence. - -For Docker and Podman, policy values take precedence independently. An omitted -`run_as_user` or `run_as_group` falls back to the corresponding identity from -the image. The supervisor resolves names from the image's `/etc/passwd` and -`/etc/group` before readiness, preserves declared name or numeric components, -and uses the same privilege-drop path for direct and SSH children. When a -declaration omits the group, the supervisor fills it with the user's numeric -primary GID. It does not rewrite the account files. - -Docker uses an absolute OCI working directory as the workspace. An -empty, root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which -OpenShell creates and owns as a compatibility workspace. Any other workdir must already -exist in the immutable image without symlink components. The completed -identity, including supplementary groups, must already be able to traverse -every parent and write and enter the workdir; OpenShell does not change that -directory's ownership or mode. A one-shot validator drops to that identity and -uses kernel effective-access checks so POSIX ACL and LSM decisions are honored. -Path checks reserve the standard OCI runtime namespaces under `/proc`, `/sys`, -and `/dev`, while separate collision checks are derived from actual OpenShell -control paths. -Docker performs the check in the final container before workload launch and -rejects image `VOLUME` declarations that would mask the workdir ancestry. The -resolved workspace is the child cwd and `HOME`; when -`filesystem.include_workdir` is enabled, it becomes the automatic writable -policy path. Podman, Kubernetes/OpenShift, and VM retain their existing -`/sandbox` workspace behavior. - -Sandbox creation fails before the workload becomes ready when a required image -identity is absent, malformed, unknown, ambiguous, or resolves to UID/GID 0. -The supervisor itself remains root so it can establish isolation before -starting unprivileged children. - -Kubernetes can run the supervisor in the default combined topology or in a -sidecar topology. Combined mode keeps network and process supervision in the -agent container. Sidecar mode runs network enforcement, the proxy, and gateway -session in a dedicated sidecar, while the agent container runs only the -process-supervision leaf and launches the user workload after the sidecar -serves bootstrap state over a local control socket. The network sidecar owns -gateway credentials and sends policy plus workload-facing provider environment -state to the process leaf over that socket. It also streams provider -environment updates after settings polls so future process sessions see -updated provider env without giving the process leaf gateway access. The -pre-workload process supervisor is the only accepted control client: the -network sidecar verifies its UID, GID, and PID with peer credentials, removes -the listener after accepting it, and ignores workload-supplied relay targets. -SSH relays use a Linux abstract socket and verify its peer PID against that -authenticated process-supervisor connection, so workload filesystem access -cannot replace the relay endpoint. Either supervisor exits when this control -connection closes. This couples their restart lifecycle and prevents a workload -that survives an isolated network-sidecar restart from becoming the next -authoritative control client. In sidecar mode, an init container performs the -privileged pod-network nftables setup with -`NET_ADMIN`. The default binary-aware network sidecar runs as UID 0 without -`NET_ADMIN` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` so it can resolve -cross-UID workload process/binary identity through shared `/proc`. Operators -can set the sidecar `process_binary_aware_network_policy` flag false to run the -sidecar as the configured non-root proxy UID, omit both inspection capabilities, -and downgrade network policy to endpoint/L7 matching without `policy.binaries`. -The init path applies nftables as individual commands so optional conntrack and -log expressions can fail without rolling back the required table, chain, and -reject rules. -The agent container runs as the resolved sandbox UID/GID with no added Linux -capabilities. Sidecar mode preserves gateway session and SSH behavior, but -treats the process leaf as network-only: Landlock filesystem policy and child -seccomp still apply where supported, while process privilege dropping and -supervisor identity mount isolation do not run because the agent container is -already unprivileged. Sidecar pods use a shared process namespace so the -network sidecar can resolve workload process and binary identity through -`/proc/`. +The gateway preserves whether each policy process field was omitted and passes +the admitted selectors to the driver. The driver resolves one exact UID, GID, +and supplementary-group set before creating the immutable workload: + +- Docker pins the image ID, resolves policy selectors against the image's + `/etc/passwd` and `/etc/group`, and validates its OCI working directory. +- Kubernetes uses platform-resolved numeric values, including OpenShift + namespace ranges. +- VM uses the configured numeric guest identity. + +UID/GID zero and `u32::MAX` are invalid. The sandbox and every child start with +the resolved identity and zero capability masks; neither process performs an +in-workload UID transition. Identity-changing policy updates require sandbox +recreation, while other policy updates remain live. + +Docker uses an absolute OCI working directory as the workspace. Empty, root, +and explicit `/sandbox` values select `/sandbox`; other paths must already +exist without symlink or reserved-mount collisions and must be usable by the +resolved identity. Kubernetes and VM use `/sandbox`. + +Kubernetes uses only the proxy-pod topology. The driver creates the empty-egress +workload fence before a suspended Sandbox CR, then provisions split immutable +bootstrap Secrets, the boundary Service, and the supervisor Deployment. A +non-root init container stages `openshell-sandbox` and one-use bootstrap files +into memory volumes. The workload Pod never mounts supervisor or gateway +credentials. The driver removes its scheduling gate only after the companions +exist; measured confirmation and supervisor-session registration gate public +readiness. ## Images diff --git a/architecture/sandbox.md b/architecture/sandbox.md index ba5b3c59b0..22edd7c515 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -1,39 +1,60 @@ # Sandbox -A sandbox is the runtime boundary where agent code executes. It is created by a -compute runtime and managed inside the workload by `openshell-sandbox`, the -sandbox supervisor. +A sandbox is the runtime boundary where agent code executes. A compute driver +creates it and connects two dedicated components: `openshell-sandbox` inside +the workload boundary and `openshell-supervisor` outside it. ## Runtime Model -Each sandbox workload has two trust levels: +Each sandbox has three trust levels: -| Process | Role | +| Component | Role | |---|---| -| Supervisor | Starts as root inside the workload, prepares isolation, runs the proxy, fetches config, injects credentials, serves the relay socket, and launches child processes. | -| Agent child | Runs as an unprivileged user with filesystem, process, and network restrictions applied. | - -The supervisor keeps enough privilege to manage the sandbox, but the agent child -loses that privilege before user code runs. On Linux, child setup clears the -capability bounding set during privilege drop so later execs cannot regain -container-granted capabilities. This is fail-closed: the supervisor retains -`CAP_SETPCAP` solely to perform the clear, and spawning the workload or SSH shell -aborts unless the bounding set ends up empty. A `setpcap` `EPERM` is tolerated -only when the set is already empty; any other outcome fails the spawn. +| Supervisor | Owns gateway credentials, admitted policy, L7 proxying, SSH, and gateway relays. It never executes inside the agent workload. | +| Sandbox | Runs as the same non-root identity as the agent, installs the workload seccomp listener, applies the Landlock baseline, owns child processes, and mediates the protected supervisor channel. | +| Agent child | Inherits the sandbox network listener and runs with zero capabilities, `no_new_privs`, Landlock, and the final syscall filter. | + +The runtime grants neither trusted component nor agent child any Linux +capability inside the workload. Drivers resolve one exact non-root UID, GID, +and supplementary-group set before launch. The sandbox and all of its children +use that immutable identity, so no in-workload privilege transition is needed. +The supervisor uses its own driver-defined identity and has no workload-creation +or backend-admin authority. + +The compute driver provisions separate protected configurations and mutual TLS +over a private Unix socket, Kubernetes TCP Service, or VM vsock channel. +NetworkPolicy is an outer reachability fence, not a confidentiality boundary. +Each sandbox generation receives a fresh CA and distinct server/client leaves; +both endpoints bind the same workload identity and immutable driver resource +claims. Driver crates do not appear in generic process, network, SSH, or +session code. + +The supervisor exposes readiness only after the sandbox is confirmed and the +gateway access plane is registered. Driver-owned channel directories limit +reachability, while mutual authentication and channel epochs prevent endpoint +replacement from granting authority. ## Startup Flow -1. The compute runtime starts the workload with sandbox identity, callback - endpoint, TLS or secret material, image metadata, and initial command. -2. The supervisor loads policy and runtime settings from local files or the - gateway, depending on mode. -3. It prepares filesystem access, process restrictions, network namespace - routing, trust stores, provider credential resolution, and inference routes. -4. It launches the persisted canonical main-process argv and retains its PTY - or pipes in the main-session multiplexer. -5. It starts the policy proxy and local SSH server. -6. It opens a supervisor session back to the gateway for connect, exec, file - sync, config polling, and log push. +1. The driver resolves the immutable workload identity, installs the outer + network fence, and starts `openshell-sandbox` with one-use bootstrap state. +2. The sandbox consumes and unlinks bootstrap material, proves the admitted + runtime posture, and listens on the protected driver channel. It does not + run untrusted code yet. +3. `openshell-supervisor` loads policy and runtime settings from the gateway, + attaches to the sandbox, and verifies the driver's generation and evidence. +4. The sandbox installs its seccomp notification broker and Landlock baseline, + then reports measured confirmation. The supervisor must accept that evidence + before it sends the launch permit. +5. The sandbox starts the canonical process through its single workload + launcher. The supervisor starts SSH and registers its gateway session. +6. Exec, signaling, PTY, DNS, TCP, and loopback-forwarding operations cross the + authenticated channel for the lifetime of the sandbox generation. + +When the admitted main process exits, its status and retained terminal output +remain available. The confirmed sandbox and supervisor-owned access plane continue +to serve policy-authorized exec and loopback forwarding until explicit stop or +delete tears down the boundary and terminates any remaining workload processes. ## Isolation Layers @@ -42,9 +63,9 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: | Layer | Purpose | |---|---| | Filesystem policy | Landlock restricts the paths the agent can read or write. | -| Process policy | The child process runs as a non-root user with reduced privileges. | -| Seccomp | Blocks dangerous syscalls, including raw socket paths that bypass the proxy. | -| Network namespace | Forces ordinary agent egress through the local CONNECT proxy. | +| Process policy | Sandbox and children run as one immutable non-root identity with zero capabilities. | +| Seccomp notification | Virtualizes supported INET sockets and sends DNS/TCP decisions to the supervisor without nftables or proxy environment variables. | +| Driver outer fence | Docker `network_mode=none`, a NIC-less VM, or Kubernetes NetworkPolicy prevents any missed or unsupported kernel path from escaping. | | Policy proxy | Evaluates destination, binary identity, TLS/L7 rules, SSRF checks, and inference interception. | The supervisor may enrich baseline filesystem allowances for runtime-required @@ -55,12 +76,27 @@ paths, such as proxy support files or GPU device paths when a GPU is present. See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, their ownership, terminal behavior, and known gaps. -All ordinary agent egress is routed through the sandbox proxy. The proxy -identifies the calling binary, checks trust-on-first-use binary identity, rejects -unsafe internal destinations, and evaluates the active policy. On Linux, it -maps an accepted proxy connection back to the workload socket by matching the -complete local-to-remote TCP tuple before resolving every process that owns the -socket inode. +The sandbox installs one seccomp user-notification listener on a dedicated +launcher thread. Every canonical and exec process inherits that listener. It +virtualizes supported INET sockets before they enter the agent FD table, copies +bounded syscall inputs from the notifying task, resolves the calling binary, +and blocks external `connect` until the supervisor returns a policy decision +and relay stream. Connected data stays on ordinary kernel sockets, so the +notification path is limited to socket setup and pointer-bearing operations. +This topology requires Linux 5.19 or newer: the sandbox treats +`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` as mandatory so cancelled +notifications cannot race task-memory writes. + +DNS uses an exact sandbox-local resolver at `127.0.0.53:53`. The driver sets the +nameserver and permits an unprivileged bind to port 53. UDP and TCP DNS requests +are attributed to the calling binary and forwarded through the supervisor; the +kernel delivers replies from the configured nameserver address, including for +strict musl and c-ares resolvers. No proxy environment variable, nftables rule, +or workload network namespace setup is part of enforcement. + +The outer fence remains mandatory. If notification handling misses a syscall, +loses the supervisor, exceeds a bound, or encounters an unsupported socket +type, the request fails and the driver-owned fence still blocks direct egress. CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same egress pipeline. Each adapter normalizes its request into an egress intent, and @@ -86,29 +122,6 @@ captured before the bypass fence, mapped back to its workload process, authorize through the same egress pipeline, and dialed only through the pinned addresses. Omitted protocol endpoints retain explicit-proxy behavior. -The DNS store is in-memory and sandbox-local. A combined-supervisor restart also -restarts its workload; before execution, the supervisor advances a persisted -boot epoch and installs only that epoch's synthetic capture ranges. An address -cached from the preceding epoch therefore falls through to the bypass fence -instead of inheriting a new mapping. Policy reload, expiry, wrong ports, direct real-IP access, missing -mappings, or pool exhaustion fail closed. Resolver injection, DNS listeners, -capture rules, and the transparent listener are all ready before workload -execution. A runtime that cannot provide the complete contract rejects a policy -containing explicit TCP endpoints rather than partially activating it. Because -that substrate is startup infrastructure, a sandbox created without explicit -TCP endpoints rejects a hot reload that introduces one and keeps its complete -previous policy active; recreating the sandbox installs the substrate before -the workload starts. A sandbox that started with the substrate may continue to -remove and re-add TCP endpoints through ordinary atomic policy reloads. -Workload DNS targets port 53, while nftables redirects eligible IPv4 DNS traffic -to an unprivileged supervisor listener. The filter admits DNS and transparent -TCP only when the kernel records the traffic as DNATed to the corresponding -supervisor listener, so direct dials to either unprivileged listener port remain -fenced. `SO_ORIGINAL_DST`, synthetic mapping lookup, endpoint correlation, and -generation-pinned authorization form the transparent TCP security boundary. -Docker and Podman do not currently advertise usable IPv6 egress for this -substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. - Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static credential resolves only when the request host, port, and path match an endpoint @@ -121,12 +134,12 @@ partially active or last-known-good static set. Invalid metadata preserves the supplied dynamic snapshot, while a fetch failure preserves the currently active dynamic snapshot. -In the Kubernetes sidecar topology, the provider environment revision remains +Across the protected sandbox/supervisor channel, the provider environment revision remains an opaque content fingerprint and has no numeric ordering semantics. The network supervisor assigns a separate, connection-local monotonic generation to each distinct environment it publishes. The process supervisor applies only newer generations, which accepts descending fingerprint values while rejecting -duplicate or delayed sidecar messages. +duplicate or delayed supervisor messages. Gateway-managed refresh credentials use an opaque workload handle derived from the sandbox, provider identity, credential key, refresh authorization epoch, @@ -263,8 +276,9 @@ last resort for proxies whose ACLs filter on hostnames and reject IP CONNECT targets — with it, the proxy resolves the name itself and its ACLs become the effective egress control for proxied TLS. (Resolving through the proxy's own DNS view, e.g. DoH tunneled via CONNECT, is a possible future -enhancement and out of scope.) The workload child's proxy variables are -unaffected — they are always rewritten to point at the local policy proxy. +enhancement and out of scope.) Workload proxy variables are removed from the +protected launch environment; transparent socket mediation does not depend on +them. Template environment is treated like user-provided sandbox environment. It can shape the workload child, but it cannot override driver-controlled identity, @@ -283,10 +297,9 @@ sandbox-create time through validators shared with the supervisor (`openshell_core::driver_utils::parse_upstream_proxy_url` and `parse_upstream_proxy_credential`). -An optional operator CA bundle (`--upstream-proxy-ca-bundle`, a PEM path the -driver bind-mounts read-only into the sandbox) extends the trust boundary for -corporate proxies. A CA certificate is not secret, so unlike the auth file it -travels as a plain read-only bind mount rather than a driver secret. It is +An optional operator CA bundle (`--upstream-proxy-ca-bundle`, a supervisor-only +PEM path) extends the trust boundary for corporate proxies. A CA certificate is +not secret, but the supervisor is still its only configuration authority. It is trusted in two places: the TLS handshake with an `https://` proxy, and — because a TLS-intercepting proxy (mitmproxy, squid `ssl-bump`) re-signs tunneled server certificates with the same CA — the sandbox combined trust @@ -301,48 +314,27 @@ plain HTTP) and is fail-closed: an unreadable or certificate-free file is fatal. Proxy credentials are never embedded in the URL: an inline `user:pass@` is rejected because it would be stored in `gateway.toml` and exposed in container metadata. Operators supply credentials via `proxy_auth_file`; the driver -stages them as a root-only secret mounted at a fixed path and passes only +stages them as a supervisor-only secret mounted at a fixed path and passes only that path on the supervisor's command line. The supervisor reads the file and builds the `Proxy-Authorization: Basic` header; a credential that is empty, contains control characters, or is not in `user:pass` form is fatal on both sides. -The VM driver has no argv seam of its own: its guest init script runs as PID 1 -and execs a fixed supervisor command line, and the libkrun and QEMU launch -backends both reach the supervisor through that script. Driver-owned -supervisor arguments therefore travel in a per-sandbox file the driver writes -into the overlay upperdir at a fixed guest path, one argument per line, which -the guest reads verbatim (no word splitting or globbing) and appends to every -supervisor exec. The file is written on **every** launch, including an empty -file when there is nothing to pass: the upperdir copy always shadows the -read-only image layer, so a sandbox image can neither supply its own -supervisor arguments by baking a file at that path nor disable the operator's -by omitting one. This mirrors the driver-authored `init.d` manifest, which -solves the same trust problem for guest init drop-ins. - -A microVM has no bind mounts or container secrets, so the VM driver stages the -credential and the CA bundle into the per-sandbox overlay disk instead — the -credential root-only, the CA world-readable, both at fixed `/opt/openshell` -paths and both removed with the sandbox state directory. The consequence, -which differs from the Podman secret model, is that the credential is at rest -inside that overlay image on the gateway host; the per-sandbox gateway JWT -already travels the same path. Proxy reachability differs by VM backend. libkrun-backed -sandboxes egress through gvproxy, so a proxy on the gateway host's loopback is -reachable through the host alias `host.openshell.internal`, which gvproxy NATs -to the host's `127.0.0.1`. QEMU/TAP sandboxes (GPU) have no equivalent: that -alias resolves to the TAP host address, and the driver's nftables `input` -chain accepts only the gateway port from the guest, so no gateway-host proxy -is reachable. The driver rejects a gateway-host proxy URL on the QEMU path at -launch rather than producing CONNECT timeouts. The guest's gateway callback is -unaffected in both backends and never traverses the proxy. - -For Kubernetes sandboxes, the operator configures a Secret name and key rather -than a gateway-host file path. Kubernetes projects that Secret only into the -container that runs network supervision. Proxy credential Secrets require the -sidecar topology, which gives them a separate container boundary from the -workload. Combined topology is rejected because Kubernetes `fsGroup` volume -permission handling can make a shared credential mount readable by the sandbox -group. +The VM driver runs `openshell-supervisor` on the host. Corporate-proxy +credentials, private CA keys, policy, and gateway credentials never enter the +guest. The NIC-less guest reaches the host supervisor only through the +authenticated vsock channel; the host supervisor performs DNS and upstream +connections. + +The Docker driver runs `openshell-supervisor` in a separate companion container. +Its private named volume contains supervisor bootstrap and channel material. +The workload container receives only `openshell-sandbox`, public interception +CA material, and the other sandbox half of the authenticated channel. + +For Kubernetes, the operator configures a Secret name and key rather than a +gateway-host file path. Kubernetes projects that Secret only into the separate +supervisor Deployment. The sandbox Pod never mounts corporate-proxy credentials +or the interception CA private key. The Basic header travels over the plain-TCP connection to the `http://` proxy, so it is readable on the network path between sandbox host and proxy. @@ -361,16 +353,14 @@ agent process and SSH child processes. Driver-controlled environment variables override template values so sandbox images cannot spoof identity, callback, or relay settings. -Supervisor bootstrap identity is not inherited by agent child processes. When -provider token grants mount a SPIFFE Workload API socket, the socket path must -live under a dedicated directory. Children also enter a private mount namespace -where that socket directory is hidden before privilege drop. +Supervisor bootstrap identity and provider workload-identity sockets never +enter the sandbox workload. The authenticated channel carries only the +policy-authorized provider environment intended for child launch and public +trust material intended for TLS clients. -Credential placeholders in proxied HTTP requests can be resolved by the proxy -when policy allows the target endpoint. For GCP providers, a loopback metadata -server inside the network namespace serves placeholders to SDKs that bypass the -proxy (e.g. Go's `cloud.google.com/go/compute/metadata`). Secrets must not be -logged in OCSF or plain tracing output. The supervisor uses revision-scoped +Credential placeholders in mediated HTTP requests can be resolved by the proxy +when policy allows the target endpoint. Secrets must not be logged in OCSF or +plain tracing output. The supervisor uses revision-scoped placeholders for unmanaged rotating credentials and identity-stable opaque handles for gateway-managed refresh credentials. Provider environment keys beginning with `v_` or `s<64 lowercase hex characters>_` are reserved @@ -514,22 +504,26 @@ refreshes and cannot permanently lose the initial acknowledgement. Only sandbox-scoped revisions (`PolicySource::Sandbox`, version greater than zero) are acknowledged. Global policies and local-file development policies do not use the sandbox revision API and produce no acknowledgement. When explicit -local Rego and data files are configured, the supervisor continues polling the -gateway for settings and provider refreshes but never replaces the local OPA -engine with a gateway policy revision. +local Rego and data files are provisioned into the supervisor, it continues +polling the gateway for settings and provider refreshes but never replaces the +local OPA engine with a gateway policy revision. Workload image files and +environment variables do not configure the separately isolated supervisor. ## Failure Behavior - If gateway config polling fails, the sandbox keeps its last-known-good policy. - If a live policy or middleware-registry update is invalid, the supervisor - rejects the combined update and keeps the current runtime pair. + rejects the update and keeps the current runtime pair. - If an operator-run middleware call fails, the selected config's `on_error` behavior decides whether to deny the request or continue without that stage. - Existing raw byte streams are connection scoped. Dynamic policy changes apply to new connections or the next parsed HTTP request where the proxy can safely re-evaluate. - If the supervisor relay drops, the sandbox can keep running, but connect and - exec operations fail until the supervisor registers again. + exec operations fail until the supervisor registers again. A replacement + supervisor replays the identical sandbox lifecycle and receives the existing + process handle. The sandbox rejects changed launch inputs and releases the + single main-process attachment when the old supervisor transport closes. - If the canonical main process exits, the supervisor durably reports the normalized result immediately. A foreground create declares a one-shot main attachment, so the supervisor accepts it even after a fast process exits, diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 62f5837e70..96e57df17b 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -160,13 +160,15 @@ policy without provenance applies neither the raw-tunnel refusal nor the WebSocket binary-frame refusal. The request-body backstop still applies, because it keys off the presence of a secret resolver rather than endpoint provenance. -Two paths load a policy without provenance. A supervisor booting from a -container-image policy is a bounded window: that policy is resynchronized to the -gateway, which then serves a stamped effective policy. An explicit local Rego and -data override is permanent, because gateway revisions are observed for settings -and providers but never replace the local policy. When that override is combined -with injected provider credentials, the supervisor emits a high-severity -detection finding at startup naming the inactive controls. +Two supervisor-local paths load a policy without provenance. A supervisor +booting from an explicitly provisioned policy file has a bounded window before +that policy is resynchronized to the gateway, which then serves a stamped +effective policy. An explicit supervisor Rego and data override is permanent, +because gateway revisions are observed for settings and providers but never +replace the local policy. Workload-image files and environment variables cannot +configure the separately isolated supervisor. When a supervisor override is +combined with injected provider credentials, the supervisor emits a +high-severity detection finding at startup naming the inactive controls. ## Live Updates diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index 2b1537a21b..d9055fa319 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -359,6 +359,17 @@ impl ProviderCredentialState { /// here so SDKs can read them at startup. /// 3. Everything else stays as placeholders for proxy-time resolution. pub fn child_env_with_gcp_resolved(&self) -> HashMap { + self.child_env_snapshot_with_gcp_resolved().1 + } + + /// Return the current revision and its workload-facing environment from + /// one state snapshot. + /// + /// Remote isolation boundaries use the pair as a revisioned update. The + /// revision must describe the exact environment sent across the boundary, + /// so callers must not obtain the two values through separate lock + /// acquisitions. + pub fn child_env_snapshot_with_gcp_resolved(&self) -> (u64, HashMap) { use crate::google_cloud; let inner = self @@ -376,7 +387,7 @@ impl ProviderCredentialState { .any(|key| env.contains_key(*key) && inner.non_secret_environment_keys.contains(*key)); if !has_gcp_metadata && !has_gcp_config { - return env; + return (inner.current.revision, env); } if has_gcp_metadata { @@ -414,7 +425,44 @@ impl ProviderCredentialState { } } - env + (inner.current.revision, env) + } + + /// Compare and install a workload-facing environment snapshot. + /// + /// Provider environment revisions are opaque content identities, not + /// ordered counters. The expected revision makes retries idempotent while + /// rejecting updates based on a stale view of the boundary state. + pub fn compare_and_install_child_env_snapshot( + &self, + expected_revision: u64, + revision: u64, + mut child_env: HashMap, + ) -> u64 { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + if revision == inner.current.revision || expected_revision != inner.current.revision { + return inner.current.revision; + } + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.non_secret_environment_keys.clear(); + inner.static_credential_bindings.clear(); + inner.known_static_credential_keys.clear(); + inner.static_credential_identity_epochs.clear(); + revision } /// Return the GCP token placeholder and its remaining lifetime in seconds. @@ -2117,6 +2165,40 @@ mod tests { ); } + #[test] + fn child_env_snapshot_update_uses_opaque_revision_cas() { + let state = ProviderCredentialState::from_child_env_snapshot( + 4, + HashMap::from([("TOKEN".to_string(), "four".to_string())]), + ); + + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 6, + HashMap::from([("TOKEN".to_string(), "six".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot( + 4, + 5, + HashMap::from([("TOKEN".to_string(), "stale".to_string())]), + ), + 6 + ); + assert_eq!( + state.compare_and_install_child_env_snapshot(6, 2, HashMap::new()), + 2, + "opaque revisions may move numerically backwards" + ); + + let (revision, env) = state.child_env_snapshot_with_gcp_resolved(); + assert_eq!(revision, 2); + assert!(env.is_empty(), "an empty snapshot must revoke the old env"); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 2ce8e4b058..f9cd03c539 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -134,6 +134,13 @@ pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; /// `"sidecar"`; the default combined supervisor path omits it. pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; +/// The isolation backend admitted by the deployment configuration (RFC 0012). +/// +/// Delivered on a channel separate from the topology descriptor so descriptor +/// verification against the admitted backend is not self-referential. Required +/// whenever a topology descriptor is supplied. +pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND"; + /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; @@ -166,6 +173,17 @@ pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; /// by workload child processes. pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; +/// Optional path to a durable PEM-encoded interception CA certificate. +/// Must be configured together with [`PROXY_CA_KEY`]. +pub const PROXY_CA_CERT: &str = "OPENSHELL_PROXY_CA_CERT"; + +/// Optional path to the private key for [`PROXY_CA_CERT`]. +/// Must be configured together with the certificate path. +pub const PROXY_CA_KEY: &str = "OPENSHELL_PROXY_CA_KEY"; + +/// Whether the control-owned SSH Unix socket is shared across trusted UIDs. +pub const SSH_SOCKET_SHARED: &str = "OPENSHELL_SSH_SOCKET_SHARED"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-isolation-interface/Cargo.toml b/crates/openshell-isolation-interface/Cargo.toml index 2384732773..6e680f6e87 100644 --- a/crates/openshell-isolation-interface/Cargo.toml +++ b/crates/openshell-isolation-interface/Cargo.toml @@ -13,10 +13,23 @@ repository.workspace = true [dependencies] openshell-core = { path = "../openshell-core", default-features = false } async-trait = "0.1" +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } tokio = { workspace = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } +socket2 = { workspace = true } +tracing = { workspace = true } +rcgen = { workspace = true } +sha2 = { workspace = true } +uuid = { workspace = true } -[target.'cfg(target_os = "linux")'.dependencies] +[target.'cfg(unix)'.dependencies] libc = "0.2" + +[target.'cfg(target_os = "linux")'.dependencies] rustix = { workspace = true, features = ["fs", "process"] } [dev-dependencies] diff --git a/crates/openshell-isolation-interface/src/boundary_protocol.rs b/crates/openshell-isolation-interface/src/boundary_protocol.rs new file mode 100644 index 0000000000..0c539ac20a --- /dev/null +++ b/crates/openshell-isolation-interface/src/boundary_protocol.rs @@ -0,0 +1,1183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Versioned control protocol shared by every remote isolation boundary. +//! +//! Drivers choose and provision the transport, but they do not redefine the +//! process lifecycle, streaming, identity, or authentication messages. The +//! control and boundary roles exchange these length-delimited JSON frames over +//! a private Unix socket, authenticated TCP connection, or virtio-vsock stream. + +use std::fmt; +use std::io; +use std::io::{Read, Write}; +use std::path::PathBuf; + +use crate::AgentSpec; +use crate::contract::Sha256Digest; +use crate::contract::{ + BackendError, BinaryIdentity, BoundaryExitStatus, BoundarySignal, DriverFenceEvidence, + ExecSpec, ResolveError, SandboxConfirmEvidence, TopologyDescriptor, +}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use rcgen::{CertificateParams, DnType, ExtendedKeyUsagePurpose, IsCa, KeyPair, KeyUsagePurpose}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; +pub const STREAM_STDIN: u8 = 0; +pub const STREAM_STDOUT: u8 = 1; +pub const STREAM_STDERR: u8 = 2; +pub const STREAM_EXIT: u8 = 3; +pub const STREAM_STDIN_CLOSED: u8 = 4; +/// Supervisor decision for a staged seccomp-mediated TCP open. +pub const STREAM_NETWORK_DECISION: u8 = 5; +/// Supervisor response for one sandbox-local DNS relay exchange. +pub const STREAM_DNS_RESPONSE: u8 = 6; +/// Boundary acknowledgement that a mediated DNS response was committed. +pub const STREAM_DNS_ACK: u8 = 7; +pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; +/// Control-side endpoint for a driver-provisioned boundary. +/// Supervisor-side mutual-TLS identity for one sandbox generation. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryClientTls { + /// DNS identity required from the sandbox certificate. + pub server_name: String, + /// Per-generation trust anchor for the sandbox certificate. + pub ca_certificate_pem: String, + /// Supervisor-only client certificate chain. + pub certificate_chain_pem: String, + /// Supervisor-only client private key. + pub private_key_pem: String, +} + +impl fmt::Debug for BoundaryClientTls { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryClientTls") + .field("server_name", &self.server_name) + .field("ca_certificate_pem", &"") + .field("certificate_chain_pem", &"") + .field("private_key_pem", &"") + .finish() + } +} + +/// Sandbox-side mutual-TLS files staged by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryServerTls { + /// Sandbox server certificate chain. + pub certificate_chain_path: PathBuf, + /// Sandbox server private key. + pub private_key_path: PathBuf, + /// Trust anchor used to require the generation-specific supervisor leaf. + pub client_ca_certificate_path: PathBuf, +} + +/// Complete per-generation material returned only to a trusted driver. +#[derive(Clone)] +pub struct BoundaryMutualTlsMaterial { + pub server_name: String, + pub ca_certificate_pem: String, + pub sandbox_certificate_pem: String, + pub sandbox_private_key_pem: String, + pub supervisor_certificate_pem: String, + pub supervisor_private_key_pem: String, +} + +/// Generate distinct server- and client-authentication leaves under a fresh CA. +pub fn generate_boundary_mutual_tls_material() -> Result { + const SERVER_NAME: &str = "sandbox.openshell.internal"; + let ca_key = KeyPair::generate() + .map_err(|error| BackendError::Descriptor(format!("generate boundary CA key: {error}")))?; + let mut ca_params = CertificateParams::default(); + ca_params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + ca_params + .distinguished_name + .push(DnType::CommonName, "OpenShell sandbox channel CA"); + ca_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + let ca = ca_params.self_signed(&ca_key).map_err(|error| { + BackendError::Descriptor(format!("generate boundary CA certificate: {error}")) + })?; + + let sandbox_key = KeyPair::generate().map_err(|error| { + BackendError::Descriptor(format!("generate sandbox channel key: {error}")) + })?; + let mut sandbox_params = CertificateParams::new(vec![SERVER_NAME.to_string()]) + .map_err(|error| BackendError::Descriptor(format!("build sandbox certificate: {error}")))?; + sandbox_params + .distinguished_name + .push(DnType::CommonName, "OpenShell sandbox"); + sandbox_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let sandbox = sandbox_params + .signed_by(&sandbox_key, &ca, &ca_key) + .map_err(|error| { + BackendError::Descriptor(format!("sign sandbox channel certificate: {error}")) + })?; + + let supervisor_key = KeyPair::generate().map_err(|error| { + BackendError::Descriptor(format!("generate supervisor channel key: {error}")) + })?; + let mut supervisor_params = CertificateParams::default(); + supervisor_params + .distinguished_name + .push(DnType::CommonName, "OpenShell supervisor"); + supervisor_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + let supervisor = supervisor_params + .signed_by(&supervisor_key, &ca, &ca_key) + .map_err(|error| { + BackendError::Descriptor(format!("sign supervisor channel certificate: {error}")) + })?; + + Ok(BoundaryMutualTlsMaterial { + server_name: SERVER_NAME.to_string(), + ca_certificate_pem: ca.pem(), + sandbox_certificate_pem: sandbox.pem(), + sandbox_private_key_pem: sandbox_key.serialize_pem(), + supervisor_certificate_pem: supervisor.pem(), + supervisor_private_key_pem: supervisor_key.serialize_pem(), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryTransport { + /// Mutual TLS over a private Unix socket, including libkrun's host mapping. + Unix { + socket_path: PathBuf, + tls: BoundaryClientTls, + }, + /// Mutual TLS over a runtime-scoped TCP endpoint. + TlsTcp { + address: std::net::SocketAddr, + tls: BoundaryClientTls, + }, + /// Mutual TLS over Linux host `AF_VSOCK`. + Vsock { + guest_cid: u32, + control_port: u32, + tls: BoundaryClientTls, + }, +} + +/// Boundary-side listener provisioned by a compute driver. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoundaryListener { + /// Mutual TLS over a private Unix socket shared with a companion. + Unix { + socket_path: PathBuf, + tls: BoundaryServerTls, + }, + /// Mutual TLS over TCP. An unspecified IP is valid for the sandbox bind. + TlsTcp { + address: std::net::SocketAddr, + tls: BoundaryServerTls, + }, + /// Mutual TLS over guest `AF_VSOCK`. + Vsock { + control_port: u32, + tls: BoundaryServerTls, + }, +} + +/// Protected descriptor consumed by `openshell-supervisor`. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryTopology { + /// Stable identity of the boundary, normally the sandbox ID. + pub boundary_id: String, + /// Immutable driver-owned workload generation. + pub generation: String, + /// Fresh session epoch shared with the sandbox bootstrap. + pub session_epoch: String, + /// Immutable numeric identity already applied to the sandbox workload. + pub workload_identity: crate::contract::ResolvedWorkloadIdentity, + /// Driver-provisioned control endpoint. + pub transport: BoundaryTransport, + /// Trusted dial target for well-known host-gateway aliases, when the + /// network supervisor cannot use the boundary's resolver view. + #[serde(default)] + pub host_gateway_ip: Option, + /// Driver-specific immutable resource coordinates bound at attach (for + /// example pod UID, VM generation, or container ID). + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Concrete outer-fence evidence validated by the driver. + pub driver_fence: DriverFenceEvidence, + /// Per-boundary authentication secret; never exposed to workload code. + pub bootstrap_token: String, +} + +impl fmt::Debug for BoundaryTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryTopology") + .field("boundary_id", &self.boundary_id) + .field("generation", &self.generation) + .field("session_epoch", &"") + .field("transport", &self.transport) + .field("host_gateway_ip", &self.host_gateway_ip) + .field("resource_claims", &self.resource_claims) + .field("driver_fence", &self.driver_fence) + .field("bootstrap_token", &"") + .finish() + } +} + +impl BoundaryTopology { + /// Encode this topology as the shared RFC 0012 descriptor admitted for + /// `backend_name`. + pub fn descriptor( + &self, + backend_name: impl Into, + ) -> Result { + let payload = serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))?; + Ok(TopologyDescriptor { + backend_name: backend_name.into(), + payload, + }) + } +} + +/// Protected bootstrap configuration consumed by `openshell-sandbox`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BoundaryConfig { + /// Stable identity expected in every authenticated request. + pub boundary_id: String, + /// Immutable driver-owned workload generation. + pub generation: String, + /// Fresh session epoch for this sandbox/supervisor relationship. + pub session_epoch: String, + /// Per-boundary authentication secret. + pub bootstrap_token: String, + /// Driver-provisioned listener. + pub listener: BoundaryListener, + /// Immutable coordinates the boundary requires from the control-side + /// topology descriptor before accepting attachment. + #[serde(default)] + pub resource_claims: std::collections::BTreeMap, + /// Driver-provisioned, read-only runtime evidence for resource claims. + /// + /// Each entry maps a claim key to an absolute file whose trimmed contents + /// must equal the corresponding value in `resource_claims` before the + /// boundary opens its listener. Kubernetes uses this to bind a one-use + /// bootstrap bundle to the admitted workload Pod UID exposed by the + /// Downward API. Other drivers may leave the map empty. + #[serde(default)] + pub resource_claim_files: std::collections::BTreeMap, + /// Exact identity already applied by the runtime to the sandbox process. + pub workload_identity: crate::contract::ResolvedWorkloadIdentity, + /// Concrete outer-fence evidence validated by the driver. + pub driver_fence: DriverFenceEvidence, + /// Driver-resolved environment exposed only to workload processes. + #[serde(default)] + pub child_env: std::collections::HashMap, +} + +impl fmt::Debug for BoundaryConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BoundaryConfig") + .field("boundary_id", &self.boundary_id) + .field("generation", &self.generation) + .field("session_epoch", &"") + .field("bootstrap_token", &"") + .field("listener", &self.listener) + .field("resource_claims", &self.resource_claims) + .field("resource_claim_files", &self.resource_claim_files) + .field("workload_identity", &self.workload_identity) + .field("driver_fence", &self.driver_fence) + .field("child_env_keys", &self.child_env.keys().collect::>()) + .finish() + } +} + +impl BoundaryConfig { + /// Serialize the protected driver-owned boundary configuration. + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode boundary config: {error}"))) + } +} + +/// Validate driver-specific immutable coordinates before a boundary binds them. +/// +/// Claim values are opaque to the common protocol, but empty or +/// whitespace-bearing identifiers cannot safely distinguish runtime objects. +pub fn validate_resource_claims( + claims: &std::collections::BTreeMap, +) -> Result<(), BackendError> { + for (key, value) in claims { + if key.is_empty() || key.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor( + "boundary resource-claim keys must be non-empty and contain no whitespace" + .to_string(), + )); + } + if value.is_empty() || value.chars().any(char::is_whitespace) { + return Err(BackendError::Descriptor(format!( + "boundary resource claim {key:?} must be non-empty and contain no whitespace" + ))); + } + } + Ok(()) +} + +pub async fn write_stream_frame( + writer: &mut (impl AsyncWrite + Unpin), + channel: u8, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame exceeds limit", + )); + } + writer.write_u8(channel).await?; + writer + .write_u32(payload.len().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "boundary stream frame length overflow", + ) + })?) + .await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_stream_frame( + reader: &mut (impl AsyncRead + Unpin), +) -> io::Result)>> { + let channel = match reader.read_u8().await { + Ok(channel) => channel, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + }; + let declared = reader.read_u32().await? as usize; + if declared > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("boundary stream frame is too large: {declared} bytes"), + )); + } + let mut payload = vec![0; declared]; + reader.read_exact(&mut payload).await?; + Ok(Some((channel, payload))) +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + /// Cryptographically random idempotency key scoped to one sandbox generation. + pub request_id: String, + /// SHA-256 of the canonically serialized request payload. + pub payload_digest: String, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl RequestEnvelope { + /// Build a request envelope with a fresh idempotency key and normalized + /// payload digest. + pub fn new( + boundary_id: String, + bootstrap_token: String, + request: Request, + ) -> Result { + let payload_digest = request_payload_digest(&request)?; + Ok(Self { + request_id: uuid::Uuid::new_v4().to_string(), + payload_digest, + boundary_id, + bootstrap_token, + request, + }) + } + + /// Verify that the request body still matches the immutable digest bound + /// to this idempotency key. + pub fn validate_payload_digest(&self) -> Result<(), FrameError> { + let actual = request_payload_digest(&self.request)?; + if actual == self.payload_digest { + Ok(()) + } else { + Err(FrameError::PayloadDigestMismatch) + } + } +} + +fn request_payload_digest(request: &Request) -> Result { + // Round-tripping through Value canonicalizes every JSON object by key. In + // particular, this makes HashMap-backed provider environments stable + // across process restarts and independently serialized retries. + let normalized = serde_json::to_value(request).map_err(FrameError::Serialize)?; + let payload = serde_json::to_vec(&normalized).map_err(FrameError::Serialize)?; + let digest = Sha256::digest(payload); + Ok(format!("{digest:x}")) +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach { + policy: Box, + resource_claims: std::collections::BTreeMap, + }, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + }, + UpdateProviderEnvironment { + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + }, + AttachProcess { + process_id: String, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, + Exec { + spec: ExecSpecWire, + }, + ExecSignal { + process_id: String, + signal: SignalWire, + }, + Resize { + process_id: String, + cols: u16, + rows: u16, + }, + PortForward { + host: std::net::IpAddr, + port: u16, + }, + AcceptNetwork, + AcceptDns, +} + +impl Request { + /// Whether this control-path request changes generation-owned sandbox + /// state and therefore must be replayed from the idempotency ledger. + #[must_use] + pub const fn is_replayable_mutation(&self) -> bool { + matches!( + self, + Self::Attach { .. } + | Self::Confirm + | Self::StartAgent { .. } + | Self::UpdateProviderEnvironment { .. } + | Self::Exec { .. } + | Self::Signal { .. } + | Self::Terminate { .. } + | Self::ExecSignal { .. } + | Self::Resize { .. } + ) + } +} + +impl fmt::Debug for Request { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Attach { + policy: _, + resource_claims, + } => formatter + .debug_struct("Attach") + .field("policy", &"") + .field("resource_claims", resource_claims) + .finish(), + Self::Confirm => formatter.write_str("Confirm"), + Self::StartAgent { + sandbox_id, + spec, + policy: _, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => formatter + .debug_struct("StartAgent") + .field("sandbox_id", sandbox_id) + .field("spec", spec) + .field("policy", &"") + .field("ca_cert_present", &ca_cert.is_some()) + .field("ca_bundle_present", &ca_bundle.is_some()) + .field("provider_env_revision", provider_env_revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => formatter + .debug_struct("UpdateProviderEnvironment") + .field("expected_revision", expected_revision) + .field("revision", revision) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::Wait { process_id } => formatter + .debug_struct("Wait") + .field("process_id", process_id) + .finish(), + Self::AttachProcess { process_id } => formatter + .debug_struct("AttachProcess") + .field("process_id", process_id) + .finish(), + Self::Signal { process_id, signal } => formatter + .debug_struct("Signal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Terminate { process_id } => formatter + .debug_struct("Terminate") + .field("process_id", process_id) + .finish(), + Self::Exec { spec } => formatter.debug_tuple("Exec").field(spec).finish(), + Self::ExecSignal { process_id, signal } => formatter + .debug_struct("ExecSignal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Resize { + process_id, + cols, + rows, + } => formatter + .debug_struct("Resize") + .field("process_id", process_id) + .field("cols", cols) + .field("rows", rows) + .finish(), + Self::PortForward { host, port } => formatter + .debug_struct("PortForward") + .field("host", host) + .field("port", port) + .finish(), + Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), + Self::AcceptDns => formatter.write_str("AcceptDns"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: String, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached { + snapshot: SessionSnapshotWire, + }, + Confirmed { + /// Measured capability-free posture produced before workload launch. + evidence: Box, + }, + Started { + process_id: String, + provider_env_revision: u64, + }, + ProviderEnvironmentUpdated { + revision: u64, + }, + ProcessAttached { + terminal: bool, + }, + Exited { + status: ExitStatusWire, + }, + Signaled, + Terminated, + ExecStarted { + process_id: String, + pty: bool, + }, + Resized, + PortConnected, + NetworkConnected { + identity: BinaryIdentityWire, + destination: std::net::SocketAddr, + socket: crate::contract::NetworkSocketMetadata, + policy_generation: u64, + }, + DnsQuery { + request: Vec, + transport: crate::contract::DnsTransport, + identity: BinaryIdentityWire, + }, + Error { + kind: String, + message: String, + }, +} + +/// Boundary-owned process/session state returned on every supervisor attach. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SessionSnapshotWire { + pub generation: String, + pub processes: Vec, +} + +/// Stable generation-scoped process state available to a replacement supervisor. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessSnapshotWire { + pub process_id: String, + pub kind: ProcessKindWire, + pub terminal: bool, + pub status: Option, + pub retained_output: OutputWindowWire, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessKindWire { + Main, + Exec, +} + +/// Sequence range retained by the sandbox output ring. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OutputWindowWire { + pub first_sequence: u64, + pub next_sequence: u64, + pub truncated: bool, +} + +/// Completion of one sandbox-local DNS relay exchange. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", content = "value", rename_all = "snake_case")] +pub enum DnsQueryResultWire { + Response(Vec), + Error(String), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryIdentityWire { + pub binary_path: Option, + pub binary_digest: Option, + pub ancestors: Vec, + pub cmdline_paths: Vec, + pub resolve_error: Option, +} + +impl From> for BinaryIdentityWire { + fn from(identity: Result) -> Self { + match identity { + Ok(identity) => Self { + binary_path: Some(identity.binary_path), + binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + ancestors: identity.ancestors, + cmdline_paths: identity.cmdline_paths, + resolve_error: None, + }, + Err(error) => Self { + binary_path: None, + binary_digest: None, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: Some(error.to_string()), + }, + } + } +} + +impl BinaryIdentityWire { + pub fn into_result(self) -> Result { + if let Some(error) = self.resolve_error { + return Err(ResolveError::Failed(error)); + } + let binary_path = self.binary_path.ok_or_else(|| { + ResolveError::Failed("boundary identity omitted binary path".to_string()) + })?; + let binary_digest = self + .binary_digest + .map(|digest| digest.parse::()) + .transpose()?; + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors: self.ancestors, + cmdline_paths: self.cmdline_paths, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecSpecWire { + pub program: String, + pub args: Vec, + pub env: Vec<(String, String)>, + pub workdir: Option, + pub pty: bool, +} + +impl From for ExecSpecWire { + fn from(spec: ExecSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +impl From for ExecSpec { + fn from(spec: ExecSpecWire) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + // Exhaustively destructure the policy so adding a `SandboxPolicy` + // field is a compile error here instead of a silently dropped field + // across the host-to-guest trust boundary. + let SandboxPolicy { + version, + filesystem, + network, + landlock, + process, + } = policy; + let FilesystemPolicy { + read_only, + read_write, + include_workdir, + } = filesystem; + let NetworkPolicy { mode, proxy } = network; + let LandlockPolicy { compatibility } = landlock; + let ProcessPolicy { + run_as_user, + run_as_group, + } = process; + Self { + version, + read_only, + read_write, + include_workdir, + network: NetworkModeWire::from(mode), + proxy_addr: proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(compatibility), + run_as_user, + run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +impl From for BoundarySignal { + fn from(signal: SignalWire) -> Self { + match signal { + SignalWire::Term => Self::Term, + SignalWire::Kill => Self::Kill, + SignalWire::Int => Self::Int, + SignalWire::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +impl From for ExitStatusWire { + fn from(status: BoundaryExitStatus) -> Self { + match status { + BoundaryExitStatus::Exited(code) => Self::Exited(code), + BoundaryExitStatus::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("control request payload digest does not match its envelope")] + PayloadDigestMismatch, + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: "4e94636d-54f8-4d85-8e4e-58954fb5af0a".to_string(), + payload_digest: String::new(), + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some(b"test certificate".to_vec()), + ca_bundle: Some(b"test bundle".to_vec()), + provider_env_revision: 7, + provider_env: std::collections::HashMap::from([( + "OPENAI_API_KEY".to_string(), + "test credential".to_string(), + )]), + }, + }; + let request = RequestEnvelope { + payload_digest: request_payload_digest(&request.request).expect("request digest"), + ..request + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + assert!(!debug.contains("test credential")); + assert!(!debug.contains("test certificate")); + assert!(!debug.contains("test bundle")); + assert!(debug.contains("OPENAI_API_KEY")); + assert!(request.validate_payload_digest().is_ok()); + } + + #[test] + fn request_digest_is_stable_across_map_order_and_detects_mutation() { + let mut first = std::collections::HashMap::new(); + first.insert("B".to_string(), "2".to_string()); + first.insert("A".to_string(), "1".to_string()); + let mut second = std::collections::HashMap::new(); + second.insert("A".to_string(), "1".to_string()); + second.insert("B".to_string(), "2".to_string()); + let build = |provider_env| Request::UpdateProviderEnvironment { + expected_revision: 1, + revision: 2, + provider_env, + }; + assert_eq!( + request_payload_digest(&build(first)).expect("first digest"), + request_payload_digest(&build(second)).expect("second digest") + ); + + let mut envelope = RequestEnvelope::new( + "sandbox-1".to_string(), + "token".to_string(), + build(std::collections::HashMap::new()), + ) + .expect("request envelope"); + envelope.request = Request::Terminate { + process_id: "different".to_string(), + }; + assert!(matches!( + envelope.validate_payload_digest(), + Err(FrameError::PayloadDigestMismatch) + )); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } + + #[test] + fn resource_claims_reject_empty_or_ambiguous_identities() { + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + String::new() + ),])) + .is_err() + ); + assert!( + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod uid".to_string(), + "uid-1".to_string() + ),])) + .is_err() + ); + validate_resource_claims(&std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "uid-1".to_string(), + )])) + .expect("opaque resource identity should be valid"); + } +} diff --git a/crates/openshell-isolation-interface/src/contract.rs b/crates/openshell-isolation-interface/src/contract.rs index f47b6625dc..dbbd489c03 100644 --- a/crates/openshell-isolation-interface/src/contract.rs +++ b/crates/openshell-isolation-interface/src/contract.rs @@ -37,6 +37,7 @@ use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::sync::oneshot; @@ -188,7 +189,7 @@ impl VerifiedTopologyDescriptor { } /// Exact non-root identity selected before the immutable workload is created. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ResolvedWorkloadIdentity { /// Effective and real user ID used by sandbox and all workload children. pub uid: u32, @@ -221,6 +222,7 @@ impl ResolvedWorkloadIdentity { "workload identity source and resource digest are required".to_string(), )); } + supplementary_gids.retain(|supplementary_gid| *supplementary_gid != gid); supplementary_gids.sort_unstable(); supplementary_gids.dedup(); Ok(Self { @@ -379,7 +381,7 @@ pub trait BoundBoundary: Send { } /// Capability masks measured from `/proc//status`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct CapabilityEvidence { pub inheritable: u64, pub permitted: u64, @@ -401,7 +403,7 @@ impl CapabilityEvidence { } /// Active seccomp notification and socket-broker evidence. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[allow( clippy::struct_excessive_bools, reason = "each independently measured kernel operation is reported explicitly" @@ -418,8 +420,88 @@ pub struct SeccompEvidence { pub cancellation: bool, } +/// Driver-owned evidence that the mandatory outer network fence is installed. +/// +/// The sandbox cannot observe the Docker daemon, Kubernetes API, or VM device +/// model directly. Drivers therefore bind the exact fence they validated into +/// both protected bootstrap halves. The sandbox reports that value back during +/// confirmation, and the supervisor rejects any mismatch before agent launch. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "backend", rename_all = "kebab-case", deny_unknown_fields)] +pub enum DriverFenceEvidence { + Docker { + container_id: String, + network_mode: String, + unexpected_networks: Vec, + }, + Kubernetes { + network_policy_uid: String, + network_policy_resource_version: String, + ingress_isolated: bool, + egress_isolated: bool, + egress_rule_count: u32, + }, + Vm { + generation: String, + network_device_count: u32, + }, +} + +impl DriverFenceEvidence { + #[must_use] + pub const fn backend_name(&self) -> &'static str { + match self { + Self::Docker { .. } => "docker", + Self::Kubernetes { .. } => "kubernetes-proxy-pod", + Self::Vm { .. } => "vm", + } + } + + /// Validate the concrete fence properties and bind them to the selected + /// isolation backend. + pub fn validate_for_backend(&self, backend_name: &str) -> Result<(), BackendError> { + let valid = match self { + Self::Docker { + container_id, + network_mode, + unexpected_networks, + } => { + backend_name == "docker" + && !container_id.is_empty() + && network_mode == "none" + && unexpected_networks.is_empty() + } + Self::Kubernetes { + network_policy_uid, + network_policy_resource_version, + ingress_isolated, + egress_isolated, + egress_rule_count, + } => { + backend_name == "kubernetes-proxy-pod" + && !network_policy_uid.is_empty() + && !network_policy_resource_version.is_empty() + && *ingress_isolated + && *egress_isolated + && *egress_rule_count == 0 + } + Self::Vm { + generation, + network_device_count, + } => backend_name == "vm" && !generation.is_empty() && *network_device_count == 0, + }; + if valid { + Ok(()) + } else { + Err(BackendError::Confirm(format!( + "driver fence evidence is incomplete or does not match backend {backend_name:?}" + ))) + } + } +} + /// Measured sandbox-owned evidence produced before agent launch. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[allow( clippy::struct_excessive_bools, reason = "confirmation preserves independently measured security results" @@ -443,13 +525,15 @@ pub struct SandboxConfirmEvidence { pub tcp_deny_round_trip: bool, pub authenticated_supervisor: bool, pub session_epoch: String, - pub direct_egress_blocked: bool, + pub driver_fence: DriverFenceEvidence, pub resource_claims: BTreeMap, } impl SandboxConfirmEvidence { /// Validate the security-critical evidence required before launch. pub fn validate(&self, expected: &ResolvedWorkloadIdentity) -> Result<(), BackendError> { + self.driver_fence + .validate_for_backend(self.driver_fence.backend_name())?; let complete = &self.identity == expected && self.capabilities.is_empty() && self.no_new_privileges @@ -472,7 +556,6 @@ impl SandboxConfirmEvidence { && self.tcp_allow_round_trip && self.tcp_deny_round_trip && self.authenticated_supervisor - && self.direct_egress_blocked && !self.generation.is_empty() && !self.session_epoch.is_empty(); if complete { @@ -568,6 +651,13 @@ pub enum BoundarySignal { /// however many times it is called; a local PID is never the process handle. #[async_trait] pub trait BoundaryProcess: Send + Sync { + /// Attach to the admitted process's retained standard I/O. The boundary + /// remains the process owner and may permit only one control attachment. + async fn attach(&self) -> Result { + Err(BackendError::Unsupported( + "process attachment is not supported".to_string(), + )) + } /// Await terminal status (stable across repeated calls). async fn wait(&self) -> Result; /// Deliver a signal to the process or its group. @@ -581,6 +671,18 @@ pub type BoundaryInput = Box; /// A boxed async reader from a boundary process's stdout or stderr. pub type BoundaryOutput = Box; +/// A control-side attachment to the admitted process's retained I/O. +pub struct ProcessAttachment { + /// Stdin writer. + pub stdin: BoundaryInput, + /// Stdout reader, or the PTY-merged output stream. + pub stdout: BoundaryOutput, + /// Stderr reader, distinct from stdout for non-PTY processes. + pub stderr: Option, + /// PTY control, present when the admitted process owns a terminal. + pub terminal: Option>, +} + /// A PTY attached to an exec session. #[async_trait] pub trait BoundaryTerminal: Send + Sync { @@ -747,7 +849,7 @@ impl FromStr for Sha256Digest { } /// Immutable socket metadata supplied with a pending external TCP open. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct NetworkSocketMetadata { /// Kernel socket cookie captured for the exact open-file description. pub socket_cookie: u64, @@ -758,7 +860,7 @@ pub struct NetworkSocketMetadata { } /// Typed supervisor decision for one pending TCP open. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum NetworkOpenResult { /// L4 authorization and a bounded relay handler are ready. L7 policy still /// applies to bytes after the local connection commits. @@ -791,8 +893,8 @@ pub struct PendingNetworkOpen { /// mediation service wherever that service runs. /// /// It may wrap a dedicated listener or a demultiplexed view over shared -/// transport; how it reaches a co-located proxy, a sidecar, or a shared -/// mediation service is backend-private. A trusted backend component associates +/// transport; how it reaches the separate supervisor is backend-private. A +/// trusted backend component associates /// every returned connection with its active boundary without relying solely on /// a transport tuple or workload-provided identifier. An `Err` from `accept` /// means the source itself is unusable and fails the boundary closed. @@ -803,7 +905,7 @@ pub trait NetworkMediationSource: Send + Sync { } /// DNS transport used by one workload exchange. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum DnsTransport { /// One DNS wire datagram without a TCP length prefix. Udp, diff --git a/crates/openshell-isolation-interface/src/contract/tests.rs b/crates/openshell-isolation-interface/src/contract/tests.rs index d16a4aa895..cc90afc4c9 100644 --- a/crates/openshell-isolation-interface/src/contract/tests.rs +++ b/crates/openshell-isolation-interface/src/contract/tests.rs @@ -370,11 +370,50 @@ fn confirmation_evidence() -> SandboxConfirmEvidence { tcp_deny_round_trip: true, authenticated_supervisor: true, session_epoch: "epoch-1".to_string(), - direct_egress_blocked: true, + driver_fence: DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + }, resource_claims: BTreeMap::new(), } } +#[test] +fn driver_fence_evidence_is_backend_specific_and_fail_closed() { + let docker = DriverFenceEvidence::Docker { + container_id: "sha256:container".to_string(), + network_mode: "none".to_string(), + unexpected_networks: Vec::new(), + }; + let kubernetes = DriverFenceEvidence::Kubernetes { + network_policy_uid: "policy-uid".to_string(), + network_policy_resource_version: "42".to_string(), + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + }; + let vm = DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + }; + + assert!(docker.validate_for_backend("docker").is_ok()); + assert!( + kubernetes + .validate_for_backend("kubernetes-proxy-pod") + .is_ok() + ); + assert!(vm.validate_for_backend("vm").is_ok()); + assert!(docker.validate_for_backend("vm").is_err()); + + let drifted = DriverFenceEvidence::Docker { + container_id: "sha256:container".to_string(), + network_mode: "bridge".to_string(), + unexpected_networks: vec!["bridge".to_string()], + }; + assert!(drifted.validate_for_backend("docker").is_err()); +} + /// The backend-independent supervisor sequence. Identical for every backend: /// this is the proof that adding a backend needs no supervisor lifecycle change. async fn drive( @@ -727,7 +766,7 @@ fn workload_identity_rejects_root_and_normalizes_groups() { let identity = ResolvedWorkloadIdentity::new( 1000, 1001, - vec![1003, 1002, 1003], + vec![1003, 1001, 1002, 1003], "policy".into(), "digest".into(), ) diff --git a/crates/openshell-isolation-interface/src/lib.rs b/crates/openshell-isolation-interface/src/lib.rs index d5dff7b06f..3cc8f9aaf7 100644 --- a/crates/openshell-isolation-interface/src/lib.rs +++ b/crates/openshell-isolation-interface/src/lib.rs @@ -7,9 +7,8 @@ //! the supervisor role drives it through one contract. The supervisor-facing //! contract lives in [`contract`]: an object-safe, runtime-selectable backend //! plus a fixed chain of boxed lifecycle states the supervisor advances without -//! branching on where the boundary sits. The same calls work whether the -//! boundary lives in the agent's container (the in-pod backend) or further out -//! (a microVM, a node daemon, a separate pod). +//! branching on placement. Each driver places a sandbox boundary beside the +//! workload and connects it to a separate supervisor. //! //! The backend establishes standing enforcement before untrusted code runs and //! ensures launch-time controls are in force before each process's first @@ -47,7 +46,11 @@ pub struct AgentSpec { pub interactive: bool, } +/// Versioned control-to-boundary wire types shared by every backend. +pub mod boundary_protocol; pub mod contract; +/// Reusable control-side implementation for a remote boundary endpoint. +pub mod remote; /// Linux-only primitives shared by capability-free sandbox implementations. #[cfg(target_os = "linux")] diff --git a/crates/openshell-isolation-interface/src/linux/landlock.rs b/crates/openshell-isolation-interface/src/linux/landlock.rs index 33c84bf2c4..28823ecba5 100644 --- a/crates/openshell-isolation-interface/src/linux/landlock.rs +++ b/crates/openshell-isolation-interface/src/linux/landlock.rs @@ -3,6 +3,7 @@ //! Race-resistant handles for an explicit Landlock root allow-list. +#![allow(unsafe_code)] use std::collections::BTreeSet; use std::ffi::{OsStr, OsString}; use std::io; @@ -12,6 +13,27 @@ use std::path::Path; use rustix::fs::{AtFlags, Mode, OFlags, Stat, fstat, open, openat, statat}; +const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1; + +/// Query the Landlock ABI admitted by the active kernel and outer seccomp +/// profile without installing a ruleset. +pub fn abi_version() -> io::Result { + // SAFETY: the VERSION operation requires a null ruleset pointer and zero + // size and returns one scalar ABI version. + let result = unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0, + LANDLOCK_CREATE_RULESET_VERSION, + ) + }; + if result < 0 { + Err(io::Error::last_os_error()) + } else { + u32::try_from(result).map_err(|_| io::Error::other("Landlock ABI does not fit u32")) + } +} /// One verified immediate child of the sandbox root. pub struct RootEntryHandle { name: OsString, diff --git a/crates/openshell-isolation-interface/src/linux/proc_fd.rs b/crates/openshell-isolation-interface/src/linux/proc_fd.rs index 8fee46b680..13f1856e54 100644 --- a/crates/openshell-isolation-interface/src/linux/proc_fd.rs +++ b/crates/openshell-isolation-interface/src/linux/proc_fd.rs @@ -9,6 +9,51 @@ use std::fs; use std::io; use std::os::fd::RawFd; +/// Snapshot socket inodes installed in process descriptor tables other than +/// `excluded_pid`. +/// +/// Inaccessible or concurrently disappearing entries are +/// skipped. Callers use this only to reclaim bounded mediation metadata; a +/// later operation on an unregistered descriptor fails closed. +pub fn installed_socket_inodes_excluding( + excluded_pid: u32, +) -> io::Result> { + let mut inodes = std::collections::BTreeSet::new(); + for process in fs::read_dir("/proc")? { + let Ok(process) = process else { continue }; + let Some(name) = process.file_name().to_str().map(str::to_owned) else { + continue; + }; + let Ok(pid) = name.parse::() else { + continue; + }; + if pid == excluded_pid { + continue; + } + let Ok(descriptors) = fs::read_dir(process.path().join("fd")) else { + continue; + }; + for descriptor in descriptors.flatten() { + let Ok(target) = fs::read_link(descriptor.path()) else { + continue; + }; + let Some(target) = target.to_str() else { + continue; + }; + let Some(digits) = target + .strip_prefix("socket:[") + .and_then(|value| value.strip_suffix(']')) + else { + continue; + }; + if let Ok(inode) = digits.parse::() { + inodes.insert(inode); + } + } + } + Ok(inodes) +} + /// Return the socket inode currently installed at `fd` in `tid`'s descriptor /// table. /// @@ -86,4 +131,35 @@ mod tests { io::ErrorKind::InvalidInput ); } + + #[test] + fn installed_socket_snapshot_can_exclude_the_broker() { + let mut pair = [-1; 2]; + // SAFETY: pair points to storage for exactly two returned descriptors. + let result = unsafe { + libc::socketpair( + libc::AF_UNIX, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + 0, + pair.as_mut_ptr(), + ) + }; + assert_eq!(result, 0, "socketpair: {}", io::Error::last_os_error()); + // SAFETY: successful socketpair returned two independently owned FDs. + let left = unsafe { OwnedFd::from_raw_fd(pair[0]) }; + // SAFETY: successful socketpair returned two independently owned FDs. + let _right = unsafe { OwnedFd::from_raw_fd(pair[1]) }; + let inode = socket_inode(std::process::id(), left.as_raw_fd()).unwrap(); + + assert!( + installed_socket_inodes_excluding(u32::MAX) + .unwrap() + .contains(&inode) + ); + assert!( + !installed_socket_inodes_excluding(std::process::id()) + .unwrap() + .contains(&inode) + ); + } } diff --git a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs index 8e20fd070b..081224abac 100644 --- a/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs +++ b/crates/openshell-isolation-interface/src/linux/seccomp_notify.rs @@ -330,13 +330,16 @@ pub fn install_listener(syscalls: &[i64]) -> io::Result { verify_notification_sizes()?; set_no_new_privileges()?; - match install_listener_with_flags(syscalls, true) { - Ok(listener) => Ok(listener), - Err(error) if error.raw_os_error() == Some(libc::EINVAL) => { - install_listener_with_flags(syscalls, false) + install_listener_with_flags(syscalls, true).map_err(|error| { + if error.raw_os_error() == Some(libc::EINVAL) { + io::Error::new( + io::ErrorKind::Unsupported, + "seccomp WAIT_KILLABLE_RECV is required (Linux 5.19 or newer)", + ) + } else { + error } - Err(error) => Err(error), - } + }) } /// Install the capability-free workload networking listener on the calling @@ -357,7 +360,6 @@ pub fn install_workload_listener() -> io::Result { libc::SYS_sendmsg, libc::SYS_sendmmsg, libc::SYS_getpeername, - libc::SYS_getsockname, libc::SYS_setsockopt, ]) } @@ -492,55 +494,22 @@ fn probe_addfd_send() -> io::Result<()> { fn probe_task_memory_copy() -> io::Result<()> { let source = 0x1122_3344_5566_7788_u64; - let mut copied = 0_u64; - let local = libc::iovec { - iov_base: std::ptr::addr_of_mut!(copied).cast(), - iov_len: size_of::(), - }; - let remote = libc::iovec { - iov_base: std::ptr::addr_of!(source).cast_mut().cast(), - iov_len: size_of::(), - }; - // SAFETY: both iovecs point to live same-process u64 values for the full - // call. This is an admission probe, not the cross-task production codec. - let read = unsafe { - libc::process_vm_readv( - libc::getpid(), - std::ptr::addr_of!(local), - 1, - std::ptr::addr_of!(remote), - 1, - 0, - ) - }; - let word_size = isize::try_from(size_of::()).expect("u64 size fits isize"); - if read != word_size || copied != source { - return Err(io::Error::last_os_error()); + let tid = std::process::id(); + let mut source_bytes = [0_u8; size_of::()]; + super::task_memory::read_exact(tid, std::ptr::addr_of!(source) as u64, &mut source_bytes)?; + let mut copied = u64::from_ne_bytes(source_bytes); + if copied != source { + return Err(io::Error::other("task-memory probe read wrong value")); } let replacement = 0xaabb_ccdd_eeff_0011_u64; - let local = libc::iovec { - iov_base: std::ptr::addr_of!(replacement).cast_mut().cast(), - iov_len: size_of::(), - }; - let remote = libc::iovec { - iov_base: std::ptr::addr_of_mut!(copied).cast(), - iov_len: size_of::(), - }; - // SAFETY: both iovecs point to live same-process u64 values for the full - // call. The write is bounded to the destination value. - let written = unsafe { - libc::process_vm_writev( - libc::getpid(), - std::ptr::addr_of!(local), - 1, - std::ptr::addr_of!(remote), - 1, - 0, - ) - }; - if written != word_size || copied != replacement { - return Err(io::Error::last_os_error()); + super::task_memory::write_exact( + tid, + std::ptr::addr_of_mut!(copied) as u64, + &replacement.to_ne_bytes(), + )?; + if copied != replacement { + return Err(io::Error::other("task-memory probe wrote wrong value")); } Ok(()) } diff --git a/crates/openshell-isolation-interface/src/linux/socket_registry.rs b/crates/openshell-isolation-interface/src/linux/socket_registry.rs index c5a39d2ce5..966612144e 100644 --- a/crates/openshell-isolation-interface/src/linux/socket_registry.rs +++ b/crates/openshell-isolation-interface/src/linux/socket_registry.rs @@ -205,6 +205,20 @@ impl SocketRegistry { self.entries.is_empty() } + /// Whether another socket would exceed the configured bound. + #[must_use] + pub fn is_full(&self) -> bool { + self.entries.len() >= self.capacity + } + + /// Retain only sockets that remain installed in a workload descriptor + /// table. The trusted broker's temporary source descriptors are excluded + /// from `installed` by the caller. + pub fn retain_installed(&mut self, installed: &std::collections::BTreeSet) { + self.entries + .retain(|inode, _entry| installed.contains(inode)); + } + /// Stage a newly created source descriptor without publishing it. pub fn stage(&self, source: OwnedFd, metadata: SocketMetadata) -> io::Result { if self.entries.len() >= self.capacity { @@ -226,6 +240,19 @@ impl SocketRegistry { /// Publish a tentative socket only after ADDFD-SEND has succeeded. pub fn commit(&mut self, tentative: TentativeSocket) -> io::Result { + self.commit_with_state(tentative, SocketState::Created) + } + + /// Publish a tentative socket in a caller-proven initial state. + /// + /// Accepted sockets are created and classified by the trusted broker, so + /// they enter the registry directly as [`SocketState::AcceptedLocal`] + /// rather than pretending to be unconnected. + pub fn commit_with_state( + &mut self, + tentative: TentativeSocket, + state: SocketState, + ) -> io::Result { if self.entries.len() >= self.capacity { return Err(io::Error::from_raw_os_error(libc::EMFILE)); } @@ -238,13 +265,17 @@ impl SocketRegistry { )); } let identity = tentative.identity; + let retain_source = matches!( + state, + SocketState::Created | SocketState::Bound { .. } | SocketState::Listening { .. } + ); self.entries.insert( identity.inode, SocketEntry { identity, metadata: tentative.metadata, - state: SocketState::Created, - retained_preconnect: Some(tentative.source), + state, + retained_preconnect: retain_source.then_some(tentative.source), }, ); Ok(identity) @@ -409,4 +440,22 @@ mod tests { identity ); } + + #[test] + fn collection_reclaims_only_uninstalled_socket_metadata() { + let mut registry = SocketRegistry::new(12, 2).unwrap(); + let first = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + let second = registry + .commit(registry.stage(tcp_socket(), metadata()).unwrap()) + .unwrap(); + assert!(registry.is_full()); + + registry.retain_installed(&std::collections::BTreeSet::from([first.inode])); + + assert_eq!(registry.len(), 1); + assert!(registry.remove_inode(first.inode)); + assert!(!registry.remove_inode(second.inode)); + } } diff --git a/crates/openshell-isolation-interface/src/linux/task_memory.rs b/crates/openshell-isolation-interface/src/linux/task_memory.rs index 00c5ab921f..4a0422a006 100644 --- a/crates/openshell-isolation-interface/src/linux/task_memory.rs +++ b/crates/openshell-isolation-interface/src/linux/task_memory.rs @@ -10,6 +10,9 @@ #![allow(unsafe_code)] use std::io; +use std::mem::size_of; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::fs::FileExt as _; /// Maximum number of task-memory bytes copied by one operation. pub const MAX_TASK_MEMORY_COPY: usize = 64 * 1024; @@ -50,8 +53,14 @@ pub fn read_exact(tid: u32, address: u64, destination: &mut [u8]) -> io::Result< 1, 0, ) - })?; - require_exact(copied, destination.len(), "task-memory read") + }); + match copied { + Ok(copied) => require_exact(copied, destination.len(), "task-memory read"), + Err(error) if syscall_profile_denied(&error) => { + read_exact_from_proc_mem(tid, address, destination) + } + Err(error) => Err(error), + } } /// Write exactly all of `source` to `address` in `tid`. @@ -90,10 +99,178 @@ pub fn write_exact(tid: u32, address: u64, source: &[u8]) -> io::Result<()> { 1, 0, ) - })?; - require_exact(copied, source.len(), "task-memory write") + }); + match copied { + Ok(copied) => require_exact(copied, source.len(), "task-memory write"), + Err(error) if syscall_profile_denied(&error) => { + write_exact_to_proc_mem(tid, address, source) + } + Err(error) => Err(error), + } +} + +fn syscall_profile_denied(error: &io::Error) -> bool { + matches!( + error.raw_os_error(), + Some(libc::EPERM | libc::EACCES | libc::ENOSYS) + ) +} + +fn read_exact_from_proc_mem(tid: u32, address: u64, destination: &mut [u8]) -> io::Result<()> { + let file = std::fs::File::open(format!("/proc/{tid}/mem"))?; + let copied = file.read_at(destination, address)?; + require_exact(copied, destination.len(), "proc task-memory read") +} + +fn write_exact_to_proc_mem(tid: u32, address: u64, source: &[u8]) -> io::Result<()> { + let file = std::fs::OpenOptions::new() + .write(true) + .open(format!("/proc/{tid}/mem"))?; + let copied = file.write_at(source, address)?; + require_exact(copied, source.len(), "proc task-memory write") +} + +/// Prove same-UID parent-to-child read and write access under the active Yama, +/// LSM, and outer seccomp posture. +/// +/// Call this only from a single-threaded probe process. The child executes +/// raw, allocation-free syscalls between `fork` and `_exit`. +pub fn probe_child_access() -> io::Result<()> { + const INITIAL: u64 = 0x1122_3344_5566_7788; + const REPLACEMENT: u64 = 0xaabb_ccdd_eeff_0011; + // SAFETY: mmap creates one private anonymous page owned by this process. + let mapping = unsafe { + libc::mmap( + std::ptr::null_mut(), + size_of::(), + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + return Err(io::Error::last_os_error()); + } + let mapping_address = mapping as u64; + // SAFETY: mapping spans at least one aligned u64-sized region. + unsafe { mapping.cast::().write(INITIAL) }; + + // SAFETY: eventfd returns independently owned descriptors on success. + let ready = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if ready < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + // SAFETY: successful eventfd returned one owned descriptor. + let ready = unsafe { OwnedFd::from_raw_fd(ready) }; + // SAFETY: eventfd returns independently owned descriptors on success. + let proceed = unsafe { libc::eventfd(0, libc::EFD_CLOEXEC) }; + if proceed < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + // SAFETY: successful eventfd returned one owned descriptor. + let proceed = unsafe { OwnedFd::from_raw_fd(proceed) }; + + // SAFETY: the caller promises this probe process is single-threaded. The + // child performs only raw syscalls and memory operations before `_exit`. + let child = unsafe { libc::fork() }; + if child < 0 { + // SAFETY: mapping is the live region returned above. + unsafe { libc::munmap(mapping, size_of::()) }; + return Err(io::Error::last_os_error()); + } + if child == 0 { + // The sandbox remains nondumpable, but an exec'd workload must be + // observable by its same-UID ancestor. This child contains no trusted + // parent address space secrets beyond this synthetic probe value. + // SAFETY: these calls use live inherited eventfds and scalar prctl + // arguments. No Rust cleanup runs in the child. + unsafe { + if libc::prctl(libc::PR_SET_DUMPABLE, 1, 0, 0, 0) < 0 + || write_eventfd(ready.as_raw_fd()).is_err() + || read_eventfd(proceed.as_raw_fd()).is_err() + || mapping.cast::().read() != REPLACEMENT + { + libc::_exit(1); + } + libc::_exit(0); + } + } + + let outcome = (|| { + read_eventfd(ready.as_raw_fd())?; + let mut observed = [0_u8; size_of::()]; + read_exact( + u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?, + mapping_address, + &mut observed, + )?; + if u64::from_ne_bytes(observed) != INITIAL { + return Err(io::Error::other( + "cross-child memory read returned wrong data", + )); + } + write_exact( + u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?, + mapping_address, + &REPLACEMENT.to_ne_bytes(), + )?; + write_eventfd(proceed.as_raw_fd())?; + let mut status = 0; + // SAFETY: child is a live direct child and status points to storage. + if unsafe { libc::waitpid(child, std::ptr::addr_of_mut!(status), 0) } != child { + return Err(io::Error::last_os_error()); + } + if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 { + return Err(io::Error::other("cross-child memory probe failed in child")); + } + Ok(()) + })(); + + if outcome.is_err() { + // SAFETY: a failed parent-side operation may leave this direct child + // blocked on eventfd. SIGKILL and waitpid guarantee cleanup. + unsafe { + libc::kill(child, libc::SIGKILL); + libc::waitpid(child, std::ptr::null_mut(), 0); + } + } + // SAFETY: mapping is the live region returned above and no child remains. + unsafe { libc::munmap(mapping, size_of::()) }; + outcome +} + +fn read_eventfd(fd: libc::c_int) -> io::Result<()> { + let mut value = 0_u64; + // SAFETY: eventfd reads exactly one u64 into live storage. + let result = unsafe { libc::read(fd, std::ptr::addr_of_mut!(value).cast(), size_of::()) }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + require_exact( + usize::try_from(result).map_err(|_| io::Error::other("eventfd read length invalid"))?, + size_of::(), + "eventfd read", + ) } +fn write_eventfd(fd: libc::c_int) -> io::Result<()> { + let value = 1_u64; + // SAFETY: eventfd reads exactly one u64 from live storage. + let result = unsafe { libc::write(fd, std::ptr::addr_of!(value).cast(), size_of::()) }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + require_exact( + usize::try_from(result).map_err(|_| io::Error::other("eventfd write length invalid"))?, + size_of::(), + "eventfd write", + ) +} fn validate_request(tid: u32, address: u64, length: usize) -> io::Result<()> { if tid == 0 { return Err(io::Error::new( @@ -177,6 +354,28 @@ mod tests { assert_eq!(destination, replacement); } + #[test] + fn proc_mem_fallback_reads_and_writes_exact_memory() { + let source = 0x0102_0304_0506_0708_u64; + let mut destination = 0_u64; + let mut bytes = [0_u8; size_of::()]; + read_exact_from_proc_mem( + std::process::id(), + std::ptr::addr_of!(source) as u64, + &mut bytes, + ) + .expect("read through proc mem"); + assert_eq!(u64::from_ne_bytes(bytes), source); + + write_exact_to_proc_mem( + std::process::id(), + std::ptr::addr_of_mut!(destination) as u64, + &source.to_ne_bytes(), + ) + .expect("write through proc mem"); + assert_eq!(destination, source); + } + #[test] fn rejects_invalid_ranges_before_syscall() { let mut byte = [0_u8; 1]; @@ -209,6 +408,4 @@ mod tests { io::ErrorKind::InvalidInput ); } - - use std::mem::size_of; } diff --git a/crates/openshell-isolation-interface/src/remote.rs b/crates/openshell-isolation-interface/src/remote.rs new file mode 100644 index 0000000000..6c09a7d8b8 --- /dev/null +++ b/crates/openshell-isolation-interface/src/remote.rs @@ -0,0 +1,1806 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned remote boundary. + +#![allow(unsafe_code)] + +#[cfg(target_os = "linux")] +use std::mem::size_of; +#[cfg(target_os = "linux")] +use std::os::fd::{FromRawFd as _, IntoRawFd as _}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use crate::AgentSpec; +use crate::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, + BoundaryTerminal, ConfirmedBoundary, DnsMediationSource, ExecSession, ExecSpec, + IsolationBackend, LoopbackTarget, MediatedDnsQuery, NetworkMediationSource, NetworkOpenResult, + PendingNetworkOpen, ProcessAttachment, ReadyBoundary, RunningBoundary, SandboxContext, + VerifiedTopologyDescriptor, +}; +use async_trait::async_trait; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; + +use crate::boundary_protocol::{ + AgentSpecWire, BoundaryClientTls, BoundaryTopology, BoundaryTransport, DnsQueryResultWire, + ExecSpecWire, ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, + ResponseEnvelope, STREAM_DNS_ACK, STREAM_DNS_RESPONSE, STREAM_EXIT, STREAM_STDERR, + STREAM_STDIN, STREAM_STDIN_CLOSED, STREAM_STDOUT, SandboxPolicyWire, SignalWire, decode_frame, + encode_frame, read_stream_frame, validate_resource_claims, write_stream_frame, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// How long one control call keeps retrying boundary connect attempts. Boot-time +/// callers retry whole calls above this; past boot, exhausting this window +/// means the remote boundary (or its launcher) is gone rather than still starting. +const CONNECT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; + +/// Host-side remote boundary implementation registered with the supervisor. +#[derive(Debug)] +pub struct RemoteIsolationBackend { + backend_name: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +impl RemoteIsolationBackend { + pub fn new( + backend_name: impl Into, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + ) -> Self { + Self { + backend_name: backend_name.into(), + ca_file_paths, + provider_credentials, + } + } +} + +#[async_trait] +impl IsolationBackend for RemoteIsolationBackend { + fn backend_name(&self) -> &str { + &self.backend_name + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: BoundaryTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox, &self.backend_name)?; + let host_gateway_ip = topology.host_gateway_ip; + let resource_claims = topology.resource_claims.clone(); + let generation = topology.generation.clone(); + let session_epoch = topology.session_epoch.clone(); + let driver_fence = topology.driver_fence.clone(); + let client = Arc::new(BoundaryClient::new(topology)); + let response = client + .call_idempotent(Request::Attach { + policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())), + resource_claims: resource_claims.clone(), + }) + .await?; + let Response::Attached { snapshot } = response else { + return Err(unexpected_response("attached", &response)); + }; + if snapshot.generation != generation { + return Err(BackendError::Confirm( + "sandbox session snapshot generation does not match topology".to_string(), + )); + } + Ok(Box::new(RemoteBound { + client: client.clone(), + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(RemoteNetworkMediation { + client: client.clone(), + }), + dns_mediation: Arc::new(RemoteDnsMediation { client }), + host_gateway_ip, + ca_file_paths: self.ca_file_paths.clone(), + provider_credentials: self.provider_credentials.clone(), + identity: sandbox.identity, + generation, + session_epoch, + resource_claims, + driver_fence, + })) + } +} + +fn validate_topology( + topology: &BoundaryTopology, + sandbox: &SandboxContext, + backend_name: &str, +) -> Result<(), BackendError> { + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.generation.is_empty() || topology.session_epoch.is_empty() { + return Err(BackendError::Descriptor( + "boundary generation and session epoch must not be empty".to_string(), + )); + } + if topology.workload_identity != sandbox.identity { + return Err(BackendError::Descriptor( + "topology workload identity does not match admitted sandbox identity".to_string(), + )); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "boundary bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + validate_resource_claims(&topology.resource_claims)?; + topology.driver_fence.validate_for_backend(backend_name)?; + let tls = match &topology.transport { + BoundaryTransport::Unix { socket_path, tls } => { + validate_socket_path(socket_path)?; + tls + } + BoundaryTransport::TlsTcp { address, tls } => { + validate_tcp_address(*address)?; + tls + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + tls, + } => { + if *guest_cid < 3 { + return Err(BackendError::Descriptor( + "boundary CID must be at least 3".to_string(), + )); + } + validate_control_port(*control_port)?; + tls + } + }; + validate_client_tls(tls)?; + Ok(()) +} + +fn validate_tcp_address(address: std::net::SocketAddr) -> Result<(), BackendError> { + if address.port() == 0 || address.ip().is_unspecified() { + Err(BackendError::Descriptor( + "boundary TCP address must have a concrete IP and nonzero port".to_string(), + )) + } else { + Ok(()) + } +} + +fn validate_client_tls(tls: &BoundaryClientTls) -> Result<(), BackendError> { + rustls::pki_types::ServerName::try_from(tls.server_name.clone()).map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {:?} is invalid: {error}", + tls.server_name + )) + })?; + tls_client_config(tls).map(|_| ()) +} + +fn tls_client_config(tls: &BoundaryClientTls) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificates = rustls_pemfile::certs(&mut tls.ca_certificate_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse boundary TLS CA certificate: {error}")) + })?; + if certificates.is_empty() { + return Err(BackendError::Descriptor( + "boundary TLS CA certificate PEM contains no certificates".to_string(), + )); + } + let mut roots = rustls::RootCertStore::empty(); + for certificate in certificates { + roots.add(certificate).map_err(|error| { + BackendError::Descriptor(format!("load boundary TLS CA certificate: {error}")) + })?; + } + let certificate_chain = rustls_pemfile::certs(&mut tls.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .map_err(|error| { + BackendError::Descriptor(format!("parse supervisor TLS certificate: {error}")) + })?; + if certificate_chain.is_empty() { + return Err(BackendError::Descriptor( + "supervisor TLS certificate PEM contains no certificates".to_string(), + )); + } + let private_key = rustls_pemfile::private_key(&mut tls.private_key_pem.as_bytes()) + .map_err(|error| { + BackendError::Descriptor(format!("parse supervisor TLS private key: {error}")) + })? + .ok_or_else(|| { + BackendError::Descriptor( + "supervisor TLS private-key PEM contains no private key".to_string(), + ) + })?; + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_client_auth_cert(certificate_chain, private_key) + .map_err(|error| { + BackendError::Descriptor(format!("build supervisor mutual-TLS config: {error}")) + }) +} + +fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { + if path.is_absolute() { + Ok(()) + } else { + Err(BackendError::Descriptor( + "boundary control Unix socket path must be absolute".to_string(), + )) + } +} + +fn validate_control_port(port: u32) -> Result<(), BackendError> { + if port == 0 { + Err(BackendError::Descriptor( + "boundary control port must be nonzero".to_string(), + )) + } else { + Ok(()) + } +} + +struct RemoteBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, + dns_mediation: Arc, + host_gateway_ip: Option, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + identity: crate::contract::ResolvedWorkloadIdentity, + generation: String, + session_epoch: String, + resource_claims: std::collections::BTreeMap, + driver_fence: crate::contract::DriverFenceEvidence, +} + +#[async_trait] +impl BoundBoundary for RemoteBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + fn dns_mediation_source(&self) -> Option> { + Some(self.dns_mediation.clone()) + } + + fn host_gateway_ip(&self) -> Option { + self.host_gateway_ip + } + + async fn confirm(self: Box) -> Result { + let response = self.client.call_idempotent(Request::Confirm).await?; + let Response::Confirmed { evidence } = response else { + return Err(unexpected_response("confirmed_with_evidence", &response)); + }; + evidence.validate(&self.identity)?; + if evidence.generation != self.generation + || evidence.session_epoch != self.session_epoch + || evidence.resource_claims != self.resource_claims + || evidence.driver_fence != self.driver_fence + { + return Err(BackendError::Confirm( + "sandbox confirmation generation, session, resource claims, or driver fence do not match topology" + .to_string(), + )); + } + Ok(ConfirmedBoundary::new( + Box::new(RemoteReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + ca_file_paths: self.ca_file_paths, + provider_credentials: self.provider_credentials, + }), + *evidence, + )) + } +} + +struct RemoteReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + ca_file_paths: Arc>>, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, +} + +#[async_trait] +impl ReadyBoundary for RemoteReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let ca_paths = self + .ca_file_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { + let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) + })?; + let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; + (Some(ca_cert), Some(ca_bundle)) + } else { + (None, None) + }; + let (provider_env_revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + }) + .await?; + let Response::Started { + process_id, + provider_env_revision, + } = response + else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(RemoteProcess { + client: self.client.clone(), + process_id, + }); + Ok(Box::new(RemoteRunning { + process, + exec: Arc::new(RemoteExec { + client: self.client.clone(), + provider_credentials: self.provider_credentials, + boundary_revision: tokio::sync::Mutex::new(provider_env_revision), + }), + port_forward: Arc::new(RemotePortForward { + client: self.client, + }), + })) + } +} + +struct RemoteRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for RemoteRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct RemoteProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for RemoteProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await + .map_err(|error| match error { + // A wait that can no longer reach the boundary leaf means the + // boundary is gone, not that a retry could still observe the + // exit status; report boundary loss per the contract. + BackendError::Unavailable(message) => { + BackendError::Terminated(format!("boundary lost during wait: {message}")) + } + error => error, + })?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +async fn open_process_attachment( + client: Arc, + process_id: String, +) -> Result { + let (stream, response) = client + .call_stream(Request::AttachProcess { + process_id: process_id.clone(), + }) + .await?; + let Response::ProcessAttached { + terminal: has_terminal, + } = response + else { + return Err(unexpected_response("process_attached", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_process_responses( + network_reader, + stdout_pump, + stderr_pump, + )); + let terminal: Option> = if has_terminal { + let terminal: Arc = Arc::new(RemoteTerminal { client, process_id }); + Some(terminal) + } else { + None + }; + let stderr: Option = if has_terminal { + None + } else { + let stderr: BoundaryOutput = Box::new(stderr); + Some(stderr) + }; + Ok(ProcessAttachment { + stdin: Box::new(stdin), + stdout: Box::new(stdout), + stderr, + terminal, + }) +} + +async fn pump_process_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + return; + } + } + Ok(Some((STREAM_EXIT, _)) | None) | Err(_) => return, + Ok(Some((_channel, _))) => return, + } + } +} + +struct RemoteExec { + client: Arc, + provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, + boundary_revision: tokio::sync::Mutex, +} + +#[async_trait] +impl BoundaryExec for RemoteExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let mut boundary_revision = self.boundary_revision.lock().await; + for _ in 0..3 { + let (revision, provider_env) = self + .provider_credentials + .child_env_snapshot_with_gcp_resolved(); + let response = self + .client + .call_idempotent(Request::UpdateProviderEnvironment { + expected_revision: *boundary_revision, + revision, + provider_env, + }) + .await?; + let Response::ProviderEnvironmentUpdated { + revision: effective_revision, + } = response + else { + return Err(unexpected_response( + "provider_environment_updated", + &response, + )); + }; + *boundary_revision = effective_revision; + if effective_revision == revision { + return open_exec_session(self.client.clone(), spec).await; + } + } + Err(BackendError::Process( + "boundary provider environment changed concurrently during reconciliation".to_string(), + )) + } +} + +struct RemotePortForward { + client: Arc, +} + +#[async_trait] +impl BoundaryPortForward for RemotePortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + let (stream, response) = self + .client + .call_stream(Request::PortForward { + host: target.host(), + port: target.port(), + }) + .await?; + match response { + Response::PortConnected => Ok(stream), + response => Err(unexpected_response("port_connected", &response)), + } + } +} + +struct RemoteExecProcess { + client: Arc, + process_id: String, + exit: Arc, +} + +struct RemoteExit { + result: std::sync::Mutex>>, + changed: Notify, +} + +impl RemoteExit { + fn new() -> Self { + Self { + result: std::sync::Mutex::new(None), + changed: Notify::new(), + } + } + + fn set(&self, result: Result) { + let mut current = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + *current = Some(result); + self.changed.notify_waiters(); + } + } + + async fn wait(&self) -> Result { + loop { + let changed = self.changed.notified(); + let result = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Terminated); + } + changed.await; + } + } +} + +#[async_trait] +impl BoundaryProcess for RemoteExecProcess { + async fn attach(&self) -> Result { + open_process_attachment(self.client.clone(), self.process_id.clone()).await + } + + async fn wait(&self) -> Result { + self.exit.wait().await + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + expect_response( + self.client + .call_idempotent(Request::ExecSignal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?, + "signaled", + ) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct RemoteTerminal { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryTerminal for RemoteTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + let response = self + .client + .call_idempotent(Request::Resize { + process_id: self.process_id.clone(), + cols, + rows, + }) + .await?; + if matches!(response, Response::Resized) { + Ok(()) + } else { + Err(unexpected_response("resized", &response)) + } + } +} + +async fn open_exec_session( + client: Arc, + spec: ExecSpec, +) -> Result { + let (stream, response) = client + .call_stream_idempotent(Request::Exec { + spec: ExecSpecWire::from(spec), + }) + .await?; + let Response::ExecStarted { process_id, pty } = response else { + return Err(unexpected_response("exec_started", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + let exit = Arc::new(RemoteExit::new()); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_exec_responses( + network_reader, + stdout_pump, + stderr_pump, + exit.clone(), + )); + + let process: Arc = Arc::new(RemoteExecProcess { + client: client.clone(), + process_id: process_id.clone(), + exit, + }); + let terminal: Option> = if pty { + Some(Arc::new(RemoteTerminal { client, process_id })) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if pty { None } else { Some(Box::new(stderr)) }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) +} + +async fn pump_exec_input( + mut input: tokio::io::DuplexStream, + mut network: tokio::io::WriteHalf, +) { + let mut buffer = vec![0; 16 * 1024]; + loop { + match input.read(&mut buffer).await { + Ok(0) => { + let _ = write_stream_frame(&mut network, STREAM_STDIN_CLOSED, &[]).await; + return; + } + Ok(read) => { + if write_stream_frame(&mut network, STREAM_STDIN, &buffer[..read]) + .await + .is_err() + { + return; + } + } + Err(_) => return, + } + } +} + +async fn pump_exec_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, + exit: Arc, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stdout consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + exit.set(Err("boundary exec stderr consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_EXIT, payload))) => { + let result = serde_json::from_slice::(&payload) + .map(BoundaryExitStatus::from) + .map_err(|error| format!("decode boundary exec exit: {error}")); + exit.set(result); + return; + } + Ok(Some((channel, _))) => { + exit.set(Err(format!( + "boundary exec returned unexpected stream channel {channel}" + ))); + return; + } + Ok(None) => { + exit.set(Err( + "boundary exec stream closed before exit status".to_string() + )); + return; + } + Err(error) => { + exit.set(Err(format!("read boundary exec stream: {error}"))); + return; + } + } + } +} + +/// Pulls boundary proxy connections over one authenticated vsock stream each. +struct RemoteNetworkMediation { + client: Arc, +} + +#[async_trait] +impl NetworkMediationSource for RemoteNetworkMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptNetwork).await?; + let Response::NetworkConnected { + identity, + destination, + socket, + policy_generation, + } = response + else { + return Err(unexpected_response("network_connected", &response)); + }; + let (result, completion) = tokio::sync::oneshot::channel(); + let (proxy_stream, transport_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(complete_network_open(stream, transport_stream, completion)); + Ok(PendingNetworkOpen { + stream: Box::new(proxy_stream), + binary_identity: identity.into_result(), + destination, + socket, + policy_generation, + result, + }) + } +} + +/// Pulls sandbox DNS wire exchanges over authenticated control streams. +struct RemoteDnsMediation { + client: Arc, +} + +#[async_trait] +impl DnsMediationSource for RemoteDnsMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptDns).await?; + let Response::DnsQuery { + request, + transport, + identity, + } = response + else { + return Err(unexpected_response("dns_query", &response)); + }; + let (response_tx, response_rx) = tokio::sync::oneshot::channel(); + tokio::spawn(complete_dns_query(stream, response_rx)); + Ok(MediatedDnsQuery { + request, + transport, + binary_identity: identity.into_result(), + response: response_tx, + }) + } +} + +async fn complete_dns_query( + mut boundary: BoundaryDuplexStream, + response: tokio::sync::oneshot::Receiver, BackendError>>, +) { + let result = match response.await { + Ok(Ok(response)) => DnsQueryResultWire::Response(response), + Ok(Err(error)) => DnsQueryResultWire::Error(error.to_string()), + Err(_) => DnsQueryResultWire::Error("DNS mediation was cancelled".to_string()), + }; + let payload = match serde_json::to_vec(&result) { + Ok(payload) => payload, + Err(error) => { + tracing::warn!(%error, "encode mediated DNS response failed: {error}"); + return; + } + }; + if let Err(error) = write_stream_frame(&mut boundary, STREAM_DNS_RESPONSE, &payload).await { + tracing::warn!(%error, "write mediated DNS response failed: {error}"); + return; + } + match tokio::time::timeout(REQUEST_TIMEOUT, read_stream_frame(&mut boundary)).await { + Ok(Ok(Some((STREAM_DNS_ACK, payload)))) if payload.is_empty() => {} + Ok(Ok(Some((channel, _)))) => { + tracing::warn!(channel, "unexpected mediated DNS acknowledgement channel"); + } + Ok(Ok(None)) => tracing::warn!("boundary closed before acknowledging DNS response"), + Ok(Err(error)) => { + tracing::warn!(%error, "read mediated DNS acknowledgement failed: {error}"); + } + Err(_) => tracing::warn!("timed out waiting for mediated DNS acknowledgement"), + } +} + +async fn complete_network_open( + mut boundary: BoundaryDuplexStream, + mut transport: tokio::io::DuplexStream, + completion: tokio::sync::oneshot::Receiver, +) { + let decision = completion.await.unwrap_or(NetworkOpenResult::Denied { + errno: cancellation_errno(), + }); + let Ok(payload) = serde_json::to_vec(&decision) else { + return; + }; + if write_stream_frame( + &mut boundary, + crate::boundary_protocol::STREAM_NETWORK_DECISION, + &payload, + ) + .await + .is_err() + { + return; + } + if matches!(decision, NetworkOpenResult::RelayReady) { + let _ = tokio::io::copy_bidirectional(&mut boundary, &mut transport).await; + } +} + +const fn cancellation_errno() -> i32 { + #[cfg(unix)] + { + libc::ECANCELED + } + #[cfg(not(unix))] + { + 125 + } +} + +struct BoundaryClient { + topology: BoundaryTopology, +} + +impl BoundaryClient { + fn new(topology: BoundaryTopology) -> Self { + Self { topology } + } + + async fn call_idempotent(&self, request: Request) -> Result { + let envelope = self.prepare_request(request)?; + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable( + "boundary idempotent control request timed out while waiting for remote boundary boot".to_string(), + ) + })? + } + + async fn call_wait(&self, request: Request) -> Result { + const WAIT_RECONNECT_ATTEMPTS: usize = 3; + let envelope = self.prepare_request(request)?; + for attempt in 1..=WAIT_RECONNECT_ATTEMPTS { + match self.exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) if attempt < WAIT_RECONNECT_ATTEMPTS => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + unreachable!("bounded wait reconnect loop always returns") + } + + async fn call_stream( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + tokio::time::timeout(REQUEST_TIMEOUT, self.open_exchange(request)) + .await + .map_err(|_| { + BackendError::Unavailable("boundary stream request timed out".to_string()) + })? + } + + async fn call_stream_idempotent( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let envelope = self.prepare_request(request)?; + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.open_exchange_envelope(&envelope).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable("boundary idempotent stream request timed out".to_string()) + })? + } + + #[cfg(test)] + async fn exchange(&self, request: Request) -> Result { + let (_, response) = self.open_exchange(request).await?; + Ok(response) + } + + fn prepare_request(&self, request: Request) -> Result { + RequestEnvelope::new( + self.topology.boundary_id.clone(), + self.topology.bootstrap_token.clone(), + request, + ) + .map_err(|error| BackendError::Process(format!("encode control request: {error}"))) + } + + async fn open_exchange( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let envelope = self.prepare_request(request)?; + self.open_exchange_envelope(&envelope).await + } + + async fn exchange_envelope( + &self, + envelope: &RequestEnvelope, + ) -> Result { + let (_, response) = self.open_exchange_envelope(envelope).await?; + Ok(response) + } + + async fn open_exchange_envelope( + &self, + envelope: &RequestEnvelope, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let request_id = envelope.request_id.clone(); + let mut stream = self.connect_boundary().await?; + let frame = encode_frame(envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write boundary control request: {error}")) + })?; + // `tokio-rustls` may retain part of a large plaintext frame in its + // internal TLS buffer. Flush before waiting for the response so the + // synchronous boundary reader can receive the complete request. + stream.flush().await.map_err(|error| { + BackendError::Unavailable(format!("flush boundary control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "boundary control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read boundary control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "boundary response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + let response = match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + }?; + Ok((stream, response)) + } + + async fn connect_boundary(&self) -> Result { + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match self.connect_boundary_once().await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } + + async fn connect_boundary_once(&self) -> Result { + let (stream, tls): (BoundaryDuplexStream, &BoundaryClientTls) = + match &self.topology.transport { + BoundaryTransport::Unix { socket_path, tls } => { + let stream = UnixStream::connect(socket_path).await.map_err(|error| { + BackendError::Unavailable(format!( + "connect to mapped boundary control socket {}: {error}", + socket_path.display() + )) + })?; + (Box::new(stream), tls) + } + BoundaryTransport::TlsTcp { address, tls } => { + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[*address]) + .await + .map_err(|error| { + BackendError::Unavailable(format!( + "connect to boundary TLS endpoint {address}: {error}" + )) + })?; + enable_boundary_tcp_keepalive(&stream); + (Box::new(stream), tls) + } + BoundaryTransport::Vsock { + guest_cid, + control_port, + tls, + } => (connect_host_vsock(*guest_cid, *control_port)?, tls), + }; + let server_name = rustls::pki_types::ServerName::try_from(tls.server_name.clone()) + .map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {:?} is invalid: {error}", + tls.server_name + )) + })?; + let connector = tokio_rustls::TlsConnector::from(Arc::new(tls_client_config(tls)?)); + let stream = connector + .connect(server_name, stream) + .await + .map_err(|error| { + BackendError::Unavailable(format!("authenticate sandbox channel: {error}")) + })?; + Ok(Box::new(stream)) + } +} + +fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(30)) + .with_interval(Duration::from_secs(10)); + let _ = socket2::SockRef::from(stream).set_tcp_keepalive(&keepalive); +} + +#[cfg(target_os = "linux")] +fn connect_host_vsock( + guest_cid: u32, + control_port: u32, +) -> Result { + let fd = unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(BackendError::Unavailable(format!( + "create host vsock: {}", + std::io::Error::last_os_error() + ))); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }; + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address family: {error}")) + })?; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: control_port, + svm_cid: guest_cid, + svm_zero: [0; 4], + }; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address length: {error}")) + })?; + let result = unsafe { + libc::connect( + std::os::fd::AsRawFd::as_raw_fd(&fd), + (&raw const address).cast::(), + address_length, + ) + }; + if result != 0 { + return Err(BackendError::Unavailable(format!( + "connect host vsock CID {guest_cid} port {control_port}: {}", + std::io::Error::last_os_error() + ))); + } + let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + BackendError::Unavailable(format!("set host vsock nonblocking: {error}")) + })?; + let stream = UnixStream::from_std(stream).map_err(|error| { + BackendError::Unavailable(format!("register host vsock with Tokio: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[cfg(not(target_os = "linux"))] +fn connect_host_vsock( + _guest_cid: u32, + _control_port: u32, +) -> Result { + Err(BackendError::Unavailable( + "host AF_VSOCK transport is supported only on Linux".to_string(), + )) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached { .. }, "attached") + | (Response::Confirmed { .. }, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected boundary response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("boundary process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::boundary_protocol::generate_boundary_mutual_tls_material; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + + fn test_driver_fence() -> crate::contract::DriverFenceEvidence { + crate::contract::DriverFenceEvidence::Vm { + generation: "test-generation".to_string(), + network_device_count: 0, + } + } + + #[tokio::test] + async fn boundary_tcp_connections_enable_keepalive() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let connected = tokio::spawn(async move { tokio::net::TcpStream::connect(address).await }); + let (_server, _) = listener.accept().await.unwrap(); + let client = connected.await.unwrap().unwrap(); + + enable_boundary_tcp_keepalive(&client); + + assert!(socket2::SockRef::from(&client).keepalive().unwrap()); + } + + #[tokio::test] + async fn remote_dns_exchange_returns_supervisor_response() { + let socket_path = std::env::temp_dir().join(format!( + "openshell-dns-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let certificate = test_certificate(); + let server_config = certificate.server_config.clone(); + let listener = tokio::net::UnixListener::bind(&socket_path).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .unwrap(); + let declared = stream.read_u32().await.unwrap() as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&u32::try_from(declared).unwrap().to_be_bytes()); + stream.read_exact(&mut frame[4..]).await.unwrap(); + let request: RequestEnvelope = decode_frame(&frame).unwrap(); + assert_eq!(request.request, Request::AcceptDns); + let response = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response: Response::DnsQuery { + request: vec![1, 2, 3], + transport: crate::contract::DnsTransport::Udp, + identity: crate::boundary_protocol::BinaryIdentityWire { + binary_path: Some(PathBuf::from("/usr/bin/dig")), + binary_digest: Some("a".repeat(64)), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: None, + }, + }, + }) + .unwrap(); + stream.write_all(&response).await.unwrap(); + let (channel, payload) = read_stream_frame(&mut stream).await.unwrap().unwrap(); + assert_eq!(channel, STREAM_DNS_RESPONSE); + assert_eq!( + serde_json::from_slice::(&payload).unwrap(), + DnsQueryResultWire::Response(vec![4, 5, 6]) + ); + write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) + .await + .unwrap(); + }); + let client = Arc::new(BoundaryClient::new(BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: socket_path.clone(), + tls: certificate.client_tls, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "a".repeat(32), + })); + let source = RemoteDnsMediation { client }; + let query = source.accept().await.unwrap(); + assert_eq!(query.request, [1, 2, 3]); + assert_eq!(query.transport, crate::contract::DnsTransport::Udp); + assert_eq!( + query.binary_identity.unwrap().binary_path, + PathBuf::from("/usr/bin/dig") + ); + query.response.send(Ok(vec![4, 5, 6])).unwrap(); + server.await.unwrap(); + let _ = std::fs::remove_file(socket_path); + } + + struct TestCertificate { + client_tls: BoundaryClientTls, + server_config: Arc, + } + + fn test_certificate() -> TestCertificate { + let _ = rustls::crypto::ring::default_provider().install_default(); + let material = generate_boundary_mutual_tls_material().expect("generate test material"); + let certificates = rustls_pemfile::certs(&mut material.sandbox_certificate_pem.as_bytes()) + .collect::, _>>() + .expect("parse server certificate"); + let private_key = + rustls_pemfile::private_key(&mut material.sandbox_private_key_pem.as_bytes()) + .expect("parse server private key") + .expect("server private key"); + let client_ca = rustls_pemfile::certs(&mut material.ca_certificate_pem.as_bytes()) + .collect::, _>>() + .expect("parse client CA"); + let mut client_roots = rustls::RootCertStore::empty(); + for certificate in client_ca { + client_roots.add(certificate).expect("add client CA"); + } + let verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .expect("build client verifier"); + let server_config = rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(certificates, private_key) + .expect("build test TLS server config"); + TestCertificate { + client_tls: BoundaryClientTls { + server_name: material.server_name, + ca_certificate_pem: material.ca_certificate_pem, + certificate_chain_pem: material.supervisor_certificate_pem, + private_key_pem: material.supervisor_private_key_pem, + }, + server_config: Arc::new(server_config), + } + } + + fn tls_topology( + address: std::net::SocketAddr, + tls: BoundaryClientTls, + token: &str, + ) -> BoundaryTopology { + BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { address, tls }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: token.to_string(), + } + } + + async fn spawn_tls_boundary( + certificate: Arc, + expected_token: String, + ) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test TLS boundary"); + let address = listener.local_addr().expect("read test listener address"); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let Ok(mut stream) = tokio_rustls::TlsAcceptor::from(certificate) + .accept(stream) + .await + else { + return; + }; + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let response = if request.boundary_id == "sandbox-1" + && request.bootstrap_token == expected_token + { + Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + } + } else { + Response::Error { + kind: "denied".to_string(), + message: "control authentication failed".to_string(), + } + }; + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + (address, task) + } + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + identity: crate::contract::ResolvedWorkloadIdentity::new( + 10_001, + 10_001, + Vec::new(), + "test".to_string(), + "sha256:test".to_string(), + ) + .expect("identity"), + } + } + + fn test_confirmation_evidence() -> crate::contract::SandboxConfirmEvidence { + crate::contract::SandboxConfirmEvidence { + generation: "test-generation".to_string(), + identity: sandbox().identity, + capabilities: crate::contract::CapabilityEvidence { + inheritable: 0, + permitted: 0, + effective: 0, + bounding: 0, + ambient: 0, + }, + no_new_privileges: true, + sandbox_dumpable: false, + child_dumpable: true, + core_limit_zero: true, + native_architecture: std::env::consts::ARCH.to_string(), + kernel_release: "test".to_string(), + seccomp: crate::contract::SeccompEvidence { + new_listener: true, + notification_round_trip: true, + id_validation: true, + addfd_send: true, + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: true, + task_memory_write: true, + cancellation: true, + }, + landlock_abi: 1, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + authenticated_supervisor: true, + session_epoch: "test-session".to_string(), + driver_fence: test_driver_fence(), + resource_claims: std::collections::BTreeMap::new(), + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + tls: test_certificate().client_tls, + }, + host_gateway_ip: Some(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)), + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = BoundaryTopology { + boundary_id: "other".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::Unix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + tls: test_certificate().client_tls, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_rejects_an_unspecified_tcp_target() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { + address: "0.0.0.0:5500".parse().expect("valid address"), + tls: test_certificate().client_tls, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn topology_accepts_a_concrete_tcp_target() { + let topology = BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: sandbox().identity, + transport: BoundaryTransport::TlsTcp { + address: "10.42.0.7:5500".parse().expect("valid address"), + tls: test_certificate().client_tls, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + validate_topology(&topology, &sandbox(), "vm").expect("TCP topology should be valid"); + } + + #[test] + fn topology_rejects_invalid_tls_configuration() { + let topology = tls_topology( + "127.0.0.1:5500".parse().expect("valid address"), + BoundaryClientTls { + server_name: "not a dns name!".to_string(), + ca_certificate_pem: "not a certificate".to_string(), + certificate_chain_pem: "not a certificate".to_string(), + private_key_pem: "not a key".to_string(), + }, + "0123456789abcdef0123456789abcdef", + ); + assert!(matches!( + validate_topology(&topology, &sandbox(), "vm"), + Err(BackendError::Descriptor(_)) + )); + } + + #[tokio::test] + async fn tls_tcp_round_trip_verifies_server_certificate() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + &"a".repeat(32), + )); + + assert_eq!( + client + .exchange(Request::Confirm) + .await + .expect("TLS request"), + Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + } + ); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_flushes_large_control_requests_before_reading_response() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary(certificate.server_config, "a".repeat(32)).await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + &"a".repeat(32), + )); + let context = sandbox(); + + assert!(matches!( + client + .exchange(Request::StartAgent { + sandbox_id: context.sandbox_id, + spec: AgentSpecWire::from(context.agent), + policy: Box::new(SandboxPolicyWire::from(context.policy)), + ca_cert: Some(vec![b'c'; 16 * 1024]), + ca_bundle: Some(vec![b'b'; 256 * 1024]), + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }) + .await + .expect("large TLS request"), + Response::Confirmed { .. } + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_unix_flushes_large_control_requests_before_reading_response() { + let socket_path = std::env::temp_dir().join(format!( + "openshell-large-control-{}-{}.sock", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time") + .as_nanos() + )); + let certificate = test_certificate(); + let server_config = certificate.server_config; + let listener = tokio::net::UnixListener::bind(&socket_path).expect("bind test socket"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept TLS control client"); + let mut stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .expect("accept TLS session"); + let declared_u32 = stream.read_u32().await.expect("read request length"); + let declared = declared_u32 as usize; + let mut frame = vec![0_u8; 4 + declared]; + frame[..4].copy_from_slice(&declared_u32.to_be_bytes()); + stream + .read_exact(&mut frame[4..]) + .await + .expect("read request frame"); + let request: RequestEnvelope = decode_frame(&frame).expect("decode request"); + let frame = encode_frame(&ResponseEnvelope { + request_id: request.request_id, + response: Response::Confirmed { + evidence: Box::new(test_confirmation_evidence()), + }, + }) + .expect("encode response"); + stream.write_all(&frame).await.expect("write response"); + }); + let context = sandbox(); + let client = BoundaryClient::new(BoundaryTopology { + boundary_id: "sandbox-1".to_string(), + generation: "test-generation".to_string(), + session_epoch: "test-session".to_string(), + workload_identity: context.identity.clone(), + transport: BoundaryTransport::Unix { + socket_path: socket_path.clone(), + tls: certificate.client_tls, + }, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + driver_fence: test_driver_fence(), + bootstrap_token: "a".repeat(32), + }); + + assert!(matches!( + tokio::time::timeout( + Duration::from_secs(2), + client.exchange(Request::StartAgent { + sandbox_id: context.sandbox_id, + spec: AgentSpecWire::from(context.agent), + policy: Box::new(SandboxPolicyWire::from(context.policy)), + ca_cert: Some(vec![b'c'; 16 * 1024]), + ca_bundle: Some(vec![b'b'; 256 * 1024]), + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }) + ) + .await + .expect("large Unix TLS request timed out") + .expect("large Unix TLS request"), + Response::Confirmed { .. } + )); + server.await.expect("TLS test server"); + let _ = std::fs::remove_file(socket_path); + } + + #[tokio::test] + async fn tls_tcp_preserves_boundary_token_authentication() { + let certificate = test_certificate(); + let (address, server) = spawn_tls_boundary( + certificate.server_config, + "expected-token-expected-token-12".to_string(), + ) + .await; + let client = BoundaryClient::new(tls_topology( + address, + certificate.client_tls, + "incorrect-token-incorrect-token", + )); + + assert!(matches!( + client.exchange(Request::Confirm).await, + Err(BackendError::Denied(_)) + )); + server.await.expect("TLS test server"); + } + + #[tokio::test] + async fn tls_tcp_rejects_an_untrusted_server_certificate() { + let presented = test_certificate(); + let trusted = test_certificate(); + let (address, server) = spawn_tls_boundary(presented.server_config, "a".repeat(32)).await; + let client = + BoundaryClient::new(tls_topology(address, trusted.client_tls, &"a".repeat(32))); + + assert!(matches!( + client.connect_boundary_once().await, + Err(BackendError::Unavailable(_)) + )); + // The server observes the client's fatal alert and may fail its accept; + // completing the task is sufficient for this rejection test. + let _ = server.await; + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 3463f03767..32e2d3537b 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -16,70 +16,59 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } -openshell-extension-core = { path = "../openshell-extension-core" } +openshell-binary-identity = { path = "../openshell-binary-identity" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } -openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } -openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } -openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } -openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +anyhow = { workspace = true } +async-trait = "0.1" +bytes = { workspace = true } +hex = "0.4" +ipnet = "2" +rand = "0.10" +sha2 = { workspace = true } # Async runtime tokio = { workspace = true } -# gRPC (tonic::Status downcast in error mapping) -tonic = { workspace = true, features = ["channel", "tls-native-roots"] } -prost-types = { workspace = true } - # CLI clap = { workspace = true } # Error handling miette = { workspace = true } -# Unix ownership for Kubernetes sidecar init setup +# Unix identity and bootstrap ownership nix = { workspace = true } # TLS crypto provider install (main.rs) rustls = { workspace = true } +rustls-pemfile = { workspace = true } +tokio-rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -prost = { workspace = true } # Logging tracing = { workspace = true } -uuid = { workspace = true } tracing-subscriber = { workspace = true } -tracing-appender = { workspace = true } -[features] -default = ["telemetry", "bundled-ca-roots"] -## Convenience alias: all defaults except bundled CA roots. Use -## `--no-default-features --features system-ca-roots` to build a supervisor -## that uses the platform trust store with telemetry intact. -system-ca-roots = ["telemetry"] -## Convenience alias: every default feature except `telemetry`. Build a -## telemetry-free supervisor with -## `--no-default-features --features defaults-without-telemetry` and stay -## correct as new default features are added. Cargo cannot subtract a single -## default feature, so this alias must be paired with `--no-default-features`; -## enabling it alongside `telemetry` is a compile error rather than a silent -## telemetry-on build. Kept in sync with `default` by -## `rust:verify:defaults-without-telemetry`. Do not pair it with -## `system-ca-roots`, which re-enables `telemetry`; a build with neither -## telemetry nor bundled CA roots is plain `--no-default-features`. -defaults-without-telemetry = ["bundled-ca-roots"] +[target.'cfg(unix)'.dependencies] +libc = "0.2" +rustix = { workspace = true } -telemetry = ["openshell-core/telemetry"] -bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] +[target.'cfg(target_os = "linux")'.dependencies] +capctl = "0.2.4" +landlock = "0.4" +seccompiler = "0.5" +socket2 = { workspace = true } +tempfile = "3" [dev-dependencies] +rcgen = { workspace = true } tempfile = "3" -temp-env = "0.3" -tokio-tungstenite = { workspace = true } -futures = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs similarity index 86% rename from crates/openshell-supervisor-process/src/boundary_exec.rs rename to crates/openshell-sandbox/src/boundary_exec.rs index eea12cac38..db69d52092 100644 --- a/crates/openshell-supervisor-process/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Co-located implementation of RFC 0012 in-boundary exec. +//! Workload-side implementation of RFC 0012 sandbox exec. use std::collections::HashMap; use std::os::fd::{AsRawFd, OwnedFd}; @@ -19,51 +19,36 @@ use openshell_isolation_interface::contract::{ BoundarySignal, BoundaryTerminal, ExecSession, ExecSpec, }; -use crate::process::{ProcessEnforcementMode, ResolvedProcessIdentity}; - -/// The co-located executor. Every spawn reuses the same admitted policy and +/// The sandbox executor. Every spawn reuses the same admitted policy and /// execution-environment controls while taking a fresh provider credential /// snapshot. #[derive(Clone)] pub struct LocalBoundaryExec { policy: SandboxPolicy, base_workdir: Option, - netns_fd: Option>, - proxy_url: Option, ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, runtime: Arc, } impl LocalBoundaryExec { - /// Construct one executor for an active co-located boundary. - #[allow(clippy::too_many_arguments)] + /// Construct the executor owned by an active sandbox boundary. #[must_use] pub fn new( policy: SandboxPolicy, base_workdir: Option, - netns_fd: Option>, - proxy_url: Option, ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, runtime: Arc, ) -> Self { Self { policy, base_workdir, - netns_fd, - proxy_url, ca_file_paths, provider_credentials, user_environment, - resolved_identity, - enforcement_mode, runtime, } } @@ -77,16 +62,31 @@ impl LocalBoundaryExec { let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); let (session_user, session_home) = crate::process::session_user_and_home(&self.policy, effective_workdir); - crate::ssh::apply_child_env( - &mut command, - &session_home, - &session_user, - if spec.pty { "xterm-256color" } else { "dumb" }, - self.proxy_url.as_deref(), - self.ca_file_paths.as_deref(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - ); + let path = std::env::var("PATH").unwrap_or_else(|_| "/usr/local/bin:/usr/bin:/bin".into()); + command + .env_clear() + .env(openshell_core::sandbox_env::SANDBOX, "1") + .env("HOME", session_home) + .env("USER", session_user) + .env("SHELL", "/bin/bash") + .env("PATH", path) + .env("TERM", if spec.pty { "xterm-256color" } else { "dumb" }); + for (key, value) in &self.user_environment { + if !key.starts_with("OPENSHELL_") { + command.env(key, value); + } + } + if let Some((ca_cert_path, combined_bundle_path)) = self.ca_file_paths.as_deref() { + for (key, value) in crate::child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { + command.env(key, value); + } + } + for (key, value) in self.provider_credentials.child_env_with_gcp_resolved() { + if !crate::process::is_supervisor_only_env_var(&key) { + command.env(key, value); + } + } + crate::process::strip_proxy_env_std(&mut command); for (key, value) in &spec.env { if !key.starts_with("OPENSHELL_") { command.env(key, value); @@ -103,10 +103,10 @@ impl LocalBoundaryExec { &self, workdir: Option<&str>, ) -> Result, BackendError> { - if self.enforcement_mode.enforces_child_sandbox() { - crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); - } - crate::process::prepare_child_sandbox(&self.policy, workdir, self.enforcement_mode) + crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); + let runtime_read_only = + crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); + crate::process::prepare_child_sandbox(&self.policy, workdir, &runtime_read_only) .map_err(|error| BackendError::Process(error.to_string())) } @@ -120,19 +120,27 @@ impl LocalBoundaryExec { let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); #[cfg(target_os = "linux")] let prepared = self.prepare_sandbox(effective_workdir)?; - crate::ssh::unsafe_pty::install_dedicated_process_group(&mut command); - crate::ssh::unsafe_pty::install_pre_exec_no_pty( + #[cfg(target_os = "linux")] + let child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))?; + crate::pty::install_dedicated_process_group(&mut command); + crate::pty::install_pre_exec_no_pty( &mut command, self.policy.clone(), effective_workdir.map(str::to_string), - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - self.resolved_identity, - self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + #[cfg(target_os = "linux")] + child_hardening, + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); + #[cfg(target_os = "linux")] + let mut child = crate::process::spawn_std_command_with_workload_launcher(command) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(not(target_os = "linux"))] let mut child = command .spawn() .map_err(|error| BackendError::Process(error.to_string()))?; @@ -222,19 +230,27 @@ impl LocalBoundaryExec { let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); #[cfg(target_os = "linux")] let prepared = self.prepare_sandbox(effective_workdir)?; - crate::ssh::unsafe_pty::install_pre_exec( + #[cfg(target_os = "linux")] + let child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| BackendError::Process(error.to_string()))?; + crate::pty::install_pre_exec( &mut command, self.policy.clone(), effective_workdir.map(str::to_string), slave_fd, - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - self.resolved_identity, - self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + #[cfg(target_os = "linux")] + child_hardening, + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); + #[cfg(target_os = "linux")] + let mut child = crate::process::spawn_std_command_with_workload_launcher(command) + .map_err(|error| BackendError::Process(error.to_string()))?; + #[cfg(not(target_os = "linux"))] let mut child = command .spawn() .map_err(|error| BackendError::Process(error.to_string()))?; @@ -331,7 +347,7 @@ struct LocalTerminal { #[async_trait] impl BoundaryTerminal for LocalTerminal { async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { - crate::ssh::unsafe_pty::set_winsize( + crate::pty::set_winsize( self.master.as_raw_fd(), Winsize { ws_row: rows.max(1), @@ -479,9 +495,23 @@ impl BoundaryProcess for LocalExecProcess { #[cfg(test)] mod tests { use super::*; + use std::sync::Once; use tokio::io::{AsyncReadExt, AsyncWriteExt}; fn executor() -> LocalBoundaryExec { + static LAUNCHER: Once = Once::new(); + LAUNCHER.call_once(|| { + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start() + .expect("start test workload launcher"); + std::thread::spawn(move || { + while let Ok(notification) = listener.receive() { + let _ = listener.respond_errno(notification.id, libc::EPERM); + } + }); + crate::process::configure_workload_launcher(launcher) + .expect("configure test workload launcher"); + }); LocalBoundaryExec::new( SandboxPolicy { version: 1, @@ -492,8 +522,6 @@ mod tests { }, None, None, - None, - None, ProviderCredentialState::from_environment( 0, HashMap::new(), @@ -501,8 +529,6 @@ mod tests { HashMap::new(), ), HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, crate::boundary_io::BoundaryRuntimeState::new(), ) } diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-sandbox/src/boundary_io.rs similarity index 78% rename from crates/openshell-supervisor-process/src/boundary_io.rs rename to crates/openshell-sandbox/src/boundary_io.rs index fab37a0062..27613f0df1 100644 --- a/crates/openshell-supervisor-process/src/boundary_io.rs +++ b/crates/openshell-sandbox/src/boundary_io.rs @@ -1,27 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! The in-pod [`BoundaryPortForward`] interface (RFC 0012 runtime contract). -//! -//! This is the live in-boundary port-forward for the in-pod placement. It lives -//! in this crate on purpose: the SSH server and supervisor session that consume -//! it are here, and so is the primitive it wraps -//! ([`connect_in_netns`](crate::ssh::connect_in_netns)). The interface trait -//! lives in the lower `openshell-isolation-interface` crate, so this crate -//! depends on the trait (process -> interface -> core, acyclic) and the SSH server drives a -//! `&dyn BoundaryPortForward` without depending on the backend. -//! -//! The SSH server and supervisor session are wired to this through the -//! `RunningBoundary::port_forward()` accessor: swapping in a kernel-separated -//! backend swaps this implementation (where `connect` tunnels into the guest) -//! and touches no consumer code. +//! Sandbox-local [`BoundaryPortForward`] implementation. use async_trait::async_trait; use openshell_isolation_interface::contract::{ BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, }; use std::collections::HashMap; -use std::os::fd::{AsRawFd, OwnedFd}; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; @@ -181,36 +167,28 @@ impl RegisteredProcessGroup { } } -/// In-pod loopback port-forward: connects to a loopback target from inside the -/// workload's network namespace via [`connect_in_netns`](crate::ssh::connect_in_netns). -pub struct NetnsPortForward { - /// File descriptor of the boundary's network namespace, or `None` to - /// connect from the supervisor's own namespace. - netns_fd: Option>, +/// Loopback port-forward owned by the sandbox process. +pub struct LocalPortForward { runtime: Option>, } -impl NetnsPortForward { +impl LocalPortForward { #[must_use] - pub fn new(netns_fd: Option>, runtime: Option>) -> Self { - Self { netns_fd, runtime } + pub fn new(runtime: Option>) -> Self { + Self { runtime } } } #[async_trait] -impl BoundaryPortForward for NetnsPortForward { +impl BoundaryPortForward for LocalPortForward { async fn connect(&self, target: LoopbackTarget) -> Result { if let Some(runtime) = &self.runtime { runtime.ensure_active()?; } let addr = std::net::SocketAddr::new(target.host(), target.port()); - let addr_string = addr.to_string(); - let stream = crate::ssh::connect_in_netns( - &addr_string, - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - ) - .await - .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + let stream = openshell_core::net::connect_tcp_nodelay_best_effort(&[addr]) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; if let Some(runtime) = &self.runtime { runtime.ensure_active()?; } @@ -225,9 +203,7 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; /// Stands in for the SSH server's port-forward path: connect through the - /// interface, write, read the echo. With `netns_fd: None` the connect happens in - /// the supervisor's namespace, so this exercises the real primitive without - /// requiring a network namespace. + /// interface, write, and read the echo. #[tokio::test] async fn port_forward_connects_and_round_trips() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -239,7 +215,7 @@ mod tests { sock.write_all(&buf).await.unwrap(); }); - let pf = NetnsPortForward::new(None, None); + let pf = LocalPortForward::new(None); let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).expect("loopback target"); let mut conn = pf.connect(target).await.expect("connect through interface"); @@ -261,7 +237,7 @@ mod tests { tokio::spawn(async move { let _ = listener.accept().await; }); - let pf = NetnsPortForward::new(None, None); + let pf = LocalPortForward::new(None); let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), addr.port()).unwrap(); assert!(forward_one(&pf, target).await); } @@ -269,7 +245,7 @@ mod tests { #[tokio::test] async fn port_forward_rejects_after_boundary_end() { let runtime = BoundaryRuntimeState::new(); - let pf = NetnsPortForward::new(None, Some(runtime.clone())); + let pf = LocalPortForward::new(Some(runtime.clone())); runtime.deactivate(); let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), 1).unwrap(); assert!(matches!( @@ -280,14 +256,11 @@ mod tests { #[tokio::test] async fn failed_port_forward_keeps_boundary_active() { - let listener = tokio::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .await - .unwrap(); - let port = listener.local_addr().unwrap().port(); - drop(listener); let runtime = BoundaryRuntimeState::new(); - let pf = NetnsPortForward::new(None, Some(runtime.clone())); - let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), port).unwrap(); + let pf = LocalPortForward::new(Some(runtime.clone())); + // Port zero is never a connectable TCP destination. Reserving an ephemeral + // port and dropping its listener races other parallel tests that may bind it. + let target = LoopbackTarget::new(Ipv4Addr::LOCALHOST.into(), 0).unwrap(); assert!(matches!( pf.connect(target).await, Err(BackendError::Process(_)) @@ -314,4 +287,25 @@ mod tests { runtime.unregister_process_group(pid, &second_terminal); assert_eq!(runtime.registered_process_group_count(), 0); } + + #[test] + fn canonical_process_completion_does_not_end_boundary_runtime() { + let runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let terminal = Arc::new(std::sync::atomic::AtomicBool::new(true)); + runtime + .register_process_group(42, terminal.clone(), Arc::new(Mutex::new(()))) + .expect("register canonical process"); + + runtime.unregister_process_group(42, &terminal); + + runtime + .ensure_active() + .expect("canonical completion must preserve exec and forwarding"); + assert_eq!(runtime.registered_process_group_count(), 0); + runtime.deactivate(); + assert!(matches!( + runtime.ensure_active(), + Err(BackendError::Terminated(_)) + )); + } } diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs new file mode 100644 index 0000000000..0d793ceb92 --- /dev/null +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -0,0 +1,3300 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared implementation of the capability-free `openshell-sandbox` runtime. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the control role authorizes `start_agent`, it invokes the existing process +//! supervisor inside the driver-provisioned boundary. + +#![allow(unsafe_code)] + +use std::path::Path; + +#[cfg(target_os = "linux")] +mod linux { + use super::Path; + use std::fs::File; + use std::io::{self, Read, Write}; + use std::mem::size_of; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _, PermissionsExt as _}; + use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + use crate::boundary_io::BoundaryRuntimeState; + use crate::delegated::{AgentSignaler, spawn_workload}; + use crate::identity::{DriverIdentity, resolve_process_identity}; + use crate::main_session::{MainOutput, MainSession}; + use crate::network_broker::NetworkBroker; + use crate::process::ProcessStatus; + use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_isolation_interface::contract::{ + BoundaryExec, BoundaryPortForward, BoundaryProcess, BoundaryTerminal, CapabilityEvidence, + ExecSession, LoopbackTarget, ResolvedWorkloadIdentity, SandboxConfirmEvidence, + }; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + use openshell_isolation_interface::boundary_protocol::{ + AgentSpecWire, BinaryIdentityWire, BoundaryConfig, + BoundaryListener as BoundaryListenerConfig, DnsQueryResultWire, ExecSpecWire, + ExitStatusWire, OutputWindowWire, ProcessKindWire, ProcessSnapshotWire, Request, + RequestEnvelope, Response, ResponseEnvelope, STREAM_DNS_ACK, STREAM_DNS_RESPONSE, + STREAM_EXIT, STREAM_NETWORK_DECISION, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, + STREAM_STDOUT, SandboxPolicyWire, SessionSnapshotWire, SignalWire, encode_frame, + read_frame, read_stream_frame, validate_resource_claims, write_frame, write_stream_frame, + }; + + const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + const MAX_CONTROL_CONNECTIONS: usize = 128; + const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096; + const MAX_RETAINED_EXEC_PROCESSES: usize = 64; + + struct ControlConnectionSlot(Arc); + + impl Drop for ControlConnectionSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + fn acquire_control_connection_slot(active: &Arc) -> Option { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < MAX_CONTROL_CONNECTIONS).then_some(current + 1) + }) + .ok() + .map(|_| ControlConnectionSlot(active.clone())) + } + static BOUNDARY_TERMINATION_REQUESTED: AtomicBool = AtomicBool::new(false); + + extern "C" fn request_boundary_termination(_signal: libc::c_int) { + BOUNDARY_TERMINATION_REQUESTED.store(true, Ordering::Release); + } + + pub fn run_boundary( + config_path: &Path, + qualification: crate::RuntimeQualification, + ) -> Result<(), String> { + install_boundary_signal_handlers()?; + make_boundary_nondumpable()?; + disable_core_dumps()?; + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read boundary config {}: {error}", config_path.display()))?; + let config: BoundaryConfig = serde_json::from_slice(&bytes).map_err(|error| { + format!("decode boundary config {}: {error}", config_path.display()) + })?; + validate_config(&config)?; + validate_runtime_resource_claims(&config)?; + validate_running_identity(&config.workload_identity)?; + std::fs::remove_file(config_path).map_err(|error| { + format!("consume boundary config {}: {error}", config_path.display()) + })?; + let child_env = serde_json::to_string(&config.child_env) + .map_err(|error| format!("encode boundary workload environment: {error}"))?; + // This runs before the Tokio runtime or control threads exist. The process + // supervisor consumes the serialized map and applies values only to + // workload children. + unsafe { + std::env::set_var(openshell_core::sandbox_env::USER_ENVIRONMENT, child_env); + } + crate::sandbox::apply_supervisor_startup_hardening() + .map_err(|error| format!("install sandbox process prelude: {error}"))?; + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .map_err(|error| format!("start sandbox workload launcher: {error}"))?; + crate::process::configure_workload_launcher(launcher.clone()) + .map_err(|error| format!("configure sandbox workload launcher: {error}"))?; + let network_broker = NetworkBroker::start(listener) + .map_err(|error| format!("start sandbox network broker: {error}"))?; + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create boundary process runtime: {error}"))?; + let runtime = Arc::new(BoundaryRuntime::new( + config.clone(), + process_runtime.handle().clone(), + network_broker, + launcher, + qualification, + )); + serve(&config.listener, runtime) + } + + fn make_boundary_nondumpable() -> Result<(), String> { + // SAFETY: PR_SET_DUMPABLE accepts one scalar flag. The sandbox keeps + // bootstrap and protected-channel keys in memory after this point. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } == 0 { + Ok(()) + } else { + Err(format!( + "make sandbox process nondumpable: {}", + io::Error::last_os_error() + )) + } + } + + fn disable_core_dumps() -> Result<(), String> { + let limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a valid immutable rlimit value. + if unsafe { libc::setrlimit(libc::RLIMIT_CORE, &raw const limit) } == 0 { + Ok(()) + } else { + Err(format!( + "disable sandbox core dumps: {}", + io::Error::last_os_error() + )) + } + } + + fn install_boundary_signal_handlers() -> Result<(), String> { + BOUNDARY_TERMINATION_REQUESTED.store(false, Ordering::Release); + let action = nix::sys::signal::SigAction::new( + nix::sys::signal::SigHandler::Handler(request_boundary_termination), + nix::sys::signal::SaFlags::empty(), + nix::sys::signal::SigSet::empty(), + ); + for signal in [ + nix::sys::signal::Signal::SIGTERM, + nix::sys::signal::Signal::SIGINT, + ] { + // SAFETY: the installed handler only performs a lock-free atomic + // store, which is async-signal-safe, and remains valid for the + // lifetime of the boundary process. + unsafe { nix::sys::signal::sigaction(signal, &action) } + .map_err(|error| format!("install boundary {signal:?} handler: {error}"))?; + } + Ok(()) + } + + fn validate_config(config: &BoundaryConfig) -> Result<(), String> { + if config.boundary_id.is_empty() { + return Err("boundary ID must not be empty".to_string()); + } + if config.generation.is_empty() || config.session_epoch.is_empty() { + return Err("boundary generation and session epoch must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("boundary bootstrap token must contain at least 32 bytes".to_string()); + } + validate_resource_claims(&config.resource_claims).map_err(|error| error.to_string())?; + config + .driver_fence + .validate_for_backend(config.driver_fence.backend_name()) + .map_err(|error| error.to_string())?; + for (claim, path) in &config.resource_claim_files { + if !config.resource_claims.contains_key(claim) { + return Err(format!( + "runtime resource-claim file refers to unknown claim {claim}" + )); + } + if !path.is_absolute() { + return Err(format!( + "runtime resource-claim file for {claim} must be absolute" + )); + } + } + match &config.listener { + BoundaryListenerConfig::Unix { socket_path, tls } + if !socket_path.is_absolute() || !tls_paths_are_absolute(tls) => + { + return Err("boundary Unix socket path must be absolute".to_string()); + } + BoundaryListenerConfig::TlsTcp { address, tls } + if address.port() == 0 || !tls_paths_are_absolute(tls) => + { + return Err( + "boundary TLS listener requires a nonzero port and absolute certificate paths" + .to_string(), + ); + } + BoundaryListenerConfig::Vsock { + control_port: 0, .. + } => { + return Err("boundary control port must be nonzero".to_string()); + } + BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::TlsTcp { .. } + | BoundaryListenerConfig::Vsock { .. } => {} + } + if config.workload_identity.uid == 0 || config.workload_identity.gid == 0 { + return Err("sandbox workload UID and GID must be nonzero".to_string()); + } + Ok(()) + } + + fn tls_paths_are_absolute( + tls: &openshell_isolation_interface::boundary_protocol::BoundaryServerTls, + ) -> bool { + tls.certificate_chain_path.is_absolute() + && tls.private_key_path.is_absolute() + && tls.client_ca_certificate_path.is_absolute() + } + + fn validate_runtime_resource_claims(config: &BoundaryConfig) -> Result<(), String> { + for (claim, path) in &config.resource_claim_files { + let expected = config + .resource_claims + .get(claim) + .expect("validated resource-claim file key"); + let observed = std::fs::read_to_string(path).map_err(|error| { + format!( + "read runtime resource claim {claim} from {}: {error}", + path.display() + ) + })?; + if observed.trim() != expected { + return Err(format!( + "runtime resource claim {claim} does not match the admitted resource" + )); + } + } + Ok(()) + } + + fn normalized_supplementary_groups(mut groups: Vec, primary_gid: u32) -> Vec { + groups.retain(|gid| *gid != primary_gid); + groups.sort_unstable(); + groups.dedup(); + groups + } + + #[allow(clippy::similar_names)] + fn validate_running_identity(expected: &ResolvedWorkloadIdentity) -> Result<(), String> { + let mut real_uid = 0; + let mut effective_uid = 0; + let mut saved_uid = 0; + let mut real_gid = 0; + let mut effective_gid = 0; + let mut saved_gid = 0; + // SAFETY: all pointers refer to live scalar output storage. + if unsafe { + libc::getresuid( + &raw mut real_uid, + &raw mut effective_uid, + &raw mut saved_uid, + ) + } != 0 + || unsafe { + libc::getresgid( + &raw mut real_gid, + &raw mut effective_gid, + &raw mut saved_gid, + ) + } != 0 + { + return Err(format!( + "measure sandbox identity: {}", + io::Error::last_os_error() + )); + } + if [real_uid, effective_uid, saved_uid] + .iter() + .any(|uid| *uid != expected.uid) + || [real_gid, effective_gid, saved_gid] + .iter() + .any(|gid| *gid != expected.gid) + { + return Err(format!( + "sandbox identity does not match resolved workload {}:{}", + expected.uid, expected.gid + )); + } + // SAFETY: a null buffer with size zero queries the group count. + let count = unsafe { libc::getgroups(0, std::ptr::null_mut()) }; + if count < 0 { + return Err(format!( + "measure sandbox supplementary groups: {}", + io::Error::last_os_error() + )); + } + let mut groups = vec![0_u32; usize::try_from(count).unwrap_or(0)]; + if count > 0 { + // SAFETY: groups has capacity for exactly `count` gid_t values. + if unsafe { libc::getgroups(count, groups.as_mut_ptr()) } != count { + return Err(format!( + "read sandbox supplementary groups: {}", + io::Error::last_os_error() + )); + } + } + let groups = normalized_supplementary_groups(groups, expected.gid); + if groups != expected.supplementary_gids { + return Err(format!( + "sandbox supplementary groups {groups:?} do not match resolved workload {:?}", + expected.supplementary_gids + )); + } + Ok(()) + } + + fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { + let listener = ControlListener::bind(config) + .map_err(|error| format!("bind boundary control listener: {error}"))?; + let active_connections = Arc::new(AtomicUsize::new(0)); + tracing::info!(?config, "Boundary control listener ready"); + loop { + if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { + runtime.shutdown(); + return Ok(()); + } + match listener.accept() { + Ok(stream) => { + let Some(slot) = acquire_control_connection_slot(&active_connections) else { + tracing::warn!( + limit = MAX_CONTROL_CONNECTIONS, + "Boundary control connection limit reached" + ); + continue; + }; + let runtime = runtime.clone(); + std::thread::spawn(move || { + let _slot = slot; + let stream = match stream.establish(&runtime.process_runtime) { + Ok(stream) => stream, + Err(error) => { + tracing::warn!(%error, "Boundary control transport handshake failed"); + return; + } + }; + if let Err(error) = serve_one(stream, &runtime) { + tracing::warn!(%error, "Boundary control request failed: {error}"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(Duration::from_millis(10)); + } + Err(error) => return Err(format!("accept boundary control connection: {error}")), + } + } + } + + fn serve_one(mut stream: ControlStream, runtime: &BoundaryRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + if !runtime.authenticate(&request) { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control authentication failed"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + if request.validate_payload_digest().is_err() { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control request payload digest mismatch"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + match request.request.clone() { + Request::Exec { spec } => { + let started = + match runtime.start_exec(&request.request_id, &request.payload_digest, spec) { + Ok(started) => started, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write exec error response: {error}")); + } + }; + if let Err(error) = write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ExecStarted { + process_id: started.process_id.clone(), + pty: started.terminal, + }, + }, + ) { + return Err(format!("write exec start response: {error}")); + } + return runtime.stream_process(stream, started.attachment); + } + Request::AttachProcess { process_id } => { + let (attachment, terminal) = match runtime.attach_process(&process_id) { + Ok(attachment) => attachment, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write process attachment error: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ProcessAttached { terminal }, + }, + ) + .map_err(|error| format!("write process attachment response: {error}"))?; + return runtime.stream_process(stream, attachment); + } + Request::PortForward { host, port } => { + let target = match LoopbackTarget::new(host, port) + .map_err(|error| format!("validate port-forward target: {error}")) + .and_then(|target| { + runtime + .connect_port(target) + .map_err(|error| format!("connect boundary loopback port: {error}")) + }) { + Ok(target) => target, + Err(error) => { + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: guest_error("failed", error), + }, + ) + .map_err(|error| format!("write port-forward error response: {error}"))?; + return Ok(()); + } + }; + let mut target = target; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::PortConnected, + }, + ) + .map_err(|error| format!("write port-forward response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge boundary loopback stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptNetwork => { + let broker = runtime.network_accept_context()?; + let request_id = request.request_id; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + let mut disconnect_probe = [0_u8; 1]; + let pending = tokio::select! { + biased; + read = stream.read(&mut disconnect_probe) => { + match read { + Ok(0) => return Ok(()), + Ok(_) => return Err("control sent data before network mediation response".to_string()), + Err(error) => return Err(format!("watch network mediation control stream: {error}")), + } + } + pending = broker.accept() => pending + .map_err(|error| format!("accept sandbox network open: {error}"))?, + }; + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::NetworkConnected { + identity: BinaryIdentityWire::from(pending.identity.clone()), + destination: pending.destination, + socket: pending.socket, + policy_generation: 0, + }, + }) + .map_err(|error| format!("encode network mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write network mediation response: {error}"))?; + let Some((channel, payload)) = read_stream_frame(&mut stream) + .await + .map_err(|error| format!("read network-open decision: {error}"))? + else { + return Err("control disconnected before network-open decision".to_string()); + }; + if channel != STREAM_NETWORK_DECISION { + return Err(format!( + "unexpected network-open decision channel {channel}" + )); + } + let decision = serde_json::from_slice(&payload) + .map_err(|error| format!("decode network-open decision: {error}"))?; + let Some(target) = pending + .complete(decision) + .await + .map_err(|error| format!("complete sandbox network open: {error}"))? + else { + return Ok(()); + }; + target + .set_nonblocking(true) + .map_err(|error| format!("set sandbox relay nonblocking: {error}"))?; + let mut target = tokio::net::TcpStream::from_std(target) + .map_err(|error| format!("register sandbox relay: {error}"))?; + openshell_core::net::set_tcp_nodelay_best_effort(&target); + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map(|_| ()) + .map_err(|error| format!("bridge sandbox network stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptDns => { + let broker = runtime.network_accept_context()?; + let request_id = request.request_id; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + let mut disconnect_probe = [0_u8; 1]; + let pending = tokio::select! { + biased; + read = stream.read(&mut disconnect_probe) => { + match read { + Ok(0) => return Ok(()), + Ok(_) => return Err("control sent data before DNS mediation response".to_string()), + Err(error) => return Err(format!("watch DNS mediation control stream: {error}")), + } + } + pending = broker.accept_dns() => pending + .map_err(|error| format!("accept sandbox DNS query: {error}"))?, + }; + let response = encode_frame(&ResponseEnvelope { + request_id, + response: Response::DnsQuery { + request: pending.request.clone(), + transport: pending.transport, + identity: BinaryIdentityWire::from(pending.identity.clone()), + }, + }) + .map_err(|error| format!("encode DNS mediation response: {error}"))?; + stream + .write_all(&response) + .await + .map_err(|error| format!("write DNS mediation response: {error}"))?; + let Some((channel, payload)) = read_stream_frame(&mut stream) + .await + .map_err(|error| format!("read DNS mediation result: {error}"))? + else { + return Err("control disconnected before DNS response".to_string()); + }; + if channel != STREAM_DNS_RESPONSE { + return Err(format!("unexpected DNS response channel {channel}")); + } + let result: DnsQueryResultWire = serde_json::from_slice(&payload) + .map_err(|error| format!("decode DNS mediation result: {error}"))?; + let result = match result { + DnsQueryResultWire::Response(response) => Ok(response), + DnsQueryResultWire::Error(error) => Err(io::Error::other(error)), + }; + pending + .complete(result) + .map_err(|error| format!("complete sandbox DNS query: {error}"))?; + write_stream_frame(&mut stream, STREAM_DNS_ACK, &[]) + .await + .map_err(|error| format!("acknowledge sandbox DNS response: {error}")) + })?; + return Ok(()); + } + _ => {} + } + let response = ResponseEnvelope { + request_id: request.request_id.clone(), + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}"))?; + Ok(()) + } + + struct BoundaryRuntime { + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, + /// The wire policy bound at first attach, so an idempotent attach retry + /// carrying a different policy is denied instead of silently keeping + /// the first policy. + attached_policy: Mutex>, + /// The complete launch request accepted by the boundary. A replacement + /// control process may replay it after reconnecting, but may not change + /// any launch input or start a second workload. + started_agent: Mutex>, + next_exec_id: AtomicU64, + exec_handles: Mutex>, + replay_ledger: Mutex, + network_broker: NetworkBroker, + workload_launcher: + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + qualification: crate::RuntimeQualification, + } + + #[derive(Clone)] + struct ExecHandle { + request_id: String, + payload_digest: String, + process: Arc, + terminal: Option>, + session: Arc, + attached: Arc, + status: Arc>>, + } + + struct StartedExec { + process_id: String, + terminal: bool, + attachment: MainAttachment, + } + + #[derive(Clone)] + struct ReplayRecord { + payload_digest: String, + response: Response, + } + + #[derive(Default)] + struct ReplayLedger { + entries: std::collections::HashMap, + order: std::collections::VecDeque, + } + + impl ReplayLedger { + fn get(&self, request_id: &str) -> Option<&ReplayRecord> { + self.entries.get(request_id) + } + + fn insert(&mut self, request_id: String, record: ReplayRecord) { + if let Some(existing) = self.entries.get_mut(&request_id) { + *existing = record; + return; + } + while self.entries.len() >= MAX_REPLAY_LEDGER_ENTRIES { + let Some(oldest) = self.order.pop_front() else { + break; + }; + self.entries.remove(&oldest); + } + self.order.push_back(request_id.clone()); + self.entries.insert(request_id, record); + } + } + + #[derive(Clone, PartialEq, Eq)] + struct StartedAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + } + + impl StartedAgent { + /// Provider environment is mutable runtime state. A replacement + /// control must replay every immutable launch input exactly, then + /// reconcile the current provider snapshot through the CAS update. + fn matches_replay(&self, other: &Self) -> bool { + self.sandbox_id == other.sandbox_id + && self.spec == other.spec + && self.policy == other.policy + && self.ca_cert == other.ca_cert + && self.ca_bundle == other.ca_bundle + } + } + + struct MainAttachment { + session: Arc, + attached: Arc, + status: AttachmentStatus, + } + + enum AttachmentStatus { + Main(Arc), + Exec(Arc>>), + } + + impl MainAttachment { + fn exit_status(&self, fallback_code: i32) -> ExitStatusWire { + match &self.status { + AttachmentStatus::Main(process) => process + .exit_status() + .unwrap_or(ExitStatusWire::Exited(fallback_code)), + AttachmentStatus::Exec(status) => { + (*lock(status)).unwrap_or(ExitStatusWire::Exited(fallback_code)) + } + } + } + } + + impl Drop for MainAttachment { + fn drop(&mut self) { + self.attached.store(false, Ordering::Release); + } + } + + #[allow(clippy::result_large_err)] + fn acquire_exec_attachment(handle: &ExecHandle) -> Result { + acquire_attachment( + handle.session.clone(), + handle.attached.clone(), + AttachmentStatus::Exec(handle.status.clone()), + ) + } + + #[allow(clippy::result_large_err)] + fn acquire_attachment( + session: Arc, + attached: Arc, + status: AttachmentStatus, + ) -> Result { + if attached + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(guest_error( + "denied", + "process already has a control attachment", + )); + } + Ok(MainAttachment { + session, + attached, + status, + }) + } + + enum RuntimeState { + AwaitingAttach, + Bound(PreparedBoundary), + Ready(PreparedBoundary), + Running(Arc), + } + + #[derive(Clone)] + struct PreparedBoundary { + network_broker: NetworkBroker, + } + + impl BoundaryRuntime { + fn new( + config: BoundaryConfig, + process_runtime: tokio::runtime::Handle, + network_broker: NetworkBroker, + workload_launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + qualification: crate::RuntimeQualification, + ) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + attached_policy: Mutex::new(None), + started_agent: Mutex::new(None), + next_exec_id: AtomicU64::new(1), + exec_handles: Mutex::new(std::collections::HashMap::new()), + replay_ledger: Mutex::new(ReplayLedger::default()), + network_broker, + workload_launcher, + qualification, + } + } + + fn shutdown(&self) { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + if let Some(process) = process { + process.boundary_runtime.deactivate(); + } + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !self.authenticate(&envelope) { + return guest_error("denied", "control authentication failed"); + } + if envelope.validate_payload_digest().is_err() { + return guest_error("denied", "control request payload digest mismatch"); + } + let replayable = envelope.request.is_replayable_mutation(); + let mut replay_ledger = replayable.then(|| lock(&self.replay_ledger)); + if let Some(record) = replay_ledger + .as_ref() + .and_then(|ledger| ledger.get(&envelope.request_id)) + { + return if record.payload_digest == envelope.payload_digest { + record.response.clone() + } else { + guest_error( + "denied", + "control request ID was reused with a different payload", + ) + }; + } + let request_id = envelope.request_id; + let payload_digest = envelope.payload_digest; + let response = match envelope.request { + Request::Attach { + policy, + resource_claims, + } => { + if resource_claims == self.config.resource_claims { + self.attach(*policy) + } else { + guest_error( + "denied", + "topology resource claims do not match the boundary configuration", + ) + } + } + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + } => self.start_agent( + sandbox_id, + spec, + *policy, + ca_cert, + ca_bundle, + provider_env_revision, + provider_env, + ), + Request::UpdateProviderEnvironment { + expected_revision, + revision, + provider_env, + } => self.update_provider_environment(expected_revision, revision, provider_env), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + Request::ExecSignal { process_id, signal } => self.signal_exec(&process_id, signal), + Request::Resize { + process_id, + cols, + rows, + } => self.resize_process(&process_id, cols, rows), + Request::Exec { .. } + | Request::AttachProcess { .. } + | Request::PortForward { .. } + | Request::AcceptNetwork + | Request::AcceptDns => { + guest_error("invalid", "streaming request used on control path") + } + }; + if let Some(ledger) = replay_ledger.as_mut() { + ledger.insert( + request_id, + ReplayRecord { + payload_digest, + response: response.clone(), + }, + ); + } + response + } + + fn authenticate(&self, envelope: &RequestEnvelope) -> bool { + constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) && constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn start_exec( + &self, + request_id: &str, + payload_digest: &str, + spec: ExecSpecWire, + ) -> Result { + let executor = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + process.boundary_exec() + }; + let mut handles = lock(&self.exec_handles); + if let Some((process_id, handle)) = handles + .iter() + .find(|(_, handle)| handle.request_id == request_id) + { + if handle.payload_digest != payload_digest { + return Err(guest_error( + "denied", + "exec request ID was reused with a different payload", + )); + } + if handle.attached.load(Ordering::Acquire) { + return Err(guest_error( + "unavailable", + "prior exec attachment is still being released", + )); + } + return Ok(StartedExec { + process_id: process_id.clone(), + terminal: handle.terminal.is_some(), + attachment: acquire_exec_attachment(handle)?, + }); + } + if handles.len() >= MAX_RETAINED_EXEC_PROCESSES { + let exited = handles + .iter() + .find(|(_, handle)| { + lock(&handle.status).is_some() && !handle.attached.load(Ordering::Acquire) + }) + .map(|(process_id, _)| process_id.clone()); + if let Some(process_id) = exited { + handles.remove(&process_id); + } else { + return Err(guest_error( + "unavailable", + "retained exec process limit reached", + )); + } + } + let session = self + .process_runtime + .block_on(executor.exec(spec.into())) + .map_err(|error| guest_error("failed", error.to_string()))?; + let process_id = format!( + "{}:exec:{}", + self.config.generation, + self.next_exec_id.fetch_add(1, Ordering::Relaxed) + ); + let ExecSession { + process, + stdin, + stdout, + stderr, + terminal, + } = session; + let Some(stdin) = stdin else { + return Err(guest_error( + "failed", + "exec process stdin pipe is unavailable", + )); + }; + let retained = { + let _runtime = self.process_runtime.enter(); + MainSession::from_boundary( + openshell_isolation_interface::contract::ProcessAttachment { + stdin, + stdout, + stderr, + terminal: terminal.clone(), + }, + process.clone(), + ) + }; + let status = Arc::new(Mutex::new(None)); + let wait_process = process.clone(); + let wait_session = retained.clone(); + let wait_status = status.clone(); + self.process_runtime.spawn(async move { + if let Ok(exit_status) = wait_process.wait().await { + *lock(&wait_status) = Some(ExitStatusWire::from(exit_status)); + let exit_code = match exit_status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited( + code, + ) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128 + signal, + }; + let _ = wait_session.finish_remote(exit_code, false).await; + } + }); + let handle = ExecHandle { + request_id: request_id.to_string(), + payload_digest: payload_digest.to_string(), + process, + terminal, + session: retained, + attached: Arc::new(AtomicBool::new(false)), + status, + }; + let terminal = handle.terminal.is_some(); + let attachment = acquire_exec_attachment(&handle)?; + handles.insert(process_id.clone(), handle); + Ok(StartedExec { + process_id, + terminal, + attachment, + }) + } + + fn signal_exec(&self, process_id: &str, signal: SignalWire) -> Response { + let process = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + let Some(process) = process else { + return guest_error("invalid", "unknown exec process ID"); + }; + match self.process_runtime.block_on(process.signal(signal.into())) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn resize_process(&self, process_id: &str, cols: u16, rows: u16) -> Response { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + if !session.terminal() { + return guest_error("invalid", "agent process has no terminal"); + } + self.process_runtime.block_on(session.resize( + u32::from(cols), + u32::from(rows), + 0, + 0, + )); + return Response::Resized; + } + let terminal = lock(&self.exec_handles) + .get(process_id) + .and_then(|handle| handle.terminal.clone()); + let Some(terminal) = terminal else { + return guest_error("invalid", "exec process has no terminal"); + }; + match self.process_runtime.block_on(terminal.resize(cols, rows)) { + Ok(()) => Response::Resized, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn connect_port( + &self, + target: LoopbackTarget, + ) -> Result { + let port_forward = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err("agent process has not been started".to_string()); + }; + process.port_forward() + }; + self.process_runtime + .block_on(port_forward.connect(target)) + .map_err(|error| error.to_string()) + } + + fn network_accept_context(&self) -> Result { + self.network_broker + .confirm_healthy() + .map_err(|error| format!("sandbox network broker unavailable: {error}"))?; + Ok(self.network_broker.clone()) + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn attach_process(&self, process_id: &str) -> Result<(MainAttachment, bool), Response> { + if let Ok(process) = self.running_process(process_id) { + let session = process.main_session(); + let terminal = session.terminal(); + let attachment = acquire_attachment( + session, + process.attached.clone(), + AttachmentStatus::Main(process), + )?; + return Ok((attachment, terminal)); + } + let handles = lock(&self.exec_handles); + let handle = handles + .get(process_id) + .ok_or_else(|| guest_error("invalid", "unknown process ID"))?; + Ok((acquire_exec_attachment(handle)?, handle.terminal.is_some())) + } + + fn stream_process( + &self, + stream: ControlStream, + attachment: MainAttachment, + ) -> Result<(), String> { + self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_main_stream(stream, attachment).await + }) + } + + fn attach(&self, policy: SandboxPolicyWire) -> Response { + let mut state = lock(&self.state); + let accepted = match &*state { + RuntimeState::AwaitingAttach => { + let prepared = match PreparedBoundary::establish(self.network_broker.clone()) { + Ok(prepared) => prepared, + Err(error) => return guest_error("failed", error), + }; + *lock(&self.attached_policy) = Some(policy); + *state = RuntimeState::Bound(prepared); + true + } + RuntimeState::Bound(_) | RuntimeState::Ready(_) | RuntimeState::Running(_) => { + // Idempotent retry of the same attach; a different policy + // must not be silently coalesced onto the bound boundary. + lock(&self.attached_policy).as_ref() == Some(&policy) + } + }; + drop(state); + if accepted { + Response::Attached { + snapshot: self.session_snapshot(), + } + } else { + guest_error("denied", "attach policy does not match the bound boundary") + } + } + + fn session_snapshot(&self) -> SessionSnapshotWire { + let process = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + RuntimeState::AwaitingAttach + | RuntimeState::Bound(_) + | RuntimeState::Ready(_) => None, + } + }; + let mut processes = process + .into_iter() + .map(|process| { + let (first_sequence, next_sequence, truncated) = + process.main_session().output_window(); + ProcessSnapshotWire { + process_id: process.process_id(), + kind: ProcessKindWire::Main, + terminal: process.main_session().terminal(), + status: process.exit_status(), + retained_output: OutputWindowWire { + first_sequence, + next_sequence, + truncated, + }, + } + }) + .collect::>(); + processes.extend(lock(&self.exec_handles).iter().map(|(process_id, handle)| { + let (first_sequence, next_sequence, truncated) = handle.session.output_window(); + ProcessSnapshotWire { + process_id: process_id.clone(), + kind: ProcessKindWire::Exec, + terminal: handle.terminal.is_some(), + status: *lock(&handle.status), + retained_output: OutputWindowWire { + first_sequence, + next_sequence, + truncated, + }, + } + })); + processes.sort_by(|left, right| left.process_id.cmp(&right.process_id)); + SessionSnapshotWire { + generation: self.config.generation.clone(), + processes, + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::Bound(prepared) => { + if let Err(error) = prepared.confirm(&self.process_runtime) { + return guest_error("failed", error); + } + let evidence = match self.measure_confirmation_evidence() { + Ok(evidence) => evidence, + Err(error) => return guest_error("failed", error), + }; + *state = RuntimeState::Ready(prepared.clone()); + Response::Confirmed { + evidence: Box::new(evidence), + } + } + RuntimeState::Ready(_) | RuntimeState::Running(_) => { + self.measure_confirmation_evidence().map_or_else( + |error| guest_error("failed", error), + |evidence| Response::Confirmed { + evidence: Box::new(evidence), + }, + ) + } + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + } + } + + fn measure_confirmation_evidence(&self) -> Result { + validate_running_identity(&self.config.workload_identity)?; + self.network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}"))?; + if !self.workload_launcher.is_alive() { + return Err("sandbox workload launcher is not running".to_string()); + } + let status = std::fs::read_to_string("/proc/self/status") + .map_err(|error| format!("read sandbox process status: {error}"))?; + let capabilities = CapabilityEvidence { + inheritable: parse_status_hex(&status, "CapInh")?, + permitted: parse_status_hex(&status, "CapPrm")?, + effective: parse_status_hex(&status, "CapEff")?, + bounding: parse_status_hex(&status, "CapBnd")?, + ambient: parse_status_hex(&status, "CapAmb")?, + }; + let no_new_privileges = parse_status_decimal(&status, "NoNewPrivs")? == 1; + // SAFETY: PR_GET_DUMPABLE reads one scalar process property. + let sandbox_dumpable = unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) } != 0; + let mut core_limit = std::mem::MaybeUninit::::uninit(); + // SAFETY: getrlimit initializes the supplied output value on success. + if unsafe { libc::getrlimit(libc::RLIMIT_CORE, core_limit.as_mut_ptr()) } != 0 { + return Err(format!( + "read sandbox core limit: {}", + io::Error::last_os_error() + )); + } + // SAFETY: successful getrlimit initialized the value. + let core_limit = unsafe { core_limit.assume_init() }; + let (native_architecture, kernel_release) = uname_values()?; + Ok(SandboxConfirmEvidence { + generation: self.config.generation.clone(), + identity: self.config.workload_identity.clone(), + capabilities, + no_new_privileges, + sandbox_dumpable, + child_dumpable: true, + core_limit_zero: core_limit.rlim_cur == 0 && core_limit.rlim_max == 0, + native_architecture, + kernel_release, + seccomp: self.qualification.seccomp, + landlock_abi: self.qualification.landlock_abi, + landlock_allow_deny: self.qualification.landlock_allow_deny, + udp_dns_round_trip: self.qualification.udp_dns_round_trip, + tcp_dns_round_trip: self.qualification.tcp_dns_round_trip, + tcp_allow_round_trip: self.qualification.tcp_allow_round_trip, + tcp_deny_round_trip: self.qualification.tcp_deny_round_trip, + authenticated_supervisor: true, + session_epoch: self.config.session_epoch.clone(), + driver_fence: self.config.driver_fence.clone(), + resource_claims: self.config.resource_claims.clone(), + }) + } + + #[allow(clippy::too_many_arguments)] + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let mut state = lock(&self.state); + let requested = StartedAgent { + sandbox_id: sandbox_id.clone(), + spec: spec.clone(), + policy: policy.clone(), + ca_cert: ca_cert.clone(), + ca_bundle: ca_bundle.clone(), + provider_env_revision, + provider_env: provider_env.clone(), + }; + if let RuntimeState::Running(process) = &*state { + return if lock(&self.started_agent) + .as_ref() + .is_some_and(|accepted| accepted.matches_replay(&requested)) + { + Response::Started { + process_id: process.process_id(), + provider_env_revision: process.provider_credentials.snapshot().revision, + } + } else { + guest_error( + "denied", + "start_agent inputs do not match the running boundary", + ) + }; + } + let RuntimeState::Ready(prepared) = &*state else { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + }; + let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { + Ok(paths) => paths, + Err(error) => return guest_error("failed", error), + }; + let mut policy = policy.into(); + let driver_identity = DriverIdentity::Resolved { + uid: self.config.workload_identity.uid, + gid: self.config.workload_identity.gid, + }; + if let Err(error) = resolve_process_identity(&mut policy, &driver_identity) { + return guest_error("failed", error.to_string()); + } + let launch = ManagedProcessLaunch { + process_id: format!("{}:main:0", self.config.generation), + sandbox_id, + spec, + policy, + provider_env_revision, + provider_env, + ca_file_paths, + }; + let process = + match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *lock(&self.started_agent) = Some(requested); + *state = RuntimeState::Running(process); + Response::Started { + process_id, + provider_env_revision, + } + } + + fn update_provider_environment( + &self, + expected_revision: u64, + revision: u64, + provider_env: std::collections::HashMap, + ) -> Response { + let process = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return guest_error( + "invalid", + "agent process must be running before provider environment updates", + ); + }; + process.clone() + }; + let revision = process + .provider_credentials + .compare_and_install_child_env_snapshot(expected_revision, revision, provider_env); + Response::ProviderEnvironmentUpdated { revision } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + #[allow( + clippy::result_large_err, + reason = "protocol errors are returned directly as complete response frames" + )] + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } + } + + fn parse_status_hex(status: &str, name: &str) -> Result { + let value = status + .lines() + .find_map(|line| { + line.strip_prefix(name) + .and_then(|value| value.strip_prefix(':')) + }) + .map(str::trim) + .ok_or_else(|| format!("sandbox process status omitted {name}"))?; + u64::from_str_radix(value, 16) + .map_err(|error| format!("parse sandbox process status {name}: {error}")) + } + + fn parse_status_decimal(status: &str, name: &str) -> Result { + let value = status + .lines() + .find_map(|line| { + line.strip_prefix(name) + .and_then(|value| value.strip_prefix(':')) + }) + .map(str::trim) + .ok_or_else(|| format!("sandbox process status omitted {name}"))?; + value + .parse::() + .map_err(|error| format!("parse sandbox process status {name}: {error}")) + } + + fn uname_values() -> Result<(String, String), String> { + let mut value = std::mem::MaybeUninit::::zeroed(); + // SAFETY: uname initializes the supplied utsname value on success. + if unsafe { libc::uname(value.as_mut_ptr()) } != 0 { + return Err(format!( + "measure sandbox kernel: {}", + io::Error::last_os_error() + )); + } + // SAFETY: successful uname initialized every fixed-size C string. + let value = unsafe { value.assume_init() }; + Ok((c_char_array(&value.machine), c_char_array(&value.release))) + } + + fn c_char_array(value: &[libc::c_char]) -> String { + let length = value + .iter() + .position(|byte| *byte == 0) + .unwrap_or(value.len()); + let bytes = value[..length] + .iter() + .map(|byte| byte.to_ne_bytes()[0]) + .collect::>(); + String::from_utf8_lossy(&bytes).into_owned() + } + + impl PreparedBoundary { + fn establish(network_broker: NetworkBroker) -> Result { + network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}"))?; + Ok(Self { network_broker }) + } + + fn confirm(&self, _runtime: &tokio::runtime::Handle) -> Result<(), String> { + self.network_broker + .confirm_healthy() + .map_err(|error| format!("verify sandbox network broker: {error}")) + } + } + + fn install_ca_material( + ca_cert: Option>, + ca_bundle: Option>, + ) -> Result, String> { + let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), + (None, None) => return Ok(None), + _ => { + return Err( + "boundary proxy CA certificate and bundle must be supplied together" + .to_string(), + ); + } + }; + install_ca_material_at(Path::new("/run/openshell-proxy-ca"), &ca_cert, &ca_bundle) + } + + fn install_ca_material_at( + directory: &Path, + ca_cert: &[u8], + ca_bundle: &[u8], + ) -> Result, String> { + use std::io::Write as _; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; + + let parent = directory + .parent() + .ok_or_else(|| "boundary proxy CA directory has no parent".to_string())?; + for path in [parent, directory] { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(format!( + "boundary proxy CA directory component is a symlink: {}", + path.display() + )); + } + Ok(metadata) if !metadata.is_dir() => { + return Err(format!( + "boundary proxy CA directory component is not a directory: {}", + path.display() + )); + } + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::NotFound => { + std::fs::create_dir(path).map_err(|error| { + format!( + "create boundary proxy CA directory {}: {error}", + path.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect boundary proxy CA directory {}: {error}", + path.display() + )); + } + } + let current_mode = std::fs::metadata(path) + .map_err(|error| { + format!( + "inspect boundary proxy CA directory permissions {}: {error}", + path.display() + ) + })? + .permissions() + .mode(); + if path == directory { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).map_err( + |error| { + format!( + "set boundary proxy CA directory permissions {}: {error}", + path.display() + ) + }, + )?; + } else if current_mode & 0o111 != 0o111 { + return Err(format!( + "boundary proxy CA parent is not traversable by workload identities: {}", + path.display() + )); + } + } + let ca_path = directory.join("ca.crt"); + let bundle_path = directory.join("ca-bundle.crt"); + for (path, contents, label) in [ + (&ca_path, ca_cert, "boundary proxy CA"), + (&bundle_path, ca_bundle, "boundary proxy CA bundle"), + ] { + let temporary = path.with_extension("tmp"); + if let Ok(metadata) = std::fs::symlink_metadata(&temporary) { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "refusing unsafe temporary {label} path: {}", + temporary.display() + )); + } + std::fs::remove_file(&temporary) + .map_err(|error| format!("remove stale temporary {label}: {error}"))?; + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o444) + .custom_flags(libc::O_NOFOLLOW) + .open(&temporary) + .map_err(|error| format!("create temporary {label}: {error}"))?; + if let Err(error) = file + .write_all(contents) + .and_then(|()| file.sync_all()) + .and_then(|()| file.set_permissions(std::fs::Permissions::from_mode(0o444))) + .and_then(|()| std::fs::rename(&temporary, path)) + { + let _ = std::fs::remove_file(&temporary); + return Err(format!("install {label}: {error}")); + } + } + Ok(Some((ca_path, bundle_path))) + } + + type ProcessExit = Result; + type SharedProcessExit = Arc<(Mutex>, Condvar)>; + + struct ManagedProcess { + process_id: String, + signaler: AgentSignaler, + exit: SharedProcessExit, + boundary_exec: Arc, + port_forward: Arc, + main_session: Arc, + attached: Arc, + boundary_runtime: Arc, + provider_credentials: ProviderCredentialState, + } + + struct ManagedProcessLaunch { + process_id: String, + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + provider_env_revision: u64, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + } + + impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + launch: ManagedProcessLaunch, + _prepared: PreparedBoundary, + ) -> Result { + let ManagedProcessLaunch { + process_id, + sandbox_id, + spec, + policy, + provider_env_revision, + provider_env, + ca_file_paths, + } = launch; + if spec.program.is_empty() { + return Err("agent program must not be empty".to_string()); + } + let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + provider_env_revision, + provider_env.clone(), + ); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + entrypoint_pid, + None, + provider_credentials.clone(), + provider_env, + ca_file_paths, + Some(boundary_runtime.clone()), + )) + .map_err(|error| format!("start process supervisor leaf: {error:?}"))?; + let signaler = spawned.signaler(); + let boundary_exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + let main_session = spawned.main_session(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + process_id, + signaler, + exit, + boundary_exec, + port_forward, + main_session, + attached: Arc::new(AtomicBool::new(false)), + boundary_runtime, + provider_credentials, + }) + } + + fn process_id(&self) -> String { + self.process_id.clone() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } + + fn exit_status(&self) -> Option { + let (state, _) = &*self.exit; + lock(state).as_ref().and_then(|result| result.clone().ok()) + } + + fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + fn main_session(&self) -> Arc { + self.main_session.clone() + } + } + + impl Drop for ManagedProcess { + fn drop(&mut self) { + self.boundary_runtime.deactivate(); + } + } + + async fn bridge_main_stream( + stream: openshell_isolation_interface::contract::BoundaryDuplexStream, + attachment: MainAttachment, + ) -> Result<(), String> { + let session = attachment.session.clone(); + let (mut reader, writer) = tokio::io::split(stream); + let writer = Arc::new(tokio::sync::Mutex::new(writer)); + let input = session.acquire_input_if_open().map_err(str::to_string)?; + let owner = input.as_ref().map(|(owner, _)| *owner); + let mut output = session.subscribe(); + let input_session = session.clone(); + let mut input_task = tokio::spawn(async move { + let mut input = input.map(|(_, input)| input); + while let Some((channel, payload)) = read_stream_frame(&mut reader).await? { + match channel { + STREAM_STDIN => { + let Some(input) = input.as_ref() else { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "main process stdin already closed", + )); + }; + input.send(payload).await.map_err(|_| { + io::Error::new(io::ErrorKind::BrokenPipe, "main process stdin closed") + })?; + } + // Keep reading after stdin closes so transport EOF still + // releases this control process's attachment lease. + STREAM_STDIN_CLOSED => { + input.take(); + if let Some(owner) = owner { + input_session.close_input(owner).await; + } + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-boundary main stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }); + let result = loop { + let output_message = tokio::select! { + input_result = &mut input_task => { + break match input_result { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(format!("read main process attachment: {error}")), + Err(error) => Err(format!("join main process input stream: {error}")), + }; + } + output_message = output.recv() => output_message, + }; + let (channel, payload) = match output_message { + Ok(MainOutput::Stdout(payload)) => (STREAM_STDOUT, payload.to_vec()), + Ok(MainOutput::Stderr(payload)) => (STREAM_STDERR, payload.to_vec()), + Ok(MainOutput::Exit(code)) => { + let status = serde_json::to_vec(&attachment.exit_status(code)) + .map_err(|error| format!("encode main process exit: {error}"))?; + break write_stream_frame(&mut *writer.lock().await, STREAM_EXIT, &status) + .await + .map_err(|error| format!("write main process exit: {error}")); + } + Err(error) => { + tracing::warn!( + skipped_chunks = error.skipped, + "main process attachment resumed after dropping retained output" + ); + continue; + } + }; + if let Err(error) = + write_stream_frame(&mut *writer.lock().await, channel, &payload).await + { + break Err(format!("write main process output: {error}")); + } + }; + input_task.abort(); + if let Some(owner) = owner { + session.release_input(owner); + } + result + } + + fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) + } + + fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } + } + + fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 + } + + enum ControlListener { + Vsock { + listener: OwnedFd, + server_config: Arc, + }, + Unix { + listener: std::os::unix::net::UnixListener, + server_config: Arc, + }, + Tcp { + listener: std::net::TcpListener, + server_config: Arc, + }, + } + + impl ControlListener { + fn bind(config: &BoundaryListenerConfig) -> io::Result { + match config { + BoundaryListenerConfig::Vsock { control_port, tls } => { + let listener = Self::bind_vsock(*control_port)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Vsock { + listener, + server_config, + }) + } + BoundaryListenerConfig::Unix { socket_path, tls } => { + remove_owned_stale_control_socket(socket_path)?; + let listener = std::os::unix::net::UnixListener::bind(socket_path)?; + // Mutual TLS makes a same-UID pathname replacement a + // detectable denial of service rather than impersonation. + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o666))?; + listener.set_nonblocking(true)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Unix { + listener, + server_config, + }) + } + BoundaryListenerConfig::TlsTcp { address, tls } => { + let listener = std::net::TcpListener::bind(address)?; + listener.set_nonblocking(true)?; + let server_config = Arc::new(load_tls_server_config(tls)?); + Ok(Self::Tcp { + listener, + server_config, + }) + } + } + } + + fn bind_vsock(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = libc::socklen_t::try_from(size_of::()) + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = unsafe { + libc::socket( + libc::AF_VSOCK, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK, + 0, + ) + }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(fd) + } + + #[cfg(test)] + fn tcp_local_addr(&self) -> io::Result { + match self { + Self::Tcp { listener, .. } => listener.local_addr(), + Self::Unix { .. } | Self::Vsock { .. } => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control listener is not TCP", + )), + } + } + + fn accept(&self) -> io::Result { + match self { + Self::Vsock { + listener, + server_config, + } => { + let raw_fd = unsafe { + libc::accept4( + listener.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Vsock(unsafe { File::from_raw_fd(raw_fd) }), + server_config: server_config.clone(), + }) + } + } + Self::Unix { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Unix(stream), + server_config: server_config.clone(), + }) + } + Self::Tcp { + listener, + server_config, + } => { + let (stream, _) = listener.accept()?; + if let Err(error) = stream.set_nodelay(true) { + tracing::debug!(%error, "Failed to set boundary TCP_NODELAY"); + } + Ok(ControlStream::PendingTls { + stream: PlainControlStream::Tcp(stream), + server_config: server_config.clone(), + }) + } + } + } + } + + fn remove_owned_stale_control_socket(socket_path: &Path) -> io::Result<()> { + let metadata = match std::fs::symlink_metadata(socket_path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + }; + if !metadata.file_type().is_socket() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!( + "refusing to replace non-socket boundary control path {}", + socket_path.display() + ), + )); + } + // The private channel directory is driver-provisioned. Requiring the + // stale inode to have been created by this exact sandbox identity + // prevents a replacement run from unlinking another principal's path. + if metadata.uid() != unsafe { libc::geteuid() } { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "refusing to replace boundary control socket {} owned by UID {}", + socket_path.display(), + metadata.uid() + ), + )); + } + std::fs::remove_file(socket_path) + } + + fn load_tls_server_config( + tls: &openshell_isolation_interface::boundary_protocol::BoundaryServerTls, + ) -> io::Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let certificate_bytes = std::fs::read(&tls.certificate_chain_path)?; + let certificates = rustls_pemfile::certs(&mut certificate_bytes.as_slice()) + .collect::, _>>()?; + if certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS certificate chain contains no certificates", + )); + } + let private_key_bytes = std::fs::read(&tls.private_key_path)?; + let private_key = rustls_pemfile::private_key(&mut private_key_bytes.as_slice())? + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS private-key file contains no private key", + ) + })?; + let client_ca_bytes = std::fs::read(&tls.client_ca_certificate_path)?; + let client_ca_certificates = rustls_pemfile::certs(&mut client_ca_bytes.as_slice()) + .collect::, _>>()?; + if client_ca_certificates.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "boundary TLS client CA contains no certificates", + )); + } + let mut client_roots = rustls::RootCertStore::empty(); + for certificate in client_ca_certificates { + client_roots + .add(certificate) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + } + let client_verifier = rustls::server::WebPkiClientVerifier::builder(Arc::new(client_roots)) + .build() + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let config = rustls::ServerConfig::builder() + .with_client_cert_verifier(client_verifier) + .with_single_cert(certificates, private_key) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + for path in [ + &tls.certificate_chain_path, + &tls.private_key_path, + &tls.client_ca_certificate_path, + ] { + std::fs::remove_file(path)?; + } + Ok(config) + } + + enum PlainControlStream { + Vsock(File), + Unix(std::os::unix::net::UnixStream), + Tcp(std::net::TcpStream), + } + + impl PlainControlStream { + fn into_tokio( + self, + ) -> io::Result { + match self { + Self::Vsock(file) => { + let stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(file.into_raw_fd()) }; + stream.set_nonblocking(true)?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream)?)) + } + Self::Unix(stream) => { + stream.set_nonblocking(true)?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream)?)) + } + Self::Tcp(stream) => { + stream.set_nonblocking(true)?; + let stream = tokio::net::TcpStream::from_std(stream)?; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + Ok(Box::new(stream)) + } + } + } + } + + enum ControlStream { + PendingTls { + stream: PlainControlStream, + server_config: Arc, + }, + Tls { + stream: Option< + Box< + tokio_rustls::server::TlsStream< + openshell_isolation_interface::contract::BoundaryDuplexStream, + >, + >, + >, + runtime: tokio::runtime::Handle, + }, + #[cfg(test)] + TestUnix(std::os::unix::net::UnixStream), + } + + impl ControlStream { + fn establish(self, runtime: &tokio::runtime::Handle) -> io::Result { + let Self::PendingTls { + stream, + server_config, + } = self + else { + return Ok(self); + }; + let stream = { + let _guard = runtime.enter(); + stream.into_tokio()? + }; + let acceptor = tokio_rustls::TlsAcceptor::from(server_config); + let stream = runtime.block_on(async { + tokio::time::timeout(CONTROL_IO_TIMEOUT, acceptor.accept(stream)) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out") + })? + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error)) + })?; + Ok(Self::Tls { + stream: Some(Box::new(stream)), + runtime: runtime.clone(), + }) + } + + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + let _ = timeout; + if matches!(self, Self::Tls { .. }) { + return Ok(()); + } + if matches!(self, Self::PendingTls { .. }) { + return Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )); + } + #[cfg(test)] + if let Self::TestUnix(stream) = self { + stream.set_read_timeout(Some(timeout))?; + return stream.set_write_timeout(Some(timeout)); + } + unreachable!("all established sandbox streams use mutual TLS") + } + + fn into_tokio( + self, + ) -> Result { + match self { + Self::Tls { mut stream, .. } => Ok(stream + .take() + .expect("boundary TLS stream can only be converted once")), + Self::PendingTls { .. } => { + Err("boundary TLS stream has not completed its handshake".to_string()) + } + #[cfg(test)] + Self::TestUnix(stream) => { + stream + .set_nonblocking(true) + .map_err(|error| format!("set test Unix stream nonblocking: {error}"))?; + Ok(Box::new(tokio::net::UnixStream::from_std(stream).map_err( + |error| format!("register test Unix stream with Tokio: {error}"), + )?)) + } + } + } + } + + impl Read for ControlStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .read(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS read timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + #[cfg(test)] + Self::TestUnix(stream) => stream.read(buffer), + } + } + } + + impl Write for ControlStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .write(buffer), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS write timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + #[cfg(test)] + Self::TestUnix(stream) => stream.write(buffer), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Tls { stream, runtime } => runtime.block_on(async { + tokio::time::timeout( + CONTROL_IO_TIMEOUT, + stream + .as_mut() + .expect("boundary TLS stream must be present") + .flush(), + ) + .await + .map_err(|_| { + io::Error::new(io::ErrorKind::TimedOut, "boundary TLS flush timed out") + })? + }), + Self::PendingTls { .. } => Err(io::Error::new( + io::ErrorKind::NotConnected, + "boundary TLS stream has not completed its handshake", + )), + #[cfg(test)] + Self::TestUnix(stream) => stream.flush(), + } + } + } + + #[cfg(test)] + mod tests { + use super::*; + use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryServerTls, generate_boundary_mutual_tls_material, + }; + + #[test] + fn replay_ledger_evicts_oldest_records_without_disabling_control() { + let mut ledger = ReplayLedger::default(); + for index in 0..=MAX_REPLAY_LEDGER_ENTRIES { + ledger.insert( + format!("request-{index}"), + ReplayRecord { + payload_digest: format!("digest-{index}"), + response: Response::Signaled, + }, + ); + } + assert!(ledger.get("request-0").is_none()); + assert!( + ledger + .get(&format!("request-{MAX_REPLAY_LEDGER_ENTRIES}")) + .is_some() + ); + assert_eq!(ledger.entries.len(), MAX_REPLAY_LEDGER_ENTRIES); + } + + fn placeholder_server_tls() -> BoundaryServerTls { + BoundaryServerTls { + certificate_chain_path: Path::new("/tmp/openshell-sandbox.crt").to_path_buf(), + private_key_path: Path::new("/tmp/openshell-sandbox.key").to_path_buf(), + client_ca_certificate_path: Path::new("/tmp/openshell-client-ca.crt").to_path_buf(), + } + } + + fn stage_test_tls( + directory: &Path, + prefix: &str, + ) -> (BoundaryServerTls, BoundaryClientTls) { + let material = generate_boundary_mutual_tls_material().expect("generate test TLS"); + let certificate_chain_path = directory.join(format!("{prefix}-sandbox.crt")); + let private_key_path = directory.join(format!("{prefix}-sandbox.key")); + let client_ca_certificate_path = directory.join(format!("{prefix}-client-ca.crt")); + std::fs::write(&certificate_chain_path, material.sandbox_certificate_pem) + .expect("write sandbox certificate"); + std::fs::write(&private_key_path, material.sandbox_private_key_pem) + .expect("write sandbox key"); + std::fs::write(&client_ca_certificate_path, &material.ca_certificate_pem) + .expect("write client CA"); + ( + BoundaryServerTls { + certificate_chain_path, + private_key_path, + client_ca_certificate_path, + }, + BoundaryClientTls { + server_name: material.server_name, + ca_certificate_pem: material.ca_certificate_pem, + certificate_chain_pem: material.supervisor_certificate_pem, + private_key_pem: material.supervisor_private_key_pem, + }, + ) + } + + fn test_client_config(tls: &BoundaryClientTls) -> rustls::ClientConfig { + let mut roots = rustls::RootCertStore::empty(); + for certificate in rustls_pemfile::certs(&mut tls.ca_certificate_pem.as_bytes()) { + roots + .add(certificate.expect("parse test CA")) + .expect("add test CA"); + } + let certificates = rustls_pemfile::certs(&mut tls.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .expect("parse test client certificate"); + let private_key = rustls_pemfile::private_key(&mut tls.private_key_pem.as_bytes()) + .expect("parse test client key") + .expect("test client key"); + rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_client_auth_cert(certificates, private_key) + .expect("build test client config") + } + + #[test] + fn boundary_config_debug_redacts_token() { + let config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn installed_proxy_ca_is_readable_by_a_non_root_workload_identity() { + use std::os::unix::fs::PermissionsExt as _; + + let root = tempfile::tempdir().expect("temporary CA root"); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let directory = root.path().join("openshell-proxy-ca"); + let (ca_path, bundle_path) = install_ca_material_at( + &directory, + b"public test certificate", + b"public test bundle", + ) + .expect("install proxy CA") + .expect("CA paths"); + + assert_eq!( + std::fs::metadata(directory.parent().unwrap()) + .unwrap() + .permissions() + .mode() + & 0o111, + 0o111, + "non-root workload identities must be able to traverse the full path" + ); + assert_eq!( + std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, + 0o755, + "non-root workload identities must be able to traverse the CA directory" + ); + for (path, expected) in [ + (&ca_path, b"public test certificate".as_slice()), + (&bundle_path, b"public test bundle".as_slice()), + ] { + assert_eq!( + std::fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o444, + "public CA material must be readable by the workload" + ); + assert_eq!(std::fs::read(path).unwrap(), expected); + } + } + + #[test] + fn proxy_ca_install_rejects_a_symlinked_directory() { + let root = tempfile::tempdir().expect("temporary CA root"); + let target = root.path().join("target"); + std::fs::create_dir(&target).unwrap(); + let parent = root.path(); + std::os::unix::fs::symlink(&target, parent.join("openshell-proxy-ca")).unwrap(); + + let error = install_ca_material_at( + &parent.join("openshell-proxy-ca"), + b"certificate", + b"bundle", + ) + .expect_err("symlinked CA directory must fail closed"); + assert!(error.contains("symlink"), "unexpected error: {error}"); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } + + #[test] + fn supplementary_group_measurement_excludes_the_primary_group() { + assert_eq!( + normalized_supplementary_groups(vec![1002, 1001, 1000, 1001], 1000), + vec![1001, 1002] + ); + } + + #[test] + fn control_connection_slots_bound_unauthenticated_threads() { + let active = Arc::new(AtomicUsize::new(MAX_CONTROL_CONNECTIONS - 1)); + let slot = acquire_control_connection_slot(&active).expect("last available slot"); + assert!(acquire_control_connection_slot(&active).is_none()); + drop(slot); + assert_eq!(active.load(Ordering::Acquire), MAX_CONTROL_CONNECTIONS - 1); + } + + fn test_workload_identity() -> ResolvedWorkloadIdentity { + let mut supplementary_gids = nix::unistd::getgroups() + .unwrap() + .into_iter() + .map(nix::unistd::Gid::as_raw) + .collect::>(); + supplementary_gids.sort_unstable(); + supplementary_gids.dedup(); + ResolvedWorkloadIdentity::new( + nix::unistd::geteuid().as_raw(), + nix::unistd::getegid().as_raw(), + supplementary_gids, + "test".to_string(), + "a".repeat(64), + ) + .unwrap() + } + + fn test_driver_fence() -> openshell_isolation_interface::contract::DriverFenceEvidence { + openshell_isolation_interface::contract::DriverFenceEvidence::Vm { + generation: "generation-1".to_string(), + network_device_count: 0, + } + } + + fn test_runtime_qualification() -> crate::RuntimeQualification { + crate::RuntimeQualification { + seccomp: openshell_isolation_interface::contract::SeccompEvidence { + new_listener: true, + notification_round_trip: true, + id_validation: true, + addfd_send: true, + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: true, + task_memory_write: true, + cancellation: true, + }, + landlock_abi: 6, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + } + } + + fn test_network_broker() -> ( + NetworkBroker, + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, + ) { + let (launcher, listener) = + openshell_isolation_interface::linux::workload_launcher::start() + .expect("start test listener"); + crate::process::configure_workload_launcher(launcher.clone()) + .expect("configure test workload launcher"); + ( + NetworkBroker::start_for_test(listener).expect("start test network broker"), + launcher, + ) + } + + #[test] + fn unix_listener_allows_authenticated_cross_uid_control() { + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let (tls, _) = stage_test_tls(directory.path(), "initial"); + let _listener = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls, + }) + .expect("bind Unix listener"); + let mode = socket_path + .metadata() + .expect("socket metadata") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o666); + } + + #[test] + fn unix_listener_replaces_only_an_owned_stale_socket() { + let directory = tempfile::tempdir().expect("temporary directory"); + let socket_path = directory.path().join("control.sock"); + let (initial_tls, _) = stage_test_tls(directory.path(), "initial"); + drop( + ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls: initial_tls, + }) + .expect("bind initial Unix listener"), + ); + let (replacement_tls, _) = stage_test_tls(directory.path(), "replacement"); + let replacement = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path: socket_path.clone(), + tls: replacement_tls, + }) + .expect("replace owned stale Unix listener"); + + drop(replacement); + std::fs::remove_file(&socket_path).expect("remove stale socket"); + std::fs::write(&socket_path, b"not a socket").expect("write collision"); + let (collision_tls, _) = stage_test_tls(directory.path(), "collision"); + let error = ControlListener::bind(&BoundaryListenerConfig::Unix { + socket_path, + tls: collision_tls, + }) + .err() + .expect("regular-file collision must fail"); + assert_eq!(error.kind(), io::ErrorKind::AlreadyExists); + } + + #[test] + fn exact_workload_identity_is_required() { + let config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "a".repeat(64), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).unwrap(); + validate_running_identity(&config.workload_identity).unwrap(); + } + + #[test] + fn runtime_resource_claim_file_must_match_admitted_claim() { + let directory = tempfile::tempdir().expect("temporary directory"); + let pod_uid_path = directory.path().join("pod-uid"); + std::fs::write(&pod_uid_path, "pod-uid-a\n").expect("write runtime claim"); + let mut config = BoundaryConfig { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_epoch: "session-1".to_string(), + bootstrap_token: "a".repeat(64), + listener: BoundaryListenerConfig::Vsock { + control_port: 5500, + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + "pod-uid-a".to_string(), + )]), + resource_claim_files: std::collections::BTreeMap::from([( + "kubernetes.pod_uid".to_string(), + pod_uid_path, + )]), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }; + + validate_config(&config).expect("valid runtime claim configuration"); + validate_runtime_resource_claims(&config).expect("matching runtime claim"); + + config.resource_claims.insert( + "kubernetes.pod_uid".to_string(), + "replacement-pod-uid".to_string(), + ); + assert!(validate_runtime_resource_claims(&config).is_err()); + } + + #[test] + fn tls_listener_preserves_session_when_control_switches_to_async_streaming() { + let directory = tempfile::tempdir().expect("temporary directory"); + let (server_tls, client_tls) = stage_test_tls(directory.path(), "stream"); + let listener = ControlListener::bind(&BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:0".parse().expect("valid address"), + tls: server_tls, + }) + .expect("bind TLS listener"); + let address = listener.tcp_local_addr().expect("TLS listener address"); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test runtime"); + let server_runtime = runtime.handle().clone(); + let server = std::thread::spawn(move || { + let mut stream = loop { + match listener.accept() { + Ok(stream) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => { + std::thread::yield_now(); + } + Err(error) => panic!("accept TLS stream: {error}"), + } + } + .establish(&server_runtime) + .expect("establish TLS stream"); + let mut first = [0_u8; 4]; + Read::read_exact(&mut stream, &mut first).expect("read blocking TLS phase"); + assert_eq!(&first, b"sync"); + Write::write_all(&mut stream, b"ack1").expect("write blocking TLS phase"); + server_runtime.block_on(async move { + let mut stream = stream.into_tokio().expect("convert negotiated TLS stream"); + let mut second = [0_u8; 5]; + stream + .read_exact(&mut second) + .await + .expect("read async TLS phase"); + assert_eq!(&second, b"async"); + stream + .write_all(b"ack2") + .await + .expect("write async TLS phase"); + }); + }); + + runtime.block_on(async { + let client_config = test_client_config(&client_tls); + let stream = tokio::net::TcpStream::connect(address) + .await + .expect("connect TLS listener"); + let server_name = rustls::pki_types::ServerName::try_from(client_tls.server_name) + .expect("valid server name"); + let mut stream = tokio_rustls::TlsConnector::from(Arc::new(client_config)) + .connect(server_name, stream) + .await + .expect("verify TLS listener"); + stream.write_all(b"sync").await.expect("write first phase"); + let mut first_ack = [0_u8; 4]; + stream + .read_exact(&mut first_ack) + .await + .expect("read first acknowledgement"); + assert_eq!(&first_ack, b"ack1"); + stream + .write_all(b"async") + .await + .expect("write second phase"); + let mut second_ack = [0_u8; 4]; + stream + .read_exact(&mut second_ack) + .await + .expect("read second acknowledgement"); + assert_eq!(&second_ack, b"ack2"); + }); + server.join().expect("TLS boundary server thread"); + } + + #[test] + fn control_restart_replays_running_lifecycle_exactly_once() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_BOUNDARY_RECONNECT_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::control_restart_replays_running_lifecycle_exactly_once", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated reconnect test"); + assert!(status.success(), "isolated reconnect test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let (network_broker, workload_launcher) = test_network_broker(); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "sandbox-reconnect".to_string(), + generation: "generation-reconnect".to_string(), + session_epoch: "session-reconnect".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().expect("control address"), + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + network_broker, + workload_launcher, + test_runtime_qualification(), + )); + let policy = SandboxPolicyWire::from(openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }); + let spec = AgentSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + workdir: None, + timeout_secs: 60, + interactive: false, + }; + + assert!(matches!( + boundary.attach(policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + let start = || { + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec.clone(), + policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ) + }; + let Response::Started { + process_id, + provider_env_revision: 0, + } = start() + else { + panic!("initial start did not succeed"); + }; + + let update = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::UpdateProviderEnvironment { + expected_revision: 0, + revision: 7, + provider_env: std::collections::HashMap::from([( + "REPLAY_TEST".to_string(), + "set-once".to_string(), + )]), + }, + ) + .expect("build replayed update"); + assert_eq!( + boundary.dispatch(update.clone()), + Response::ProviderEnvironmentUpdated { revision: 7 } + ); + assert_eq!( + boundary.dispatch(update.clone()), + Response::ProviderEnvironmentUpdated { revision: 7 }, + "the same request ID and payload must replay its recorded response" + ); + let mut changed = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::Terminate { + process_id: process_id.clone(), + }, + ) + .expect("build changed request"); + changed.request_id = update.request_id; + assert!(matches!( + boundary.dispatch(changed), + Response::Error { kind, .. } if kind == "denied" + )); + + let (first_attachment, _) = boundary + .attach_process(&process_id) + .expect("initial main-process attachment"); + assert!(boundary.attach_process(&process_id).is_err()); + let (boundary_stream, control_stream) = + std::os::unix::net::UnixStream::pair().expect("main attachment socket pair"); + let stream_boundary = boundary.clone(); + let stream_thread = std::thread::spawn(move || { + stream_boundary + .stream_process(ControlStream::TestUnix(boundary_stream), first_attachment) + }); + drop(control_stream); + stream_thread + .join() + .expect("join disconnected main attachment") + .expect("transport EOF cleanly ends main attachment"); + let (replacement_attachment, _) = boundary + .attach_process(&process_id) + .expect("replacement main-process attachment after disconnect"); + drop(replacement_attachment); + + assert!(matches!( + boundary.attach(policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + start(), + Response::Started { + process_id: process_id.clone(), + provider_env_revision: 7, + } + ); + + let mut changed_policy = policy.clone(); + changed_policy.version += 1; + assert!(matches!( + boundary.attach(changed_policy.clone()), + Response::Error { kind, .. } if kind == "denied" + )); + assert!(matches!( + boundary.start_agent( + "sandbox-reconnect".to_string(), + spec, + changed_policy, + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Error { kind, .. } if kind == "denied" + )); + + let exec_spec = ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec!["-c".to_string(), "printf reconnected".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }; + let exec_request = RequestEnvelope::new( + "sandbox-reconnect".to_string(), + "a".repeat(32), + Request::Exec { + spec: exec_spec.clone(), + }, + ) + .expect("build exec request"); + let exec = boundary + .start_exec( + &exec_request.request_id, + &exec_request.payload_digest, + exec_spec, + ) + .expect("exec after reconnect"); + let exec_id = exec.process_id.clone(); + let mut output = String::new(); + let mut cursor = exec.attachment.session.subscribe(); + process_runtime.block_on(async { + loop { + match cursor.recv().await.expect("retained exec output") { + MainOutput::Stdout(bytes) => { + output.push_str(std::str::from_utf8(&bytes).expect("UTF-8 output")); + } + MainOutput::Stderr(_) => {} + MainOutput::Exit(code) => { + assert_eq!(code, 0); + break; + } + } + } + }); + assert_eq!(output, "reconnected"); + drop(exec); + let Response::Attached { snapshot } = boundary.attach(policy.clone()) else { + panic!("reconnect attach did not return a session snapshot"); + }; + assert_eq!(snapshot.generation, "generation-reconnect"); + assert!(snapshot.processes.iter().any(|process| { + process.process_id == process_id && process.kind == ProcessKindWire::Main + })); + assert!(snapshot.processes.iter().any(|process| { + process.process_id == exec_id + && process.kind == ProcessKindWire::Exec + && process.status == Some(ExitStatusWire::Exited(0)) + && process.retained_output.next_sequence > 0 + })); + assert_eq!(boundary.terminate(&process_id), Response::Terminated); + } + + #[test] + fn canonical_exit_preserves_pending_network_accept_and_exec() { + const CHILD_MARKER: &str = "OPENSHELL_TEST_RETAINED_BOUNDARY_CHILD"; + if std::env::var_os(CHILD_MARKER).is_none() { + let status = std::process::Command::new( + std::env::current_exe().expect("current test executable"), + ) + .args([ + "--exact", + "boundary_server::linux::tests::canonical_exit_preserves_pending_network_accept_and_exec", + "--nocapture", + ]) + .env(CHILD_MARKER, "1") + .status() + .expect("run isolated retained-boundary test"); + assert!(status.success(), "isolated retained-boundary test failed"); + return; + } + + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("test process runtime"); + let policy = openshell_core::policy::SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy { + mode: openshell_core::policy::NetworkMode::Proxy, + proxy: Some(openshell_core::policy::ProxyPolicy { + http_addr: Some("127.0.0.1:3128".parse().expect("proxy address")), + }), + }, + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }; + let (network_broker, workload_launcher) = test_network_broker(); + let prepared = PreparedBoundary { + network_broker: network_broker.clone(), + }; + let agent_spec = AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: None, + timeout_secs: 5, + interactive: false, + }; + let wire_policy = SandboxPolicyWire::from(policy.clone()); + let process = Arc::new( + ManagedProcess::spawn( + process_runtime.handle(), + ManagedProcessLaunch { + process_id: "generation-retained:main:0".to_string(), + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy, + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + ca_file_paths: None, + }, + prepared, + ) + .expect("spawn canonical process"), + ); + let boundary = Arc::new(BoundaryRuntime::new( + BoundaryConfig { + boundary_id: "sandbox-retained".to_string(), + generation: "generation-retained".to_string(), + session_epoch: "session-retained".to_string(), + bootstrap_token: "a".repeat(32), + listener: BoundaryListenerConfig::TlsTcp { + address: "127.0.0.1:5500".parse().expect("control address"), + tls: placeholder_server_tls(), + }, + resource_claims: std::collections::BTreeMap::new(), + resource_claim_files: std::collections::BTreeMap::new(), + workload_identity: test_workload_identity(), + driver_fence: test_driver_fence(), + child_env: std::collections::HashMap::new(), + }, + process_runtime.handle().clone(), + network_broker, + workload_launcher, + test_runtime_qualification(), + )); + *lock(&boundary.state) = RuntimeState::Running(process.clone()); + *lock(&boundary.attached_policy) = Some(wire_policy.clone()); + *lock(&boundary.started_agent) = Some(StartedAgent { + sandbox_id: "sandbox-retained".to_string(), + spec: agent_spec.clone(), + policy: wire_policy.clone(), + ca_cert: None, + ca_bundle: None, + provider_env_revision: 0, + provider_env: std::collections::HashMap::new(), + }); + + // A replacement control process replays the durable lifecycle and + // receives the original process rather than spawning another one. + assert!(matches!( + boundary.attach(wire_policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec.clone(), + wire_policy.clone(), + None, + None, + 0, + std::collections::HashMap::new(), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 0, + } + ); + + assert_eq!( + boundary.update_provider_environment( + 0, + 2, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "refreshed".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment( + 0, + 1, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "stale".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 2 } + ); + assert_eq!( + boundary.update_provider_environment(2, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a numerically smaller opaque revision must revoke the environment" + ); + assert_eq!( + boundary.update_provider_environment( + 2, + 3, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "out-of-order".to_string(), + )]), + ), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a stale expected revision must not overwrite current state" + ); + assert_eq!( + boundary.update_provider_environment(1, 1, std::collections::HashMap::new()), + Response::ProviderEnvironmentUpdated { revision: 1 }, + "a duplicate update must be idempotent" + ); + + assert!(matches!( + boundary.attach(wire_policy.clone()), + Response::Attached { .. } + )); + assert!(matches!(boundary.confirm(), Response::Confirmed { .. })); + assert_eq!( + boundary.start_agent( + "sandbox-retained".to_string(), + agent_spec, + wire_policy, + None, + None, + 99, + std::collections::HashMap::from([( + "ROTATED_TOKEN".to_string(), + "replacement-control-snapshot".to_string(), + )]), + ), + Response::Started { + process_id: process.process_id(), + provider_env_revision: 1, + }, + "a replacement control must resume from the boundary's current revision" + ); + + let sleep_spec = ExecSpecWire { + program: "/bin/sleep".to_string(), + args: vec!["30".to_string()], + env: Vec::new(), + workdir: None, + pty: false, + }; + let sleep_request = RequestEnvelope::new( + "sandbox-retained".to_string(), + "a".repeat(32), + Request::Exec { + spec: sleep_spec.clone(), + }, + ) + .expect("build retained exec request"); + let started = boundary + .start_exec( + &sleep_request.request_id, + &sleep_request.payload_digest, + sleep_spec.clone(), + ) + .expect("start exec whose response is disconnected"); + let retained_id = started.process_id.clone(); + drop(started); + let replayed = boundary + .start_exec( + &sleep_request.request_id, + &sleep_request.payload_digest, + sleep_spec, + ) + .expect("reattach exec after response loss"); + assert_eq!(replayed.process_id, retained_id); + assert_eq!(lock(&boundary.exec_handles).len(), 1); + drop(replayed); + assert_eq!( + boundary.signal_exec(&retained_id, SignalWire::Kill), + Response::Signaled + ); + + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !process.has_exited() && std::time::Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(process.has_exited(), "canonical process did not exit"); + + let mut session = process_runtime + .block_on( + process.boundary_exec().exec( + ExecSpecWire { + program: "/bin/sh".to_string(), + args: vec![ + "-c".to_string(), + "if [ -z \"${ROTATED_TOKEN+x}\" ]; then printf revoked; else printf 'unexpected:%s' \"$ROTATED_TOKEN\"; fi" + .to_string(), + ], + env: Vec::new(), + workdir: None, + pty: false, + } + .into(), + ), + ) + .expect("exec after canonical exit"); + let mut output = String::new(); + process_runtime + .block_on(session.stdout.read_to_string(&mut output)) + .expect("read retained exec output"); + assert_eq!( + output, "revoked", + "exec after canonical exit must use the latest reconciled provider snapshot" + ); + assert!(matches!( + process_runtime.block_on(session.process.wait()), + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(0)) + )); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::run_boundary; + +#[cfg(not(target_os = "linux"))] +pub fn run_boundary( + _config_path: &Path, + _qualification: crate::RuntimeQualification, +) -> Result<(), String> { + Err("boundary mode is supported only on Linux".to_string()) +} diff --git a/crates/openshell-supervisor-process/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs similarity index 59% rename from crates/openshell-supervisor-process/src/child_env.rs rename to crates/openshell-sandbox/src/child_env.rs index 32eecbee35..50549a7439 100644 --- a/crates/openshell-supervisor-process/src/child_env.rs +++ b/crates/openshell-sandbox/src/child_env.rs @@ -3,24 +3,6 @@ use std::path::Path; -const LOCAL_NO_PROXY: &str = "127.0.0.1,localhost,::1"; - -pub fn proxy_env_vars(proxy_url: &str) -> [(&'static str, String); 9] { - [ - ("ALL_PROXY", proxy_url.to_owned()), - ("HTTP_PROXY", proxy_url.to_owned()), - ("HTTPS_PROXY", proxy_url.to_owned()), - ("NO_PROXY", LOCAL_NO_PROXY.to_owned()), - ("http_proxy", proxy_url.to_owned()), - ("https_proxy", proxy_url.to_owned()), - ("no_proxy", LOCAL_NO_PROXY.to_owned()), - ("grpc_proxy", proxy_url.to_owned()), - // Node.js only honors HTTP(S)_PROXY for built-in fetch/http clients when - // proxy support is explicitly enabled at process startup. - ("NODE_USE_ENV_PROXY", "1".to_owned()), - ] -} - pub fn tls_env_vars( ca_cert_path: &Path, combined_bundle_path: &Path, @@ -45,26 +27,6 @@ mod tests { use std::process::Command; use std::process::Stdio; - #[test] - fn apply_proxy_env_includes_node_proxy_opt_in_and_local_bypass() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - - for (key, value) in proxy_env_vars("http://10.200.0.1:3128") { - cmd.env(key, value); - } - - let output = cmd.output().expect("spawn env"); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - - assert!(stdout.contains("HTTP_PROXY=http://10.200.0.1:3128")); - assert!(stdout.contains("NO_PROXY=127.0.0.1,localhost,::1")); - assert!(stdout.contains("NODE_USE_ENV_PROXY=1")); - assert!(stdout.contains("no_proxy=127.0.0.1,localhost,::1")); - } - #[test] fn apply_tls_env_sets_node_and_bundle_paths() { let mut cmd = Command::new("/usr/bin/env"); diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs new file mode 100644 index 0000000000..a9d09a8ec5 --- /dev/null +++ b/crates/openshell-sandbox/src/delegated.rs @@ -0,0 +1,241 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process and access-plane assembly for the capability-free sandbox boundary. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::Duration; + +use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +use openshell_core::policy::SandboxPolicy; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; +use openshell_ocsf::{ + ActionId, ActivityId, DispositionId, LaunchTypeId, Process as OcsfProcess, + ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, +}; + +use crate::process::{ProcessHandle, ProcessStatus, ResolvedWorkspace}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Spawn the admitted workload without placing the gateway or policy authority +/// inside its boundary. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn spawn_workload( + program: &str, + args: &[String], + workdir: Option<&str>, + timeout_secs: u64, + interactive: bool, + _sandbox_id: Option<&str>, + _openshell_endpoint: Option<&str>, + _ssh_socket_path: Option, + _shared_ssh_socket: bool, + policy: &SandboxPolicy, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + boundary_runtime: Option>, +) -> Result { + // Driver-selected workspaces are the sandbox identity's home. This keeps + // canonical and later exec processes consistent for image WorkingDir and + // the managed /sandbox fallback without consulting privileged account + // setup inside the capability-free boundary. + let workspace = ResolvedWorkspace::new(workdir.map(str::to_string), true); + + #[cfg(target_os = "linux")] + { + let mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { + crate::process::RuntimePidLimitMode::Require + } else { + crate::process::RuntimePidLimitMode::Warn + }; + crate::process::check_runtime_pid_limit(mode).wrap_err("check runtime PID limit")?; + } + + let boundary_runtime = boundary_runtime + .unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace); + let mut user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + user_environment.retain(|key, _value| !crate::process::is_proxy_env_var(key)); + let port_forward: Arc = Arc::new( + crate::boundary_io::LocalPortForward::new(Some(boundary_runtime.clone())), + ); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + ca_file_paths.clone().map(Arc::new), + provider_credentials, + user_environment, + boundary_runtime.clone(), + )); + + #[cfg(target_os = "linux")] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + ca_file_paths.as_ref(), + &provider_env, + ) + .wrap_err("spawn delegated workload process")?; + #[cfg(not(target_os = "linux"))] + let mut handle = ProcessHandle::spawn( + program, + args, + &workspace, + interactive, + policy, + ca_file_paths.as_ref(), + &provider_env, + )?; + + entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(sender) = entrypoint_started_tx { + let _ = sender.send(handle.pid()); + } + let main_session = crate::main_session::MainSession::new(handle.take_io(), handle.pid()); + let (terminal, signal_lock) = handle.signaling_state(); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .launch_type(LaunchTypeId::Spawn) + .process(OcsfProcess::new(program, i64::from(handle.pid()))) + .message(format!("Process started: pid={}", handle.pid())) + .build() + ); + + Ok(SpawnedAgent { + handle, + timeout_secs, + terminal, + signal_lock, + main_session, + boundary_exec, + port_forward, + boundary_runtime, + }) +} + +/// Owned workload process and its live boundary capabilities. +pub struct SpawnedAgent { + handle: ProcessHandle, + timeout_secs: u64, + terminal: Arc, + signal_lock: Arc>, + main_session: Arc, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, +} + +impl SpawnedAgent { + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), + } + } + + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + /// Retained canonical-process I/O owned by the boundary. + #[must_use] + pub fn main_session(&self) -> Arc { + self.main_session.clone() + } + + /// Wait for the canonical process to exit, enforcing its admitted + /// wall-clock timeout. Completion does not end the boundary: exec and + /// loopback forwarding remain available until the boundary owner tears + /// down the retained runtime. + pub async fn wait(&mut self) -> Result { + let signaler = self.signaler(); + let status = if self.timeout_secs == 0 { + self.handle.wait().await.into_diagnostic()? + } else if let Ok(status) = + tokio::time::timeout(Duration::from_secs(self.timeout_secs), self.handle.wait()).await + { + status.into_diagnostic()? + } else { + let _ = signaler.term(); + tokio::time::sleep(Duration::from_millis(100)).await; + let _ = signaler.kill(); + self.handle.wait().await.into_diagnostic()? + }; + self.boundary_runtime + .unregister_process_group(self.handle.pid(), &self.terminal); + let _ = self.main_session.finish(status.code(), false).await; + self.main_session.mark_terminal_reported(); + Ok(status) + } +} + +/// Lock-free process-group signal handle used while another task owns `wait`. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } +} diff --git a/crates/openshell-sandbox/src/google_cloud_metadata.rs b/crates/openshell-sandbox/src/google_cloud_metadata.rs deleted file mode 100644 index 9e1e179872..0000000000 --- a/crates/openshell-sandbox/src/google_cloud_metadata.rs +++ /dev/null @@ -1,536 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! GCE metadata server emulator for sandbox credential injection. -//! -//! Implements a subset of the GCE instance metadata API so that GCP client -//! libraries (Go, Python, Node.js) can obtain `OAuth2` tokens natively inside -//! sandboxes. Tokens are served from the existing `ProviderCredentialState` -//! store — no separate refresh mechanism is needed. -//! -//! The emulator runs as a loopback HTTP server inside the sandbox network -//! namespace (see [`metadata_server`](crate::metadata_server)). GCP SDKs -//! discover it via the `GCE_METADATA_HOST` environment variable, which is -//! set to the loopback address by `child_env_with_gcp_resolved()`. - -use miette::{IntoDiagnostic, Result}; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_core::secrets; -use openshell_ocsf::{ActivityId, HttpActivityBuilder, SeverityId, StatusId, ocsf_emit}; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; - -type MetadataResponse = (u16, &'static str, String); - -const PATH_SERVICE_ACCOUNTS: &str = "/computeMetadata/v1/instance/service-accounts"; -const PATH_SERVICE_ACCOUNT_DEFAULT: &str = "/computeMetadata/v1/instance/service-accounts/default"; -const PATH_TOKEN: &str = "/computeMetadata/v1/instance/service-accounts/default/token"; -const PATH_EMAIL: &str = "/computeMetadata/v1/instance/service-accounts/default/email"; -const PATH_SCOPES: &str = "/computeMetadata/v1/instance/service-accounts/default/scopes"; -const PATH_ALIASES: &str = "/computeMetadata/v1/instance/service-accounts/default/aliases"; -const PATH_PROJECT_ID: &str = "/computeMetadata/v1/project/project-id"; - -const ENV_GCP_PROJECT_ID: &str = openshell_core::google_cloud::PROJECT_ID_ENV_VARS[0]; -const ENV_GCP_SERVICE_ACCOUNT_EMAIL: &str = - openshell_core::google_cloud::SERVICE_ACCOUNT_EMAIL_ENV_VARS[0]; - -const METADATA_FLAVOR_HEADER: &str = "metadata-flavor"; -const METADATA_FLAVOR_VALUE: &str = "Google"; -const X_FORWARDED_FOR_HEADER: &str = "x-forwarded-for"; - -#[derive(Debug, Clone)] -pub struct MetadataContext { - credentials: ProviderCredentialState, -} - -impl MetadataContext { - pub fn new(credentials: ProviderCredentialState) -> Self { - Self { credentials } - } -} - -impl crate::metadata_server::MetadataHandler for MetadataContext { - async fn handle( - &self, - method: &str, - path: &str, - request: &[u8], - stream: &mut S, - ) -> Result<()> { - handle_forward_request(self, method, path, request, stream).await - } -} - -async fn handle_forward_request( - ctx: &MetadataContext, - method: &str, - path: &str, - initial_request: &[u8], - client: &mut S, -) -> Result<()> -where - S: AsyncRead + AsyncWrite + Unpin, -{ - let headers = parse_request_headers(initial_request); - let (status, content_type, body) = route_request(ctx, method, path, &headers); - write_metadata_response(client, status, content_type, &body).await -} - -fn route_request( - ctx: &MetadataContext, - method: &str, - path: &str, - headers: &[(String, String)], -) -> MetadataResponse { - if method != "GET" { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: unsupported method {method}"), - ); - return (405, "text/html", "Method Not Allowed".to_string()); - } - - if let Err(resp) = validate_metadata_headers(headers) { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Medium, - StatusId::Failure, - &format!("metadata: header validation failed for {path}"), - ); - return resp; - } - - let (route, query) = path.split_once('?').map_or((path, ""), |(r, q)| (r, q)); - let route = route.strip_suffix('/').unwrap_or(route); - let recursive = query.split('&').any(|p| p == "recursive=true"); - - match route { - PATH_TOKEN => handle_token(ctx), - PATH_EMAIL => handle_env(ctx, ENV_GCP_SERVICE_ACCOUNT_EMAIL), - PATH_PROJECT_ID => handle_env(ctx, ENV_GCP_PROJECT_ID), - PATH_ALIASES => (200, "text/plain", "default\n".to_string()), - PATH_SCOPES => ( - 200, - "text/plain", - "https://www.googleapis.com/auth/cloud-platform".to_string(), - ), - PATH_SERVICE_ACCOUNT_DEFAULT => { - if recursive { - handle_service_account_recursive(ctx) - } else { - ( - 200, - "text/plain", - "aliases\nemail\nscopes\ntoken\n".to_string(), - ) - } - } - PATH_SERVICE_ACCOUNTS => (200, "text/plain", "default/\n".to_string()), - "" | "/" | "/computeMetadata" | "/computeMetadata/v1" => { - (200, "text/plain", "computeMetadata/\n".to_string()) - } - "/computeMetadata/v1/instance" => (200, "text/plain", "service-accounts/\n".to_string()), - _ => { - emit_metadata_event( - ActivityId::Refuse, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: unknown path {route}"), - ); - ( - 404, - "application/json", - serde_json::json!({"error": "not_found"}).to_string(), - ) - } - } -} - -fn handle_token(ctx: &MetadataContext) -> MetadataResponse { - let Some((placeholder, expires_in)) = ctx.credentials.gcp_token_response() else { - let has_resolver = ctx.credentials.resolver().is_some(); - let (msg, error_key) = if has_resolver { - ( - "metadata: no GCP access token available or expired", - "token_unavailable", - ) - } else { - ( - "metadata: token request but no credentials configured", - "credentials_unavailable", - ) - }; - emit_metadata_event(ActivityId::Fail, SeverityId::Medium, StatusId::Failure, msg); - return ( - 503, - "application/json", - serde_json::json!({"error": error_key}).to_string(), - ); - }; - - emit_metadata_event( - ActivityId::Open, - SeverityId::Informational, - StatusId::Success, - "metadata: token placeholder served", - ); - - let body = serde_json::json!({ - "access_token": placeholder, - "expires_in": expires_in, - "token_type": "Bearer" - }); - (200, "application/json", body.to_string()) -} - -fn handle_service_account_recursive(ctx: &MetadataContext) -> MetadataResponse { - let resolver = ctx.credentials.resolver(); - let email = resolver - .as_ref() - .and_then(|r| { - let p = secrets::placeholder_for_env_key(ENV_GCP_SERVICE_ACCOUNT_EMAIL); - r.resolve_placeholder(&p).map(str::to_string) - }) - .unwrap_or_default(); - - let scopes = "https://www.googleapis.com/auth/cloud-platform"; - - let body = serde_json::json!({ - "aliases": ["default"], - "email": email, - "scopes": [scopes], - }); - (200, "application/json", body.to_string()) -} - -/// Serve a non-secret config value (project ID, SA email) as plain text. -/// -/// Unlike `handle_token` which serves placeholders, this resolves to the real -/// value. This matches real GCE metadata server behavior and is safe because -/// these values are non-secret configuration (project IDs, email addresses). -fn handle_env(ctx: &MetadataContext, env_key: &str) -> MetadataResponse { - let Some(resolver) = ctx.credentials.resolver() else { - emit_metadata_event( - ActivityId::Fail, - SeverityId::Medium, - StatusId::Failure, - &format!("metadata: {env_key} request but no credentials configured"), - ); - return (503, "text/plain", String::new()); - }; - - let placeholder = secrets::placeholder_for_env_key(env_key); - resolver.resolve_placeholder(&placeholder).map_or_else( - || { - emit_metadata_event( - ActivityId::Fail, - SeverityId::Low, - StatusId::Failure, - &format!("metadata: {env_key} not configured"), - ); - ( - 404, - "application/json", - serde_json::json!({"error": "not_found"}).to_string(), - ) - }, - |value| (200, "text/plain", value.to_string()), - ) -} - -fn validate_metadata_headers(headers: &[(String, String)]) -> Result<(), MetadataResponse> { - if headers - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case(X_FORWARDED_FOR_HEADER)) - { - return Err((403, "text/html", "Forbidden".to_string())); - } - - let has_flavor = headers.iter().any(|(name, value)| { - name.eq_ignore_ascii_case(METADATA_FLAVOR_HEADER) - && value.trim().eq_ignore_ascii_case(METADATA_FLAVOR_VALUE) - }); - if !has_flavor { - return Err((403, "text/html", "Forbidden".to_string())); - } - - Ok(()) -} - -fn parse_request_headers(raw: &[u8]) -> Vec<(String, String)> { - let request = String::from_utf8_lossy(raw); - let mut headers = Vec::new(); - for line in request.split("\r\n").skip(1) { - if line.is_empty() { - break; - } - if let Some((name, value)) = line.split_once(':') { - headers.push((name.trim().to_string(), value.trim().to_string())); - } - } - headers -} - -fn status_text(status: u16) -> &'static str { - match status { - 403 => "Forbidden", - 404 => "Not Found", - 405 => "Method Not Allowed", - 503 => "Service Unavailable", - _ => "OK", - } -} - -async fn write_metadata_response( - client: &mut S, - status: u16, - content_type: &str, - body: &str, -) -> Result<()> -where - S: AsyncWrite + Unpin, -{ - let response = format!( - "HTTP/1.1 {status} {}\r\nContent-Type: {content_type}\r\nMetadata-Flavor: Google\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - status_text(status), - body.len(), - ); - client - .write_all(response.as_bytes()) - .await - .into_diagnostic()?; - client.flush().await.into_diagnostic()?; - Ok(()) -} - -fn emit_metadata_event( - activity: ActivityId, - severity: SeverityId, - status: StatusId, - message: &str, -) { - let event = HttpActivityBuilder::new(crate::ocsf_ctx()) - .activity(activity) - .severity(severity) - .status(status) - .message(message.to_string()) - .build(); - ocsf_emit!(event); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - fn make_context(env: HashMap) -> MetadataContext { - let state = - ProviderCredentialState::from_environment(0, env, HashMap::new(), HashMap::new()); - MetadataContext::new(state) - } - - fn make_context_with_expiry( - env: HashMap, - expires: HashMap, - ) -> MetadataContext { - let state = ProviderCredentialState::from_environment(0, env, expires, HashMap::new()); - MetadataContext::new(state) - } - - fn flavor_headers() -> Vec<(String, String)> { - vec![("Metadata-Flavor".to_string(), "Google".to_string())] - } - - #[test] - fn token_returns_placeholder_not_real_value() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.test-token".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "application/json"); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - let token = json["access_token"].as_str().unwrap(); - assert!( - token.starts_with("openshell:resolve:env:"), - "token should be a placeholder, got: {token}" - ); - assert!(!token.contains("ya29"), "real token must not be served"); - assert_eq!(json["token_type"], "Bearer"); - assert!(json["expires_in"].is_number()); - } - - #[test] - fn token_expires_in_computed_from_credential_expiry() { - let now_ms = openshell_core::time::now_ms(); - let expires_at = now_ms + 1_800_000; // 30 minutes from now - let ctx = make_context_with_expiry( - HashMap::from([("GCP_ADC_ACCESS_TOKEN".to_string(), "ya29.tok".to_string())]), - HashMap::from([("GCP_ADC_ACCESS_TOKEN".to_string(), expires_at)]), - ); - let (status, _, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 200); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - let expires_in = json["expires_in"].as_i64().unwrap(); - assert!( - expires_in > 1700 && expires_in <= 1800, - "expires_in={expires_in}" - ); - } - - #[test] - fn token_no_expiry_defaults_to_3600() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let (_, _, body) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - let json: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(json["expires_in"], 3600); - } - - #[test] - fn missing_metadata_flavor_header_403() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &[]); - assert_eq!(status, 403); - } - - #[test] - fn x_forwarded_for_header_403() { - let ctx = make_context(HashMap::new()); - let headers = vec![ - ("Metadata-Flavor".to_string(), "Google".to_string()), - ("X-Forwarded-For".to_string(), "10.0.0.1".to_string()), - ]; - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &headers); - assert_eq!(status, 403); - } - - #[test] - fn unknown_path_404() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request( - &ctx, - "GET", - "/computeMetadata/v1/unknown", - &flavor_headers(), - ); - assert_eq!(status, 404); - } - - #[test] - fn no_credentials_503() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 503); - } - - #[test] - fn post_method_405() { - let ctx = make_context(HashMap::new()); - let (status, _, _) = route_request(&ctx, "POST", PATH_TOKEN, &flavor_headers()); - assert_eq!(status, 405); - } - - #[test] - fn project_id_served_as_plain_text() { - let ctx = make_context(HashMap::from([( - "GCP_PROJECT_ID".to_string(), - "my-project-123".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_PROJECT_ID, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "text/plain"); - assert_eq!(body, "my-project-123"); - } - - #[test] - fn email_served_as_plain_text() { - let ctx = make_context(HashMap::from([( - "GCP_SERVICE_ACCOUNT_EMAIL".to_string(), - "sa@project.iam.gserviceaccount.com".to_string(), - )])); - let (status, ct, body) = route_request(&ctx, "GET", PATH_EMAIL, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(ct, "text/plain"); - assert_eq!(body, "sa@project.iam.gserviceaccount.com"); - } - - #[test] - fn scopes_returns_cloud_platform() { - let ctx = make_context(HashMap::new()); - let (status, _, body) = route_request(&ctx, "GET", PATH_SCOPES, &flavor_headers()); - assert_eq!(status, 200); - assert_eq!(body, "https://www.googleapis.com/auth/cloud-platform"); - } - - #[test] - fn query_parameters_ignored_for_routing() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let path = format!("{PATH_TOKEN}?scopes=cloud-platform"); - let (status, _, _) = route_request(&ctx, "GET", &path, &flavor_headers()); - assert_eq!(status, 200); - } - - #[test] - fn metadata_flavor_case_insensitive() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let headers = vec![("metadata-FLAVOR".to_string(), "google".to_string())]; - let (status, _, _) = route_request(&ctx, "GET", PATH_TOKEN, &headers); - assert_eq!(status, 200); - } - - #[test] - fn missing_env_var_returns_404() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - // project-id not set - let (status, _, _) = route_request(&ctx, "GET", PATH_PROJECT_ID, &flavor_headers()); - assert_eq!(status, 404); - } - - #[test] - fn trailing_slash_handled_for_service_account_default() { - let ctx = make_context(HashMap::from([( - "GCP_ADC_ACCESS_TOKEN".to_string(), - "ya29.tok".to_string(), - )])); - let with_slash = route_request( - &ctx, - "GET", - "/computeMetadata/v1/instance/service-accounts/default/", - &flavor_headers(), - ); - let without_slash = route_request( - &ctx, - "GET", - "/computeMetadata/v1/instance/service-accounts/default", - &flavor_headers(), - ); - assert_eq!(with_slash.0, 200); - assert_eq!(without_slash.0, 200); - assert_eq!(with_slash.2, without_slash.2); - } - - #[test] - fn parse_request_headers_extracts_correctly() { - let raw = b"GET /path HTTP/1.1\r\nHost: example.com\r\nMetadata-Flavor: Google\r\n\r\n"; - let headers = parse_request_headers(raw); - assert_eq!(headers.len(), 2); - assert_eq!(headers[0].0, "Host"); - assert_eq!(headers[0].1, "example.com"); - assert_eq!(headers[1].0, "Metadata-Flavor"); - assert_eq!(headers[1].1, "Google"); - } -} diff --git a/crates/openshell-supervisor-process/src/identity.rs b/crates/openshell-sandbox/src/identity.rs similarity index 100% rename from crates/openshell-supervisor-process/src/identity.rs rename to crates/openshell-sandbox/src/identity.rs diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 845a594207..ed98161d78 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1,6309 +1,51 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox library. -//! -//! This crate provides process sandboxing and monitoring capabilities. - -// `defaults-without-telemetry` is an alias for the default feature set minus -// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a -// default feature, so adding it on top of the defaults would otherwise produce -// a telemetry-on build that reads as telemetry-free. Fail the build instead. -#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] -compile_error!( - "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ - build a telemetry-free supervisor with `--no-default-features --features defaults-without-telemetry`" -); - -mod activity_aggregator; -mod denial_aggregator; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod google_cloud_metadata; -mod mechanistic_mapper; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod metadata_server; -mod sidecar_control; - -use miette::{IntoDiagnostic, Result, WrapErr}; -use std::future::Future; -use std::pin::Pin; -use std::sync::Arc; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU32}; -use std::time::Duration; -use tracing::{debug, info, warn}; - -use openshell_core::PolicyValidationFailureMode; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, - DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, - StateId, StatusId, ocsf_emit, -}; - -// --------------------------------------------------------------------------- -// OCSF Context -// --------------------------------------------------------------------------- -// -// The following log sites intentionally remain as plain `tracing` macros -// and are NOT migrated to OCSF builders: -// -// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) -// - Transient "about to do X" events where the result is logged separately -// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") -// - Internal SSH channel warnings (unknown channel, PTY resize failures) -// - Denial flush telemetry (the individual denials are already OCSF events) -// - Status reporting failures (sync to gateway, non-actionable) -// - Route refresh interval validation warnings -// -// These are operational plumbing that don't represent security decisions, -// policy changes, or observable sandbox behavior worth structuring. -// --------------------------------------------------------------------------- - -/// Re-export the process-wide OCSF sandbox context getter. -/// -/// The singleton lives in `openshell-ocsf` so both supervisor leaves can -/// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. -pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; - -use openshell_core::denial::DenialEvent; -use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_network::proxy::ProxyHandle; -use openshell_supervisor_process::process::ProcessEnforcementMode; -pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; -use openshell_supervisor_process::skills; -use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] -use tokio::time::timeout; - -const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; -const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; - -#[cfg(any(test, target_os = "linux"))] -fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> bool { - capabilities.is_some_and(|capabilities| { - capabilities - .split(',') - .any(|capability| capability.trim() == required) - }) -} -const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; -const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; - -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::implicit_hasher, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, - inference_routes: Option, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - network_enabled: bool, - process_enabled: bool, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, -) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; - - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; they are only significant for network - // events which happen after the netns is created. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(SandboxContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); - let process_uses_sidecar_control = - process_enabled && !network_enabled && sidecar_network_enforcement; - let mut process_control_connection = None; - let sidecar_bootstrap = if process_uses_sidecar_control { - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let (bootstrap, connection) = sidecar_control::connect_process_client( - &socket, - Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), - ) - .await?; - process_control_connection = Some(connection); - Some(bootstrap) - } else { - None - }; - - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. - let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let ( - mut policy, - opa_engine, - retained_proto, - middleware_registry_status, - loaded_policy_origin, - initial_agent_proposals_enabled, - initial_extension_authentication_enabled, - ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let (policy, opa_engine, retained_proto, loaded_policy_origin) = - load_policy_from_sidecar_bootstrap(bootstrap)?; - ( - policy, - opa_engine, - retained_proto, - MiddlewareRegistryStatus::Synchronized, - loaded_policy_origin, - bootstrap.agent_proposals_enabled, - false, - ) - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - &extension_credentials, - ) - .await? - }; - - // Normalize the active driver's identity contract once, while both the - // policy and launched image filesystem are available. Kubernetes and - // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. - #[cfg(unix)] - let (resolved_process_identity, workspace) = { - let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; - let use_workdir_as_home = matches!( - &driver_identity, - openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } - ); - let resolved = openshell_supervisor_process::identity::resolve_process_identity( - &mut policy, - &driver_identity, - )?; - ( - resolved, - openshell_supervisor_process::process::ResolvedWorkspace::new( - workdir.clone(), - use_workdir_as_home, - ), - ) - }; - #[cfg(not(unix))] - let (resolved_process_identity, workspace) = ( - openshell_supervisor_process::process::ResolvedProcessIdentity::default(), - openshell_supervisor_process::process::ResolvedWorkspace::new(workdir.clone(), false), - ); - - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = if let Some(bootstrap) = - sidecar_bootstrap.as_ref() - { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - result.static_credential_bindings, - result.non_secret_environment_keys, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Failed to fetch provider environment; no provider credentials are active: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - }; - - let dynamic_credentials_fallback = dynamic_credentials.clone(); - let provider_credentials = match ProviderCredentialState::from_bound_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) { - Ok(credentials) => credentials, - Err(error) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment bindings; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - ProviderCredentialState::from_environment( - provider_env_revision, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - dynamic_credentials_fallback, - ) - } - }; - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - if credential_gating_unavailable( - &loaded_policy_origin, - provider_credentials.resolver().is_some(), - network_enabled, - ) { - report_credential_gating_unavailable(); - } - - // Canonical-process overrides are deliberately applied only to the main - // child. Keep the provider snapshot pristine because Kubernetes forwards - // it to the process sidecar for later exec/editor/SFTP children. - - // Shared agent-proposals feature flag. Seed from the same initial settings - // snapshot that produced the policy so networking and process setup agree - // before the poll loop starts reconciling later changes. - let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); - - let process_control_writer = process_control_connection - .as_ref() - .map(|connection| connection.writer.clone()); - let process_exit_ack = Arc::new(tokio::sync::Mutex::new(None)); - let initial_provider_env_generation = sidecar_bootstrap - .as_ref() - .map_or(0, |bootstrap| bootstrap.provider_env_generation); - let mut process_control_closed = None; - if let Some(connection) = process_control_connection { - process_control_closed = Some(connection.closed); - spawn_sidecar_control_update_watcher( - connection.updates, - provider_credentials.clone(), - agent_proposals.clone(), - Arc::clone(&process_exit_ack), - initial_provider_env_generation, - ); - } - - // Shared PID: set after process spawn so the proxy can look up - // the entrypoint process's /proc/net/tcp for identity binding. - let entrypoint_pid = Arc::new(AtomicU32::new(0)); - - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - #[cfg(target_os = "linux")] - let transparent_tcp_requested = opa_engine - .as_ref() - .map(|engine| engine.policy_dns_eligibility_snapshot()) - .transpose()? - .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); - #[cfg(target_os = "linux")] - let runtime_capabilities = - std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); - #[cfg(target_os = "linux")] - let transparent_tcp_capable = has_network_runtime_capability( - runtime_capabilities.as_deref(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, - ); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_capable = false; - #[cfg(target_os = "linux")] - let transparent_runtime = if transparent_tcp_requested { - if !transparent_tcp_capable { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_runtime") - .message( - "Policy DNS and transparent TCP unavailable: runtime capability is missing" - ) - .build() - ); - return Err(miette::miette!( - "policy contains protocol: tcp endpoints, but the selected runtime does not advertise policy DNS and transparent TCP support" - )); - } - if sidecar_network_enforcement { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_topology") - .message("Policy DNS and transparent TCP unavailable: sidecar topology is unsupported") - .build() - ); - return Err(miette::miette!( - "policy DNS and transparent TCP are not yet supported by the sidecar topology" - )); - } - let namespace = netns.as_ref().ok_or_else(|| { - miette::miette!("policy DNS and transparent TCP require a workload network namespace") - })?; - let listeners = namespace - .bind_transparent_tcp_listeners() - .await - .into_diagnostic() - .wrap_err("failed to bind transparent TCP listeners")?; - let (dns_udp, dns_tcp) = namespace - .bind_policy_dns_sockets() - .await - .into_diagnostic() - .wrap_err("failed to bind policy DNS listeners")?; - let proxy_port = policy - .network - .proxy - .as_ref() - .and_then(|proxy| proxy.http_addr) - .map_or(3128, |address| address.port()); - let runtime = openshell_supervisor_network::run::TransparentRuntimeSetup::new( - listeners, - dns_udp, - dns_tcp, - sandbox_id.as_deref(), - )?; - let (ipv4_cidr, ipv6_cidr) = runtime.synthetic_cidrs(); - namespace.install_transparent_tcp_rules(proxy_port, &ipv4_cidr, &ipv6_cidr)?; - Some(runtime) - } else { - None - }; - #[cfg(target_os = "linux")] - let transparent_tcp_substrate_ready = transparent_runtime.is_some(); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_substrate_ready = false; - // The denial channel is owned by the orchestrator: the proxy (in the - // networking leaf) and the bypass monitor (in the process leaf) both - // produce DenialEvents that the denial aggregator (orchestrator-side) - // consumes via the matching receiver. Both leaves are pure producers; - // the orchestrator owns the consumer task spawned below. - let (denial_tx, denial_rx, bypass_denial_tx): ( - Option>, - _, - Option>, - ) = if sandbox_id.is_some() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_denial_tx); - - // Anonymous activity channel: same orchestrator-owned pattern as the - // denial channel. The proxy and the bypass monitor both emit per-event - // activity records; the orchestrator-side aggregator drains, sanitizes, - // and flushes anonymous summaries to the gateway. - let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { - let (tx, rx) = - tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_activity_tx); - - // Workspace watch: the policy poll loop learns the workspace from - // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local - // API read the current value so proposals target the correct workspace. - let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - - let mut networking = if network_enabled { - #[cfg(target_os = "linux")] - let proxy_bind_ip = netns - .as_ref() - .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); - #[cfg(not(target_os = "linux"))] - let proxy_bind_ip: Option = None; - - Some( - openshell_supervisor_network::run::run_networking( - &policy, - proxy_bind_ip, - opa_engine.as_ref(), - retained_proto.as_ref(), - entrypoint_pid.clone(), - process_enabled, - &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), - inference_routes.as_deref(), - denial_tx, - activity_tx, - agent_proposals.clone(), - workspace_rx.clone(), - &upstream_proxy_args, - None, - #[cfg(target_os = "linux")] - transparent_runtime, - ) - .await?, - ) - } else { - None - }; - - #[cfg(target_os = "linux")] - let sidecar_control_server = if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let proto = retained_proto.as_ref().ok_or_else(|| { - miette::miette!( - "sidecar topology requires gateway policy data for the process supervisor" - ) - })?; - let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); - Some(sidecar_control::spawn_server( - &socket, - sidecar_control::BootstrapData { - policy_proto: proto.clone(), - provider_env_revision: provider_credentials.snapshot().revision, - provider_env_generation: 0, - provider_child_env: provider_env.clone(), - agent_proposals_enabled: agent_proposals.enabled(), - proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), - proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), - }, - sidecar_expected_peer()?, - )?) - } else { - None - }; - #[cfg(not(target_os = "linux"))] - let sidecar_control_server: Option = None; - - let sidecar_control_publisher = sidecar_control_server - .as_ref() - .map(sidecar_control::ServerHandle::publisher); - - #[cfg(target_os = "linux")] - let mut sidecar_control_task = None; - - #[cfg(target_os = "linux")] - if network_enabled - && sidecar_network_enforcement - && let Some(server) = sidecar_control_server - { - let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar network topology", - openshell_core::sandbox_env::SSH_SOCKET_PATH - ) - })?; - let (entrypoint_rx, connection_task) = server.into_runtime_parts(); - sidecar_control_task = Some(connection_task); - spawn_sidecar_entrypoint_handler( - entrypoint_rx, - SidecarEntrypointHandler { - entrypoint_pid: entrypoint_pid.clone(), - opa_engine: opa_engine.clone(), - retained_proto: retained_proto.clone(), - openshell_endpoint: openshell_endpoint.clone(), - sandbox_id: sandbox_id.clone(), - trusted_ssh_socket_path: std::path::PathBuf::from(trusted_ssh_socket_path), - control_publisher: sidecar_control_publisher.clone(), - }, - ); - } - - #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { - return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" - )); - } - - // Spawn the denial-aggregator flush task. The aggregator drains denial - // events from the proxy + bypass monitor, batches them, and ships - // summaries to the gateway via `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { - // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall - // back to the ID when the name isn't set. - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - - let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); - let denial_workspace_gate = workspace_rx.clone(); - let denial_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - |summaries| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = denial_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_proposals_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summaries, - ) - .await - { - warn!(error = %e, "Failed to flush denial summaries to gateway"); - } - } - }, - move || !denial_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn the activity-aggregator flush task. The aggregator drains - // anonymous activity events from the proxy, sanitizes deny groups, - // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( - std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") - .ok() - .as_deref(), - ); - - let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); - let activity_workspace_gate = workspace_rx.clone(); - let activity_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - move |summary| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = activity_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_activity_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summary, - ) - .await - { - warn!(error = %e, "Failed to flush activity summary to gateway"); - } - } - }, - move || !activity_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_control - && let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) - { - let poll_id = id.to_string(); - let poll_endpoint = endpoint.to_string(); - let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_ocsf_schema_version = ocsf_schema_version.clone(); - let poll_pid = entrypoint_pid.clone(); - let poll_provider_credentials = provider_credentials.clone(); - let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, - sandbox_id: poll_id, - opa_engine: poll_engine, - loaded_policy_origin, - entrypoint_pid: poll_pid, - interval_secs: poll_interval_secs, - ocsf_enabled: poll_ocsf_enabled, - ocsf_schema_version: poll_ocsf_schema_version, - provider_credentials: poll_provider_credentials, - policy_local_ctx: poll_policy_local, - agent_proposals: agent_proposals.clone(), - middleware_registry_status, - sidecar_control_publisher: sidecar_control_publisher.clone(), - workspace_tx, - extension_credentials: extension_credentials.clone(), - extension_authentication_enabled: initial_extension_authentication_enabled, - middleware_connector: default_middleware_connector(), - transparent_tcp: TransparentTcpReloadState { - capable: transparent_tcp_capable, - substrate_ready: transparent_tcp_substrate_ready, - }, - }; - - tokio::spawn(async move { - if let Err(e) = run_policy_poll_loop(poll_ctx).await { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .message(format!("Policy poll loop exited with error: {e}")) - .build() - ); - } - }); - } - - // Start GCE metadata loopback server inside the network namespace so - // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) - // can reach it via direct TCP. Must start before the process leaf so SSH - // sessions also see corrected env vars on bind failure. - #[cfg(target_os = "linux")] - if let Some(ns) = netns.as_ref() - && provider_credentials - .snapshot() - .child_env - .contains_key("GCE_METADATA_HOST") - { - let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - match ns - .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) - .await - { - Ok(listener) => { - tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); - if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { - info!(addr = %addr, "GCE metadata loopback server ready"); - } else { - warn!("GCE metadata server failed to become ready, removing metadata env vars"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - Err(e) => { - warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - } - - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let main_env = provider_env.clone(); - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let proxy_exited: Pin + Send>> = if let Some(rx) = networking - .as_mut() - .and_then(|n| n.proxy.as_mut()) - .and_then(ProxyHandle::take_exit_receiver) - { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(proxy_exited); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(ssh_exited); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok((pid, instance_id)) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid, instance_id) - .await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - let sidecar_exit_tx = if process_uses_sidecar_control - && let Some(writer) = process_control_writer.clone() - { - let exit_ack = Arc::clone(&process_exit_ack); - let (tx, mut rx) = tokio::sync::mpsc::channel::< - openshell_supervisor_process::run::SidecarExitReport, - >(1); - tokio::spawn(async move { - while let Some(report) = rx.recv().await { - match report { - openshell_supervisor_process::run::SidecarExitReport::Exited { - instance_id, - exit_code, - ack, - } => { - let (durable_tx, durable_rx) = tokio::sync::oneshot::channel(); - *exit_ack.lock().await = Some((instance_id.clone(), durable_tx)); - let result = match sidecar_control::send_main_process_exited( - &writer, - instance_id, - exit_code, - ) - .await - { - Ok(()) => durable_rx.await.map_err(|_| { - "sidecar durable exit acknowledgement closed".to_string() - }), - Err(error) => Err(error.to_string()), - }; - let _ = ack.send(result); - } - openshell_supervisor_process::run::SidecarExitReport::Finalized { - instance_id, - ack, - } => { - let result = - sidecar_control::send_main_process_finalized(&writer, instance_id) - .await - .map_err(|error| error.to_string()); - let _ = ack.send(result); - } - } - } - }); - Some(tx) - } else { - None - }; - - let process = openshell_supervisor_process::run::run_process( - program, - args, - workspace, - timeout_secs, - interactive, - await_main_process_attachment, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - ssh_exit_tx, - &process_policy, - resolved_process_identity, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - sidecar_exit_tx, - provider_credentials, - main_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - result = process => result?, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If the - // sole authenticated process-supervisor control connection closes, - // exit non-zero so Kubernetes restarts the network sidecar and creates - // a fresh one-client bootstrap listener for the restarted agent. - #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - result = control_task => { - warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); - 1 - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } - #[cfg(not(target_os = "linux"))] - { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } - } - }; - - // Drop networking explicitly so the proxy + bypass monitor RAII - // handles tear down before we return. - drop(networking); - - Ok(exit_code) -} - -/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is -/// no entrypoint child whose lifetime drives the supervisor's exit. -async fn wait_for_shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - "Failed to install SIGTERM handler; waiting on SIGINT only" - ); - let _ = tokio::signal::ctrl_c().await; - return; - } - }; - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT, shutting down network-only supervisor"); - } - _ = sigterm.recv() => { - info!("Received SIGTERM, shutting down network-only supervisor"); - } - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - info!("Received Ctrl-C, shutting down network-only supervisor"); - } -} - -fn sidecar_network_enforcement_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) - .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) -} - -fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .ok() - .as_deref() - { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, - _ => ProcessEnforcementMode::Full, - } -} - -fn sidecar_control_socket() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) - .ok() - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn sidecar_expected_peer() -> Result { - fn required_numeric_env(name: &str) -> Result { - let value = std::env::var(name) - .into_diagnostic() - .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; - value.parse::().into_diagnostic().wrap_err_with(|| { - format!("{name} must be a numeric ID for sidecar control authentication") - }) - } - - Ok(sidecar_control::ExpectedPeer { - uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, - gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, - }) -} - -type LoadedPolicyBundle = ( - SandboxPolicy, - Option>, - Option, - LoadedPolicyOrigin, -); - -type MainProcessExitAckWaiter = - Arc)>>>; - -fn load_policy_from_sidecar_bootstrap( - bootstrap: &sidecar_control::BootstrapData, -) -> Result { - let proto = bootstrap.policy_proto.clone(); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); - let policy = SandboxPolicy::try_from(proto.clone())?; - info!("Loaded sidecar policy from control socket bootstrap"); - Ok(( - policy, - opa_engine, - Some(proto), - LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }, - )) -} - -fn spawn_sidecar_control_update_watcher( - mut updates: tokio::sync::mpsc::UnboundedReceiver, - provider_credentials: ProviderCredentialState, - agent_proposals: AgentProposals, - exit_ack: MainProcessExitAckWaiter, - mut provider_env_generation: u64, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - while let Some(update) = updates.recv().await { - match update { - sidecar_control::ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - if generation <= provider_env_generation { - continue; - } - let env_count = provider_credentials - .install_child_env_snapshot(revision, provider_child_env); - provider_env_generation = generation; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("provider_env_revision", serde_json::json!(revision)) - .unmapped("provider_env_generation", serde_json::json!(generation)) - .message(format!( - "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" - )) - .build() - ); - } - sidecar_control::ControlUpdate::Policy { - policy_proto, - policy_hash, - config_revision, - } => { - debug!( - version = policy_proto.version, - policy_hash, - config_revision, - "Received sidecar policy update for process supervisor" - ); - } - sidecar_control::ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - apply_agent_proposals_enabled( - &agent_proposals, - enabled, - "sidecar control", - Some(config_revision), - None, - skills::install_static_skills, - ); - } - sidecar_control::ControlUpdate::MainProcessExitAck { instance_id } => { - let mut waiter = exit_ack.lock().await; - if waiter - .as_ref() - .is_some_and(|(expected, _)| expected == &instance_id) - && let Some((_, ack)) = waiter.take() - { - let _ = ack.send(()); - } - } - } - } - }) -} - +//! Capability-free in-workload sandbox boundary. + +pub mod boundary_exec; +pub mod boundary_io; +mod boundary_server; +pub mod child_env; +pub(crate) mod delegated; +#[cfg(unix)] +pub mod identity; +pub mod main_session; +pub mod managed_children; #[cfg(target_os = "linux")] -struct SidecarEntrypointHandler { - entrypoint_pid: Arc, - opa_engine: Option>, - retained_proto: Option, - openshell_endpoint: Option, - sandbox_id: Option, - trusted_ssh_socket_path: std::path::PathBuf, - control_publisher: Option, -} +mod network_broker; +pub mod process; +mod pty; +pub mod sandbox; +/// Results of actively qualifying the admitted workload runtime before the +/// sandbox consumes protected bootstrap material. #[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - handler: SidecarEntrypointHandler, -) { - tokio::spawn(async move { - let SidecarEntrypointHandler { - entrypoint_pid, - opa_engine, - retained_proto, - openshell_endpoint, - sandbox_id, - trusted_ssh_socket_path, - control_publisher, - } = handler; - let mut session_started = false; - let mut session_task: Option> = None; - let mut trusted_supervisor_pid = None; - let terminating = Arc::new(AtomicBool::new(false)); - while let Some(started) = entrypoint_rx.recv().await { - if started.finalized { - if let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let mut delay = Duration::from_millis(250); - loop { - match openshell_supervisor_process::supervisor_session::finalize_main_process_exit( - endpoint, - id, - &started.instance_id, - ) - .await - { - Ok(()) => break, - Err(error) => { - warn!(%error, "sidecar main-process finalization failed; retrying"); - tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); - } - } - } - } - terminating.store(true, Ordering::Release); - if let Some(task) = session_task.take() { - task.abort(); - } - break; - } - if let Some(exit_code) = started.exit_code { - if let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let mut delay = Duration::from_millis(250); - loop { - match openshell_supervisor_process::supervisor_session::report_main_process_exit( - endpoint, - id, - &started.instance_id, - exit_code, - ) - .await - { - Ok(()) => break, - Err(error) => { - warn!(%error, "sidecar main-process exit report failed; retrying"); - tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); - } - } - } - if let Some(publisher) = control_publisher.as_ref() { - publisher.publish_main_process_exit_ack(started.instance_id.clone()); - } - } - continue; - } - entrypoint_pid.store(started.pid, Ordering::Release); - if started.start_session { - info!( - pid = started.pid, - ssh_socket = %trusted_ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - } else { - trusted_supervisor_pid = Some(started.pid); - info!( - pid = started.pid, - "Sidecar process supervisor reported initial process anchor" - ); - } - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( - pid = started.pid, - "Policy binary symlink resolution complete for sidecar process anchor" - ), - Err(err) => warn!( - error = %err, - pid = started.pid, - "Failed to rebuild OPA engine with sidecar process anchor PID" - ), - } - } - - if started.start_session - && !session_started - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let Some(supervisor_pid) = trusted_supervisor_pid else { - warn!( - pid = started.pid, - "Ignoring sidecar entrypoint event before authenticated supervisor anchor" - ); - continue; - }; - session_task = Some(openshell_supervisor_process::supervisor_session::spawn( - endpoint.clone(), - id.clone(), - trusted_ssh_socket_path.clone(), - None, - Some(supervisor_pid), - Arc::clone(&terminating), - started.instance_id.clone(), - )); - session_started = true; - info!("sidecar supervisor session task spawned"); - } - } - terminating.store(true, Ordering::Release); - }); -} - -fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); - let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); - let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); - (cert.exists() && bundle.exists()).then_some((cert, bundle)) -} - -fn process_policy_for_topology( - policy: &SandboxPolicy, - sidecar_network_enforcement: bool, -) -> Result { - let mut process_policy = policy.clone(); - if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { - let proxy = process_policy - .network - .proxy - .get_or_insert(ProxyPolicy { http_addr: None }); - if proxy.http_addr.is_none() { - proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); - } - } - Ok(process_policy) -} - -/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. -async fn flush_proposals_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summaries: Vec, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialSummary, L7RequestSample}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summaries: Vec = summaries - .into_iter() - .map(|s| DenialSummary { - sandbox_id: String::new(), - host: s.host, - port: u32::from(s.port), - binary: s.binary, - ancestors: s.ancestors, - deny_reason: s.deny_reason, - first_seen_ms: s.first_seen_ms, - last_seen_ms: s.last_seen_ms, - count: s.count, - suppressed_count: 0, - total_count: s.count, - sample_cmdlines: s.sample_cmdlines, - binary_sha256: String::new(), - persistent: false, - denial_stage: s.denial_stage, - l7_request_samples: s - .l7_samples - .into_iter() - .map(|l| L7RequestSample { - method: l.method, - path: l.path, - decision: "deny".to_string(), - count: l.count, - }) - .collect(), - l7_inspection_active: false, - }) - .collect(); - - // Run the mechanistic mapper sandbox-side to generate proposals. - // The gateway is a thin persistence + validation layer — it never - // generates proposals itself. - let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); - - info!( - sandbox_name = %sandbox_name, - summaries = proto_summaries.len(), - proposals = proposals.len(), - "Flushed denial analysis to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - proto_summaries, - proposals, - Vec::new(), - "mechanistic", - ) - .await?; - - Ok(()) -} - -/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. -async fn flush_activity_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summary: activity_aggregator::FlushableActivitySummary, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summary = NetworkActivitySummary { - network_activity_count: summary.network_activity_count, - denied_action_count: summary.denied_action_count, - denials_by_group: summary - .denials_by_group - .into_iter() - .map(|(group, count)| DenialGroupCount { - deny_group: group, - denied_count: count, - }) - .collect(), - }; - - info!( - sandbox_name = %sandbox_name, - network_activity_count = proto_summary.network_activity_count, - denied_action_count = proto_summary.denied_action_count, - "Flushed activity summary to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - Vec::new(), - Vec::new(), - vec![proto_summary], - "activity", - ) - .await?; - - Ok(()) -} - -// ============================================================================ -// Baseline filesystem path enrichment -// ============================================================================ - -/// Minimum read-only paths required for a proxy-mode sandbox child process to -/// function: dynamic linker, shared libraries, DNS resolution, CA certs, -/// Python venv, openshell logs, process info, and random bytes. -/// -/// `/proc` and `/dev/urandom` are included here for the same reasons they -/// appear in `restrictive_default_policy()`: virtually every process needs -/// them. Before the Landlock per-path fix (#677) these were effectively free -/// because a missing path silently disabled the entire ruleset; now they must -/// be explicit. -const PROXY_BASELINE_READ_ONLY: &[&str] = &[ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/proc", - "/dev/urandom", -]; - -/// Minimum read-write paths required for a proxy-mode sandbox child process. -/// The active workspace is granted separately through `include_workdir`. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; - -/// GPU read-only paths. -/// -/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced -/// socket at init time. If the directory exists but Landlock denies traversal -/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` -/// even though the daemon is optional. Only read/traversal access is needed. -/// -/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, -/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` -/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may -/// not be covered by the parent-directory Landlock rule when the mount crosses -/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal -/// is permitted regardless of Landlock's cross-mount behaviour. -const GPU_BASELINE_READ_ONLY: &[&str] = &[ - "/run/nvidia-persistenced", - "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory -]; - -/// GPU read-write paths (static). -/// -/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, -/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native -/// Linux. Landlock restricts `open(2)` on device files even when DAC allows -/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. -/// These devices do not exist on WSL2 and will be skipped by the existence -/// check in `enrich_proto_baseline_paths()`. -/// -/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver -/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects -/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux -/// and will be skipped there by the existence check. -/// -/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` -/// to set thread names. Without write access, `cuInit()` returns error 304. -/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to -/// inodes and child processes have different procfs inodes than the parent. -/// -/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by -/// `enumerate_gpu_device_nodes()` since the count varies. -const GPU_BASELINE_READ_WRITE: &[&str] = &[ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) - "/proc", -]; - -/// Returns true if GPU devices are present in the container. -/// -/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and -/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these -/// depending on the host kernel; the other will not exist. -fn has_gpu_devices() -> bool { - std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() -} - -/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). -fn enumerate_gpu_device_nodes() -> Vec { - let mut paths = Vec::new(); - if let Ok(entries) = std::fs::read_dir("/dev") { - for entry in entries.flatten() { - let name = entry.file_name(); - let name = name.to_string_lossy(); - if let Some(suffix) = name.strip_prefix("nvidia") { - if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { - continue; - } - paths.push(entry.path().to_string_lossy().into_owned()); - } - } - } - paths -} - -fn push_unique(paths: &mut Vec, path: String) { - if !paths.iter().any(|p| p == &path) { - paths.push(path); - } -} - -fn collect_baseline_enrichment_paths( - include_proxy: bool, - include_gpu: bool, - gpu_device_nodes: Vec, -) -> (Vec, Vec) { - let mut ro = Vec::new(); - let mut rw = Vec::new(); - - if include_proxy { - for &path in PROXY_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in PROXY_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - } - - if include_gpu { - for &path in GPU_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in GPU_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - for path in gpu_device_nodes { - push_unique(&mut rw, path); - } - } - - // A path promoted to read_write (e.g. /proc for GPU) should not also - // appear in read_only — Landlock handles the overlap correctly but the - // duplicate is confusing when inspecting the effective policy. - ro.retain(|p| !rw.contains(p)); - - (ro, rw) -} - -fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { - let include_gpu = has_gpu_devices(); - let gpu_device_nodes = if include_gpu { - enumerate_gpu_device_nodes() - } else { - Vec::new() - }; - collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) -} - -/// Collect all active baseline paths for tests and diagnostics. -/// Returns `(read_only, read_write)` as owned `String` vecs. -#[cfg(test)] -fn baseline_enrichment_paths() -> (Vec, Vec) { - active_baseline_enrichment_paths(true) -} - -fn enrich_proto_baseline_paths_with( - proto: &mut openshell_core::proto::SandboxPolicy, - ro: &[String], - rw: &[String], - path_exists: F, -) -> bool -where - F: Fn(&str) -> bool, -{ - if ro.is_empty() && rw.is_empty() { - return false; - } - - let fs = proto - .filesystem - .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { - include_workdir: true, - ..Default::default() - }); - - let mut modified = false; - for path in ro { - if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { - if !path_exists(path) { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - fs.read_only.push(path.clone()); - modified = true; - } - } - for path in rw { - if fs.read_write.iter().any(|p| p == path) { - continue; - } - if !path_exists(path) { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - if fs.read_only.iter().any(|p| p == path) { - if path == "/proc" { - info!( - path, - "Promoting /proc from read-only to read-write for GPU runtime compatibility" - ); - fs.read_only.retain(|p| p != path); - fs.read_write.push(path.clone()); - modified = true; - } - continue; - } - fs.read_write.push(path.clone()); - modified = true; - } - - modified -} - -/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths -/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if -/// missing; user-specified paths are never removed. -/// -/// Returns `true` if the policy was modified (caller may want to sync back). -fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); - - // Baseline paths are system-injected, not user-specified. Skip paths - // that do not exist in this container image to avoid noisy warnings from - // Landlock and, more critically, to prevent a single missing baseline - // path from abandoning the entire Landlock ruleset under best-effort - // mode (see issue #664). - let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { - std::path::Path::new(path).exists() - }); - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } - - modified -} - -fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - openshell_policy::strip_provider_rule_names(proto) -} - -fn proto_sync_payload_for_enriched_policy( - proto: &openshell_core::proto::SandboxPolicy, - enriched: bool, -) -> Option { - if !enriched { - return None; - } - - let mut sync_policy = proto.clone(); - strip_proto_provider_policy_entries(&mut sync_policy); - Some(sync_policy) -} - -/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem -/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the -/// local-file code path where no proto is available. -fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { - let (ro, rw) = - active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); - if ro.is_empty() && rw.is_empty() { - return; - } - - let mut modified = false; - for path in &ro { - let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_only.push(p); - modified = true; - } - } - for path in &rw { - let p = std::path::PathBuf::from(path); - if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { - continue; - } - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; - } - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } -} - -#[cfg(test)] +#[derive(Clone, Copy, Debug)] #[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." + clippy::struct_excessive_bools, + reason = "qualification preserves independently exercised security results" )] -mod baseline_tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - use std::path::PathBuf; - - #[test] - fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { - // When GPU devices are present, /proc is promoted to read_write - // (CUDA needs to write /proc//task//comm). It should - // NOT also appear in read_only. - if !has_gpu_devices() { - // Can't test GPU dedup without GPU devices; skip silently. - return; - } - let (ro, rw) = baseline_enrichment_paths(); - assert!( - rw.contains(&"/proc".to_string()), - "/proc should be in read_write when GPU is present" - ); - assert!( - !ro.contains(&"/proc".to_string()), - "/proc should NOT be in read_only when it is already in read_write" - ); - } - - #[test] - fn proc_in_read_only_without_gpu() { - if has_gpu_devices() { - // On a GPU host we can't test the non-GPU path; skip silently. - return; - } - let (ro, _rw) = baseline_enrichment_paths(); - assert!( - ro.contains(&"/proc".to_string()), - "/proc should be in read_only when GPU is not present" - ); - } - - #[test] - fn baseline_read_write_does_not_hardcode_sandbox() { - let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/tmp".to_string())); - assert!(!rw.contains(&"/sandbox".to_string())); - } - - #[test] - fn enumerate_gpu_device_nodes_skips_bare_nvidia() { - // "nvidia" (without a trailing digit) is a valid /dev entry on some - // systems but is not a per-GPU device node. The enumerator must - // not match it. - let nodes = enumerate_gpu_device_nodes(); - assert!( - !nodes.contains(&"/dev/nvidia".to_string()), - "bare /dev/nvidia should not be enumerated: {nodes:?}" - ); - } - - #[test] - fn no_duplicate_paths_in_baseline() { - let (ro, rw) = baseline_enrichment_paths(); - // No path should appear in both lists. - for path in &ro { - assert!( - !rw.contains(path), - "path {path} appears in both read_only and read_write" - ); - } - } - - #[test] - fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { - read_only: vec!["/tmp".to_string()], - read_write: vec![], - include_workdir: false, - }); - policy.network_policies.insert( - "test".into(), - openshell_core::proto::NetworkPolicyRule { - name: "test-rule".into(), - endpoints: vec![openshell_core::proto::NetworkEndpoint { - host: "example.com".into(), - port: 443, - ..Default::default() - }], - ..Default::default() - }, - ); - - enrich_proto_baseline_paths(&mut policy); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - filesystem.read_only.contains(&"/tmp".to_string()), - "explicit read_only baseline path should be preserved" - ); - assert!( - !filesystem.read_write.contains(&"/tmp".to_string()), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - assert!(strip_proto_provider_policy_entries(&mut policy)); - assert!( - !policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(policy.network_policies.contains_key("sandbox_only")); - assert!(!strip_proto_provider_policy_entries(&mut policy)); - } - - #[test] - fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - - assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "provider-derived rules alone must not trigger sync or mutate runtime policy" - ); - } - - #[test] - fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - runtime_policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) - .expect("enrichment should create a sync payload"); - - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "runtime policy must retain provider-derived rules for OPA input" - ); - assert!( - !sync_policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(sync_policy.network_policies.contains_key("sandbox_only")); - } - - #[test] - fn proto_gpu_enrichment_promotes_proc_without_network_policy() { - let mut policy = openshell_policy::restrictive_default_policy(); - assert!( - policy.network_policies.is_empty(), - "regression setup must exercise the no-network default path" - ); - let (ro, rw) = - collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); - - let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { - matches!(path, "/proc" | "/dev/nvidia0") - }); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - enriched, - "GPU enrichment should not require network policies" - ); - assert!( - filesystem.read_write.contains(&"/dev/nvidia0".to_string()), - "GPU enrichment should add enumerated device nodes without network policies" - ); - assert!( - !filesystem.read_only.contains(&"/proc".to_string()), - "GPU enrichment should remove /proc from read_only" - ); - assert!( - filesystem.read_write.contains(&"/proc".to_string()), - "GPU enrichment should promote /proc to read_write" - ); - } - - #[test] - fn gpu_baseline_read_write_contains_dxg() { - // /dev/dxg must be present so WSL2 sandboxes get the Landlock - // read-write rule for the CDI-injected DXG device. The existence - // check in enrich_proto_baseline_paths() skips it on native Linux. - assert!( - GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), - "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" - ); - } - - #[test] - fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![PathBuf::from("/tmp")], - read_write: vec![], - include_workdir: false, - }, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - }; - - enrich_sandbox_baseline_paths(&mut policy); - - assert!( - policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), - "explicit read_only baseline path should be preserved" - ); - assert!( - !policy - .filesystem - .read_write - .contains(&PathBuf::from("/tmp")), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn gpu_baseline_read_only_contains_usr_lib_wsl() { - // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library - // bind-mounts are accessible under Landlock. Skipped on native Linux. - assert!( - GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), - "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" - ); - } - - #[test] - fn has_gpu_devices_reflects_dxg_or_nvidiactl() { - // Verify the OR logic: result must match the manual disjunction of - // the two path checks. Passes in all environments. - let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); - let dxg = std::path::Path::new("/dev/dxg").exists(); - assert_eq!( - has_gpu_devices(), - nvidiactl || dxg, - "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" - ); - } -} - -/// Returns `true` if the error is transient and worth retrying. -/// -/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If -/// found, only the gRPC codes that represent transient failures are retryable. -/// If no `tonic::Status` is present (e.g. a raw connection error), assume the -/// failure is transient. -fn is_retryable_error(err: &miette::Report) -> bool { - let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); - while let Some(e) = source { - if let Some(status) = e.downcast_ref::() { - return matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - | tonic::Code::Internal - | tonic::Code::Unknown - ); - } - source = e.source(); - } - true -} - -/// Retry a gRPC operation with exponential backoff (capped at 4 s). -/// -/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, -/// `PERMISSION_DENIED`) are returned immediately without retrying. -async fn grpc_retry(op_name: &str, f: F) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let mut last_err = None; - for attempt in 1..=5u32 { - match f().await { - Ok(val) => return Ok(val), - Err(e) => { - if !is_retryable_error(&e) { - return Err(e); - } - if attempt < 5 { - warn!( - attempt, - max_attempts = 5, - error = %e, - "{op_name} failed, retrying" - ); - let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); - tokio::time::sleep(backoff).await; - } - last_err = Some(e); - } - } - } - Err(miette::miette!( - "{op_name} failed after 5 attempts: {}", - last_err.expect("loop executed at least once") - )) -} - -/// Load sandbox policy from local files or gRPC. -/// -/// Priority: -/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files -/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC -/// 3. If the server returns no policy, discover from disk or use restrictive default -/// 4. Otherwise, return an error -/// -/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto -/// policy. The proto is retained so the OPA engine can be rebuilt with symlink -/// resolution after the container entrypoint starts. -async fn load_policy( - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - extension_credentials: &openshell_extension_core::ExtensionCredentialStore, -) -> Result<( - SandboxPolicy, - Option>, - Option, - MiddlewareRegistryStatus, - LoadedPolicyOrigin, - bool, - bool, -)> { - // File mode: load OPA engine from rego rules + YAML data (dev override) - if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "loading") - .unmapped("policy_rules", serde_json::json!(policy_file)) - .unmapped("policy_data", serde_json::json!(data_file)) - .message(format!( - "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" - )) - .build()); - let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { - openshell_supervisor_middleware_builtins::validate_config(implementation, config) - .map_err(|error| error.to_string()) - }; - let engine = OpaEngine::from_files_with_middleware_config( - std::path::Path::new(policy_file), - std::path::Path::new(data_file), - Some(&validate_middleware_config), - )?; - let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; - let config = engine.query_sandbox_config()?; - let mut policy = SandboxPolicy { - version: 1, - filesystem: config.filesystem, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: config.landlock, - process: config.process, - }; - enrich_sandbox_baseline_paths(&mut policy); - // File mode has no operator-registered middleware to connect. - return Ok(( - policy, - Some(Arc::new(engine)), - None, - MiddlewareRegistryStatus::Synchronized, - LoadedPolicyOrigin::LocalOverride, - false, - false, - )); - } - - // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - info!( - sandbox_id = %id, - endpoint = %endpoint, - "Fetching sandbox policy via gRPC" - ); - let mut snapshot = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) - }) - .await?; - - let mut proto_policy = if let Some(p) = snapshot.policy.clone() { - p - } else { - // No policy configured on the server. Discover from disk or - // fall back to the restrictive default, then sync to the - // gateway so it becomes the authoritative baseline. - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "discovery") - .message("Server returned no policy; attempting local discovery") - .build() - ); - let mut discovered = discover_policy_from_disk_or_default(); - // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); - strip_proto_provider_policy_entries(&mut discovered); - let sandbox = sandbox.as_deref().ok_or_else(|| { - miette::miette!( - "Cannot sync discovered policy: sandbox not available.\n\ - Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." - ) - })?; - - // Sync and re-fetch over a single connection to avoid extra - // TLS handshakes. - let ws = snapshot.workspace.clone(); - snapshot = grpc_retry("Policy discovery sync", || { - openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox, - &discovered, - &ws, - ) - }) - .await?; - snapshot.policy.clone().ok_or_else(|| { - miette::miette!("Server still returned no policy after sync — this is a bug") - })? - }; - - // True only while `snapshot` describes the exact policy that will be - // constructed below. If enrichment cannot be synced and re-fetched, - // the policy remains enforceable but cannot be acknowledged by - // inferred structural equality. - let mut policy_bound_to_snapshot = true; - - // Ensure baseline filesystem paths are present for proxy-mode - // sandboxes. If the policy was enriched, sync the updated version - // back to the gateway so users can see the effective policy. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); - let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); - if let Some(sync_policy) = sync_policy { - if let Some(sandbox_name) = sandbox.as_deref() { - match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox_name, - &sync_policy, - &snapshot.workspace, - ) - .await - { - Ok(canonical) => { - if let Some(policy) = canonical.policy.clone() { - proto_policy = policy; - snapshot = canonical; - } else { - policy_bound_to_snapshot = false; - warn!( - "Gateway returned no policy after enrichment sync; initial revision will be reconciled" - ); - } - } - Err(e) => { - policy_bound_to_snapshot = false; - warn!( - error = %e, - "Failed to sync enriched policy back to gateway; initial revision will be reconciled" - ); - } - } - } else { - policy_bound_to_snapshot = false; - } - } - - let mut loaded_policy_revision = - policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); - - // Build OPA engine from baked-in rules + typed proto data. - // In cluster mode, proxy networking is always enabled so OPA is - // always required for allow/deny decisions. - // The initial load uses pid=0 (no symlink resolution) because the - // container hasn't started yet. After the entrypoint spawns, the - // engine is rebuilt with the real PID for symlink resolution. - info!("Creating OPA engine from proto policy data"); - let mut has_last_valid_policy = true; - let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => Arc::new(engine), - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - let validation_error = e.to_string(); - let candidate_version = snapshot.version; - let candidate_hash = snapshot.policy_hash.clone(); - // There is no in-memory last-known-good generation during - // startup, so both configured modes necessarily fail closed. - // Load the restrictive default atomically and keep the - // rejected revision unacknowledged for poll reconciliation. - has_last_valid_policy = false; - proto_policy = openshell_policy::restrictive_default_policy(); - let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); - let disposition = apply_policy_validation_failure( - &engine, - snapshot.policy_validation_failure_mode, - has_last_valid_policy, - candidate_version, - &validation_error, - )?; - emit_policy_validation_failure( - &disposition, - candidate_version, - &candidate_hash, - &validation_error, - ); - loaded_policy_revision = None; - engine - } - }; - - // Install the in-process catalog before any external connection can - // fail. A newly started sandbox must always be able to resolve built-in - // bindings, even while operator-run services are unavailable. - install_builtin_middleware_registry(&engine).await?; - - // Connect operator-registered middleware services. A connect/describe - // failure keeps the built-in registry active so each request's - // `on_error` policy governs matched traffic. The policy poll loop - // retries the install without waiting for a config change. - let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = if middleware_services.is_empty() { - MiddlewareRegistryStatus::Synchronized - } else if let Err(error) = grpc_retry("Middleware connect", || { - let middleware_services = middleware_services.clone(); - let extension_credentials = extension_credentials.clone(); - let extension_authentication_enabled = snapshot.extension_authentication_enabled; - async move { - let credentials = if extension_authentication_enabled { - // Share the supervisor's store so the slots installed here - // are the ones the policy poll loop later rotates in place. - openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - endpoint, - extension_credentials, - ) - .await? - .refresh_extension_credentials(&middleware_services) - .await? - } else { - std::collections::HashMap::new() - }; - connect_middleware_registry( - &middleware_services, - &MiddlewareAuthentication { - credentials, - enabled: extension_authentication_enabled, - }, - ) - .await - } - }) - .await - .and_then(|registry| engine.replace_middleware_registry(registry)) - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } else { - MiddlewareRegistryStatus::Synchronized - }; - let opa_engine = Some(engine); - - let policy = match SandboxPolicy::try_from(proto_policy.clone()) { - Ok(policy) => policy, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - return Ok(( - policy, - opa_engine, - Some(proto_policy), - middleware_registry_status, - LoadedPolicyOrigin::Gateway { - revision: loaded_policy_revision, - has_last_valid_policy, - }, - agent_proposals_enabled_from_settings(&snapshot.settings), - snapshot.extension_authentication_enabled, - )); - } - - // No policy source available - Err(miette::miette!( - "Sandbox policy required. Provide one of:\n\ - - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ - - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" - )) -} - -/// Try to discover a sandbox policy from the well-known disk path, falling -/// back to the legacy path, then to the hardcoded restrictive default. -fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { - let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); - if primary.exists() { - return discover_policy_from_path(primary); - } - let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); - if legacy.exists() { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "legacy_path", - serde_json::json!(legacy.display().to_string()) - ) - .unmapped("new_path", serde_json::json!(primary.display().to_string())) - .message(format!( - "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", - legacy.display(), - primary.display() - )) - .build() - ); - return discover_policy_from_path(legacy); - } - discover_policy_from_path(primary) -} - -/// Try to read a sandbox policy YAML from `path`, falling back to the -/// hardcoded restrictive default if the file is missing or invalid. -fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { - use openshell_policy::{ - parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, - }; - - let Ok(yaml) = std::fs::read_to_string(path) else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "default") - .message(format!( - "No policy file on disk, using restrictive default [path:{}]", - path.display() - )) - .build() - ); - return restrictive_default_policy(); - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Loaded sandbox policy from container disk [path:{}]", - path.display() - )) - .build() - ); - match parse_sandbox_policy(&yaml) { - Ok(policy) => { - // Validate the disk-loaded policy for safety. - if let Err(violations) = validate_sandbox_policy(&policy) { - let messages: Vec = violations.iter().map(ToString::to_string).collect(); - ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Medium) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .finding_info( - FindingInfo::new( - "unsafe-disk-policy", - "Unsafe Disk Policy Content", - ) - .with_desc(&format!( - "Disk policy at {} contains unsafe content: {}", - path.display(), - messages.join("; "), - )), - ) - .message(format!( - "Disk policy contains unsafe content, using restrictive default [path:{}]", - path.display() - )) - .build()); - return restrictive_default_policy(); - } - policy - } - Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "fallback") - .message(format!( - "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", - path.display() - )) - .build()); - restrictive_default_policy() - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum MiddlewareRegistryStatus { - Synchronized, - NeedsReconciliation, -} - -#[derive(Debug)] -enum GatewayRuntimeReloadError { - PolicyValidation(miette::Report), - TransparentTcpPrerequisite(miette::Report), - MiddlewareRegistry(miette::Report), -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum GatewayRuntimeFailureClass { - PolicyValidation, - TransparentTcpPrerequisite, - MiddlewareRegistry, -} - -impl GatewayRuntimeReloadError { - fn class(&self) -> GatewayRuntimeFailureClass { - match self { - Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, - Self::TransparentTcpPrerequisite(_) => { - GatewayRuntimeFailureClass::TransparentTcpPrerequisite - } - Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, - } - } -} - -#[derive(Debug, PartialEq, Eq)] -struct FailedRuntimeRevision { - config_revision: u64, - policy_hash: String, - failure_class: GatewayRuntimeFailureClass, -} - -impl FailedRuntimeRevision { - fn new(config_revision: u64, policy_hash: &str, failure: &GatewayRuntimeReloadError) -> Self { - Self { - config_revision, - policy_hash: policy_hash.to_string(), - failure_class: failure.class(), - } - } -} - -struct MiddlewareReloadContext<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: &'a MiddlewareAuthentication, - registry_changed: bool, - connector: &'a MiddlewareConnector, -} - -async fn reload_gateway_policy_runtime( - engine: &OpaEngine, - policy: Option<&openshell_core::proto::SandboxPolicy>, - entrypoint_pid: u32, - middleware: MiddlewareReloadContext<'_>, - transparent_tcp: TransparentTcpReloadState, -) -> std::result::Result<(), GatewayRuntimeReloadError> { - if let Some(policy) = policy - && policy_contains_explicit_tcp(policy) - { - if !transparent_tcp.capable { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" - ), - )); - } - if !transparent_tcp.substrate_ready { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" - ), - )); - } - } - match policy { - Some(policy) if middleware.registry_changed => { - let registry = (middleware.connector)( - middleware.desired_services.to_vec(), - middleware.authentication.clone(), - ) - .await - .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; - engine - .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) - .map_err(GatewayRuntimeReloadError::PolicyValidation) - } - // Policy-only change: the installed registry already matches the - // delivered service set, so swap the engine alone. This must not - // require middleware reachability. - Some(policy) => engine - .reload_from_proto_with_pid(policy, entrypoint_pid) - .map_err(GatewayRuntimeReloadError::PolicyValidation), - None => Err(GatewayRuntimeReloadError::PolicyValidation( - miette::miette!("runtime reload requires a policy payload but none was returned"), - )), - } -} - -fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { - policy.network_policies.values().any(|rule| { - rule.endpoints - .iter() - .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) - }) -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct TransparentTcpReloadState { - capable: bool, - substrate_ready: bool, -} - -/// True when the installed middleware registry no longer matches the desired -/// service set and must be rebuilt (reconnecting every delivered service). -/// -/// A policy-only change never requires a rebuild: middleware configs were -/// validated at gateway admission and the installed registry's manifests -/// already cover the unchanged service set, so requiring the services to be -/// reachable would only let a middleware outage block the policy update. -fn middleware_registry_needs_rebuild( - registry_status: MiddlewareRegistryStatus, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> bool { - registry_status == MiddlewareRegistryStatus::NeedsReconciliation - || current_services != desired_services -} - -fn gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy: bool, - current_policy_hash: &str, - desired_policy_hash: &str, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - registry_status: MiddlewareRegistryStatus, -) -> bool { - reloads_gateway_policy - && (current_policy_hash != desired_policy_hash - || middleware_registry_needs_rebuild( - registry_status, - current_services, - desired_services, - )) +pub struct RuntimeQualification { + pub seccomp: openshell_isolation_interface::contract::SeccompEvidence, + pub landlock_abi: u32, + pub landlock_allow_deny: bool, + pub udp_dns_round_trip: bool, + pub tcp_dns_round_trip: bool, + pub tcp_allow_round_trip: bool, + pub tcp_deny_round_trip: bool, } -/// Identity returned with the exact policy snapshot used to construct OPA. -#[derive(Clone, Debug, PartialEq, Eq)] -struct LoadedPolicyRevision { - version: u32, - policy_hash: String, - config_revision: u64, - policy_source: openshell_core::proto::PolicySource, -} - -/// Identifies where the policy currently loaded into OPA came from. +/// Run the authenticated boundary-local sandbox. /// -/// A missing gateway revision means the policy was loaded from the gateway but -/// could not be bound to an authoritative snapshot (for example, enrichment -/// sync failed). That state must reconcile on the first successful poll. A -/// local-file override is different: gateway policy revisions are observed for -/// settings/provider refreshes but must never replace the explicit local OPA -/// policy. -#[derive(Clone, Debug, PartialEq, Eq)] -enum LoadedPolicyOrigin { - LocalOverride, - Gateway { - revision: Option, - has_last_valid_policy: bool, - }, -} - -impl LoadedPolicyOrigin { - fn allows_gateway_policy_reload(&self) -> bool { - matches!(self, Self::Gateway { .. }) - } - - fn has_last_valid_policy(&self) -> bool { - match self { - Self::LocalOverride => true, - Self::Gateway { - has_last_valid_policy, - .. - } => *has_last_valid_policy, - } - } -} - -impl LoadedPolicyRevision { - fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { - Self { - version: snapshot.version, - policy_hash: snapshot.policy_hash.clone(), - config_revision: snapshot.config_revision, - policy_source: snapshot.policy_source, - } - } -} - -/// A sandbox-scoped policy revision that was constructed successfully at -/// startup and must be acknowledged to the gateway exactly once. -#[derive(Clone, Debug, PartialEq, Eq)] -struct InitialPolicyAck { - version: u32, - policy_hash: String, - config_revision: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PolicyStatusUpdate { - version: u32, - loaded: bool, - error: String, - success_event: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum PolicyStatusSuccessEvent { - InitialAcknowledgement { policy_hash: String }, - UnchangedAcknowledgement { policy_hash: String }, -} - -impl PolicyStatusUpdate { - fn initial_loaded(ack: &InitialPolicyAck) -> Self { - Self { - version: ack.version, - loaded: true, - error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::InitialAcknowledgement { - policy_hash: ack.policy_hash.clone(), - }), - } - } - - fn loaded(version: u32) -> Self { - Self { - version, - loaded: true, - error: String::new(), - success_event: None, - } - } - - fn unchanged_loaded(version: u32, policy_hash: String) -> Self { - Self { - version, - loaded: true, - error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash }), - } - } - - fn failed(version: u32, error: String) -> Self { - Self { - version, - loaded: false, - error, - success_event: None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum InitialPollDisposition { - Acknowledge(InitialPolicyAck), - Reconcile, - TrackOnly, -} - -/// Determine whether the initially loaded policy corresponds to an -/// authoritative sandbox-scoped revision that must be acknowledged. +/// # Errors /// -/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose -/// captured gateway identity matches the current version and hash. Global -/// policies, local-file development policies, version zero, and changed -/// identities yield `None`, so those paths never emit a sandbox-revision -/// acknowledgement. -fn initial_policy_ack_candidate( - loaded: Option<&LoadedPolicyRevision>, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - let loaded = loaded?; - if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox - || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox - { - return None; - } - if loaded.version == 0 || canonical.version == 0 { - return None; - } - if loaded.version != canonical.version - || loaded.policy_hash != canonical.policy_hash - || canonical.config_revision < loaded.config_revision - { - return None; - } - Some(InitialPolicyAck { - version: loaded.version, - policy_hash: loaded.policy_hash.clone(), - config_revision: canonical.config_revision, - }) -} - -fn initial_poll_disposition( - origin: &LoadedPolicyOrigin, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> InitialPollDisposition { - match origin { - LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision, .. } => { - initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( - InitialPollDisposition::Reconcile, - InitialPollDisposition::Acknowledge, - ) - } - } -} - -fn unchanged_policy_revision_candidate( - reloads_gateway_policy: bool, - recovering_rejected_policy: bool, - current_policy_version: u32, - current_policy_hash: &str, - result: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - (reloads_gateway_policy - && !recovering_rejected_policy - && !current_policy_hash.is_empty() - && result.policy_source == openshell_core::proto::PolicySource::Sandbox - && result.version > current_policy_version - && result.policy_hash == current_policy_hash) - .then_some(result.version) -} - -fn unchanged_policy_revision_ready_to_ack( - candidate: Option, - policy_runtime_changed: bool, - policy_runtime_reconciled: bool, -) -> Option { - candidate.filter(|_| !policy_runtime_changed || policy_runtime_reconciled) -} - -/// Whether the credential-provenance gates cannot apply to the loaded policy. -/// -/// The gateway derives `provider_credentialed` and deliberately keeps it out of -/// the policy YAML schema, so a local-file policy never carries it and never -/// will: gateway revisions are observed for settings and providers but must not -/// replace the local OPA policy. Provider credentials still arrive from the -/// gateway on that path, so the raw-tunnel and WebSocket binary-frame refusals -/// have nothing to match on. The request-body backstop is unaffected because it -/// keys off the secret resolver rather than endpoint provenance. -fn credential_gating_unavailable( - origin: &LoadedPolicyOrigin, - has_resolver: bool, - network_enabled: bool, -) -> bool { - network_enabled && has_resolver && matches!(origin, LoadedPolicyOrigin::LocalOverride) -} - -/// Report that credential provenance is unavailable for the loaded policy. -/// -/// Carries no credential name, host, or value: the finding states which -/// controls are inactive, nothing about what they would have protected. -fn report_credential_gating_unavailable() { - ocsf_emit!( - DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::High) - .confidence(ConfidenceId::High) - .is_alert(true) - .finding_info( - FindingInfo::new( - "credential-gating-unavailable", - "Credential Provenance Unavailable", - ) - .with_desc( - "Provider credentials are injected, but the loaded policy comes from local \ - files and carries no gateway-derived credential provenance. Uninspected \ - credentialed tunnels and WebSocket binary frames are not refused. Load \ - policy from the gateway to enable these controls." - ), - ) - .evidence_pairs(&[ - ("policy_source", "local-override"), - ("uninspected_connect_gate", "inactive"), - ("websocket_binary_gate", "inactive"), - ("request_body_backstop", "active"), - ]) - .remediation( - "Remove the local policy override so the gateway-delivered effective policy \ - applies, or detach provider credentials from this sandbox." - ) - .message( - "Credential provenance unavailable for local-file policy; uninspected credential gates inactive" - ) - .build() - ); -} - -/// Deliver policy status updates independently from policy reconciliation. -/// -/// The channel is FIFO, so a delayed older status can never arrive after a -/// newer status and move the gateway's active version backward. Delivery uses -/// the existing bounded retry, but failures never delay policy enforcement. -#[tonic::async_trait] -trait PolicyGatewayClient: Clone + Send + Sync + 'static { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result; - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()>; - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - Ok(()) - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - Ok(std::collections::HashMap::new()) - } - - fn workspace(&self) -> String; -} - -#[tonic::async_trait] -impl PolicyGatewayClient for openshell_core::grpc_client::CachedOpenShellClient { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - self.refresh_installed_extension_credentials().await - } - - async fn extension_credentials_for( - &self, - services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - self.extension_credentials_for(services).await - } - - fn workspace(&self) -> String { - self.workspace() - } -} - -async fn run_policy_status_reporter( - client: C, - sandbox_id: String, - mut updates: tokio::sync::mpsc::UnboundedReceiver, -) { - 'updates: while let Some(update) = updates.recv().await { - let operation = if matches!( - update.success_event, - Some(PolicyStatusSuccessEvent::InitialAcknowledgement { .. }) - ) { - "Initial policy acknowledgement" - } else { - "Policy status report" - }; - let mut attempt = 1_u32; - loop { - let sandbox_id = sandbox_id.clone(); - let error = update.error.clone(); - let client = client.clone(); - match client - .report_policy_status(&sandbox_id, update.version, update.loaded, &error) - .await - { - Ok(()) => break, - Err(error) if is_retryable_error(&error) => { - let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); - warn!( - %error, - attempt, - version = update.version, - loaded = update.loaded, - retry_in_secs = backoff.as_secs(), - "{operation} failed transiently; retaining ordered update" - ); - tokio::time::sleep(backoff).await; - attempt = attempt.saturating_add(1); - } - Err(error) => { - warn!( - %error, - version = update.version, - loaded = update.loaded, - "Discarding terminal policy status update" - ); - continue 'updates; - } - } - } - - if let Some(event) = update.success_event { - let (policy_hash, message) = match event { - PolicyStatusSuccessEvent::InitialAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - ), - ), - PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged unchanged policy revision as loaded [version:{}]", - update.version - ), - ), - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("version", serde_json::json!(update.version)) - .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(message) - .build() - ); - } - } -} - -fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { - let version = update.version; - if let Err(error) = sender.send(update) { - warn!( - %error, - version, - "Policy status reporter unavailable during shutdown" - ); - } -} - -/// Best-effort `FAILED` acknowledgement when initial policy construction or -/// conversion fails. -/// -/// Uses the revision identity captured with the policy that failed to build, -/// and preserves the original construction error as the reported message. A -/// delivery failure here is swallowed so it can never mask that error. -async fn report_initial_policy_failure( - endpoint: &str, - sandbox_id: &str, - revision: Option<&LoadedPolicyRevision>, - error: &miette::Report, -) { - let Some(revision) = revision.filter(|revision| { - revision.version > 0 - && revision.policy_source == openshell_core::proto::PolicySource::Sandbox - }) else { - return; - }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { - Ok(client) => client, - Err(e) => { - warn!(error = %e, "Failed to connect to report initial policy failure"); - return; - } - }; - let message = error.to_string(); - if let Err(e) = grpc_retry("Initial policy failure report", || { - let client = client.clone(); - let message = message.clone(); - async move { - client - .report_policy_status(sandbox_id, revision.version, false, &message) - .await - } - }) - .await - { - warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); - } -} - -/// Background loop that polls the server for policy updates. -/// -/// When a new version is detected, attempts to reload the OPA engine via -/// `reload_from_proto_with_pid()`. Reports load success/failure back to the -/// server. On failure, the previous engine is untouched (LKG behavior). -/// -/// When the entrypoint PID is available, policy reloads include symlink -/// resolution for binary paths via the container filesystem. -struct PolicyPollLoopContext { - endpoint: String, - sandbox_id: String, - opa_engine: Arc, - /// Source of the policy currently loaded into OPA. This distinguishes an - /// explicit local-file override from an unbound gateway revision so the - /// former is never replaced by policy polling. - loaded_policy_origin: LoadedPolicyOrigin, - entrypoint_pid: Arc, - interval_secs: u64, - ocsf_enabled: Arc, - ocsf_schema_version: Arc>, - provider_credentials: ProviderCredentialState, - policy_local_ctx: Option>, - agent_proposals: AgentProposals, - middleware_registry_status: MiddlewareRegistryStatus, - sidecar_control_publisher: Option, - workspace_tx: tokio::sync::watch::Sender, - extension_credentials: openshell_extension_core::ExtensionCredentialStore, - extension_authentication_enabled: bool, - middleware_connector: MiddlewareConnector, - /// Immutable driver capability and startup substrate state. - transparent_tcp: TransparentTcpReloadState, -} - -type MiddlewareConnector = Arc< - dyn Fn( - Vec, - MiddlewareAuthentication, - ) -> Pin< - Box< - dyn std::future::Future< - Output = Result, - > + Send, - >, - > + Send - + Sync, ->; - -#[derive(Clone, Default)] -struct MiddlewareAuthentication { - credentials: std::collections::HashMap, - enabled: bool, -} - -fn default_middleware_connector() -> MiddlewareConnector { - Arc::new(|services, authentication| { - Box::pin(async move { connect_middleware_registry(&services, &authentication).await }) - }) -} - -async fn connect_middleware_registry( - services: &[openshell_core::proto::SupervisorMiddlewareService], - authentication: &MiddlewareAuthentication, -) -> Result { - if authentication.enabled { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services_authenticated( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - &authentication.credentials, - ) - .await - } else { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await - } -} - -async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - opa_engine.replace_middleware_registry(registry) -} - -/// Wait the configured poll interval, but never past the point at which an -/// installed extension credential must be rotated. -fn next_poll_delay( - store: &openshell_extension_core::ExtensionCredentialStore, - interval: Duration, -) -> Duration { - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) - }); - store.next_refresh_delay(interval, now_ms) -} - -/// Drop credentials for services no longer in the installed registry. -/// -/// Call only after a registry swap succeeds, so a failed candidate cannot -/// invalidate the last-known-good clients. -fn retain_extension_credentials( - store: &openshell_extension_core::ExtensionCredentialStore, - installed: &[openshell_core::proto::SupervisorMiddlewareService], - extension_authentication_enabled: bool, -) { - let retained = if extension_authentication_enabled { - installed - .iter() - .map(|service| service.name.as_str()) - .collect() - } else { - std::collections::HashSet::default() - }; - store.retain(&retained); -} - -struct MiddlewareRegistryReconciliation<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: MiddlewareAuthentication, - registry_changed: bool, - extension_credentials: &'a openshell_extension_core::ExtensionCredentialStore, - current_services: &'a mut Vec, - status: &'a mut MiddlewareRegistryStatus, -} - -async fn reconcile_middleware_registry( - opa_engine: &OpaEngine, - middleware_connector: &MiddlewareConnector, - reconciliation: MiddlewareRegistryReconciliation<'_>, -) { - if !reconciliation.registry_changed { - return; - } - - match middleware_connector( - reconciliation.desired_services.to_vec(), - reconciliation.authentication.clone(), - ) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - retain_extension_credentials( - reconciliation.extension_credentials, - reconciliation.desired_services, - reconciliation.authentication.enabled, - ); - reconciliation.current_services.clear(); - reconciliation - .current_services - .extend_from_slice(reconciliation.desired_services); - *reconciliation.status = MiddlewareRegistryStatus::Synchronized; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(reconciliation.current_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - reconciliation.current_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during an outage. - if *reconciliation.status == MiddlewareRegistryStatus::Synchronized { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - *reconciliation.status = MiddlewareRegistryStatus::NeedsReconciliation; - } - } -} - -#[derive(Debug, PartialEq, Eq)] -struct PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode, - mode: PolicyValidationFailureMode, - previous_policy_active: bool, - active_generation: u64, -} - -struct RejectedPolicyGeneration { - version: u32, - policy_hash: String, - validation_error: String, - configured_mode: PolicyValidationFailureMode, -} - -enum GatewayRuntimeFailureDisposition { - PolicyRejected { - error: String, - disposition: PolicyValidationFailureDisposition, - }, - MiddlewareUnavailable { - error: String, - }, - TransparentTcpExpansionRejected { - error: String, - active_generation: u64, - }, -} - -fn apply_gateway_runtime_reload_failure( - engine: &OpaEngine, - failure: GatewayRuntimeReloadError, - configured_mode: PolicyValidationFailureMode, - has_last_valid_policy: bool, - version: u32, -) -> Result { - match failure { - GatewayRuntimeReloadError::PolicyValidation(error) => { - let error = error.to_string(); - let disposition = apply_policy_validation_failure( - engine, - configured_mode, - has_last_valid_policy, - version, - &error, - )?; - Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) - } - GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error: error.to_string(), - active_generation: engine.current_generation(), - }, - ), - GatewayRuntimeReloadError::MiddlewareRegistry(error) => { - Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { - error: error.to_string(), - }) - } - } -} - -fn emit_transparent_tcp_expansion_rejection( - version: u32, - policy_hash: &str, - active_generation: u64, - error: &str, -) { - let message = format!( - "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Enabled, "retained_previous_policy") - .unmapped("candidate_version", serde_json::json!(version)) - .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) - .unmapped("previous_policy_active", serde_json::json!(true)) - .unmapped("active_generation", serde_json::json!(active_generation)) - .unmapped("validation_error", serde_json::json!(error)) - .message(message) - .build() - ); -} - -fn apply_policy_validation_failure( - engine: &OpaEngine, - configured_mode: PolicyValidationFailureMode, - has_last_valid_policy: bool, - version: u32, - error: &str, -) -> Result { - let mode = if has_last_valid_policy { - configured_mode - } else { - PolicyValidationFailureMode::FailClosed - }; - match mode { - PolicyValidationFailureMode::FailClosed => { - let reason = format!( - "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" - ); - let active_generation = engine.enter_fail_closed(reason)?; - Ok(PolicyValidationFailureDisposition { - configured_mode, - mode, - previous_policy_active: false, - active_generation, - }) - } - PolicyValidationFailureMode::RetainLastValid => { - let active_generation = engine.exit_fail_closed()?; - Ok(PolicyValidationFailureDisposition { - configured_mode, - mode, - previous_policy_active: true, - active_generation, - }) - } - } -} - -fn policy_validation_failure_events( - disposition: &PolicyValidationFailureDisposition, - version: u32, - policy_hash: &str, - error: &str, -) -> [OcsfEvent; 2] { - let previous_policy_state = if disposition.previous_policy_active { - "IS active" - } else { - "IS NOT active" - }; - let state = if disposition.previous_policy_active { - (StateId::Enabled, "retained_last_valid") - } else { - (StateId::Disabled, "fail_closed") - }; - let message = format!( - "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", - disposition.configured_mode.as_str(), - disposition.mode.as_str(), - disposition.active_generation, - ); - let finding_uid = format!("policy-validation-failed-{version}"); - let version_string = version.to_string(); - let config = ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(state.0, state.1) - .unmapped("candidate_version", serde_json::json!(version)) - .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) - .unmapped( - "validation_failure_mode", - serde_json::json!(disposition.mode.as_str()), - ) - .unmapped( - "configured_validation_failure_mode", - serde_json::json!(disposition.configured_mode.as_str()), - ) - .unmapped( - "previous_policy_active", - serde_json::json!(disposition.previous_policy_active), - ) - .unmapped( - "active_generation", - serde_json::json!(disposition.active_generation), - ) - .unmapped("validation_error", serde_json::json!(error)) - .message(message.clone()) - .build(); - let finding = DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .is_alert(true) - .finding_info( - FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), - ) - .evidence_pairs(&[ - ("candidate_version", &version_string), - ("candidate_policy_hash", policy_hash), - ("validation_failure_mode", disposition.mode.as_str()), - ( - "configured_validation_failure_mode", - disposition.configured_mode.as_str(), - ), - ( - "previous_policy_active", - if disposition.previous_policy_active { - "true" - } else { - "false" - }, - ), - ]) - .remediation("Submit a valid, unambiguous policy generation") - .message(message) - .build(); - [config, finding] -} - -fn emit_policy_validation_failure( - disposition: &PolicyValidationFailureDisposition, - version: u32, - policy_hash: &str, - error: &str, -) { - for event in policy_validation_failure_events(disposition, version, policy_hash, error) { - ocsf_emit!(event); - } -} - -async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, - ctx.extension_credentials.clone(), - ) - .await?; - run_policy_poll_loop_with_client(ctx, client).await -} - -async fn run_policy_poll_loop_with_client( - ctx: PolicyPollLoopContext, - client: C, -) -> Result<()> { - use openshell_core::proto::PolicySource; - use std::sync::atomic::Ordering; - - let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn(run_policy_status_reporter( - client.clone(), - ctx.sandbox_id.clone(), - status_receiver, - )); - - let mut current_config_revision: u64 = 0; - let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_version: u32 = 0; - let mut current_policy_hash = String::new(); - let mut current_middleware_services = Vec::new(); - let mut current_extension_authentication_enabled = ctx.extension_authentication_enabled; - let mut middleware_registry_status = ctx.middleware_registry_status; - let mut current_settings: std::collections::HashMap< - String, - openshell_core::proto::EffectiveSetting, - > = std::collections::HashMap::new(); - let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option = None; - let mut rejected_policy_generation: Option = None; - let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); - - // A first poll that does not match the policy already loaded into OPA must - // pass through the normal reconciliation path immediately. It must never - // seed the applied-state trackers before OPA actually loads it. - let mut pending_result = None; - - // Initialize revision from the first poll and acknowledge the initial - // policy revision the supervisor actually loaded. A mismatched result is - // reconciled below instead of being recorded as already applied. - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { - InitialPollDisposition::Acknowledge(candidate) => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(candidate.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = candidate.config_revision; - current_policy_version = candidate.version; - current_policy_hash.clone_from(&candidate.policy_hash); - current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; - current_settings = result.settings; - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::initial_loaded(&candidate), - ); - debug!( - config_revision = current_config_revision, - "Settings poll: initial policy matches loaded revision" - ); - } - InitialPollDisposition::Reconcile => pending_result = Some(result), - InitialPollDisposition::TrackOnly => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash.clone(); - current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; - current_settings = result.settings; - debug!( - config_revision = current_config_revision, - "Settings poll: tracking gateway config while preserving local policy override" - ); - } - } - } - Err(e) => { - warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); - } - } - - let interval = Duration::from_secs(ctx.interval_secs); - loop { - let result = if let Some(result) = pending_result.take() { - result - } else { - tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - result - } - Err(e) => { - debug!(error = %e, "Settings poll: server unreachable, will retry"); - if current_extension_authentication_enabled - && let Err(refresh_error) = - client.refresh_installed_extension_credentials().await - { - warn!( - error = %refresh_error, - "Settings poll: extension credential refresh failed while configuration was unavailable" - ); - } - continue; - } - } - }; - - // Reuse installed per-service credentials, rotating only when one is - // missing or due. Rotation happens on the existing gateway channel and - // updates slots in place, so it is independent of config revision and - // registry equality. - let middleware_credentials = if result.extension_authentication_enabled { - match client - .extension_credentials_for(&result.supervisor_middleware_services) - .await - { - Ok(credentials) => credentials, - Err(error) => { - warn!(error = %error, "Settings poll: extension credential refresh failed"); - std::collections::HashMap::new() - } - } - } else { - std::collections::HashMap::new() - }; - - let config_changed = result.config_revision != current_config_revision; - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - let policy_changed = result.policy_hash != current_policy_hash; - let extension_authentication_changed = - current_extension_authentication_enabled != result.extension_authentication_enabled; - let middleware_registry_changed = extension_authentication_changed - || middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); - // A valid candidate may intentionally restore byte-for-byte policy - // content that was active before a rejected update. Its hash then - // equals `current_policy_hash`, but the runtime is still quarantined - // and must reload (or it would remain deny-all indefinitely). - let recovering_rejected_policy = reloads_gateway_policy - && rejected_policy_generation - .as_ref() - .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); - let policy_runtime_changed = recovering_rejected_policy - || extension_authentication_changed - || gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); - // Recovery already has its own acknowledgement path below. Giving it - // precedence here prevents a restored last-known-good policy from - // also being acknowledged as an ordinary same-hash revision. - let unchanged_policy_revision = unchanged_policy_revision_candidate( - reloads_gateway_policy, - recovering_rejected_policy, - current_policy_version, - ¤t_policy_hash, - &result, - ); - let mut policy_runtime_reconciled = false; - - // A local policy override is not coupled to the gateway policy - // snapshot, so its service registry can still be reconciled alone. - // Gateway policy snapshots, however, must install policy and registry - // as one generation below. - if !reloads_gateway_policy { - reconcile_middleware_registry( - &ctx.opa_engine, - &ctx.middleware_connector, - MiddlewareRegistryReconciliation { - desired_services: &result.supervisor_middleware_services, - authentication: MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - extension_credentials: &ctx.extension_credentials, - current_services: &mut current_middleware_services, - status: &mut middleware_registry_status, - }, - ) - .await; - if middleware_registry_status == MiddlewareRegistryStatus::Synchronized { - current_extension_authentication_enabled = result.extension_authentication_enabled; - } - } - - if !config_changed - && !provider_env_changed - && !policy_runtime_changed - && unchanged_policy_revision.is_none() - { - continue; - } - - if config_changed || provider_env_changed { - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); - - // A posture change after a rejected update takes effect immediately. - // The compiled last-known-good engine remains available beneath a - // fail-closed quarantine, so an explicit retain_last_valid selection - // can reactivate it without accepting any part of the invalid policy. - if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { - let mode = result.policy_validation_failure_mode; - if mode != rejected.configured_mode { - let disposition = apply_policy_validation_failure( - &ctx.opa_engine, - mode, - has_last_valid_policy, - rejected.version, - &rejected.validation_error, - )?; - emit_policy_validation_failure( - &disposition, - rejected.version, - &rejected.policy_hash, - &rejected.validation_error, - ); - rejected.configured_mode = mode; - } - } - - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); - } - - if provider_env_changed { - match openshell_core::grpc_client::fetch_provider_environment( - &ctx.endpoint, - &ctx.sandbox_id, - ) - .await - { - Ok(env_result) => { - let provider_env_revision = env_result.provider_env_revision; - let install_result = ctx.provider_credentials.install_bound_environment( - provider_env_revision, - env_result.environment, - env_result.credential_expires_at_ms, - env_result.dynamic_credentials, - env_result.static_credential_bindings, - env_result.non_secret_environment_keys, - ); - if let Err(error) = install_result { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment refresh; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - } else { - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher - .publish_provider_env(provider_env_revision, child_env.clone()); - } - current_provider_env_revision = provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{provider_env_revision} env_count:{env_count}]" - )) - .build() - ); - } - } - Err(e) => { - ctx.provider_credentials - .revoke_static_provider_environment(result.provider_env_revision); - warn!( - error = %e, - provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment; static provider credentials were revoked; previous dynamic token grants remain active" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message( - "Provider environment refresh failed; static provider credentials were revoked; previous dynamic token grants remain active" - ) - .build() - ); - } - } - } - - if policy_runtime_changed { - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = reload_gateway_policy_runtime( - &ctx.opa_engine, - result.policy.as_ref(), - pid, - MiddlewareReloadContext { - desired_services: &result.supervisor_middleware_services, - authentication: &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - connector: &ctx.middleware_connector, - }, - ctx.transparent_tcp, - ) - .await; - - match runtime_result { - Ok(()) => { - policy_runtime_reconciled = true; - let policy = result - .policy - .as_ref() - .expect("successful runtime reload requires a policy payload"); - has_last_valid_policy = true; - rejected_policy_generation = None; - if policy_changed { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; - } - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_policy( - policy.clone(), - result.policy_hash.clone(), - result.config_revision, - ); - } - if result.global_policy_version > 0 { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) - .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version - )) - .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - current_policy_version = result.version; - } - } else if recovering_rejected_policy - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - current_policy_version = result.version; - } - - if middleware_registry_changed { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(result.supervisor_middleware_services.len()) - ) - .message(format!( - "Supervisor policy runtime reloaded atomically [service_count:{}]", - result.supervisor_middleware_services.len() - )) - .build()); - } - - current_policy_hash.clone_from(&result.policy_hash); - current_middleware_services.clone_from(&result.supervisor_middleware_services); - current_extension_authentication_enabled = - result.extension_authentication_enabled; - retain_extension_credentials( - &ctx.extension_credentials, - &result.supervisor_middleware_services, - result.extension_authentication_enabled, - ); - middleware_registry_status = MiddlewareRegistryStatus::Synchronized; - last_failed_runtime_revision = None; - } - Err(failure) => { - let failed_revision = FailedRuntimeRevision::new( - result.config_revision, - &result.policy_hash, - &failure, - ); - if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - let failure_mode = result.policy_validation_failure_mode; - match apply_gateway_runtime_reload_failure( - &ctx.opa_engine, - failure, - failure_mode, - has_last_valid_policy, - result.version, - )? { - GatewayRuntimeFailureDisposition::PolicyRejected { - error, - disposition, - } => { - emit_policy_validation_failure( - &disposition, - result.version, - &result.policy_hash, - &error, - ); - rejected_policy_generation = Some(RejectedPolicyGeneration { - version: result.version, - policy_hash: result.policy_hash.clone(), - validation_error: error.clone(), - configured_mode: failure_mode, - }); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, error), - ); - } - } - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error } => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(&error)) - .unmapped("previous_policy_active", serde_json::json!(true)) - .message(format!( - "Supervisor middleware registry unavailable, keeping last-known-good policy runtime active [version:{} error:{error}]", - result.version - )) - .build()); - } - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error, - active_generation, - } => { - emit_transparent_tcp_expansion_rejection( - result.version, - &result.policy_hash, - active_generation, - &error, - ); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, error), - ); - } - } - } - } - last_failed_runtime_revision = Some(failed_revision); - // Nothing was installed, so the registry status still - // describes the live registry. The retry is driven by the - // persisting hash/service-set mismatch (or an existing - // NeedsReconciliation), not by degrading the status here. - } - } - } - - if let Some(version) = unchanged_policy_revision_ready_to_ack( - unchanged_policy_revision, - policy_runtime_changed, - policy_runtime_reconciled, - ) { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::unchanged_loaded(version, result.policy_hash.clone()), - ); - current_policy_version = version; - } - - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_ocsf_schema_version_setting(&ctx.ocsf_schema_version, &result.settings); - - // Apply the agent-proposals feature toggle. On a false→true transition - // we lazily install the skill so a sandbox that started with the flag - // off picks up the surface without a recreate. We never uninstall on - // a true→false transition: stale skill content on disk is harmless - // because route_request and agent_next_steps both gate on the live - // shared flag, so the agent that reads the skill will see 404s and an - // empty `next_steps` array regardless. - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - - current_config_revision = result.config_revision; - if !reloads_gateway_policy { - current_policy_hash = result.policy_hash; - } - current_settings = result.settings; - } -} - -fn apply_ocsf_json_setting( - enabled: &AtomicBool, - settings: &std::collections::HashMap, -) { - use std::sync::atomic::Ordering; - - let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); - let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); - if new_ocsf != prev_ocsf { - info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); - } -} - -/// Extract a bool value from an effective setting, if present. -fn extract_bool_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) -} - -fn apply_ocsf_schema_version_setting( - version: &std::sync::Mutex, - settings: &std::collections::HashMap, -) { - let new_version = extract_string_setting(settings, "ocsf_schema_version").unwrap_or_default(); - if let Ok(mut current) = version.lock() - && *current != new_version - { - info!( - ocsf_schema_version = %new_version, - "OCSF schema version target changed" - ); - *current = new_version; - } -} - -fn extract_string_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::StringValue(s) => Some(s.clone()), - _ => None, - }) -} - -fn agent_proposals_enabled_from_settings( - settings: &std::collections::HashMap, -) -> bool { - extract_bool_setting( - settings, - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, - ) - .unwrap_or(false) -} - -fn apply_agent_proposals_enabled( - agent_proposals: &AgentProposals, - enabled: bool, - source: &'static str, - config_revision: Option, - sidecar_control_publisher: Option<&sidecar_control::Publisher>, - install_static_skills: impl FnOnce() -> Result, -) { - let previously_enabled = agent_proposals.swap_enabled(enabled); - if enabled == previously_enabled { - return; - } - - info!( - agent_policy_proposals_enabled = enabled, - source, config_revision, "agent-driven policy proposals toggled" - ); - - if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { - publisher.publish_agent_proposals(enabled, config_revision); - } - - if enabled && !previously_enabled { - match install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill on toggle-on" - ), - Err(error) => warn!( - error = %error, - "Failed to install sandbox agent skill on toggle-on" - ), - } - } -} - -/// Log individual setting changes between two snapshots. -fn log_setting_changes( - old: &std::collections::HashMap, - new: &std::collections::HashMap, -) { - for (key, new_es) in new { - let new_val = format_setting_value(new_es); - match old.get(key) { - Some(old_es) => { - let old_val = format_setting_value(old_es); - if old_val != new_val { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "updated") - .unmapped("key", serde_json::json!(key)) - .unmapped("old", serde_json::json!(old_val.clone())) - .unmapped("new", serde_json::json!(new_val.clone())) - .message(format!( - "Setting changed [key:{key} old:{old_val} new:{new_val}]" - )) - .build() - ); - } - } - None => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enabled") - .unmapped("key", serde_json::json!(key)) - .unmapped("value", serde_json::json!(new_val.clone())) - .message(format!("Setting added [key:{key} value:{new_val}]")) - .build() - ); - } - } - } - for key in old.keys() { - if !new.contains_key(key) { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Disabled, "disabled") - .unmapped("key", serde_json::json!(key)) - .message(format!("Setting removed [key:{key}]")) - .build() - ); - } - } -} - -/// Format an `EffectiveSetting` value for log display. -fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { - use openshell_core::proto::setting_value; - match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { - None => "".to_string(), - Some(setting_value::Value::StringValue(v)) => v.clone(), - Some(setting_value::Value::BoolValue(v)) => v.to_string(), - Some(setting_value::Value::IntValue(v)) => v.to_string(), - Some(setting_value::Value::BytesValue(_)) => "".to_string(), - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod tests { - use super::*; - - #[test] - fn transparent_tcp_capability_requires_exact_driver_marker() { - let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; - assert!(!has_network_runtime_capability(None, required)); - assert!(!has_network_runtime_capability(Some(""), required)); - assert!(!has_network_runtime_capability( - Some("policy-dns-transparent-tcp-extra"), - required - )); - assert!(has_network_runtime_capability( - Some("other, policy-dns-transparent-tcp"), - required - )); - } - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - fn proxy_policy(http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::BoolValue( - value, - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn sidecar_process_policy_sets_loopback_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, true).unwrap(); - - let http_addr = process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .expect("sidecar process policy should set proxy address"); - assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); - assert!( - policy - .network - .proxy - .as_ref() - .expect("original policy should keep proxy config") - .http_addr - .is_none(), - "process policy normalization must not mutate the network policy" - ); - } - - #[test] - fn non_sidecar_process_policy_preserves_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, false).unwrap(); - - assert!( - process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .is_none() - ); - } - - #[tokio::test] - async fn sidecar_control_provider_env_update_orders_by_generation() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - u64::MAX, - std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), - ); - let agent_proposals = AgentProposals::new(true); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials.clone(), - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 10, - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "new".to_string(), - )]), - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 1 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 1); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "duplicate-generation".to_string(), - )]), - }) - .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 1, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - assert_eq!( - provider_credentials - .snapshot() - .child_env - .get("TOKEN") - .map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 12, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "newest".to_string(), - )]), - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: u64::MAX, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: true, - config_revision: 2, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while !agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("newest") - ); - handle.abort(); - } - - #[tokio::test] - async fn sidecar_control_agent_proposals_update_flips_shared_state() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); - let agent_proposals = AgentProposals::new(true); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials, - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 0, - ); - - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 5, - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if !agent_proposals.enabled() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - handle.abort(); - } - - #[test] - fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { - let agent_proposals = AgentProposals::default(); - let installs = AtomicUsize::new(0); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(!agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - } - - #[test] - fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { - let enabled = AtomicBool::new(false); - let mut settings = std::collections::HashMap::new(); - settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(enabled.load(Ordering::Relaxed)); - } - - #[test] - fn apply_ocsf_json_setting_disables_when_setting_is_unset() { - let enabled = AtomicBool::new(true); - let settings = std::collections::HashMap::new(); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(!enabled.load(Ordering::Relaxed)); - } - - #[test] - fn agent_proposals_setting_enables_from_initial_settings_snapshot() { - let mut settings = std::collections::HashMap::new(); - settings.insert( - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), - effective_bool(true), - ); - - assert!(agent_proposals_enabled_from_settings(&settings)); - } - - #[test] - fn agent_proposals_setting_defaults_false_when_unset() { - let settings = std::collections::HashMap::new(); - - assert!(!agent_proposals_enabled_from_settings(&settings)); - } - - // ---- Policy disk discovery tests ---- - - #[test] - fn discover_policy_from_nonexistent_path_returns_restrictive_default() { - let path = std::path::Path::new("/nonexistent/policy.yaml"); - let policy = discover_policy_from_path(path); - // Restrictive default has no network policies. - assert!(policy.network_policies.is_empty()); - // It keeps filesystem restrictions while leaving identity to the - // active compute driver. - assert!(policy.filesystem.is_some()); - assert!(policy.process.is_none()); - } - - #[test] - fn discover_policy_from_valid_yaml_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr - read_write: - - /tmp -network_policies: - test: - name: test - endpoints: - - { host: example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - assert_eq!(policy.network_policies.len(), 1); - assert!(policy.network_policies.contains_key("test")); - let fs = policy.filesystem.unwrap(); - assert!(!fs.include_workdir); - } - - #[test] - fn discover_policy_from_invalid_yaml_returns_restrictive_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default. - assert!(policy.network_policies.is_empty()); - assert!(policy.filesystem.is_some()); - } - - #[test] - fn discover_policy_from_unsafe_yaml_falls_back_to_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -process: - run_as_user: root - run_as_group: root -filesystem_policy: - include_workdir: true - read_only: - - /usr - read_write: - - /tmp -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default because of root user. - assert!(policy.process.is_none()); - } - - #[test] - fn discover_policy_restrictive_default_blocks_network() { - // In cluster mode we keep proxy mode enabled so `inference.local` - // can always be routed through proxy/OPA controls. - let proto = openshell_policy::restrictive_default_policy(); - let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); - assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); - } - - // ---- Initial policy acknowledgement tests ---- - - fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::restrictive_default_policy() - } - - fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::parse_sandbox_policy( - r#" -version: 1 -network_policies: - redis: - name: redis - endpoints: - - host: redis.example.com - port: 6379 - protocol: tcp - binaries: - - path: /usr/bin/redis-cli -"#, - ) - .expect("parse TCP policy") - } - - fn settings_poll_result( - policy: Option, - version: u32, - source: openshell_core::proto::PolicySource, - ) -> openshell_core::grpc_client::SettingsPollResult { - openshell_core::grpc_client::SettingsPollResult { - policy, - version, - policy_hash: format!("hash-v{version}"), - config_revision: u64::from(version) * 100, - policy_source: source, - settings: std::collections::HashMap::new(), - global_policy_version: 0, - provider_env_revision: 0, - supervisor_middleware_services: Vec::new(), - workspace: String::new(), - policy_validation_failure_mode: PolicyValidationFailureMode::default(), - extension_authentication_enabled: false, - } - } - - #[derive(Clone)] - struct ScriptedPolicyGateway { - polls: Arc< - tokio::sync::Mutex< - tokio::sync::mpsc::UnboundedReceiver< - openshell_core::grpc_client::SettingsPollResult, - >, - >, - >, - reports: UnboundedSender<(u32, bool, String)>, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for ScriptedPolicyGateway { - async fn poll_settings( - &self, - _sandbox_id: &str, - ) -> Result { - self.polls - .lock() - .await - .recv() - .await - .ok_or_else(|| miette::miette!("scripted policy poll channel closed")) - } - - async fn report_policy_status( - &self, - _sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.reports - .send((version, loaded, error.to_string())) - .map_err(|_| miette::miette!("scripted policy report channel closed")) - } - - fn workspace(&self) -> String { - "test-workspace".to_string() - } - } - - #[derive(Clone)] - struct CredentialRejectingPolicyGateway { - inner: ScriptedPolicyGateway, - credential_requests: Arc, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for CredentialRejectingPolicyGateway { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.inner.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.inner - .report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> - { - self.credential_requests.fetch_add(1, Ordering::SeqCst); - Err(miette::miette!( - "gateway extension authentication is unavailable" - )) - } - - fn workspace(&self) -> String { - self.inner.workspace() - } - } - - fn scripted_policy_gateway() -> ( - ScriptedPolicyGateway, - UnboundedSender, - tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - let (poll_tx, poll_rx) = tokio::sync::mpsc::unbounded_channel(); - let (report_tx, report_rx) = tokio::sync::mpsc::unbounded_channel(); - ( - ScriptedPolicyGateway { - polls: Arc::new(tokio::sync::Mutex::new(poll_rx)), - reports: report_tx, - }, - poll_tx, - report_rx, - ) - } - - fn policy_poll_test_context( - opa_engine: Arc, - loaded_policy_origin: LoadedPolicyOrigin, - middleware_connector: MiddlewareConnector, - ) -> PolicyPollLoopContext { - let (workspace_tx, _workspace_rx) = tokio::sync::watch::channel(String::new()); - PolicyPollLoopContext { - endpoint: String::new(), - sandbox_id: "sandbox-test".to_string(), - opa_engine, - loaded_policy_origin, - entrypoint_pid: Arc::new(AtomicU32::new(0)), - interval_secs: 0, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - ocsf_schema_version: Arc::new(std::sync::Mutex::new(String::new())), - provider_credentials: ProviderCredentialState::from_child_env_snapshot( - 0, - std::collections::HashMap::new(), - ), - policy_local_ctx: None, - agent_proposals: AgentProposals::default(), - middleware_registry_status: MiddlewareRegistryStatus::Synchronized, - sidecar_control_publisher: None, - workspace_tx, - extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), - extension_authentication_enabled: false, - middleware_connector, - transparent_tcp: TransparentTcpReloadState::default(), - } - } - - async fn expect_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - version: u32, - ) { - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("policy report timed out") - .expect("policy reporter stopped"); - assert_eq!(report, (version, true, String::new())); - } - - async fn expect_no_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - assert!( - timeout(Duration::from_millis(50), reports.recv()) - .await - .is_err(), - "unexpected policy status report" - ); - } - - #[tokio::test] - async fn same_hash_poll_revision_is_acknowledged_once_without_opa_reload() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - expect_policy_report(&mut reports, 2).await; - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - - assert_eq!( - engine.current_generation(), - 0, - "same-hash acknowledgement must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_tcp_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let active_generation = engine.current_generation(); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let mut ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - ctx.transparent_tcp = TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }; - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("TCP rejection report timed out") - .expect("policy reporter stopped"); - - assert_eq!(report.0, 2); - assert!(!report.1); - assert!(report.2.contains("recreate the sandbox"), "{}", report.2); - assert!(report.2.contains("previous policy remains active")); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "scripted-guard".to_string(), - grpc_endpoint: "http://scripted.invalid".to_string(), - ..Default::default() - }]; - - let connector_attempts = Arc::new(AtomicUsize::new(0)); - let (attempt_tx, mut attempt_rx) = tokio::sync::mpsc::unbounded_channel(); - let middleware_connector: MiddlewareConnector = { - let connector_attempts = connector_attempts.clone(); - Arc::new(move |_services, _authentication| { - let attempt = connector_attempts.fetch_add(1, Ordering::SeqCst) + 1; - attempt_tx.send(attempt).unwrap(); - Box::pin(async move { - if attempt == 1 { - Err(miette::miette!("scripted middleware connection failure")) - } else { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - } - }) - }) - }; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - middleware_connector, - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(1) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(engine.current_generation(), 0); - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(2) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(engine.current_generation(), 1); - - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!(connector_attempts.load(Ordering::SeqCst), 2); - handle.abort(); - } - - #[tokio::test] - async fn no_signer_capability_uses_legacy_middleware_connector_without_credentials() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "legacy-guard".to_string(), - grpc_endpoint: "http://legacy.invalid".to_string(), - ..Default::default() - }]; - assert!(!v2.extension_authentication_enabled); - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, false)) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 0); - handle.abort(); - } - - #[tokio::test] - async fn enabled_extension_authentication_keeps_credential_failure_fail_closed() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.extension_authentication_enabled = true; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "authenticated-guard".to_string(), - grpc_endpoint: "https://guard.invalid".to_string(), - ..Default::default() - }]; - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - if authentication.enabled && authentication.credentials.is_empty() { - Err(miette::miette!( - "missing authenticated middleware credential" - )) - } else { - connect_middleware_registry(&[], &authentication).await - } - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, true)) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 1); - handle.abort(); - } - - async fn assert_poll_does_not_use_same_hash_acknowledgement( - initial: openshell_core::grpc_client::SettingsPollResult, - next: openshell_core::grpc_client::SettingsPollResult, - origin: LoadedPolicyOrigin, - initial_report: Option, - ) { - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context(engine.clone(), origin, default_middleware_connector()); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(initial).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - if let Some(version) = initial_report { - expect_policy_report(&mut reports, version).await; - } else { - expect_no_policy_report(&mut reports).await; - } - - polls.send(next).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!( - engine.current_generation(), - 0, - "negative same-hash scope must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_poll_loop_rejects_local_global_empty_equal_and_older_scopes() { - let mut sandbox_v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - sandbox_v1.policy_hash = "same-policy".to_string(); - let loaded_v1 = LoadedPolicyRevision::from_snapshot(&sandbox_v1); - let mut sandbox_v2 = sandbox_v1.clone(); - sandbox_v2.version = 2; - sandbox_v2.config_revision = 200; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v2.clone(), - LoadedPolicyOrigin::LocalOverride, - None, - ) - .await; - - let mut global_v2 = sandbox_v2.clone(); - global_v2.policy_source = openshell_core::proto::PolicySource::Global; - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - global_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let mut empty_v1 = sandbox_v1.clone(); - empty_v1.policy_hash.clear(); - let empty_loaded = LoadedPolicyRevision::from_snapshot(&empty_v1); - let mut empty_v2 = sandbox_v2.clone(); - empty_v2.policy_hash.clear(); - assert_poll_does_not_use_same_hash_acknowledgement( - empty_v1, - empty_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(empty_loaded), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v1.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let loaded_v2 = LoadedPolicyRevision::from_snapshot(&sandbox_v2); - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v2, - sandbox_v1, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v2), - has_last_valid_policy: true, - }, - Some(2), - ) - .await; - } - - #[tokio::test] - async fn changed_hash_poll_uses_normal_opa_reload_and_status_path() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - expect_policy_report(&mut reports, 2).await; - assert_eq!( - engine.current_generation(), - 1, - "changed policy content must still reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn failed_external_startup_registry_build_preserves_installed_builtins() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let builtins_generation = engine.current_generation(); - assert_eq!(builtins_generation, 1); - - let invalid_external = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_payload_bytes: 1024, - ..Default::default() - }; - connect_middleware_registry(&[invalid_external], &MiddlewareAuthentication::default()) - .await - .expect_err("unavailable external service must not replace built-ins"); - - assert_eq!(engine.current_generation(), builtins_generation); - } - - #[tokio::test] - async fn unavailable_middleware_reload_keeps_last_known_good_runtime_active() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let active_generation = engine.current_generation(); - let unavailable_service = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_payload_bytes: 1024, - ..Default::default() - }; - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[unavailable_service], - authentication: &MiddlewareAuthentication::default(), - registry_changed: true, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), - ) - .await - .expect_err("unavailable middleware must fail candidate preparation"); - let disposition = apply_gateway_runtime_reload_failure( - &engine, - failure, - PolicyValidationFailureMode::FailClosed, - true, - 2, - ) - .expect("middleware failure handling must succeed"); - - assert!(matches!( - disposition, - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } - )); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[tokio::test] - async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - let active_generation = engine.current_generation(); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }, - ) - .await - .expect_err("TCP expansion must require startup substrate"); - let disposition = apply_gateway_runtime_reload_failure( - &engine, - failure, - PolicyValidationFailureMode::FailClosed, - true, - 2, - ) - .expect("runtime prerequisite failure handling must succeed"); - - assert!(matches!( - disposition, - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - active_generation: generation, - .. - } if generation == active_generation - )); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[tokio::test] - async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), - ) - .await - .expect_err("unsupported runtime must reject TCP expansion"); - - assert!(matches!( - failure, - GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) - )); - assert_eq!(engine.current_generation(), 0); - } - - #[test] - fn policy_rejection_after_middleware_outage_is_not_deduplicated() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let middleware_failure = GatewayRuntimeReloadError::MiddlewareRegistry(miette::miette!( - "middleware service unavailable" - )); - let first_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &middleware_failure); - let middleware_disposition = apply_gateway_runtime_reload_failure( - &engine, - middleware_failure, - PolicyValidationFailureMode::FailClosed, - true, - 7, - ) - .unwrap(); - - assert!(matches!( - middleware_disposition, - GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } - )); - assert!(engine.fail_closed_reason().is_none()); - - let policy_failure = GatewayRuntimeReloadError::PolicyValidation(miette::miette!( - "conflicting endpoint metadata" - )); - let second_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &policy_failure); - assert_ne!( - first_failure, second_failure, - "a changed failure class for the same candidate must be handled" - ); - - let policy_disposition = apply_gateway_runtime_reload_failure( - &engine, - policy_failure, - PolicyValidationFailureMode::FailClosed, - true, - 7, - ) - .unwrap(); - assert!(matches!( - policy_disposition, - GatewayRuntimeFailureDisposition::PolicyRejected { .. } - )); - assert!(engine.fail_closed_reason().is_some()); - } - - #[test] - fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { - let services = Vec::new(); - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - } - - #[test] - fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &no_services, - &no_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &no_services, - &desired_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - false, - "local-policy", - "hash-v2", - &no_services, - &desired_services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - } - - #[test] - fn policy_only_change_does_not_rebuild_middleware_registry() { - let services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - // The runtime must reconcile, but the registry (and therefore - // middleware reachability) is not part of that reconciliation. - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &services, - &services, - )); - } - - #[test] - fn registry_rebuild_requires_service_set_change_or_degraded_registry() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &no_services, - &desired_services, - )); - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::NeedsReconciliation, - &desired_services, - &desired_services, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &desired_services, - &desired_services, - )); - } - - #[test] - fn initial_ack_candidate_matches_sandbox_revision() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) - .expect("sandbox-sourced matching revision should be acknowledged"); - - assert_eq!(ack.version, 2); - assert_eq!(ack.policy_hash, "hash-v2"); - assert_eq!(ack.config_revision, 200); - } - - #[test] - fn initial_ack_candidate_ignores_global_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Global, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_version_zero() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 0, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_local_file_mode() { - // Local-file mode retains no proto policy, so there is nothing to - // acknowledge to the gateway. - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(None, &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_rejects_mismatched_identity() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let mut newer = proto_policy_fixture(); - newer.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule::default(), - ); - let canonical = - settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); - let canonical = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "hash-provider-change".to_string(), - config_revision: loaded.config_revision + 1, - ..canonical - }; - - assert_eq!( - initial_poll_disposition( - &LoadedPolicyOrigin::Gateway { - revision: Some(loaded), - has_last_valid_policy: true, - }, - &canonical, - ), - InitialPollDisposition::Reconcile - ); - } - - #[test] - fn initial_poll_tracks_local_override_without_reconciliation() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert_eq!( - initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), - InitialPollDisposition::TrackOnly - ); - assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); - } - - #[test] - fn initial_poll_reconciles_unbound_gateway_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let origin = LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }; - - assert_eq!( - initial_poll_disposition(&origin, &canonical), - InitialPollDisposition::Reconcile - ); - assert!(origin.allows_gateway_policy_reload()); - } - - #[test] - fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { - let sandbox_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ) - }; - - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &sandbox_result), - Some(2) - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 2, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate( - true, - false, - 1, - "different-policy", - &sandbox_result, - ), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(false, false, 1, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, true, 1, "same-policy", &sandbox_result), - None - ); - - let global_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Global, - ) - }; - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &global_result), - None - ); - } - - #[test] - fn unchanged_policy_revision_waits_for_required_runtime_reconciliation() { - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), false, false), - Some(2), - "a same-hash revision needs no OPA reload" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, false), - None, - "failed runtime reconciliation must keep the revision pending" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, true), - Some(2), - "successful runtime reconciliation permits acknowledgement" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(None, false, true), - None, - "runtime success cannot manufacture a revision candidate" - ); - } - - #[test] - fn credential_gating_unavailable_for_local_override_with_credentials() { - assert!(credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - true - )); - } - - #[test] - fn credential_gating_available_without_local_override_or_credentials() { - // A gateway policy is stamped with provenance, so the gates apply. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }, - true, - true - )); - // No provider credentials means there is nothing to leak. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - false, - true - )); - // Without networking the proxy never evaluates endpoint provenance. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - false - )); - } - - #[test] - fn policy_status_outbox_preserves_all_revision_order() { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - for version in 1..=128 { - enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); - } - - for version in 1..=128 { - assert_eq!( - receiver.try_recv().unwrap(), - PolicyStatusUpdate::loaded(version) - ); - } - } - - #[test] - fn settings_snapshot_carries_workspace_for_policy_sync() { - let mut snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - snapshot.workspace = "beta".to_string(); - - let revision = LoadedPolicyRevision::from_snapshot(&snapshot); - assert_eq!(revision.version, 1); - assert_eq!( - snapshot.workspace, "beta", - "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" - ); - } - #[test] - fn fail_closed_validation_failure_deactivates_previous_generation() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let previous_generation = engine.current_generation(); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::FailClosed, - true, - 7, - "conflicting tls metadata", - ) - .unwrap(); - - assert!(!disposition.previous_policy_active); - assert!(disposition.active_generation > previous_generation); - assert!( - engine - .fail_closed_reason() - .expect("quarantine reason") - .contains("candidate version 7 rejected") - ); - } - - #[test] - fn retain_validation_failure_keeps_previous_generation_active() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - let previous_generation = engine.current_generation(); - - let quarantined = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::FailClosed, - true, - 6, - "conflicting tls metadata", - ) - .unwrap(); - assert!(!quarantined.previous_policy_active); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::RetainLastValid, - true, - 7, - "conflicting tls metadata", - ) - .unwrap(); - - assert!(disposition.previous_policy_active); - assert!(disposition.active_generation > quarantined.active_generation); - assert!(disposition.active_generation > previous_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[test] - fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { - let engine = OpaEngine::from_strings( - include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .unwrap(); - - let disposition = apply_policy_validation_failure( - &engine, - PolicyValidationFailureMode::RetainLastValid, - false, - 1, - "conflicting tls metadata", - ) - .unwrap(); - - assert_eq!( - disposition.configured_mode, - PolicyValidationFailureMode::RetainLastValid - ); - assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); - assert!(!disposition.previous_policy_active); - assert!(engine.fail_closed_reason().is_some()); - - let [config, _] = policy_validation_failure_events( - &disposition, - 1, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); - assert_eq!( - config["unmapped"]["configured_validation_failure_mode"], - "retain_last_valid" - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS NOT active") - ); - } - - #[test] - fn validation_failure_ocsf_states_whether_previous_policy_is_active() { - let fail_closed = PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode::FailClosed, - mode: PolicyValidationFailureMode::FailClosed, - previous_policy_active: false, - active_generation: 9, - }; - let [config, finding] = policy_validation_failure_events( - &fail_closed, - 8, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["class_uid"], 5019); - assert_eq!(config["status"], "Failure"); - assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); - assert_eq!( - config["unmapped"]["configured_validation_failure_mode"], - "fail_closed" - ); - assert_eq!(config["unmapped"]["previous_policy_active"], false); - assert_eq!( - config["unmapped"]["validation_error"], - "conflicting tls metadata" - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS NOT active") - ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("error:conflicting tls metadata") - ); - - let finding = finding.to_json().unwrap(); - assert_eq!(finding["class_uid"], 2004); - assert_eq!(finding["action"], "Denied"); - assert_eq!(finding["disposition"], "Blocked"); - - let retained = PolicyValidationFailureDisposition { - configured_mode: PolicyValidationFailureMode::RetainLastValid, - mode: PolicyValidationFailureMode::RetainLastValid, - previous_policy_active: true, - active_generation: 4, - }; - let [config, _] = policy_validation_failure_events( - &retained, - 8, - "sha256:test", - "conflicting tls metadata", - ); - let config = config.to_json().unwrap(); - assert_eq!(config["unmapped"]["previous_policy_active"], true); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("previous policy IS active") - ); - } +/// Returns an error when the protected bootstrap is invalid or the boundary +/// listener cannot be established. +pub fn run( + config_path: &std::path::Path, + qualification: RuntimeQualification, +) -> miette::Result<()> { + boundary_server::run_boundary(config_path, qualification) + .map_err(|error| miette::miette!(error)) } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 1ad69e1070..4145d6cd00 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -1,288 +1,1510 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Sandbox - process sandbox and monitor. +//! `OpenShell` capability-free in-workload sandbox boundary. +use std::mem::size_of; use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; use clap::Parser; use miette::{IntoDiagnostic, Result}; -use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; +use openshell_ocsf::OcsfShorthandLayer; use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; -use openshell_sandbox::run_sandbox; - -/// Subcommand name used to self-copy the supervisor binary into a shared volume. +/// Subcommand name used to self-copy the sandbox binary into a shared volume. /// /// Init containers invoke the binary directly instead of relying on `sh`/`cp` /// to copy the binary out. Invoking the binary itself with this argument /// performs the copy in pure Rust. const COPY_SELF_SUBCOMMAND: &str = "copy-self"; +const BOOTSTRAP_SUBCOMMAND: &str = "bootstrap"; +const SEED_WORKSPACE_SUBCOMMAND: &str = "seed-workspace"; +const BOOTSTRAP_INPUT_ROOT: &str = "/.openshell/bootstrap-input"; +const SANDBOX_RUNTIME_ROOT: &str = "/.openshell/runtime"; +const SANDBOX_STATE_ROOT: &str = "/.openshell/state"; -/// Subcommand for one-shot debug RPCs from inside a sandbox container. -/// -/// Reads the same token sources as the supervisor (env, file, K8s SA -/// bootstrap) and issues a single gRPC call against the gateway. Useful -/// for end-to-end verification: e.g. `docker exec` into a sandbox, then -/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` -/// to confirm the cross-sandbox IDOR guard fires. -const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; - -/// Default `--mode` value: run both supervisor leaves in a single binary. -const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +const CAPABILITY_PROBE_SUBCOMMAND: &str = "capability-probe"; +const CAPABILITY_PROBE_LAUNCH_SUBCOMMAND: &str = "capability-probe-launch"; +const CAPABILITY_SOCKET_CHILD_SUBCOMMAND: &str = "capability-socket-child"; +const CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND: &str = "capability-landlock-child"; +const CAPABILITY_FREE_LAUNCH_SUBCOMMAND: &str = "launch-capability-free"; #[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR; +const PROBE_DENIED_TCP_PEER: &str = "203.0.113.1:9"; #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +const PROBE_SOCKADDR_IN_LEN: usize = size_of::(); #[cfg(target_os = "linux")] -const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +const LINUX_SIGNAL_LIMIT: i32 = 65; + +#[derive(Parser, Debug)] +#[command(name = "openshell-sandbox")] +#[command(version = openshell_core::VERSION)] +#[command(about = "OpenShell in-workload isolation boundary")] +struct BoundaryArgs { + /// Protected one-use bootstrap configuration staged by the driver. + #[arg(long)] + bootstrap: std::path::PathBuf, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, +} + +/// Internal one-shot command used by trusted driver bootstrap to validate an +/// image-provided workdir as the final sandbox identity. +#[derive(Parser, Debug)] +#[command(name = "validate-workspace", hide = true)] +struct ValidateWorkspaceArgs { + #[arg(long)] + workdir: String, + #[arg(long)] + expected_uid: u32, + #[arg(long)] + expected_gid: u32, +} + #[cfg(target_os = "linux")] -const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +fn validate_workspace(args: &[String]) -> Result<()> { + let args = ValidateWorkspaceArgs::try_parse_from( + std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), + ) + .into_diagnostic()?; + let actual = ( + nix::unistd::geteuid().as_raw(), + nix::unistd::getegid().as_raw(), + ); + if actual != (args.expected_uid, args.expected_gid) { + return Err(miette::miette!( + "workspace validator privilege drop failed: expected {}:{}, got {}:{}", + args.expected_uid, + args.expected_gid, + actual.0, + actual.1 + )); + } + openshell_sandbox::process::validate_oci_workspace_as_effective_identity(Path::new( + &args.workdir, + )) +} + +#[cfg(not(target_os = "linux"))] +fn validate_workspace(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "workspace validation is only supported on Unix" + )) +} + +/// Run the active Phase 0 probe inside the exact workload runtime profile. #[cfg(target_os = "linux")] -const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[allow(unsafe_code)] +fn run_capability_probe() -> Result<()> { + let (qualification, report) = qualify_runtime()?; + debug_assert!(qualification.seccomp.notification_round_trip); + println!("{report}"); + Ok(()) +} + +/// Actively qualify every kernel primitive used by the capability-free +/// sandbox. Callers decide whether to emit the resulting diagnostic report. #[cfg(target_os = "linux")] -const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[allow(unsafe_code)] +fn qualify_runtime() -> Result<(openshell_sandbox::RuntimeQualification, serde_json::Value)> { + use miette::Context as _; + + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "capability-free sandbox probe requires non-root UID and GID, got {uid}:{gid}" + )); + } + let status = std::fs::read_to_string("/proc/self/status") + .into_diagnostic() + .wrap_err("read /proc/self/status")?; + for field in ["CapInh", "CapPrm", "CapEff", "CapBnd", "CapAmb"] { + let value = proc_status_hex(&status, field)?; + if value != 0 { + return Err(miette::miette!( + "capability-free sandbox probe found {field}=0x{value:x}" + )); + } + } + // SAFETY: PR_GET_NO_NEW_PRIVS reads one scalar process property. + let no_new_privileges = unsafe { libc::prctl(libc::PR_GET_NO_NEW_PRIVS, 0, 0, 0, 0) }; + if no_new_privileges != 1 { + return Err(miette::miette!( + "capability-free sandbox probe requires no_new_privs=1" + )); + } + + // The trusted sandbox must be nondumpable before it handles bootstrap or + // channel secrets. Perform the parent-to-child observation probe after + // tightening the parent; the synthetic child explicitly becomes dumpable. + // SAFETY: PR_SET_DUMPABLE accepts one scalar and only tightens this process. + if unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "set sandbox probe nondumpable: {}", + std::io::Error::last_os_error() + )); + } + openshell_isolation_interface::linux::task_memory::probe_child_access() + .into_diagnostic() + .wrap_err("same-UID task-memory probe")?; + probe_landlock_allow_deny().wrap_err("Landlock allow/deny probe")?; + let notification = + openshell_isolation_interface::linux::seccomp_notify::probe_notification_api() + .into_diagnostic() + .wrap_err("seccomp notification probe")?; + probe_socket_virtualization().wrap_err("socket virtualization probe")?; + probe_dns_relay_bind().wrap_err("DNS relay bind probe")?; + let landlock_abi = openshell_isolation_interface::linux::landlock::abi_version() + .into_diagnostic() + .wrap_err("Landlock ABI probe")?; + if landlock_abi == 0 { + return Err(miette::miette!("Landlock ABI version is zero")); + } + + let groups = nix::unistd::getgroups() + .into_diagnostic()? + .into_iter() + .map(nix::unistd::Gid::as_raw) + .collect::>(); + let report = serde_json::json!({ + "qualified": true, + "uid": uid, + "gid": gid, + "supplementary_groups": groups, + "capabilities_zero": true, + "no_new_privileges": true, + "sandbox_dumpable": false, + "child_dumpable": true, + "child_core_limit_zero": true, + "same_uid_self_protection": true, + "landlock_abi": landlock_abi, + "landlock_allow_deny": true, + "seccomp_notification": notification.notification_round_trip(), + "seccomp_addfd_send": notification.addfd_send(), + "task_memory_copy": notification.task_memory_copy(), + "connected_send_fast_path": notification.connected_send_fast_path(), + "socket_virtualization": true, + "dns_relay_bind": true, + "udp_dns_round_trip": true, + "tcp_dns_round_trip": true, + "tcp_allow_round_trip": true, + "tcp_deny_round_trip": true, + "wait_killable_recv": notification.wait_killable_recv, + }); + let qualification = openshell_sandbox::RuntimeQualification { + seccomp: openshell_isolation_interface::contract::SeccompEvidence { + new_listener: notification.notification_round_trip(), + notification_round_trip: notification.notification_round_trip(), + id_validation: notification.notification_round_trip(), + addfd_send: notification.addfd_send(), + retained_socket_operation: true, + proc_fd_identity: true, + task_memory_read: notification.task_memory_copy(), + task_memory_write: notification.task_memory_copy(), + cancellation: notification.wait_killable_recv, + }, + landlock_abi, + landlock_allow_deny: true, + udp_dns_round_trip: true, + tcp_dns_round_trip: true, + tcp_allow_round_trip: true, + tcp_deny_round_trip: true, + }; + Ok((qualification, report)) +} + #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +fn probe_dns_relay_bind() -> Result<()> { + use std::net::{TcpListener, UdpSocket}; + + let unprivileged_port_start = + std::fs::read_to_string("/proc/sys/net/ipv4/ip_unprivileged_port_start") + .into_diagnostic()? + .trim() + .parse::() + .into_diagnostic()?; + if unprivileged_port_start != 0 { + return Err(miette::miette!( + "DNS relay requires net.ipv4.ip_unprivileged_port_start=0, got {unprivileged_port_start}" + )); + } + let tcp = TcpListener::bind("127.0.0.53:53").into_diagnostic()?; + let udp = UdpSocket::bind("127.0.0.53:53").into_diagnostic()?; + drop((tcp, udp)); + Ok(()) +} + +/// Prove that the exact unprivileged runtime can install a hard Landlock +/// allow-list which admits one path and rejects an adjacent path. Landlock is +/// irreversible, so the restriction is exercised in a fresh trusted child. +#[cfg(target_os = "linux")] +fn probe_landlock_allow_deny() -> Result<()> { + use miette::Context as _; + + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic()? + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "openshell-landlock-probe-{}-{nonce}", + std::process::id() + )); + let allowed = root.join("allowed"); + let denied = root.join("denied"); + std::fs::create_dir(&root) + .into_diagnostic() + .wrap_err("create Landlock probe root")?; + let probe_result = (|| -> Result<()> { + std::fs::create_dir(&allowed).into_diagnostic()?; + std::fs::create_dir(&denied).into_diagnostic()?; + std::fs::write(allowed.join("sentinel"), b"allowed").into_diagnostic()?; + std::fs::write(denied.join("sentinel"), b"denied").into_diagnostic()?; + let status = std::process::Command::new(std::env::current_exe().into_diagnostic()?) + .arg(CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND) + .arg(&allowed) + .arg(&denied) + .env_clear() + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()) + .status() + .into_diagnostic() + .wrap_err("run Landlock probe child")?; + if !status.success() { + return Err(miette::miette!( + "Landlock probe child exited with status {status}" + )); + } + Ok(()) + })(); + let cleanup_result = std::fs::remove_dir_all(&root).into_diagnostic(); + probe_result?; + cleanup_result.wrap_err("remove Landlock probe root") +} + #[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; +fn run_capability_landlock_child(args: &[String]) -> Result<()> { + use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkPolicy, ProcessPolicy, + SandboxPolicy, + }; + + let [allowed, denied] = args else { + return Err(miette::miette!( + "usage: {CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND} " + )); + }; + let allowed = Path::new(allowed); + let denied = Path::new(denied); + let policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![allowed.to_path_buf()], + read_write: Vec::new(), + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy { + compatibility: LandlockCompatibility::HardRequirement, + }, + process: ProcessPolicy::default(), + }; + let prepared = openshell_sandbox::sandbox::linux::prepare_capability_free(&policy, None)?; + openshell_sandbox::sandbox::linux::enforce(prepared)?; + if std::fs::read(allowed.join("sentinel")).into_diagnostic()? != b"allowed" { + return Err(miette::miette!("Landlock probe allowed-path mismatch")); + } + match std::fs::read(denied.join("sentinel")) { + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => Err(error).into_diagnostic(), + Ok(_) => Err(miette::miette!( + "Landlock probe unexpectedly read the denied path" + )), + } +} -/// Which supervisor leaves are enabled in this process. +#[cfg(not(target_os = "linux"))] +fn run_capability_landlock_child(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "Landlock qualification is supported only on Linux" + )) +} + +/// Exercise the production listener inheritance and socket-time ADDFD shape. /// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. -#[derive(Clone, Copy, Debug)] -struct Mode { - network: bool, - process: bool, - network_init: bool, -} - -impl std::str::FromStr for Mode { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut mode = Self { - network: false, - process: false, - network_init: false, - }; - for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { - match part { - "network" => mode.network = true, - "process" => mode.process = true, - "network-init" => mode.network_init = true, - other => { - return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" - )); +/// One dedicated launcher thread installs the non-TSYNC listener, moves the +/// listener FD to this unfiltered broker through an in-process channel, then +/// execs the child. The child proves that the injected open-file description +/// survives dup and epoll registration before connect and that the broker can +/// return the original peer rather than the local relay endpoint. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_socket_virtualization() -> Result<()> { + use std::io::{Read as _, Write as _}; + use std::net::{Ipv4Addr, SocketAddr, TcpListener, UdpSocket}; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + use std::os::unix::process::CommandExt as _; + use std::sync::mpsc; + + use miette::Context as _; + use openshell_isolation_interface::linux::seccomp_notify::NotificationListener; + use openshell_isolation_interface::linux::socket_registry::{ + InetFamily, InetKind, SocketMetadata, SocketRegistry, SocketState, + }; + + let relay = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .into_diagnostic() + .wrap_err("bind socket probe relay")?; + let original_peer = relay.local_addr().into_diagnostic()?; + let relay_thread = std::thread::Builder::new() + .name("openshell-probe-relay".to_string()) + .spawn(move || -> std::io::Result<()> { + let (mut stream, _) = relay.accept()?; + stream.set_nodelay(true)?; + let mut request = [0_u8; 4]; + stream.read_exact(&mut request)?; + if &request != b"ping" { + return Err(std::io::Error::other("socket probe payload mismatch")); + } + stream.write_all(b"pong") + }) + .into_diagnostic()?; + let dns_relay_addr = "127.0.0.53:53" + .parse::() + .expect("fixed DNS relay address is valid"); + let dns_relay = UdpSocket::bind(dns_relay_addr) + .into_diagnostic() + .wrap_err("bind socket probe DNS relay")?; + let dns_thread = std::thread::Builder::new() + .name("openshell-probe-dns".to_string()) + .spawn(move || -> std::io::Result<()> { + let mut query = [0_u8; 512]; + let (length, peer) = dns_relay.recv_from(&mut query)?; + let response = build_probe_dns_response(&query[..length])?; + let sent = dns_relay.send_to(&response, peer)?; + if sent != response.len() { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "short DNS probe response", + )); + } + Ok(()) + }) + .into_diagnostic()?; + let dns_tcp_relay = TcpListener::bind(dns_relay_addr) + .into_diagnostic() + .wrap_err("bind socket probe TCP DNS relay")?; + let dns_tcp_thread = std::thread::Builder::new() + .name("openshell-probe-dns-tcp".to_string()) + .spawn(move || -> std::io::Result<()> { + let (mut stream, _) = dns_tcp_relay.accept()?; + let mut length = [0_u8; 2]; + stream.read_exact(&mut length)?; + let length = usize::from(u16::from_be_bytes(length)); + if length == 0 || length > 512 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid TCP DNS probe length", + )); + } + let mut query = vec![0_u8; length]; + stream.read_exact(&mut query)?; + let response = build_probe_dns_response(&query)?; + stream.write_all( + &u16::try_from(response.len()) + .expect("probe DNS response fits u16") + .to_be_bytes(), + )?; + stream.write_all(&response) + }) + .into_diagnostic()?; + + let executable = std::env::current_exe() + .into_diagnostic() + .wrap_err("resolve socket probe executable")?; + let sandbox_tgid = std::process::id(); + let mut child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(sandbox_tgid) + .into_diagnostic() + .wrap_err("prepare socket probe child hardening")?; + let (listener_tx, listener_rx) = mpsc::sync_channel::>(1); + let (child_tx, child_rx) = mpsc::sync_channel::>(1); + let launcher = std::thread::Builder::new() + .name("openshell-probe-launcher".to_string()) + .spawn(move || { + if let Err(error) = block_launcher_signals() { + let _ = listener_tx.send(Err(error)); + return; + } + let listener = + openshell_isolation_interface::linux::seccomp_notify::install_listener(&[ + libc::SYS_socket, + libc::SYS_connect, + libc::SYS_getpeername, + libc::SYS_sendto, + ]); + let Ok(listener) = listener else { + let _ = listener_tx.send(listener); + return; + }; + if listener_tx.send(Ok(listener)).is_err() { + return; + } + let mut command = std::process::Command::new(executable); + command + .arg(CAPABILITY_SOCKET_CHILD_SUBCOMMAND) + .arg(original_peer.to_string()) + .arg(sandbox_tgid.to_string()) + .env_clear() + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::inherit()); + // SAFETY: the hook uses only raw signal/process syscalls and the + // prebuilt, allocation-free seccomp installation path. + unsafe { + command.pre_exec(move || { + if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + set_child_core_limit()?; + reset_child_signal_dispositions()?; + child_hardening.install()?; + install_child_signal_mask() + }); + } + let child = command.spawn(); + let _ = child_tx.send(child); + }) + .into_diagnostic()?; + + let listener = listener_rx + .recv() + .into_diagnostic() + .wrap_err("socket probe launcher stopped before listener handoff")? + .into_diagnostic() + .wrap_err("install socket probe listener")?; + let mut child = child_rx + .recv() + .into_diagnostic() + .wrap_err("socket probe launcher stopped before child spawn")? + .into_diagnostic() + .wrap_err("spawn socket probe child")?; + let mut registry = SocketRegistry::new(1, 8).into_diagnostic()?; + let mut observed_tcp_sockets = 0_u8; + let mut observed_dns_socket = false; + let mut observed_connect = false; + let mut observed_dns_tcp_connect = false; + let mut observed_denied_connect = false; + let mut observed_peer = false; + let mut observed_dns_send = false; + + while !(observed_tcp_sockets == 3 + && observed_dns_socket + && observed_connect + && observed_dns_tcp_connect + && observed_denied_connect + && observed_peer + && observed_dns_send) + { + let notification = listener + .receive() + .into_diagnostic() + .wrap_err("receive socket probe notification")?; + match i64::from(notification.syscall) { + libc::SYS_socket => { + if notification.args[0] != u64::try_from(libc::AF_INET).unwrap() { + listener + .respond_errno(notification.id, libc::EPROTONOSUPPORT) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe request")); + } + let requested_type = i32::try_from(notification.args[1]) + .map_err(|_| miette::miette!("socket type does not fit i32"))?; + let base_type = requested_type & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK); + let protocol = i32::try_from(notification.args[2]) + .map_err(|_| miette::miette!("socket protocol does not fit i32"))?; + let (kind, canonical_protocol) = match (base_type, protocol) { + (libc::SOCK_STREAM, 0 | libc::IPPROTO_TCP) if observed_tcp_sockets < 3 => { + observed_tcp_sockets += 1; + (InetKind::Tcp, libc::IPPROTO_TCP) + } + (libc::SOCK_DGRAM, 0 | libc::IPPROTO_UDP) if !observed_dns_socket => { + observed_dns_socket = true; + (InetKind::DnsUdp, libc::IPPROTO_UDP) + } + _ => { + listener + .respond_errno(notification.id, libc::EPROTONOSUPPORT) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe request")); + } + }; + // SAFETY: scalar validated AF_INET/TCP arguments return one + // newly owned descriptor on success. + let source = + unsafe { libc::socket(libc::AF_INET, requested_type, canonical_protocol) }; + if source < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); } + // SAFETY: successful socket returned one newly owned FD. + let source = unsafe { OwnedFd::from_raw_fd(source) }; + let close_on_exec = requested_type & libc::SOCK_CLOEXEC != 0; + let tentative = registry + .stage( + source, + SocketMetadata { + family: InetFamily::V4, + kind, + close_on_exec, + nonblocking: requested_type & libc::SOCK_NONBLOCK != 0, + creator_generation: 1, + }, + ) + .into_diagnostic()?; + listener + .add_fd_and_send(notification.id, tentative.source_fd(), close_on_exec) + .into_diagnostic()?; + registry.commit(tentative).into_diagnostic()?; + } + libc::SYS_connect => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("connect FD does not fit i32"))?; + let destination = read_probe_sockaddr( + notification.tid, + notification.args[1], + notification.args[2], + )?; + let denied_peer = PROBE_DENIED_TCP_PEER + .parse::() + .expect("fixed denied peer is valid"); + if destination == denied_peer && !observed_denied_connect { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + observed_denied_connect = true; + continue; + } + if destination != original_peer && destination != dns_relay_addr { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe destination")); + } + if (destination == original_peer && observed_connect) + || (destination == dns_relay_addr && observed_dns_tcp_connect) + { + listener + .respond_errno(notification.id, libc::EALREADY) + .into_diagnostic()?; + return Err(miette::miette!("duplicate socket probe connect")); + } + let entry = registry + .resolve_mut(notification.tid, fd) + .into_diagnostic()?; + entry.validate_retained_identity().into_diagnostic()?; + let (sockaddr, length) = encode_probe_sockaddr(destination)?; + // SAFETY: the retained FD is the registered injected socket; + // `sockaddr` is live for the declared IPv4 length. + let connected = unsafe { + libc::connect( + entry.retained_preconnect().into_diagnostic()?.as_raw_fd(), + sockaddr.as_ptr().cast(), + length, + ) + }; + if connected != 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + if destination == dns_relay_addr { + entry.set_state(SocketState::DnsTcp { + relay: dns_relay_addr, + }); + observed_dns_tcp_connect = true; + } else { + entry.set_state(SocketState::Connected { original_peer }); + observed_connect = true; + } + entry.release_preconnect(); + listener + .respond_value(notification.id, 0) + .into_diagnostic()?; + } + libc::SYS_getpeername => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("peer FD does not fit i32"))?; + let entry = registry.resolve(notification.tid, fd).into_diagnostic()?; + let SocketState::Connected { original_peer } = entry.state() else { + listener + .respond_errno(notification.id, libc::ENOTCONN) + .into_diagnostic()?; + return Err(miette::miette!("peer query preceded mediated connect")); + }; + write_probe_sockaddr( + notification.tid, + notification.args[1], + notification.args[2], + *original_peer, + )?; + listener + .respond_value(notification.id, 0) + .into_diagnostic()?; + observed_peer = true; + } + libc::SYS_sendto => { + let fd = i32::try_from(notification.args[0]) + .map_err(|_| miette::miette!("sendto FD does not fit i32"))?; + let length = usize::try_from(notification.args[2]) + .map_err(|_| miette::miette!("DNS payload length does not fit usize"))?; + if length == 0 || length > 512 || notification.args[3] != 0 { + listener + .respond_errno(notification.id, libc::EMSGSIZE) + .into_diagnostic()?; + return Err(miette::miette!("unexpected DNS probe payload shape")); + } + let destination = read_probe_sockaddr( + notification.tid, + notification.args[4], + notification.args[5], + )?; + if observed_dns_send || destination != dns_relay_addr { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("unexpected DNS probe destination")); + } + let mut payload = vec![0_u8; length]; + openshell_isolation_interface::linux::task_memory::read_exact( + notification.tid, + notification.args[1], + &mut payload, + ) + .into_diagnostic()?; + validate_probe_dns_query(&payload)?; + let entry = registry + .resolve_mut(notification.tid, fd) + .into_diagnostic()?; + if entry.metadata().kind != InetKind::DnsUdp + || !matches!(entry.state(), SocketState::Created) + { + listener + .respond_errno(notification.id, libc::EACCES) + .into_diagnostic()?; + return Err(miette::miette!("DNS probe socket is not eligible")); + } + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(destination)?; + let retained = entry.retained_preconnect().into_diagnostic()?; + // SAFETY: the retained source is the exact injected OFD and + // both copied buffers remain live for their declared lengths. + if unsafe { + libc::connect( + retained.as_raw_fd(), + sockaddr.as_ptr().cast(), + sockaddr_length, + ) + } != 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let sent = unsafe { + libc::send( + retained.as_raw_fd(), + payload.as_ptr().cast(), + payload.len(), + libc::MSG_NOSIGNAL, + ) + }; + if sent != isize::try_from(payload.len()).expect("DNS payload fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + entry.set_state(SocketState::DnsUdp { + relay: dns_relay_addr, + }); + entry.release_preconnect(); + listener + .respond_value( + notification.id, + i64::try_from(length).expect("length fits i64"), + ) + .into_diagnostic()?; + observed_dns_send = true; + } + _ => { + listener + .respond_errno(notification.id, libc::EPERM) + .into_diagnostic()?; + return Err(miette::miette!("unexpected socket probe syscall")); } } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); + } + + let status = child + .wait() + .into_diagnostic() + .wrap_err("wait for socket probe child")?; + launcher + .join() + .map_err(|_| miette::miette!("socket probe launcher panicked"))?; + relay_thread + .join() + .map_err(|_| miette::miette!("socket probe relay panicked"))? + .into_diagnostic()?; + dns_thread + .join() + .map_err(|_| miette::miette!("socket probe DNS relay panicked"))? + .into_diagnostic()?; + dns_tcp_thread + .join() + .map_err(|_| miette::miette!("socket probe TCP DNS relay panicked"))? + .into_diagnostic()?; + if !status.success() { + return Err(miette::miette!( + "socket probe child exited with status {status}" + )); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn validate_probe_dns_query(query: &[u8]) -> Result<()> { + const EXPECTED_QUESTION: &[u8] = b"\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + if query.len() != 12 + EXPECTED_QUESTION.len() + || query[2] & 0x80 != 0 + || query[4..6] != [0, 1] + || &query[12..] != EXPECTED_QUESTION + { + return Err(miette::miette!("DNS probe query is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn build_probe_dns_response(query: &[u8]) -> std::io::Result> { + validate_probe_dns_query(query).map_err(std::io::Error::other)?; + let mut response = query.to_vec(); + response[2..4].copy_from_slice(&[0x81, 0x80]); + response[6..8].copy_from_slice(&[0, 1]); + response.extend_from_slice(&[0xc0, 0x0c, 0, 1, 0, 1, 0, 0, 0, 30, 0, 4, 203, 0, 113, 7]); + Ok(response) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn block_launcher_signals() -> std::io::Result<()> { + let mut signals = std::mem::MaybeUninit::::uninit(); + // SAFETY: `signals` points to writable sigset storage and pthread_sigmask + // copies it during the call. + if unsafe { libc::sigfillset(signals.as_mut_ptr()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: sigfillset initialized the value above. + let signals = unsafe { signals.assume_init() }; + // SAFETY: changing the mask affects only the dedicated launcher thread. + let result = + unsafe { libc::pthread_sigmask(libc::SIG_BLOCK, &raw const signals, std::ptr::null_mut()) }; + if result != 0 { + return Err(std::io::Error::from_raw_os_error(result)); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +unsafe fn set_child_core_limit() -> std::io::Result<()> { + let limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + // SAFETY: `limit` is a live fixed-size rlimit and this child-only update + // permanently disables core dumps before any untrusted instruction. + if unsafe { libc::setrlimit(libc::RLIMIT_CORE, &raw const limit) } < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +unsafe fn reset_child_signal_dispositions() -> std::io::Result<()> { + // SAFETY: an all-zero sigaction is a valid base before the explicit + // default handler and empty mask are installed below. + let mut action = unsafe { std::mem::zeroed::() }; + action.sa_sigaction = libc::SIG_DFL; + // SAFETY: action.sa_mask points to writable sigset storage. + if unsafe { libc::sigemptyset(&raw mut action.sa_mask) } < 0 { + return Err(std::io::Error::last_os_error()); + } + for signal in 1..LINUX_SIGNAL_LIMIT { + if signal == libc::SIGKILL || signal == libc::SIGSTOP { + continue; } - if !mode.network && !mode.process && !mode.network_init { - return Err( - "--mode must enable at least one of: network, process, network-init".into(), - ); + // SAFETY: action contains the default disposition and the null output + // pointer requests no previous action. + if unsafe { libc::sigaction(signal, &raw const action, std::ptr::null_mut()) } < 0 { + let error = std::io::Error::last_os_error(); + // glibc reserves two real-time signals for its threading runtime; + // Linux rejects sigaction for those numbers with EINVAL. + if error.raw_os_error() != Some(libc::EINVAL) { + return Err(error); + } } - Ok(mode) } + Ok(()) } -/// `OpenShell` Sandbox - process isolation and monitoring. -// CLI flags are naturally boolean switches; grouping them into structs would -// only obscure the clap definition. -#[allow(clippy::struct_excessive_bools)] -#[derive(Parser, Debug)] -#[command(name = "openshell-sandbox")] -#[command(version = openshell_core::VERSION)] -#[command(about = "Process sandbox and monitor", long_about = None)] -struct Args { - /// Command to execute in the sandbox. - /// Defaults to a login shell if neither this nor the driver specification is - /// provided: `/bin/bash -l` when available, otherwise a shell detected in the - /// sandbox image (e.g. `/bin/sh` on Alpine). - #[arg(trailing_var_arg = true)] - command: Vec, - - /// Working directory for the sandboxed process. - #[arg(long, short)] - workdir: Option, - - /// Timeout in seconds (0 = no timeout). - #[arg(long, short, default_value = "0")] - timeout: u64, - - /// Run in interactive mode (inherit process group for terminal control). - #[arg(long, short = 'i')] - interactive: bool, - - /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. - /// Requires --openshell-endpoint to be set. - #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] - sandbox_id: Option, - - /// Sandbox (used for policy sync when the sandbox discovers policy - /// from disk or falls back to the restrictive default). - #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] - sandbox: Option, - - /// `OpenShell` server gRPC endpoint for fetching policy. - /// Required when using --sandbox-id. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: Option, - - /// Path to Rego policy file for OPA-based network access control. - /// Requires --policy-data to also be set. - #[arg(long, env = "OPENSHELL_POLICY_RULES")] - policy_rules: Option, - - /// Path to YAML data file containing network policies and sandbox config. - /// Requires --policy-rules to also be set. - #[arg(long, env = "OPENSHELL_POLICY_DATA")] - policy_data: Option, +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn install_child_signal_mask() -> std::io::Result<()> { + let mut signals = std::mem::MaybeUninit::::uninit(); + // SAFETY: `signals` points to writable sigset storage. + if unsafe { libc::sigemptyset(signals.as_mut_ptr()) } < 0 { + return Err(std::io::Error::last_os_error()); + } + // SAFETY: sigemptyset initialized the value above. + let signals = unsafe { signals.assume_init() }; + // SAFETY: this installs the declared empty target mask immediately before + // exec, after copied sandbox dispositions have been reset. + let result = unsafe { + libc::pthread_sigmask(libc::SIG_SETMASK, &raw const signals, std::ptr::null_mut()) + }; + if result != 0 { + return Err(std::io::Error::from_raw_os_error(result)); + } + Ok(()) +} - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, +#[cfg(target_os = "linux")] +fn read_probe_sockaddr(tid: u32, address: u64, length: u64) -> Result { + let length = + usize::try_from(length).map_err(|_| miette::miette!("sockaddr length too large"))?; + if length != PROBE_SOCKADDR_IN_LEN { + return Err(miette::miette!("socket probe requires an IPv4 sockaddr")); + } + let mut bytes = vec![0_u8; length]; + openshell_isolation_interface::linux::task_memory::read_exact(tid, address, &mut bytes) + .into_diagnostic()?; + decode_probe_sockaddr(&bytes) +} - /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning - /// with `@` selects an abstract socket in the network namespace. - /// The supervisor bridges `RelayStream` traffic from the gateway onto - /// this socket; nothing else should connect to it. - #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] - ssh_socket_path: Option, +#[cfg(target_os = "linux")] +fn encode_probe_sockaddr( + address: std::net::SocketAddr, +) -> Result<([u8; PROBE_SOCKADDR_IN_LEN], libc::socklen_t)> { + let std::net::SocketAddr::V4(address) = address else { + return Err(miette::miette!("socket probe requires IPv4")); + }; + let mut bytes = [0_u8; PROBE_SOCKADDR_IN_LEN]; + bytes[0..2].copy_from_slice( + &libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t") + .to_ne_bytes(), + ); + bytes[2..4].copy_from_slice(&address.port().to_be_bytes()); + bytes[4..8].copy_from_slice(&address.ip().octets()); + Ok(( + bytes, + libc::socklen_t::try_from(PROBE_SOCKADDR_IN_LEN) + .expect("sockaddr_in length fits socklen_t"), + )) +} - /// Path to YAML inference routes for standalone routing. - /// When set, inference routes are loaded from this file instead of - /// fetching a bundle from the gateway. - #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] - inference_routes: Option, +#[cfg(target_os = "linux")] +fn decode_probe_sockaddr(bytes: &[u8]) -> Result { + if bytes.len() != PROBE_SOCKADDR_IN_LEN { + return Err(miette::miette!("socket probe requires an IPv4 sockaddr")); + } + let family = libc::sa_family_t::from_ne_bytes([bytes[0], bytes[1]]); + if i32::from(family) != libc::AF_INET { + return Err(miette::miette!("socket probe sockaddr is not IPv4")); + } + Ok(std::net::SocketAddr::V4(std::net::SocketAddrV4::new( + std::net::Ipv4Addr::new(bytes[4], bytes[5], bytes[6], bytes[7]), + u16::from_be_bytes([bytes[2], bytes[3]]), + ))) +} - /// Enable health check endpoint. - #[arg(long)] - health_check: bool, - - /// Port for health check endpoint. - #[arg(long, default_value = "8080")] - health_port: u16, - - /// Which supervisor components to run. Comma-separated list of - /// "network" and/or "process". Defaults to both (single-binary - /// topology). Use --mode=network for a network-only sidecar, or - /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. Use --mode=network-init only in - /// the Kubernetes init container that prepares sidecar nftables. - #[arg(long, default_value = DEFAULT_MODE)] - mode: Mode, - - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] - proxy_uid: u32, - - /// GID assigned to shared sidecar state directories. Defaults to - /// `--proxy-uid` when omitted. - #[arg(long, env = "OPENSHELL_PROXY_GID")] - proxy_gid: Option, - - /// Shared state directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] - sidecar_state_dir: String, - - /// Shared TLS work directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] - sidecar_tls_dir: String, - - // Corporate upstream proxy. Operator-owned egress boundary: accepted - // only as command-line arguments (no `env =`), because the driver - // controls the supervisor's argv while a sandbox image could bake - // matching `ENV` values. - /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. - #[arg(long)] - upstream_proxy: Option, +#[cfg(target_os = "linux")] +fn write_probe_sockaddr( + tid: u32, + address: u64, + length_address: u64, + peer: std::net::SocketAddr, +) -> Result<()> { + use std::mem::size_of; + + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(peer)?; + let mut requested_length = [0_u8; size_of::()]; + openshell_isolation_interface::linux::task_memory::read_exact( + tid, + length_address, + &mut requested_length, + ) + .into_diagnostic()?; + let requested_length = libc::socklen_t::from_ne_bytes(requested_length); + if requested_length < sockaddr_length { + return Err(miette::miette!("peer sockaddr buffer is too small")); + } + openshell_isolation_interface::linux::task_memory::write_exact(tid, address, &sockaddr) + .into_diagnostic()?; + openshell_isolation_interface::linux::task_memory::write_exact( + tid, + length_address, + &sockaddr_length.to_ne_bytes(), + ) + .into_diagnostic()?; + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn run_capability_socket_child(args: &[String]) -> Result<()> { + use std::io::Read as _; + use std::net::SocketAddr; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + let [expected_peer, sandbox_tgid] = args else { + return Err(miette::miette!( + "usage: {CAPABILITY_SOCKET_CHILD_SUBCOMMAND} " + )); + }; + let expected_peer = expected_peer + .parse::() + .into_diagnostic() + .map_err(|error| miette::miette!("parse socket probe peer: {error}"))?; + let sandbox_tgid = sandbox_tgid + .parse::() + .into_diagnostic() + .map_err(|error| miette::miette!("parse sandbox TGID: {error}"))?; + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(expected_peer)?; + + // SAFETY: this call is intentionally intercepted and completed with one + // newly injected socket descriptor. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: dup creates an alias of the same open-file description. + let alias = unsafe { libc::dup(socket.as_raw_fd()) }; + if alias < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful dup returned one newly owned descriptor. + let alias = unsafe { OwnedFd::from_raw_fd(alias) }; + + // SAFETY: epoll_create1 returns one owned descriptor; epoll_ctl consumes + // only the live event value for this call. + let epoll = unsafe { libc::epoll_create1(libc::EPOLL_CLOEXEC) }; + if epoll < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful epoll_create1 returned one newly owned descriptor. + let epoll = unsafe { OwnedFd::from_raw_fd(epoll) }; + let mut event = libc::epoll_event { + events: u32::try_from(libc::EPOLLIN | libc::EPOLLOUT).expect("epoll flags fit u32"), + u64: 1, + }; + // SAFETY: descriptors and event pointer are live for this call. + if unsafe { + libc::epoll_ctl( + epoll.as_raw_fd(), + libc::EPOLL_CTL_ADD, + socket.as_raw_fd(), + std::ptr::addr_of_mut!(event), + ) + } < 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + // SAFETY: the sockaddr is live and the alias references the mediated OFD. + if unsafe { libc::connect(alias.as_raw_fd(), sockaddr.as_ptr().cast(), sockaddr_length) } != 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + let mut observed_peer = [0_u8; PROBE_SOCKADDR_IN_LEN]; + let mut observed_length = + libc::socklen_t::try_from(PROBE_SOCKADDR_IN_LEN).expect("sockaddr length fits socklen_t"); + // SAFETY: the output objects are live for the full declared length. + if unsafe { + libc::getpeername( + socket.as_raw_fd(), + observed_peer.as_mut_ptr().cast(), + std::ptr::addr_of_mut!(observed_length), + ) + } != 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let observed = decode_probe_sockaddr(&observed_peer)?; + if observed != expected_peer { + return Err(miette::miette!( + "socket probe peer mismatch: expected {expected_peer}, got {observed}" + )); + } + + let request = b"ping"; + // SAFETY: null destination on a connected socket follows the cBPF fast + // path and reads only the live request buffer. + let sent = unsafe { + libc::sendto( + alias.as_raw_fd(), + request.as_ptr().cast(), + request.len(), + libc::MSG_NOSIGNAL, + std::ptr::null(), + 0, + ) + }; + if sent != isize::try_from(request.len()).expect("request length fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let mut ready = libc::epoll_event { events: 0, u64: 0 }; + // SAFETY: event points to storage for one returned event. + if unsafe { libc::epoll_wait(epoll.as_raw_fd(), std::ptr::addr_of_mut!(ready), 1, 5_000) } <= 0 + { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let mut stream = std::net::TcpStream::from(socket); + let mut response = [0_u8; 4]; + stream.read_exact(&mut response).into_diagnostic()?; + if &response != b"pong" { + return Err(miette::miette!("socket probe response mismatch")); + } + probe_dns_socket_round_trip()?; + probe_tcp_dns_socket_round_trip()?; + probe_tcp_denial()?; + probe_child_self_protection(sandbox_tgid, alias.as_raw_fd())?; + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_dns_socket_round_trip() -> Result<()> { + use std::net::SocketAddr; + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + const DNS_QUERY: &[u8] = + b"\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + let relay = "127.0.0.53:53" + .parse::() + .expect("fixed DNS relay address is valid"); + let (sockaddr, sockaddr_length) = encode_probe_sockaddr(relay)?; + // SAFETY: the syscall is intercepted and completed with a DNS-only + // registered socket descriptor. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_UDP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: the destination and query buffers are live for the call. The + // broker copies and emulates this send before replying to the notification. + let sent = unsafe { + libc::sendto( + socket.as_raw_fd(), + DNS_QUERY.as_ptr().cast(), + DNS_QUERY.len(), + 0, + sockaddr.as_ptr().cast(), + sockaddr_length, + ) + }; + if sent != isize::try_from(DNS_QUERY.len()).expect("DNS query length fits isize") { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + + let mut response = [0_u8; 512]; + let mut source = [0_u8; PROBE_SOCKADDR_IN_LEN]; + let mut source_length = + libc::socklen_t::try_from(source.len()).expect("sockaddr length fits socklen_t"); + // SAFETY: all output buffers are live for their declared lengths. + let received = unsafe { + libc::recvfrom( + socket.as_raw_fd(), + response.as_mut_ptr().cast(), + response.len(), + 0, + source.as_mut_ptr().cast(), + std::ptr::addr_of_mut!(source_length), + ) + }; + if received < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + let source = decode_probe_sockaddr(&source)?; + if source != relay { + return Err(miette::miette!( + "DNS response source mismatch: expected {relay}, got {source}" + )); + } + let received = usize::try_from(received).expect("positive recv length fits usize"); + if received < 16 + || response[0..2] != DNS_QUERY[0..2] + || response[2] & 0x80 == 0 + || response[received - 4..received] != [203, 0, 113, 7] + { + return Err(miette::miette!("DNS probe response is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn probe_tcp_dns_socket_round_trip() -> Result<()> { + use std::io::{Read as _, Write as _}; + + const DNS_QUERY: &[u8] = + b"\x56\x78\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x05probe\x09openshell\x04test\x00\x00\x01\x00\x01"; + let mut stream = std::net::TcpStream::connect("127.0.0.53:53").into_diagnostic()?; + stream.set_nodelay(true).into_diagnostic()?; + stream + .write_all( + &u16::try_from(DNS_QUERY.len()) + .expect("probe DNS query fits u16") + .to_be_bytes(), + ) + .into_diagnostic()?; + stream.write_all(DNS_QUERY).into_diagnostic()?; + let mut length = [0_u8; 2]; + stream.read_exact(&mut length).into_diagnostic()?; + let length = usize::from(u16::from_be_bytes(length)); + if length == 0 || length > 512 { + return Err(miette::miette!("TCP DNS probe response length is invalid")); + } + let mut response = vec![0_u8; length]; + stream.read_exact(&mut response).into_diagnostic()?; + if length < 16 + || response[0..2] != DNS_QUERY[0..2] + || response[2] & 0x80 == 0 + || response[length - 4..] != [203, 0, 113, 7] + { + return Err(miette::miette!("TCP DNS probe response is malformed")); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_tcp_denial() -> Result<()> { + use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd}; + + let peer = PROBE_DENIED_TCP_PEER + .parse::() + .expect("fixed denied peer is valid"); + let (sockaddr, length) = encode_probe_sockaddr(peer)?; + // SAFETY: socket creation is intercepted and returns one injected FD. + let socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_STREAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_TCP, + ) + }; + if socket < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + // SAFETY: successful socket returned one newly owned descriptor. + let socket = unsafe { OwnedFd::from_raw_fd(socket) }; + // SAFETY: both the injected FD and encoded sockaddr are live. + let result = unsafe { libc::connect(socket.as_raw_fd(), sockaddr.as_ptr().cast(), length) }; + require_probe_errno( + isize::try_from(result).expect("connect result fits isize"), + libc::EACCES, + "denied TCP connect", + ) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn probe_child_self_protection(sandbox_tgid: libc::pid_t, socket: libc::c_int) -> Result<()> { + // SAFETY: PR_GET_DUMPABLE reads one scalar property. Normal exec of the + // trusted child image must make it observable to the same-UID sandbox. + if unsafe { libc::prctl(libc::PR_GET_DUMPABLE, 0, 0, 0, 0) } != 1 { + return Err(miette::miette!("workload child is not dumpable after exec")); + } + let mut core_limit = libc::rlimit { + rlim_cur: libc::rlim_t::MAX, + rlim_max: libc::rlim_t::MAX, + }; + // SAFETY: `core_limit` is writable storage for the current limit. + if unsafe { libc::getrlimit(libc::RLIMIT_CORE, &raw mut core_limit) } < 0 { + return Err(std::io::Error::last_os_error()).into_diagnostic(); + } + if core_limit.rlim_cur != 0 || core_limit.rlim_max != 0 { + return Err(miette::miette!("workload child core limit is not zero")); + } + let mut local = 0_u8; + let remote = 0_u8; + let local_iov = libc::iovec { + iov_base: std::ptr::addr_of_mut!(local).cast(), + iov_len: 1, + }; + let remote_iov = libc::iovec { + iov_base: std::ptr::addr_of!(remote).cast_mut().cast(), + iov_len: 1, + }; + // SAFETY: live one-byte iovecs are supplied. The child filter must reject + // the operation before the kernel inspects the remote pointer. + let read = unsafe { + libc::process_vm_readv( + sandbox_tgid, + &raw const local_iov, + 1, + &raw const remote_iov, + 1, + 0, + ) + }; + require_probe_errno(read, libc::EPERM, "process_vm_readv sandbox")?; + // SAFETY: signal zero would only probe process existence if the filter did + // not reject the trusted sandbox target. + require_probe_errno( + isize::try_from(unsafe { libc::kill(sandbox_tgid, 0) }).expect("kill result fits isize"), + libc::EPERM, + "kill sandbox", + )?; + // SAFETY: scalar syscall arguments request a read-only resource query; + // the child filter rejects non-self targets. + require_probe_errno( + isize::try_from(unsafe { + libc::syscall(libc::SYS_prlimit64, sandbox_tgid, libc::RLIMIT_CORE, 0, 0) + }) + .expect("prlimit result fits isize"), + libc::EPERM, + "prlimit sandbox", + )?; + require_probe_errno( + isize::try_from(unsafe { libc::kill(-sandbox_tgid, 0) }).expect("kill result fits isize"), + libc::EPERM, + "process-group signal", + )?; + require_probe_errno( + isize::try_from(unsafe { libc::fcntl(socket, libc::F_SETOWN, sandbox_tgid) }) + .expect("fcntl result fits isize"), + libc::EPERM, + "fcntl F_SETOWN", + )?; + let mut owner = sandbox_tgid; + require_probe_errno( + isize::try_from(unsafe { + libc::syscall(libc::SYS_ioctl, socket, 0x8901_u32, &raw mut owner) + }) + .expect("ioctl result fits isize"), + libc::EPERM, + "ioctl FIOSETOWN", + )?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn require_probe_errno(result: isize, expected: i32, operation: &str) -> Result<()> { + if result == -1 && std::io::Error::last_os_error().raw_os_error() == Some(expected) { + Ok(()) + } else { + Err(miette::miette!( + "{operation} was not rejected with errno {expected}" + )) + } +} + +/// Exercise the trusted VM bootstrap transition before running the Phase 0 +/// capability probe. This command is intentionally hidden: it exists so the +/// VM conformance lane can prove that a privileged guest init can hand off to +/// a non-root, capability-free sandbox without relying on a shell utility. +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn enter_capability_free_identity(uid: u32, gid: u32) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; + + #[repr(C)] + struct CapabilityHeader { + version: u32, + pid: i32, + } + #[repr(C)] + #[derive(Clone, Copy)] + struct CapabilityData { + effective: u32, + permitted: u32, + inheritable: u32, + } + + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "capability-free launch requires non-root UID and GID" + )); + } + if nix::unistd::geteuid().as_raw() != 0 { + return Err(miette::miette!("capability-free launch must start as root")); + } + + let cap_last_cap = std::fs::read_to_string("/proc/sys/kernel/cap_last_cap") + .into_diagnostic() + .wrap_err("read cap_last_cap")? + .trim() + .parse::() + .into_diagnostic() + .wrap_err("parse cap_last_cap")?; + for capability in 0..=cap_last_cap { + // SAFETY: PR_CAPBSET_DROP only removes one capability from the current + // process' bounding set. The loop runs while guest init still has the + // authority required to perform the transition. + if unsafe { libc::prctl(libc::PR_CAPBSET_DROP, capability, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "drop capability {capability} from bounding set: {}", + std::io::Error::last_os_error() + )); + } + } - /// Comma-separated `NO_PROXY` list for the corporate proxy. - #[arg(long)] - upstream_no_proxy: Option, + // SAFETY: the process is single-threaded at this pre-clap bootstrap path; + // the null pointer is valid for a zero-length supplementary group list. + if unsafe { libc::setgroups(0, std::ptr::null()) } < 0 { + return Err(miette::miette!( + "clear supplementary groups: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: scalar credential transition to the operator-selected guest + // identity. All saved IDs are changed so the process cannot regain root. + if unsafe { libc::setresgid(gid, gid, gid) } < 0 { + return Err(miette::miette!( + "set guest GID {gid}: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: see setresgid above. + if unsafe { libc::setresuid(uid, uid, uid) } < 0 { + return Err(miette::miette!( + "set guest UID {uid}: {}", + std::io::Error::last_os_error() + )); + } - /// Path to the root-only file holding corporate proxy credentials (`user:pass`). - #[arg(long)] - upstream_proxy_auth_file: Option, + let mut header = CapabilityHeader { + version: 0x2008_0522, + pid: 0, + }; + let data = [CapabilityData { + effective: 0, + permitted: 0, + inheritable: 0, + }; 2]; + // SAFETY: capset reads the fixed-size header and two zeroed V3 data words. + if unsafe { libc::syscall(libc::SYS_capset, &raw mut header, data.as_ptr()) } < 0 { + return Err(miette::miette!( + "clear process capability sets: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: clears any ambient capabilities, then permanently forbids + // privilege gain across the following exec. + if unsafe { + libc::prctl( + libc::PR_CAP_AMBIENT, + libc::PR_CAP_AMBIENT_CLEAR_ALL, + 0, + 0, + 0, + ) + } < 0 + { + return Err(miette::miette!( + "clear ambient capabilities: {}", + std::io::Error::last_os_error() + )); + } + // SAFETY: PR_SET_NO_NEW_PRIVS is a one-way process hardening transition. + if unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) } < 0 { + return Err(miette::miette!( + "set no_new_privs: {}", + std::io::Error::last_os_error() + )); + } - /// Acknowledge that proxy credentials travel as cleartext Basic auth over - /// the plain-TCP connection to the `http://` proxy. - #[arg(long)] - upstream_proxy_auth_allow_insecure: bool, + Ok(()) +} - /// Send the destination hostname in CONNECT instead of a validated IP - /// (for proxies whose ACLs filter on hostnames). - #[arg(long)] - upstream_proxy_connect_by_hostname: bool, +#[cfg(target_os = "linux")] +fn launch_capability_probe(args: &[String]) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; - /// Path to a PEM CA bundle trusted for the corporate proxy: the TLS - /// handshake with an `https://` proxy and, for TLS-intercepting proxies, - /// re-signed upstream certificates and the sandbox trust bundle. - #[arg(long)] - upstream_proxy_ca_bundle: Option, + let [uid, gid] = args else { + return Err(miette::miette!( + "usage: openshell-sandbox {CAPABILITY_PROBE_LAUNCH_SUBCOMMAND} " + )); + }; + let uid = uid.parse::().into_diagnostic().wrap_err("parse UID")?; + let gid = gid.parse::().into_diagnostic().wrap_err("parse GID")?; + enter_capability_free_identity(uid, gid)?; + run_capability_probe() } -/// Internal one-shot command used by the privileged supervisor to validate an -/// image-provided workdir as the final sandbox identity. -#[derive(Parser, Debug)] -#[command(name = "validate-workspace", hide = true)] -struct ValidateWorkspaceArgs { - #[arg(long)] - workdir: String, - #[arg(long)] - expected_uid: u32, - #[arg(long)] - expected_gid: u32, +#[cfg(not(target_os = "linux"))] +fn launch_capability_probe(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "capability probe launch is supported only on Linux" + )) } #[cfg(target_os = "linux")] -fn validate_workspace(args: &[String]) -> Result<()> { - let args = ValidateWorkspaceArgs::try_parse_from( - std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), - ) - .into_diagnostic()?; - let actual = ( - nix::unistd::geteuid().as_raw(), - nix::unistd::getegid().as_raw(), - ); - if actual != (args.expected_uid, args.expected_gid) { +fn launch_capability_free(args: &[String]) -> Result<()> { + use miette::{Context as _, IntoDiagnostic as _}; + + let [uid, gid, bootstrap] = args else { return Err(miette::miette!( - "workspace validator privilege drop failed: expected {}:{}, got {}:{}", - args.expected_uid, - args.expected_gid, - actual.0, - actual.1 + "usage: openshell-sandbox {CAPABILITY_FREE_LAUNCH_SUBCOMMAND} " )); - } - openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new( - &args.workdir, + }; + let uid = uid.parse::().into_diagnostic().wrap_err("parse UID")?; + let gid = gid.parse::().into_diagnostic().wrap_err("parse GID")?; + enter_capability_free_identity(uid, gid)?; + let log_level = std::env::var(openshell_core::sandbox_env::LOG_LEVEL) + .unwrap_or_else(|_| "warn".to_string()); + run_boundary(Path::new(bootstrap), &log_level) +} + +#[cfg(not(target_os = "linux"))] +fn launch_capability_free(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "capability-free launch is only supported on Linux" )) } #[cfg(not(target_os = "linux"))] -fn validate_workspace(_args: &[String]) -> Result<()> { +fn run_capability_probe() -> Result<()> { Err(miette::miette!( - "workspace validation is only supported on Unix" + "capability-free sandbox probe is supported only on Linux" )) } +fn proc_status_hex(status: &str, field: &str) -> Result { + let value = status + .lines() + .find_map(|line| line.strip_prefix(&format!("{field}:"))) + .map(str::trim) + .ok_or_else(|| miette::miette!("/proc/self/status is missing {field}"))?; + u64::from_str_radix(value, 16) + .map_err(|error| miette::miette!("invalid {field} value {value:?}: {error}")) +} + /// Copy the running executable to `dest`, creating parent directories as /// needed and ensuring the result is executable (mode `0755`). /// @@ -323,431 +1545,398 @@ fn copy_self(dest: &str) -> Result<()> { Ok(()) } +/// Stage the immutable Kubernetes bootstrap Secret into private writable +/// memory-backed volumes. The projected Secret remains mounted only in this +/// trusted init container; the long-lived sandbox consumes and unlinks the +/// staged configuration before it starts workload code. +fn stage_kubernetes_bootstrap() -> Result<()> { + stage_kubernetes_bootstrap_at( + Path::new(BOOTSTRAP_INPUT_ROOT), + Path::new(SANDBOX_RUNTIME_ROOT), + Path::new(SANDBOX_STATE_ROOT), + ) +} + +/// Stage protected state without performing a duplicate runtime probe. The +/// long-lived sandbox actively qualifies its own exact admitted profile before +/// consuming this material. #[cfg(target_os = "linux")] -fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; +fn run_kubernetes_bootstrap() -> Result<()> { + stage_kubernetes_bootstrap() +} - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {uid}:{gid}", - path.display() - ) - })?; - Ok(()) +#[cfg(not(target_os = "linux"))] +fn run_kubernetes_bootstrap() -> Result<()> { + Err(miette::miette!( + "Kubernetes sandbox bootstrap requires Linux" + )) } -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; +fn stage_kubernetes_bootstrap_at(source: &Path, runtime: &Path, state: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::os::unix::fs::PermissionsExt as _; - let uid = Uid::current(); - let gid = Gid::current(); - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - chown(path, Some(uid), Some(gid)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {}:{}", - path.display(), - uid.as_raw(), - gid.as_raw() - ) - })?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid == 0 || gid == 0 { + return Err(miette::miette!( + "Kubernetes bootstrap staging requires a non-root UID and GID, got {uid}:{gid}" + )); + } + + fs::create_dir_all(runtime).into_diagnostic()?; + fs::create_dir_all(state).into_diagnostic()?; + + let nonce = format!( + "{}.{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .into_diagnostic()? + .as_nanos() + ); + let runtime_tmp = runtime.join(format!(".openshell-sandbox.{nonce}")); + let runtime_final = runtime.join("openshell-sandbox"); + let executable = std::env::current_exe().into_diagnostic()?; + copy_regular_file(&executable, &runtime_tmp, 0o500)?; + fs::rename(&runtime_tmp, &runtime_final).into_diagnostic()?; + + let bundle_tmp = state.join(format!(".bootstrap.{nonce}")); + let bundle_final = state.join("bootstrap"); + fs::create_dir(&bundle_tmp).into_diagnostic()?; + fs::set_permissions(&bundle_tmp, fs::Permissions::from_mode(0o700)).into_diagnostic()?; + for name in ["boundary.json", "tls.crt", "tls.key", "client-ca.crt"] { + copy_projected_secret_file(source, name, &bundle_tmp.join(name), 0o600)?; + } + fs::rename(&bundle_tmp, &bundle_final).into_diagnostic()?; + + // Flush the two directory entries before the init container exits. Both + // targets are tmpfs in production, but keeping the staging operation + // durable also makes the helper safe in local conformance tests. + OpenOptions::new() + .read(true) + .open(runtime) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; + OpenOptions::new() + .read(true) + .open(state) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; Ok(()) } -#[cfg(target_os = "linux")] -fn copy_sidecar_client_tls_if_present( - source_dir: &Path, - sidecar_tls_dir: &Path, - uid: u32, - gid: u32, +fn copy_projected_secret_file( + source_root: &Path, + name: &str, + destination: &Path, + mode: u32, ) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; + let canonical_root = std::fs::canonicalize(source_root).into_diagnostic()?; + let canonical_source = std::fs::canonicalize(source_root.join(name)).into_diagnostic()?; + if !canonical_source.starts_with(&canonical_root) { + return Err(miette::miette!( + "projected bootstrap input escapes its mounted Secret: {}", + source_root.join(name).display() + )); + } + copy_regular_file(&canonical_source, destination, mode) +} - if !source_dir.exists() { - return Ok(()); +fn copy_regular_file(source: &Path, destination: &Path, mode: u32) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::OpenOptionsExt as _; + + let metadata = fs::symlink_metadata(source).into_diagnostic()?; + if !metadata.file_type().is_file() || metadata.len() == 0 { + return Err(miette::miette!( + "bootstrap input must be a non-empty regular file: {}", + source.display() + )); + } + let mut input = OpenOptions::new() + .read(true) + .open(source) + .into_diagnostic()?; + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(mode) + .open(destination) + .into_diagnostic()?; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let length = input.read(&mut buffer).into_diagnostic()?; + if length == 0 { + break; + } + output.write_all(&buffer[..length]).into_diagnostic()?; } + output.sync_all().into_diagnostic()?; + Ok(()) +} + +/// Seed the persistent workspace from the agent image as the final workload +/// identity. This replaces the former root shell/tar init container. +fn seed_kubernetes_workspace() -> Result<()> { + seed_kubernetes_workspace_at(Path::new("/sandbox"), Path::new("/mnt/openshell-workspace")) +} - let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; - for file_name in CLIENT_TLS_FILES { - let source = source_dir.join(file_name); - if !source.exists() { +fn copy_workspace_tree(source: &Path, destination: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _, symlink}; + + for entry in fs::read_dir(source).into_diagnostic()? { + let entry = entry.into_diagnostic()?; + let source_path = entry.path(); + let destination_path = destination.join(entry.file_name()); + let metadata = fs::symlink_metadata(&source_path).into_diagnostic()?; + if metadata.file_type().is_symlink() { + symlink( + fs::read_link(&source_path).into_diagnostic()?, + &destination_path, + ) + .into_diagnostic()?; + } else if metadata.is_dir() { + fs::create_dir(&destination_path).into_diagnostic()?; + fs::set_permissions(&destination_path, fs::Permissions::from_mode(0o700)) + .into_diagnostic()?; + copy_workspace_tree(&source_path, &destination_path)?; + } else if metadata.is_file() { + let mut input = OpenOptions::new() + .read(true) + .open(&source_path) + .into_diagnostic()?; + let mode = 0o600 | (metadata.permissions().mode() & 0o100); + let mut output = OpenOptions::new() + .write(true) + .create_new(true) + .mode(mode) + .open(&destination_path) + .into_diagnostic()?; + let mut buffer = [0_u8; 16 * 1024]; + loop { + let length = input.read(&mut buffer).into_diagnostic()?; + if length == 0 { + break; + } + output.write_all(&buffer[..length]).into_diagnostic()?; + } + output.sync_all().into_diagnostic()?; + } else { return Err(miette::miette!( - "client TLS source file is missing: {}", - source.display() + "workspace seed contains unsupported file type: {}", + source_path.display() )); } - let dest = dest_dir.join(file_name); - if dest.exists() { - std::fs::remove_file(&dest) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to remove stale client TLS file {}", dest.display()) - })?; - } - std::fs::copy(&source, &dest) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to copy client TLS file {} to {}", - source.display(), - dest.display() - ) - })?; - let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); - std::fs::set_permissions(&dest, perms) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to chmod copied client TLS file {}", dest.display()) - })?; - chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown copied client TLS file {} to {uid}:{gid}", - dest.display() - ) - })?; } - - prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; - Ok(()) } -#[cfg(target_os = "linux")] -fn run_network_init( - proxy_user_id: u32, - proxy_primary_group_id: u32, - sidecar_state_dir: &str, - sidecar_tls_dir: &str, -) -> Result<()> { - validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; - - let sidecar_state_dir = Path::new(sidecar_state_dir); - let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory( - sidecar_state_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_STATE_DIR_MODE, - )?; - // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the - // TLS work directory owned by the init user until the client cert copy is - // complete, then hand it to the long-running proxy UID. - prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - )?; - prepare_sidecar_directory( - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_TLS_DIR_MODE, - )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) -} +fn seed_kubernetes_workspace_at(source: &Path, destination: &Path) -> Result<()> { + use std::fs::{self, OpenOptions}; + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; -#[cfg(target_os = "linux")] -fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 - && !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_user_id) - { - return Err(miette::miette!( - "--proxy-uid must be 0 or in range [{}, {}]", - openshell_policy::MIN_SANDBOX_PROXY_UID, - openshell_policy::MAX_SANDBOX_UID, - )); + let sentinel = destination.join(".openshell-initialized"); + if sentinel.try_exists().into_diagnostic()? { + return Ok(()); } - if !(openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_primary_group_id) - { + let destination_metadata = fs::symlink_metadata(destination).into_diagnostic()?; + if !destination_metadata.is_dir() || destination_metadata.file_type().is_symlink() { return Err(miette::miette!( - "--proxy-gid must be in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, + "workspace target must be a real directory: {}", + destination.display() )); } + + if source.try_exists().into_diagnostic()? { + let metadata = fs::symlink_metadata(source).into_diagnostic()?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(miette::miette!( + "image workspace must be a real directory: {}", + source.display() + )); + } + copy_workspace_tree(source, destination)?; + } + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&sentinel) + .into_diagnostic()?; + file.write_all(b"initialized\n").into_diagnostic()?; + file.sync_all().into_diagnostic()?; + OpenOptions::new() + .read(true) + .open(destination) + .into_diagnostic()? + .sync_all() + .into_diagnostic()?; Ok(()) } +#[cfg(target_os = "linux")] +fn run_boundary(bootstrap: &Path, log_level: &str) -> Result<()> { + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(log_level)); + let _ = tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .try_init(); + let (qualification, _) = qualify_runtime()?; + openshell_sandbox::run(bootstrap, qualification) +} + #[cfg(not(target_os = "linux"))] -fn run_network_init( - _proxy_uid: u32, - _proxy_gid: u32, - _sidecar_state_dir: &str, - _sidecar_tls_dir: &str, -) -> Result<()> { - Err(miette::miette!( - "--mode=network-init is only supported on Linux" - )) +fn run_boundary(_bootstrap: &Path, _log_level: &str) -> Result<()> { + Err(miette::miette!("openshell-sandbox requires Linux")) } fn main() -> Result<()> { - // Handle `copy-self ` before clap so it works without any of the - // sandbox flags. Kubernetes init containers invoke this path to seed an - // emptyDir volume that the agent container then executes from. - let raw_args: Vec = std::env::args().collect(); + let raw_args = std::env::args().collect::>(); if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { let dest = raw_args.get(2).ok_or_else(|| { miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") })?; return copy_self(dest); } - - // Handle `debug-rpc [args]` before clap. Uses a small - // dedicated runtime so we don't pay the supervisor's full startup cost. - if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .into_diagnostic()?; - return runtime.block_on(async move { - let _ = rustls::crypto::ring::default_provider().install_default(); - let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; - std::process::exit(exit); - }); + if raw_args.get(1).map(String::as_str) == Some(BOOTSTRAP_SUBCOMMAND) { + if raw_args.len() != 2 { + return Err(miette::miette!( + "usage: openshell-sandbox {BOOTSTRAP_SUBCOMMAND}" + )); + } + return run_kubernetes_bootstrap(); + } + if raw_args.get(1).map(String::as_str) == Some(SEED_WORKSPACE_SUBCOMMAND) { + if raw_args.len() != 2 { + return Err(miette::miette!( + "usage: openshell-sandbox {SEED_WORKSPACE_SUBCOMMAND}" + )); + } + return seed_kubernetes_workspace(); } if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { return validate_workspace(&raw_args[2..]); } - - let args = Args::parse(); - - if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); - return run_network_init( - args.proxy_uid, - proxy_gid, - &args.sidecar_state_dir, - &args.sidecar_tls_dir, - ); + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_PROBE_SUBCOMMAND) { + return run_capability_probe(); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_PROBE_LAUNCH_SUBCOMMAND) { + return launch_capability_probe(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_SOCKET_CHILD_SUBCOMMAND) { + return run_capability_socket_child(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_LANDLOCK_CHILD_SUBCOMMAND) { + return run_capability_landlock_child(&raw_args[2..]); + } + if raw_args.get(1).map(String::as_str) == Some(CAPABILITY_FREE_LAUNCH_SUBCOMMAND) { + return launch_capability_free(&raw_args[2..]); } - // Try to open a rolling log file; fall back to stderr-only logging if it fails - // (e.g., /var/log is not writable in custom workload images). - // Rotates daily, keeps the 3 most recent files to bound disk usage. - let file_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - (writer, guard) - }); - - let console_filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .into_diagnostic()?; + let args = BoundaryArgs::parse(); + run_boundary(&args.bootstrap, &args.log_level) +} - let exit_code = runtime.block_on(async move { - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), - sandbox_id.clone(), + #[test] + fn kubernetes_bootstrap_stages_private_memory_bundle() { + if nix::unistd::geteuid().is_root() || nix::unistd::getegid().as_raw() == 0 { + return; + } + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("input"); + let runtime = root.path().join("runtime"); + let state = root.path().join("state"); + std::fs::create_dir(&source).unwrap(); + let revision = source.join("..2026_09_04_00_00_00"); + std::fs::create_dir(&revision).unwrap(); + for (name, contents) in [ + ("boundary.json", b"{}".as_slice()), + ("tls.crt", b"certificate".as_slice()), + ("tls.key", b"private-key".as_slice()), + ("client-ca.crt", b"client-ca".as_slice()), + ] { + std::fs::write(revision.join(name), contents).unwrap(); + std::os::unix::fs::symlink(format!("..data/{name}"), source.join(name)).unwrap(); + } + std::os::unix::fs::symlink(revision.file_name().unwrap(), source.join("..data")).unwrap(); + + stage_kubernetes_bootstrap_at(&source, &runtime, &state).unwrap(); + + assert!(runtime.join("openshell-sandbox").is_file()); + assert_eq!( + std::fs::metadata(runtime.join("openshell-sandbox")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o500 + ); + for name in ["boundary.json", "tls.crt", "tls.key", "client-ca.crt"] { + let staged = state.join("bootstrap").join(name); + assert_eq!( + std::fs::read(&staged).unwrap(), + std::fs::read(source.join(name)).unwrap() ); - let layer = - openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - let ocsf_schema_version = Arc::new(std::sync::Mutex::new(String::new())); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer) - .with_enabled_flag(ocsf_enabled.clone()) - .with_target_version(ocsf_schema_version.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), - }; - - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stderr-only logging"); - (None, None) - }; - - // Resolve an exact canonical process. Explicit offline/test argv wins; - // drivers otherwise provide a versioned JSON transport so argument - // boundaries are never reconstructed with shell parsing. - let workdir = args.workdir.clone(); - let (command, interactive, await_main_process_attachment) = if !args.command.is_empty() { - (args.command, args.interactive, false) - } else if let Ok(json) = std::env::var(openshell_core::sandbox_env::MAIN_PROCESS_SPEC) { - let config = openshell_core::sandbox_env::MainProcessConfig::decode(&json) - .map_err(|error| miette::miette!("{error}"))?; - ( - config.command, - config.tty, - config.await_main_process_attachment, - ) - } else { - let config = openshell_core::sandbox_env::MainProcessConfig::scratch(); - ( - config.command, - config.tty, - config.await_main_process_attachment, - ) - }; - - // An omitted command (the gateway leaves the default empty rather than - // baking a shell it cannot verify) is resolved to a login shell here, in - // the supervisor, so it matches the sandbox image: bash when present, - // otherwise /bin/sh (e.g. Alpine). An explicit command is used verbatim. - let command = resolve_default_command(command); - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { - https_proxy: args.upstream_proxy, - no_proxy: args.upstream_no_proxy, - proxy_auth_file: args.upstream_proxy_auth_file, - proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - proxy_ca_bundle: args.upstream_proxy_ca_bundle, - }; + assert_eq!( + std::fs::metadata(staged).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } - run_sandbox( - command, - workdir, - args.timeout, - interactive, - await_main_process_attachment, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_socket_path, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - ocsf_schema_version, - args.mode.network, - args.mode.process, - upstream_proxy_args, - ) - .await - })?; + #[test] + fn kubernetes_workspace_seed_preserves_files_and_symlinks_without_root() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + let destination = root.path().join("destination"); + std::fs::create_dir_all(source.join("bin")).unwrap(); + std::fs::create_dir(&destination).unwrap(); + std::fs::write(source.join("README"), b"workspace").unwrap(); + std::fs::write(source.join("bin/tool"), b"tool").unwrap(); + let mut executable = std::fs::metadata(source.join("bin/tool")) + .unwrap() + .permissions(); + executable.set_mode(0o755); + std::fs::set_permissions(source.join("bin/tool"), executable).unwrap(); + std::os::unix::fs::symlink("README", source.join("latest")).unwrap(); - std::process::exit(exit_code); -} + seed_kubernetes_workspace_at(&source, &destination).unwrap(); + seed_kubernetes_workspace_at(&source, &destination).unwrap(); -/// Resolve an omitted canonical command to a login shell that exists in this -/// sandbox image. Empty means "use the default": the gateway leaves an omitted -/// command empty rather than persisting a shell it cannot verify, so the -/// supervisor picks one here against the real sandbox filesystem (bash when -/// present, otherwise `/bin/sh`). An explicit command is returned unchanged. -fn resolve_default_command(command: Vec) -> Vec { - if !command.is_empty() { - return command; + assert_eq!( + std::fs::read(destination.join("README")).unwrap(), + b"workspace" + ); + assert_eq!( + std::fs::read_link(destination.join("latest")).unwrap(), + Path::new("README") + ); + assert_ne!( + std::fs::metadata(destination.join("bin/tool")) + .unwrap() + .permissions() + .mode() + & 0o100, + 0 + ); + assert!(destination.join(".openshell-initialized").is_file()); } - let shell = openshell_core::shell::detect_login_shell(); - info!(shell = %shell, "no command specified; resolved default login shell"); - vec![shell, "-l".to_string()] -} - -#[cfg(test)] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; #[cfg(target_os = "linux")] #[test] @@ -826,56 +2015,4 @@ mod tests { let final_path = dest_dir.join("openshell-sandbox"); assert!(final_path.exists(), "binary should land inside dest dir"); } - - #[test] - fn mode_parses_network_init_standalone() { - let mode = "network-init".parse::().unwrap(); - assert!(mode.network_init); - assert!(!mode.network); - assert!(!mode.process); - } - - #[test] - fn mode_rejects_combined_network_init() { - let err = "network-init,network".parse::().unwrap_err(); - assert!(err.contains("cannot be combined")); - } - - #[test] - fn mode_rejects_empty_value() { - let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); - } - - #[cfg(target_os = "linux")] - #[test] - fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { - assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); - assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); - assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); - assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, 30).unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_still_rejects_low_non_root_proxy_uid_and_root_gid() { - let uid_err = - validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); - assert!(uid_err.to_string().contains("--proxy-uid")); - - let gid_err = validate_network_init_ids(0, 0).unwrap_err(); - assert!(gid_err.to_string().contains("--proxy-gid")); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_non_root_system_proxy_group() { - validate_network_init_ids(openshell_policy::MIN_SANDBOX_PROXY_UID, 30).unwrap(); - } } diff --git a/crates/openshell-sandbox/src/main_session.rs b/crates/openshell-sandbox/src/main_session.rs new file mode 100644 index 0000000000..898cfeaed3 --- /dev/null +++ b/crates/openshell-sandbox/src/main_session.rs @@ -0,0 +1,1067 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Retained I/O multiplexer for the canonical sandbox process. + +use std::collections::VecDeque; +use std::io::{Read, Write}; +use std::os::fd::AsRawFd; +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use nix::fcntl::{FcntlArg, OFlag, fcntl}; +use nix::pty::Winsize; +use tokio::io::unix::AsyncFd; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Notify; +use tokio::sync::watch; + +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; + +use crate::process::ProcessIo; + +const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Debug)] +pub enum MainOutput { + Stdout(Bytes), + Stderr(Bytes), + Exit(i32), +} + +impl MainOutput { + fn len(&self) -> usize { + match self { + Self::Stdout(data) | Self::Stderr(data) => data.len(), + Self::Exit(_) => 0, + } + } +} + +#[derive(Clone, Debug)] +struct SequencedOutput { + sequence: u64, + event: MainOutput, +} + +#[derive(Debug)] +struct OutputLogState { + events: VecDeque, + retained_bytes: usize, + next_sequence: u64, +} + +#[derive(Debug)] +struct OutputLog { + state: Mutex, + version: watch::Sender, + terminal_reported: AtomicBool, + terminal_reported_notify: Notify, +} + +impl OutputLog { + fn new() -> Arc { + let (version, _) = watch::channel(0); + Arc::new(Self { + state: Mutex::new(OutputLogState { + events: VecDeque::new(), + retained_bytes: 0, + next_sequence: 0, + }), + version, + terminal_reported: AtomicBool::new(false), + terminal_reported_notify: Notify::new(), + }) + } + + fn publish(&self, event: MainOutput) { + let version = { + let mut state = self.state.lock().expect("main output log lock poisoned"); + let sequence = state.next_sequence; + state.next_sequence = state + .next_sequence + .checked_add(1) + .expect("main output sequence exhausted"); + state.retained_bytes += event.len(); + state.events.push_back(SequencedOutput { sequence, event }); + while state.retained_bytes > OUTPUT_BUFFER_BYTES { + let Some(removed) = state.events.pop_front() else { + break; + }; + state.retained_bytes = state.retained_bytes.saturating_sub(removed.event.len()); + } + state.next_sequence + }; + self.version.send_replace(version); + } + + fn subscribe(self: &Arc) -> MainOutputCursor { + let version = self.version.subscribe(); + let state = self.state.lock().expect("main output log lock poisoned"); + let next_sequence = state + .events + .front() + .map_or(state.next_sequence, |retained| retained.sequence); + drop(state); + MainOutputCursor { + output: Arc::clone(self), + next_sequence, + version, + } + } +} + +#[derive(Debug)] +struct TerminalAttachmentState { + active: usize, + process_finished: bool, + expectation: AttachmentExpectation, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AttachmentExpectation { + None, + Pending, + Satisfied, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct MainOutputLagged { + pub skipped: u64, +} + +pub struct MainOutputCursor { + output: Arc, + next_sequence: u64, + version: watch::Receiver, +} + +impl MainOutputCursor { + pub async fn recv(&mut self) -> Result { + loop { + let next = { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let oldest = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + if self.next_sequence < oldest { + let skipped = oldest - self.next_sequence; + self.next_sequence = oldest; + return Err(MainOutputLagged { skipped }); + } + if self.next_sequence >= state.next_sequence { + None + } else { + let offset = usize::try_from(self.next_sequence - oldest) + .expect("main output cursor offset exceeds usize"); + let event = state + .events + .get(offset) + .expect("main output cursor references retained event") + .event + .clone(); + self.next_sequence += 1; + Some(event) + } + }; + if let Some(event) = next { + return Ok(event); + } + // The log owns a sender for the cursor lifetime, so closure is not + // expected. A changed version means there is another event to read. + let _ = self.version.changed().await; + } + } +} + +enum MainInput { + Data(Vec), + Close, +} + +#[derive(Clone)] +pub(crate) struct MainInputSender { + sender: tokio::sync::mpsc::Sender, +} + +impl MainInputSender { + pub(crate) async fn send(&self, data: Vec) -> Result<(), &'static str> { + self.sender + .send(MainInput::Data(data)) + .await + .map_err(|_| "canonical process stdin closed") + } + + async fn close(&self) { + let _ = self.sender.send(MainInput::Close).await; + } +} + +pub struct MainSession { + pid: u32, + terminal: bool, + input: MainInputSender, + output: Arc, + input_owner: Mutex>, + input_closed: AtomicBool, + next_owner: AtomicU64, + pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, + readers_remaining: AtomicUsize, + readers_done: Notify, + finished: AtomicBool, + terminal_attachments: Mutex, + terminal_attachments_done: Notify, +} + +impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + #[cfg(test)] + pub fn inert() -> Arc { + let (input, _input_rx) = tokio::sync::mpsc::channel(64); + Arc::new(Self { + pid: 1, + terminal: false, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: None, + boundary_terminal: None, + readers_remaining: AtomicUsize::new(0), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }) + } + + #[cfg(test)] + pub fn terminal_for_test() -> (Arc, std::fs::File) { + let pty = nix::pty::openpty(None, None).expect("open test PTY"); + let slave = std::fs::File::from(pty.slave); + ( + Self::new(ProcessIo::Pty(std::fs::File::from(pty.master)), 1), + slave, + ) + } + + #[cfg(test)] + #[allow(unsafe_code)] + pub fn terminal_size_for_test(&self) -> (u16, u16) { + let master = self.pty_master.as_ref().expect("terminal PTY master"); + let mut winsize: libc::winsize = unsafe { std::mem::zeroed() }; + let result = unsafe { libc::ioctl(master.as_raw_fd(), libc::TIOCGWINSZ, &mut winsize) }; + assert_eq!(result, 0, "read terminal dimensions"); + (winsize.ws_col, winsize.ws_row) + } + + #[must_use] + pub fn new(io: ProcessIo, pid: u32) -> Arc { + let terminal = matches!(io, ProcessIo::Pty(_)); + let (input, input_rx) = tokio::sync::mpsc::channel(64); + let pty_master = match &io { + ProcessIo::Pty(master) => { + set_nonblocking(master).expect("set canonical PTY master nonblocking"); + master.try_clone().ok().map(Arc::new) + } + ProcessIo::Pipes { .. } => None, + }; + let session = Arc::new(Self { + pid, + terminal, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master, + boundary_process: None, + boundary_terminal: None, + readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + Self::start_io(&session, io, input_rx); + session + } + + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input: MainInputSender { sender: input }, + output: OutputLog::new(), + input_owner: Mutex::new(None), + input_closed: AtomicBool::new(false), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(input) = input_rx.recv().await { + match input { + MainInput::Data(data) => { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + MainInput::Close => break, + } + } + }); + session + } + + fn start_io( + this: &Arc, + io: ProcessIo, + mut input_rx: tokio::sync::mpsc::Receiver, + ) { + match io { + ProcessIo::Pty(master) => { + let master = Arc::new(AsyncFd::new(master).expect("register canonical PTY master")); + let reader = Arc::clone(&master); + let output = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + let Ok(mut ready) = reader.readable().await else { + break; + }; + match ready.try_io(|inner| { + let mut file = inner.get_ref(); + file.read(&mut buffer) + }) { + Ok(Ok(0) | Err(_)) => break, + Ok(Ok(read)) => output.publish(MainOutput::Stdout( + Bytes::copy_from_slice(&buffer[..read]), + )), + Err(_would_block) => {} + } + } + output.reader_finished(); + }); + tokio::spawn(async move { + while let Some(input) = input_rx.recv().await { + let MainInput::Data(data) = input else { + return; + }; + let mut remaining = data.as_slice(); + while !remaining.is_empty() { + let Ok(mut ready) = master.writable().await else { + return; + }; + match ready.try_io(|inner| { + let mut file = inner.get_ref(); + file.write(remaining) + }) { + Ok(Ok(0) | Err(_)) => return, + Ok(Ok(written)) => remaining = &remaining[written..], + Err(_would_block) => {} + } + } + } + }); + } + ProcessIo::Pipes { + mut stdin, + mut stdout, + mut stderr, + } => { + let stdout_session = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => { + stdout_session.publish(MainOutput::Stdout(Bytes::copy_from_slice( + &buffer[..read], + ))); + } + } + } + stdout_session.reader_finished(); + }); + let stderr_session = Arc::clone(this); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => { + stderr_session.publish(MainOutput::Stderr(Bytes::copy_from_slice( + &buffer[..read], + ))); + } + } + } + stderr_session.reader_finished(); + }); + tokio::spawn(async move { + while let Some(input) = input_rx.recv().await { + match input { + MainInput::Data(data) => { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + MainInput::Close => break, + } + } + }); + } + } + } + + fn publish(&self, event: MainOutput) { + self.output.publish(event); + } + + fn reader_finished(&self) { + if self.readers_remaining.fetch_sub(1, Ordering::AcqRel) == 1 { + self.readers_done.notify_waiters(); + } + } + + /// Publish the terminal event and retain the transport only when a real + /// foreground attachment exists or the creating client declared one. + /// + /// Returns whether terminal delivery must complete before shutdown. + pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { + let notified = self.readers_done.notified(); + if self.readers_remaining.load(Ordering::Acquire) != 0 { + notified.await; + } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + let delivery_pending = { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + state.process_finished = true; + state.expectation = if attachment_expected { + if state.active == 0 && state.expectation != AttachmentExpectation::Satisfied { + AttachmentExpectation::Pending + } else { + AttachmentExpectation::Satisfied + } + } else { + AttachmentExpectation::None + }; + attachment_expected || state.active != 0 + }; + self.finished.store(true, Ordering::Release); + self.publish(MainOutput::Exit(exit_code)); + delivery_pending + } + + pub fn subscribe(&self) -> MainOutputCursor { + self.output.subscribe() + } + + /// Return the bounded output sequence range currently retained for a + /// replacement supervisor. A nonzero first sequence is an explicit + /// truncation watermark rather than silent data loss. + #[must_use] + pub fn output_window(&self) -> (u64, u64, bool) { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let first_sequence = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + (first_sequence, state.next_sequence, first_sequence != 0) + } + + /// Wait until the gateway durably acknowledges the main-process result. + pub async fn wait_for_terminal_reported(&self) { + let notified = self.output.terminal_reported_notify.notified(); + if self.output.terminal_reported.load(Ordering::Acquire) { + return; + } + notified.await; + } + + /// Release attached clients to receive their SSH exit status after the + /// durable sandbox phase and exit code have been recorded. + pub fn mark_terminal_reported(&self) { + self.output.terminal_reported.store(true, Ordering::Release); + self.output.terminal_reported_notify.notify_waiters(); + } + + /// Register a foreground main attachment while the process is live. + pub fn begin_terminal_attachment(&self) -> Result<(), &'static str> { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + if state.process_finished && state.expectation != AttachmentExpectation::Pending { + return Err("canonical main process already finished"); + } + state.active = state + .active + .checked_add(1) + .expect("terminal attachment count exhausted"); + state.expectation = AttachmentExpectation::Satisfied; + self.terminal_attachments_done.notify_waiters(); + Ok(()) + } + + /// Release a foreground main attachment after its SSH channel closes. + pub fn end_terminal_attachment(&self) { + let completed = { + let mut state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + debug_assert!(state.active != 0, "terminal attachment count underflow"); + if state.active == 0 { + return; + } + state.active -= 1; + state.active == 0 + }; + if completed { + self.terminal_attachments_done.notify_waiters(); + } + } + + /// Wait for the declared foreground attachment to start, then for every + /// accepted attachment to close naturally. + pub async fn wait_for_terminal_attachments(&self) { + loop { + let notified = self.terminal_attachments_done.notified(); + let complete = { + let state = self + .terminal_attachments + .lock() + .expect("terminal attachment lock poisoned"); + state.active == 0 && state.expectation != AttachmentExpectation::Pending + }; + if complete { + return; + } + notified.await; + } + } + + pub(crate) fn acquire_input(&self) -> Result<(u64, MainInputSender), &'static str> { + if self.input_closed.load(Ordering::Acquire) { + return Err("canonical process stdin closed"); + } + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if owner.is_some() { + return Err("canonical main process already has an input owner"); + } + let id = self.next_owner.fetch_add(1, Ordering::Relaxed); + *owner = Some(id); + Ok((id, self.input.clone())) + } + + /// Acquire canonical input when it remains open. A replacement control may + /// still attach output after a prior control intentionally closed stdin. + pub(crate) fn acquire_input_if_open( + &self, + ) -> Result, &'static str> { + match self.acquire_input() { + Ok(input) => Ok(Some(input)), + Err(_) if self.input_closed.load(Ordering::Acquire) => Ok(None), + Err(error) => Err(error), + } + } + + pub(crate) fn release_input(&self, id: u64) { + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if *owner == Some(id) { + *owner = None; + } + } + + pub(crate) async fn close_input(&self, id: u64) { + let owns_input = { + let mut owner = self.input_owner.lock().expect("main input lock poisoned"); + if *owner == Some(id) { + *owner = None; + true + } else { + false + } + }; + if owns_input && !self.input_closed.swap(true, Ordering::AcqRel) { + self.input.close().await; + } + } + + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } + let Some(master) = self.pty_master.as_ref() else { + return; + }; + let winsize = Winsize { + ws_row: u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ws_col: u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + ws_xpixel: u16::try_from(pixel_width).unwrap_or(u16::MAX), + ws_ypixel: u16::try_from(pixel_height).unwrap_or(u16::MAX), + }; + #[allow(unsafe_code)] + unsafe { + libc::ioctl(master.as_raw_fd(), libc::TIOCSWINSZ, &winsize); + } + } + + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) + } + + #[must_use] + pub const fn terminal(&self) -> bool { + self.terminal + } + + #[must_use] + pub fn finished(&self) -> bool { + self.finished.load(Ordering::Acquire) + } +} + +fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { + let flags = fcntl(file.as_raw_fd(), FcntlArg::F_GETFL)?; + let flags = OFlag::from_bits_truncate(flags); + fcntl( + file.as_raw_fd(), + FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + session.close_input(owner).await; + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + stdin_peer.read(&mut received) + ) + .await + .expect("boundary stdin close timed out") + .expect("read boundary stdin EOF"), + 0 + ); + assert!(session.acquire_input().is_err()); + assert!(session.acquire_input_if_open().unwrap().is_none()); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } + + #[test] + fn input_lease_has_one_owner_and_can_be_reacquired() { + let session = MainSession::inert(); + let (first, _) = session.acquire_input().expect("first owner"); + assert!(session.acquire_input().is_err()); + + session.release_input(first); + let (second, _) = session.acquire_input().expect("replacement owner"); + assert_ne!(first, second); + } + + #[tokio::test] + async fn subscribers_receive_replay_then_live_output() { + let session = MainSession::inert(); + session.publish(MainOutput::Stdout(Bytes::from_static(b"before"))); + + let mut output = session.subscribe(); + assert!(matches!( + output.recv().await.expect("replayed output"), + MainOutput::Stdout(data) if data == b"before"[..] + )); + + session.publish(MainOutput::Stderr(Bytes::from_static(b"after"))); + assert!(matches!( + output.recv().await.expect("live output"), + MainOutput::Stderr(data) if data == b"after"[..] + )); + } + + #[tokio::test] + async fn finish_without_attachment_does_not_defer_shutdown() { + let session = MainSession::inert(); + assert!(!session.finish(0, false).await); + assert!(session.finished()); + assert!(session.begin_terminal_attachment().is_err()); + } + + #[tokio::test] + async fn terminal_report_acknowledgement_is_independent_from_delivery() { + let session = MainSession::inert(); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_reported(), + ) + .await + .is_err(), + "draining output must not imply durable gateway persistence" + ); + + session.mark_terminal_reported(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_reported(), + ) + .await + .expect("durable report acknowledgement should wake waiter"); + } + + #[tokio::test] + async fn finish_waits_for_an_active_attachment_to_close_naturally() { + let session = MainSession::inert(); + session + .begin_terminal_attachment() + .expect("begin terminal attachment"); + assert!(session.finish(0, false).await); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_attachments(), + ) + .await + .is_err(), + "an active attachment must keep terminal delivery open" + ); + + session.end_terminal_attachment(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_attachments(), + ) + .await + .expect("closing the attachment should wake the waiter"); + } + + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + + #[tokio::test] + async fn declared_attachment_waits_for_connection_then_natural_close() { + let session = MainSession::inert(); + assert!(session.finish(0, true).await); + + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(10), + session.wait_for_terminal_attachments(), + ) + .await + .is_err(), + "declared attachment must connect before delivery is complete" + ); + + session + .begin_terminal_attachment() + .expect("declared post-exit attachment"); + session.end_terminal_attachment(); + tokio::time::timeout( + std::time::Duration::from_secs(1), + session.wait_for_terminal_attachments(), + ) + .await + .expect("natural attachment close should complete delivery"); + } + + #[tokio::test] + async fn exit_is_retained_in_the_output_log() { + let session = MainSession::inert(); + let _ = session.finish(0, false).await; + + let mut output = session.subscribe(); + assert!(matches!( + output.recv().await.expect("replayed exit"), + MainOutput::Exit(0) + )); + } + + #[tokio::test] + async fn slow_subscriber_reports_evicted_events_then_resumes() { + let session = MainSession::inert(); + let mut output = session.subscribe(); + let chunk = Bytes::from(vec![0; 4096]); + for _ in 0..=(OUTPUT_BUFFER_BYTES / chunk.len()) { + session.publish(MainOutput::Stdout(chunk.clone())); + } + + let lag = output.recv().await.expect_err("oldest event was evicted"); + assert_eq!(lag.skipped, 1); + assert!(matches!( + output.recv().await.expect("resume at oldest retained event"), + MainOutput::Stdout(data) if data.len() == chunk.len() + )); + } + + #[tokio::test] + async fn terminal_pump_reads_output_and_writes_input() { + let (session, mut slave) = MainSession::terminal_for_test(); + set_nonblocking(&slave).expect("set test PTY slave nonblocking"); + let mut output = session.subscribe(); + + slave + .write_all(b"process output") + .expect("write PTY output"); + let event = tokio::time::timeout(std::time::Duration::from_secs(1), output.recv()) + .await + .expect("PTY output timed out") + .expect("PTY output was retained"); + assert!(matches!( + event, + MainOutput::Stdout(data) if data == b"process output"[..] + )); + + let (owner, input) = session.acquire_input().expect("acquire PTY input"); + input + .send(b"client input\n".to_vec()) + .await + .expect("queue PTY input"); + let mut received = [0; 64]; + let read = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match slave.read(&mut received) { + Ok(read) => break read, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + tokio::task::yield_now().await; + } + Err(error) => panic!("read PTY input: {error}"), + } + } + }) + .await + .expect("PTY input timed out"); + assert_eq!(&received[..read], b"client input\n"); + session.release_input(owner); + } +} diff --git a/crates/openshell-supervisor-process/src/managed_children.rs b/crates/openshell-sandbox/src/managed_children.rs similarity index 100% rename from crates/openshell-supervisor-process/src/managed_children.rs rename to crates/openshell-sandbox/src/managed_children.rs diff --git a/crates/openshell-sandbox/src/metadata_server.rs b/crates/openshell-sandbox/src/metadata_server.rs deleted file mode 100644 index dcfe3e439a..0000000000 --- a/crates/openshell-sandbox/src/metadata_server.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Loopback HTTP server for cloud metadata emulators. -//! -//! Binds a TCP listener inside the sandbox network namespace so that -//! cloud SDKs that bypass `HTTP_PROXY` (e.g. Go's -//! `cloud.google.com/go/compute/metadata`) can reach the emulator via -//! direct TCP. -//! -//! The server is generic over [`MetadataHandler`] — any cloud provider -//! that needs an instance metadata emulator can implement the trait. - -use miette::Result; -use openshell_core::net::set_tcp_nodelay_best_effort; -use std::future::Future; -use std::net::SocketAddr; -use std::sync::Arc; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::net::TcpListener; -use tokio::sync::{Semaphore, oneshot}; -use tracing::{debug, warn}; - -const MAX_REQUEST_BYTES: usize = 4096; -const MAX_CONCURRENT_CONNECTIONS: usize = 32; - -/// Handler for cloud metadata HTTP requests. -/// -/// Implementors receive the parsed HTTP method, path, raw request bytes, -/// and a bidirectional stream to write the response. The handler owns the -/// response format (status, headers, body) — the server only does TCP -/// accept and HTTP request-line parsing. -pub trait MetadataHandler: Send + Sync + 'static { - fn handle( - &self, - method: &str, - path: &str, - request: &[u8], - stream: &mut S, - ) -> impl Future> + Send; -} - -/// Bind a TCP listener inside the sandbox network namespace. -/// -/// Run the metadata server accept loop. -/// -/// Signals `ready_tx` with the bound address before entering the loop. -/// Returns when the listener encounters a fatal error or the runtime shuts down. -pub async fn run( - listener: TcpListener, - handler: H, - ready_tx: oneshot::Sender, -) { - let local_addr = match listener.local_addr() { - Ok(addr) => addr, - Err(e) => { - warn!("metadata server failed to get local address: {e}"); - return; - } - }; - - let _ = ready_tx.send(local_addr); - - let handler = Arc::new(handler); - let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_CONNECTIONS)); - - loop { - let Ok(permit) = semaphore.clone().acquire_owned().await else { - break; - }; - - match listener.accept().await { - Ok((stream, _addr)) => { - // Small-request IMDS-style endpoint an agent polls for - // credentials/identity — disable Nagle to avoid delayed-ACK stalls. - set_tcp_nodelay_best_effort(&stream); - let handler = handler.clone(); - tokio::spawn(async move { - if let Err(e) = handle_connection(handler.as_ref(), stream).await { - debug!("metadata server connection error: {e}"); - } - drop(permit); - }); - } - Err(e) => { - warn!("metadata server accept error: {e}"); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - } - } -} - -const READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -async fn handle_connection( - handler: &H, - mut stream: tokio::net::TcpStream, -) -> Result<()> { - let mut buf = vec![0u8; MAX_REQUEST_BYTES]; - let mut used = 0; - let deadline = tokio::time::sleep(READ_TIMEOUT); - tokio::pin!(deadline); - loop { - tokio::select! { - result = stream.read(&mut buf[used..]) => { - let n = result.map_err(|e| miette::miette!("{e}"))?; - if n == 0 { - return Ok(()); - } - used += n; - if buf[..used].windows(4).any(|w| w == b"\r\n\r\n") { - break; - } - if used >= buf.len() { - let _ = stream - .write_all(b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n") - .await; - return Ok(()); - } - } - () = &mut deadline => { - return Ok(()); - } - } - } - let request = String::from_utf8_lossy(&buf[..used]); - let request_line = request.split("\r\n").next().unwrap_or(""); - let mut parts = request_line.split_whitespace(); - let method = parts.next().unwrap_or(""); - let path = parts.next().unwrap_or("/"); - - tokio::time::timeout( - READ_TIMEOUT, - handler.handle(method, path, &buf[..used], &mut stream), - ) - .await - .unwrap_or_else(|_| { - debug!(method, path, "metadata handler timed out"); - Ok(()) - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::sync::mpsc; - - struct RecordingHandler { - requests: mpsc::UnboundedSender<(String, String)>, - } - - impl MetadataHandler for RecordingHandler { - async fn handle( - &self, - method: &str, - path: &str, - _request: &[u8], - stream: &mut S, - ) -> Result<()> { - self.requests - .send((method.to_string(), path.to_string())) - .unwrap(); - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await - .map_err(|error| miette::miette!("{error}"))?; - Ok(()) - } - } - - async fn connection_pair() -> (tokio::net::TcpStream, tokio::net::TcpStream) { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let client = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) - .await - .unwrap(); - let (server, _) = listener.accept().await.unwrap(); - (client, server) - } - - #[tokio::test] - async fn metadata_loopback_dispatches_method_path_and_response() { - let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); - let handler = RecordingHandler { - requests: requests_tx, - }; - let (mut client, server) = connection_pair().await; - let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); - - client - .write_all(b"GET /computeMetadata/v1/instance HTTP/1.1\r\nHost: metadata\r\n\r\n") - .await - .unwrap(); - let mut response = Vec::new(); - client.read_to_end(&mut response).await.unwrap(); - server_task.await.unwrap().unwrap(); - - assert_eq!( - requests_rx.try_recv().unwrap(), - ( - "GET".to_string(), - "/computeMetadata/v1/instance".to_string() - ) - ); - assert_eq!(response, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); - } - - #[tokio::test] - async fn metadata_loopback_rejects_oversized_headers_before_handler() { - let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); - let handler = RecordingHandler { - requests: requests_tx, - }; - let (mut client, server) = connection_pair().await; - let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); - - client - .write_all(&vec![b'x'; MAX_REQUEST_BYTES]) - .await - .unwrap(); - let mut response = Vec::new(); - client.read_to_end(&mut response).await.unwrap(); - server_task.await.unwrap().unwrap(); - - assert_eq!( - response, - b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n" - ); - assert!(requests_rx.try_recv().is_err()); - } -} diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs new file mode 100644 index 0000000000..209ab879cd --- /dev/null +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -0,0 +1,1797 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Seccomp-notification broker owned by the in-workload sandbox. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +use std::io; +use std::mem::size_of; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, TcpStream, UdpSocket}; +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_binary_identity::ProcfsIdentityResolver; +use openshell_isolation_interface::contract::{ + BinaryIdentity, DnsTransport, NetworkOpenResult, NetworkSocketMetadata, ResolveError, +}; +use openshell_isolation_interface::linux::seccomp_notify::{Notification, NotificationListener}; +use openshell_isolation_interface::linux::socket_registry::{ + InetFamily, InetKind, SocketMetadata, SocketRegistry, SocketState, +}; +use openshell_isolation_interface::linux::task_memory; +use tokio::sync::{mpsc, oneshot}; + +const SOCKET_CAPACITY: usize = 4_096; +const OPEN_QUEUE_CAPACITY: usize = 256; +const ACCEPT_WORKER_CAPACITY: usize = 64; +const DNS_QUEUE_CAPACITY: usize = 256; +const DNS_WORKER_CAPACITY: usize = 256; +const DNS_QUERY_TIMEOUT: Duration = Duration::from_secs(10); +const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(250); +const DNS_RELAY_ADDRESS: SocketAddr = SocketAddr::V4(std::net::SocketAddrV4::new( + Ipv4Addr::new(127, 0, 0, 53), + 53, +)); +const RELAY_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug)] +struct PendingOpenSlot(Arc); + +impl Drop for PendingOpenSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +#[derive(Debug)] +struct PendingDnsSlot(Arc); + +impl Drop for PendingDnsSlot { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } +} + +fn acquire_pending_dns_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < DNS_WORKER_CAPACITY).then_some(current + 1) + }) + .map(|_| PendingDnsSlot(Arc::clone(active))) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN)) +} + +struct PendingAcceptSlot { + active: Arc, +} + +impl Drop for PendingAcceptSlot { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +fn acquire_pending_accept_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < ACCEPT_WORKER_CAPACITY).then_some(current + 1) + }) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN))?; + Ok(PendingAcceptSlot { + active: Arc::clone(active), + }) +} + +fn acquire_pending_open_slot(active: &Arc) -> io::Result { + active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + (current < OPEN_QUEUE_CAPACITY).then_some(current + 1) + }) + .map(|_| PendingOpenSlot(Arc::clone(active))) + .map_err(|_| io::Error::from_raw_os_error(libc::EAGAIN)) +} + +/// One external TCP open blocked in `connect(2)` until the supervisor decides. +pub struct PendingTcpOpen { + pub(crate) destination: SocketAddr, + pub(crate) identity: Result, + pub(crate) socket: NetworkSocketMetadata, + decision: std::sync::mpsc::SyncSender, + relay: oneshot::Receiver>, + _slot: PendingOpenSlot, +} + +impl PendingTcpOpen { + pub(crate) async fn complete( + self, + decision: NetworkOpenResult, + ) -> io::Result> { + self.decision + .send(decision) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "network broker stopped"))?; + if matches!(decision, NetworkOpenResult::Denied { .. }) { + return Ok(None); + } + self.relay + .await + .map_err(|_| { + io::Error::new( + io::ErrorKind::BrokenPipe, + "network relay setup was cancelled", + ) + })? + .map(Some) + } +} + +/// One DNS exchange received by the exact sandbox-local resolver endpoint. +pub struct PendingDnsQuery { + pub(crate) request: Vec, + pub(crate) transport: DnsTransport, + pub(crate) identity: Result, + response: std::sync::mpsc::SyncSender>>, +} + +impl PendingDnsQuery { + pub(crate) fn complete(self, response: io::Result>) -> io::Result<()> { + self.response + .send(response) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "DNS relay stopped")) + } +} + +#[derive(Clone)] +struct DnsRelay { + address: SocketAddr, + udp_attribution: Arc>>>, + tcp_attribution: Arc>>>, +} + +#[derive(Clone)] +struct NotificationQueues { + pending: mpsc::Sender, + dns_relay: DnsRelay, + active_opens: Arc, + active_accepts: Arc, +} + +/// Live broker handle retained by the sandbox boundary. +#[derive(Clone)] +pub struct NetworkBroker { + pending: Arc>>, + pending_dns: Arc>>, + dns_address: SocketAddr, + healthy: Arc, +} + +impl NetworkBroker { + pub(crate) fn start(listener: NotificationListener) -> io::Result { + Self::start_with_dns_address(listener, DNS_RELAY_ADDRESS) + } + + #[cfg(test)] + pub(crate) fn start_for_test(listener: NotificationListener) -> io::Result { + Self::start_with_dns_address( + listener, + SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + ) + } + + fn start_with_dns_address( + listener: NotificationListener, + dns_address: SocketAddr, + ) -> io::Result { + let listener = Arc::new(listener); + let (pending_tx, pending_rx) = mpsc::channel(OPEN_QUEUE_CAPACITY); + let (pending_dns_tx, pending_dns_rx) = mpsc::channel(DNS_QUEUE_CAPACITY); + let registry = Arc::new(Mutex::new(SocketRegistry::new(1, SOCKET_CAPACITY)?)); + let active_opens = Arc::new(AtomicUsize::new(0)); + let active_accepts = Arc::new(AtomicUsize::new(0)); + let dns_relay = start_dns_relay(dns_address, pending_dns_tx)?; + let dns_address = dns_relay.address; + let queues = NotificationQueues { + pending: pending_tx, + dns_relay, + active_opens, + active_accepts, + }; + let healthy = Arc::new(AtomicBool::new(true)); + let broker_healthy = healthy.clone(); + std::thread::Builder::new() + .name("openshell-network-broker".to_string()) + .spawn(move || { + while broker_healthy.load(Ordering::Acquire) { + let notification = match listener.receive() { + Ok(notification) => notification, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => { + tracing::error!(%error, "sandbox network broker listener failed"); + broker_healthy.store(false, Ordering::Release); + break; + } + }; + if let Err(error) = dispatch_notification( + Arc::clone(®istry), + Arc::clone(&listener), + notification, + queues.clone(), + ) { + tracing::warn!( + tid = notification.tid, + syscall = notification.syscall, + %error, + "sandbox network notification denied (tid={}, syscall={}): {error}", + notification.tid, + notification.syscall + ); + let _ = listener.respond_errno(notification.id, error_to_errno(&error)); + } + } + }) + .map_err(|error| io::Error::other(format!("start network broker: {error}")))?; + Ok(Self { + pending: Arc::new(tokio::sync::Mutex::new(pending_rx)), + pending_dns: Arc::new(tokio::sync::Mutex::new(pending_dns_rx)), + dns_address, + healthy, + }) + } + + pub(crate) async fn accept(&self) -> io::Result { + self.pending + .lock() + .await + .recv() + .await + .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "network broker queue closed")) + } + + pub(crate) async fn accept_dns(&self) -> io::Result { + self.pending_dns + .lock() + .await + .recv() + .await + .ok_or_else(|| io::Error::new(io::ErrorKind::BrokenPipe, "DNS broker queue closed")) + } + + #[cfg(test)] + fn dns_address(&self) -> SocketAddr { + self.dns_address + } + + pub(crate) fn confirm_healthy(&self) -> io::Result<()> { + if self.healthy.load(Ordering::Acquire) && self.dns_address.port() != 0 { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "network broker is not running", + )) + } + } +} + +fn start_dns_relay( + address: SocketAddr, + pending: mpsc::Sender, +) -> io::Result { + let udp = UdpSocket::bind(address)?; + let address = udp.local_addr()?; + let tcp = TcpListener::bind(address)?; + let udp_attribution = Arc::new(Mutex::new(HashMap::new())); + let tcp_attribution = Arc::new(Mutex::new(HashMap::new())); + let active_workers = Arc::new(AtomicUsize::new(0)); + let relay = DnsRelay { + address, + udp_attribution: Arc::clone(&udp_attribution), + tcp_attribution: Arc::clone(&tcp_attribution), + }; + + let udp_active_workers = Arc::clone(&active_workers); + let udp_pending = pending.clone(); + std::thread::Builder::new() + .name("openshell-dns-udp".to_string()) + .spawn(move || { + let mut request = vec![0_u8; u16::MAX as usize]; + while let Ok((length, peer)) = udp.recv_from(&mut request) { + let Some(identity) = lock(&udp_attribution).get(&peer).cloned() else { + tracing::warn!(%peer, "dropping DNS datagram from unattributed socket"); + continue; + }; + let Ok(worker_slot) = acquire_pending_dns_slot(&udp_active_workers) else { + tracing::warn!(%peer, "dropping DNS datagram because the worker quota is full"); + continue; + }; + let (response_tx, response_rx) = std::sync::mpsc::sync_channel(1); + let query = PendingDnsQuery { + request: request[..length].to_vec(), + transport: DnsTransport::Udp, + identity, + response: response_tx, + }; + if pending_try_send(&udp_pending, query).is_err() { + continue; + } + let Ok(udp_response) = udp.try_clone() else { + continue; + }; + let _ = std::thread::Builder::new() + .name("openshell-dns-udp-query".to_string()) + .spawn(move || { + let _worker_slot = worker_slot; + if let Ok(Ok(response)) = response_rx.recv_timeout(DNS_QUERY_TIMEOUT) { + let _ = udp_response.send_to(&response, peer); + } + }); + } + }) + .map_err(|error| io::Error::other(format!("start UDP DNS relay: {error}")))?; + + let tcp_active_workers = active_workers; + std::thread::Builder::new() + .name("openshell-dns-tcp".to_string()) + .spawn(move || { + for accepted in tcp.incoming() { + let Ok((stream, peer)) = accepted.and_then(|stream| { + let peer = stream.peer_addr()?; + Ok((stream, peer)) + }) else { + break; + }; + let identity = lock(&tcp_attribution).get(&peer).cloned(); + let Some(identity) = identity else { + tracing::warn!(%peer, "dropping DNS stream from unattributed socket"); + continue; + }; + let Ok(worker_slot) = acquire_pending_dns_slot(&tcp_active_workers) else { + tracing::warn!(%peer, "dropping DNS stream because the worker quota is full"); + continue; + }; + let tcp_pending = pending.clone(); + let _ = std::thread::Builder::new() + .name("openshell-dns-tcp-query".to_string()) + .spawn(move || { + let _worker_slot = worker_slot; + serve_dns_tcp(stream, identity, tcp_pending); + }); + } + }) + .map_err(|error| io::Error::other(format!("start TCP DNS relay: {error}")))?; + Ok(relay) +} + +fn pending_try_send( + pending: &mpsc::Sender, + query: PendingDnsQuery, +) -> Result<(), ()> { + pending.try_send(query).map_err(|error| { + tracing::warn!(%error, "dropping DNS query because mediation queue is unavailable"); + }) +} + +fn serve_dns_tcp( + mut stream: TcpStream, + identity: Result, + pending: mpsc::Sender, +) { + use std::io::{Read as _, Write as _}; + + let _ = stream.set_read_timeout(Some(DNS_QUERY_TIMEOUT)); + let _ = stream.set_write_timeout(Some(DNS_QUERY_TIMEOUT)); + loop { + let mut length = [0_u8; 2]; + if stream.read_exact(&mut length).is_err() { + return; + } + let message_length = usize::from(u16::from_be_bytes(length)); + let mut request = Vec::with_capacity(message_length + 2); + request.extend_from_slice(&length); + request.resize(message_length + 2, 0); + if stream.read_exact(&mut request[2..]).is_err() { + return; + } + let (response_tx, response_rx) = std::sync::mpsc::sync_channel(1); + let query = PendingDnsQuery { + request, + transport: DnsTransport::Tcp, + identity: identity.clone(), + response: response_tx, + }; + if pending_try_send(&pending, query).is_err() { + return; + } + let Ok(Ok(response)) = response_rx.recv_timeout(DNS_QUERY_TIMEOUT) else { + return; + }; + if stream.write_all(&response).is_err() { + return; + } + } +} + +fn dispatch_notification( + registry: Arc>, + listener: Arc, + notification: Notification, + queues: NotificationQueues, +) -> io::Result<()> { + let syscall = i64::from(notification.syscall); + if syscall == libc::SYS_socket { + return create_socket(®istry, &listener, notification); + } + if syscall == libc::SYS_connect { + return connect_socket( + registry, + listener, + notification, + queues.pending, + &queues.dns_relay, + queues.active_opens, + ); + } + if syscall == libc::SYS_bind { + return bind_socket(®istry, &listener, notification); + } + if syscall == libc::SYS_listen { + return listen_socket(®istry, &listener, notification); + } + if matches!(syscall, libc::SYS_accept | libc::SYS_accept4) { + return accept_socket(registry, listener, notification, queues.active_accepts); + } + if matches!( + syscall, + libc::SYS_sendto | libc::SYS_sendmsg | libc::SYS_sendmmsg + ) { + return classify_send(®istry, &listener, notification, &queues.dns_relay); + } + if syscall == libc::SYS_getpeername { + return get_peer_name(®istry, &listener, notification); + } + if syscall == libc::SYS_setsockopt { + let level = i32::try_from(notification.args[1]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + let option = i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if (level == libc::IPPROTO_TCP && option == libc::TCP_FASTOPEN_CONNECT) + || (level == libc::IPPROTO_IPV6 && option == libc::IPV6_ADDRFORM) + { + return Err(io::Error::from_raw_os_error(libc::EPERM)); + } + return listener.respond_continue(notification.id); + } + Err(io::Error::from_raw_os_error(libc::EPERM)) +} + +fn create_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let domain = i32::try_from(notification.args[0]) + .map_err(|_| io::Error::from_raw_os_error(libc::EAFNOSUPPORT))?; + if !matches!(domain, libc::AF_INET | libc::AF_INET6) { + return listener.respond_continue(notification.id); + } + let raw_kind = i32::try_from(notification.args[1]) + .map_err(|_| io::Error::from_raw_os_error(libc::EPROTONOSUPPORT))?; + let protocol = i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EPROTONOSUPPORT))?; + let base_kind = raw_kind & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK); + let kind = match (base_kind, protocol) { + (libc::SOCK_STREAM, 0 | libc::IPPROTO_TCP) => InetKind::Tcp, + (libc::SOCK_DGRAM, 0 | libc::IPPROTO_UDP) => InetKind::DnsUdp, + _ => return Err(io::Error::from_raw_os_error(libc::EPROTONOSUPPORT)), + }; + let family = if domain == libc::AF_INET { + InetFamily::V4 + } else { + InetFamily::V6 + }; + // SAFETY: arguments were reduced to the supported native INET matrix. A + // successful call returns one newly owned descriptor. + let mut source = unsafe { libc::socket(domain, raw_kind, protocol) }; + if source < 0 && io::Error::last_os_error().raw_os_error() == Some(libc::EMFILE) { + collect_closed_socket_entries(registry)?; + // SAFETY: same validated native INET socket creation after reclaiming + // broker-held descriptors for closed workload sockets. + source = unsafe { libc::socket(domain, raw_kind, protocol) }; + } + if source < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful socket returned one owned descriptor. + let source = unsafe { OwnedFd::from_raw_fd(source) }; + let metadata = SocketMetadata { + family, + kind, + close_on_exec: raw_kind & libc::SOCK_CLOEXEC != 0, + nonblocking: raw_kind & libc::SOCK_NONBLOCK != 0, + creator_generation: u64::from(notification.tid), + }; + let mut registry = lock(registry); + if registry.is_full() { + collect_closed_socket_entries_locked(&mut registry)?; + } + let tentative = registry.stage(source, metadata)?; + listener.add_fd_and_send( + notification.id, + tentative.source_fd(), + metadata.close_on_exec, + )?; + registry.commit(tentative)?; + Ok(()) +} + +fn connect_socket( + registry: Arc>, + listener: Arc, + notification: Notification, + pending: mpsc::Sender, + dns_relay: &DnsRelay, + active_opens: Arc, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + if !socket_address_is_inet(notification.tid, notification.args[1], notification.args[2])? { + if lock(®istry).resolve(notification.tid, fd).is_ok() { + // Every registered descriptor is an injected INET socket. Never + // CONTINUE based on a mutable workload sockaddr for such an FD. + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + // Native non-INET descriptors remain kernel-driven. + return listener.respond_continue(notification.id); + } + let destination = + read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; + let (kind, socket_cookie, nonblocking) = { + let registry = lock(®istry); + let entry = registry.resolve(notification.tid, fd)?; + ( + entry.metadata().kind, + entry.identity().cookie, + entry.metadata().nonblocking, + ) + }; + if destination == dns_relay.address { + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + if !matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) { + return Err(io::Error::from_raw_os_error(libc::EISCONN)); + } + let source_fd = entry.retained_preconnect()?.as_raw_fd(); + ensure_dns_source_bound(source_fd, entry.metadata().family)?; + let peer = socket_local_addr(source_fd)?; + let attribution = match kind { + InetKind::Tcp => &dns_relay.tcp_attribution, + InetKind::DnsUdp => &dns_relay.udp_attribution, + }; + lock(attribution).insert(peer, identity); + if let Err(error) = connect_exact(source_fd, destination) { + lock(attribution).remove(&peer); + return Err(error); + } + entry.set_state(match kind { + InetKind::Tcp => SocketState::DnsTcp { relay: destination }, + InetKind::DnsUdp => SocketState::DnsUdp { relay: destination }, + }); + entry.release_preconnect(); + return listener.respond_value(notification.id, 0); + } + if destination.ip().is_loopback() { + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + connect_exact(entry.retained_preconnect()?.as_raw_fd(), destination)?; + entry.set_state(SocketState::Local { peer: destination }); + entry.release_preconnect(); + return listener.respond_value(notification.id, 0); + } + if kind != InetKind::Tcp { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let (decision_tx, decision_rx) = std::sync::mpsc::sync_channel(1); + let (relay_tx, relay_rx) = oneshot::channel(); + let slot = acquire_pending_open_slot(&active_opens)?; + pending + .try_send(PendingTcpOpen { + destination, + identity, + socket: NetworkSocketMetadata { + socket_cookie, + nonblocking, + process_generation: u64::from(notification.tid), + }, + decision: decision_tx, + relay: relay_rx, + _slot: slot, + }) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => io::Error::from_raw_os_error(libc::EAGAIN), + mpsc::error::TrySendError::Closed(_) => { + io::Error::new(io::ErrorKind::BrokenPipe, "network-open queue closed") + } + })?; + let worker_listener = Arc::clone(&listener); + std::thread::Builder::new() + .name("openshell-network-open".to_string()) + .spawn(move || { + let result = decision_rx.recv().unwrap_or(NetworkOpenResult::Denied { + errno: libc::ECANCELED, + }); + match result { + NetworkOpenResult::Denied { errno } => { + let _ = worker_listener.respond_errno(notification.id, errno); + } + NetworkOpenResult::RelayReady => { + match establish_relay(®istry, notification.tid, fd, destination) { + Ok(stream) => { + let result = worker_listener + .respond_value(notification.id, 0) + .map(|()| stream); + let _ = relay_tx.send(result); + } + Err(error) => { + let _ = worker_listener + .respond_errno(notification.id, error_to_errno(&error)); + let _ = relay_tx.send(Err(error)); + } + } + } + } + }) + .map_err(|error| io::Error::other(format!("start network-open worker: {error}")))?; + Ok(()) +} + +fn ensure_dns_source_bound(fd: RawFd, family: InetFamily) -> io::Result<()> { + let address = socket_local_addr(fd)?; + if address.port() != 0 { + return Ok(()); + } + bind_exact( + fd, + match family { + InetFamily::V4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + InetFamily::V6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + }, + ) +} + +fn establish_relay( + registry: &Mutex, + tid: u32, + fd: RawFd, + destination: SocketAddr, +) -> io::Result { + let relay = TcpListener::bind(match destination { + SocketAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), + SocketAddr::V6(_) => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), + })?; + relay.set_nonblocking(false)?; + let relay_address = relay.local_addr()?; + let expected_peer = { + let mut registry = lock(registry); + let entry = registry.resolve_mut(tid, fd)?; + connect_exact(entry.retained_preconnect()?.as_raw_fd(), relay_address)?; + let expected_peer = socket_local_addr(entry.retained_preconnect()?.as_raw_fd())?; + entry.set_state(SocketState::Connected { + original_peer: destination, + }); + entry.release_preconnect(); + expected_peer + }; + relay.set_nonblocking(true)?; + let deadline = std::time::Instant::now() + RELAY_CONNECT_TIMEOUT; + let stream = loop { + let now = std::time::Instant::now(); + if now >= deadline { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + let timeout = deadline.saturating_duration_since(now); + let mut poll = libc::pollfd { + fd: relay.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let timeout = i32::try_from(timeout.as_millis()).unwrap_or(i32::MAX); + // SAFETY: poll points to one live descriptor record. + if unsafe { libc::poll(&raw mut poll, 1, timeout) } <= 0 { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + match relay.accept() { + Ok((stream, peer)) if peer == expected_peer => break stream, + Ok((_stream, peer)) => { + tracing::warn!(%peer, %expected_peer, "rejected unexpected sandbox relay peer"); + } + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => return Err(error), + } + }; + stream.set_nodelay(true)?; + Ok(stream) +} + +fn bind_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + if !socket_address_is_inet(notification.tid, notification.args[1], notification.args[2])? { + if lock(registry).resolve(notification.tid, fd).is_ok() { + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + return listener.respond_continue(notification.id); + } + let local = read_socket_addr(notification.tid, notification.args[1], notification.args[2])?; + if !local.ip().is_loopback() && !local.ip().is_unspecified() { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + let bind_result = { + let mut registry = lock(registry); + let entry = registry.resolve_mut(notification.tid, fd)?; + bind_exact(entry.retained_preconnect()?.as_raw_fd(), local) + }; + if bind_result + .as_ref() + .is_err_and(|error| error.raw_os_error() == Some(libc::EADDRINUSE)) + { + collect_closed_socket_entries(registry)?; + let mut registry = lock(registry); + let entry = registry.resolve_mut(notification.tid, fd)?; + bind_exact(entry.retained_preconnect()?.as_raw_fd(), local)?; + entry.set_state(SocketState::Bound { local }); + } else { + bind_result?; + lock(registry) + .resolve_mut(notification.tid, fd)? + .set_state(SocketState::Bound { local }); + } + listener.respond_value(notification.id, 0) +} + +fn collect_closed_socket_entries(registry: &Mutex) -> io::Result<()> { + let mut registry = lock(registry); + collect_closed_socket_entries_locked(&mut registry) +} + +fn collect_closed_socket_entries_locked(registry: &mut SocketRegistry) -> io::Result<()> { + let installed = + openshell_isolation_interface::linux::proc_fd::installed_socket_inodes_excluding( + std::process::id(), + )?; + registry.retain_installed(&installed); + Ok(()) +} + +fn listen_socket( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let backlog = i32::try_from(notification.args[1]).unwrap_or(i32::MAX); + let mut registry = lock(registry); + let Ok(entry) = registry.resolve_mut(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + // SAFETY: retained descriptor is the exact registered socket OFD. + if unsafe { libc::listen(entry.retained_preconnect()?.as_raw_fd(), backlog) } < 0 { + return Err(io::Error::last_os_error()); + } + let local = socket_local_addr(entry.retained_preconnect()?.as_raw_fd())?; + entry.set_state(SocketState::Listening { local }); + listener.respond_value(notification.id, 0) +} + +fn accept_socket( + registry: Arc>, + listener: Arc, + notification: Notification, + active_accepts: Arc, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let flags = if i64::from(notification.syscall) == libc::SYS_accept4 { + i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))? + } else { + 0 + }; + if flags & !(libc::SOCK_CLOEXEC | libc::SOCK_NONBLOCK) != 0 { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + if (notification.args[1] == 0) != (notification.args[2] == 0) { + return Err(io::Error::from_raw_os_error(libc::EFAULT)); + } + let (listener_inode, metadata, source) = { + let registry = lock(®istry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + if !matches!(entry.state(), SocketState::Listening { .. }) + || entry.metadata().kind != InetKind::Tcp + { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + let source = duplicate_close_on_exec(entry.retained_preconnect()?.as_raw_fd())?; + (entry.identity().inode, entry.metadata(), source) + }; + let slot = acquire_pending_accept_slot(&active_accepts)?; + let worker_listener = Arc::clone(&listener); + std::thread::Builder::new() + .name("openshell-local-accept".to_string()) + .spawn(move || { + let _slot = slot; + if let Err(error) = accept_and_inject( + ®istry, + &worker_listener, + notification, + flags, + listener_inode, + metadata, + source, + ) { + let _ = worker_listener.respond_errno(notification.id, error_to_errno(&error)); + } + }) + .map_err(|error| io::Error::other(format!("start local-accept worker: {error}")))?; + Ok(()) +} + +fn accept_and_inject( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, + flags: i32, + listener_inode: u64, + metadata: SocketMetadata, + source: OwnedFd, +) -> io::Result<()> { + let mut poll = libc::pollfd { + fd: source.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: F_GETFL reads the live listener OFD flags. + let current_flags = unsafe { libc::fcntl(source.as_raw_fd(), libc::F_GETFL) }; + if current_flags < 0 { + return Err(io::Error::last_os_error()); + } + let nonblocking = current_flags & libc::O_NONBLOCK != 0; + let timeout = if nonblocking { + 0 + } else { + i32::try_from(ACCEPT_POLL_INTERVAL.as_millis()).expect("accept poll interval fits i32") + }; + loop { + listener.validate_id(notification.id)?; + // SAFETY: poll references one live pollfd for this call. + let ready = unsafe { libc::poll(&raw mut poll, 1, timeout) }; + if ready < 0 { + let error = io::Error::last_os_error(); + if error.kind() == io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if ready == 0 { + if nonblocking { + return Err(io::Error::from_raw_os_error(libc::EAGAIN)); + } + continue; + } + break; + } + + let mut storage = std::mem::MaybeUninit::::zeroed(); + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr storage size fits"); + // Always keep the broker-side descriptor close-on-exec. ADDFD separately + // applies the workload's requested descriptor flag. + let accepted_flags = flags | libc::SOCK_CLOEXEC; + // SAFETY: storage and length are live outputs and source is a listening + // socket proven by the registry. + let accepted = unsafe { + libc::accept4( + source.as_raw_fd(), + storage.as_mut_ptr().cast(), + &raw mut length, + accepted_flags, + ) + }; + if accepted < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful accept4 returned one newly owned descriptor. + let accepted = unsafe { OwnedFd::from_raw_fd(accepted) }; + // SAFETY: accept4 initialized the reported prefix of storage. + let peer = decode_sockaddr( + unsafe { storage.assume_init() }, + usize::try_from(length).unwrap_or(0), + )?; + if !peer.ip().is_loopback() { + return Err(io::Error::from_raw_os_error(libc::EACCES)); + } + if notification.args[1] != 0 { + write_socket_addr( + listener, + notification.id, + notification.tid, + notification.args[1], + notification.args[2], + peer, + )?; + } + + let accepted_metadata = SocketMetadata { + family: metadata.family, + kind: InetKind::Tcp, + close_on_exec: flags & libc::SOCK_CLOEXEC != 0, + nonblocking: flags & libc::SOCK_NONBLOCK != 0, + creator_generation: u64::from(notification.tid), + }; + let mut registry = lock(registry); + let notifying_fd = raw_fd(notification.args[0])?; + if registry + .resolve(notification.tid, notifying_fd)? + .identity() + .inode + != listener_inode + { + return Err(io::Error::from_raw_os_error(libc::EBADF)); + } + if registry.is_full() { + collect_closed_socket_entries_locked(&mut registry)?; + } + let tentative = registry.stage(accepted, accepted_metadata)?; + listener.add_fd_and_send( + notification.id, + tentative.source_fd(), + accepted_metadata.close_on_exec, + )?; + registry.commit_with_state(tentative, SocketState::AcceptedLocal { peer })?; + Ok(()) +} + +fn duplicate_close_on_exec(fd: RawFd) -> io::Result { + // SAFETY: F_DUPFD_CLOEXEC returns an independent owned descriptor for the + // same open-file description. + let duplicate = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 3) }; + if duplicate < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: successful fcntl returned one newly owned descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(duplicate) }) +} + +fn classify_send( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, + dns_relay: &DnsRelay, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let syscall = i64::from(notification.syscall); + let (state, metadata) = { + let registry = lock(registry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + // Non-INET sockets are never injected into the registry. Leave + // their native sendmsg/control-message semantics to the kernel. + return listener.respond_continue(notification.id); + }; + (entry.state().clone(), entry.metadata()) + }; + if matches!( + &state, + SocketState::Connected { .. } | SocketState::AcceptedLocal { .. } + ) || (metadata.kind == InetKind::Tcp && matches!(&state, SocketState::Local { .. })) + { + return listener.respond_continue(notification.id); + } + let messages = match syscall { + libc::SYS_sendto => vec![read_sendto_message(notification)?], + libc::SYS_sendmsg => vec![read_sendmsg_message( + notification.tid, + notification.args[1], + i32::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?, + None, + )?], + libc::SYS_sendmmsg => read_sendmmsg_messages(notification)?, + _ => return Err(io::Error::from_raw_os_error(libc::ENOSYS)), + }; + + let mut registry = lock(registry); + let resolution = registry.resolve(notification.tid, fd); + match resolution { + Ok(entry) + if entry.metadata().kind == InetKind::DnsUdp + && matches!(entry.state(), SocketState::Local { .. }) => + { + if messages.iter().all(|message| message.destination.is_none()) { + listener.respond_continue(notification.id) + } else { + Err(io::Error::from_raw_os_error(libc::EACCES)) + } + } + Ok(entry) if matches!(entry.state(), SocketState::DnsUdp { .. }) => { + if messages.iter().all(|message| message.destination.is_none()) { + listener.respond_continue(notification.id) + } else { + Err(io::Error::from_raw_os_error(libc::EACCES)) + } + } + Ok(entry) + if entry.metadata().kind == InetKind::DnsUdp + && matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) + && messages.iter().all(|message| { + message + .destination + .is_some_and(|value| value == dns_relay.address) + }) => + { + let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); + let entry = registry.resolve_mut(notification.tid, fd)?; + let source_fd = entry.retained_preconnect()?.as_raw_fd(); + ensure_dns_source_bound(source_fd, entry.metadata().family)?; + let peer = socket_local_addr(source_fd)?; + lock(&dns_relay.udp_attribution).insert(peer, identity); + if let Err(error) = connect_exact(source_fd, dns_relay.address) { + lock(&dns_relay.udp_attribution).remove(&peer); + return Err(error); + } + for message in &messages { + send_dns_message(source_fd, message)?; + if let Some(length_address) = message.result_length_address { + let length = u32::try_from(message.data.len()) + .map_err(|_| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + listener.validate_id(notification.id)?; + task_memory::write_exact( + notification.tid, + length_address, + &length.to_ne_bytes(), + )?; + } + } + entry.set_state(SocketState::DnsUdp { + relay: dns_relay.address, + }); + entry.release_preconnect(); + let result = if syscall == libc::SYS_sendmmsg { + i64::try_from(messages.len()).unwrap_or(i64::MAX) + } else { + i64::try_from(messages[0].data.len()).unwrap_or(i64::MAX) + }; + listener.respond_value(notification.id, result) + } + Ok(_) => Err(io::Error::from_raw_os_error(libc::EDESTADDRREQ)), + // Non-INET sockets and accepted local sockets were never registered. + // The mandatory outer fence still prevents an external kernel route. + Err(_) => listener.respond_continue(notification.id), + } +} + +struct SendMessage { + data: Vec, + destination: Option, + flags: i32, + result_length_address: Option, +} + +fn read_sendto_message(notification: Notification) -> io::Result { + let length = usize::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + if u16::try_from(length).is_err() { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let mut data = vec![0_u8; length]; + task_memory::read_exact(notification.tid, notification.args[1], &mut data)?; + let destination = if notification.args[4] == 0 { + None + } else { + Some(read_socket_addr( + notification.tid, + notification.args[4], + notification.args[5], + )?) + }; + Ok(SendMessage { + data, + destination, + flags: i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?, + result_length_address: None, + }) +} + +fn read_sendmsg_message( + tid: u32, + address: u64, + flags: i32, + result_length_address: Option, +) -> io::Result { + let header = read_task_value::(tid, address)?; + if header.msg_controllen != 0 { + return Err(io::Error::from_raw_os_error(libc::EOPNOTSUPP)); + } + let destination = if header.msg_name.is_null() { + None + } else { + Some(read_socket_addr( + tid, + header.msg_name as u64, + u64::from(header.msg_namelen), + )?) + }; + #[cfg(target_env = "musl")] + let iov_count = usize::try_from(header.msg_iovlen) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + #[cfg(not(target_env = "musl"))] + let iov_count = header.msg_iovlen; + if iov_count > 32 { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let mut data = Vec::new(); + for index in 0..iov_count { + let offset = index + .checked_mul(size_of::()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + let iov = read_task_value::( + tid, + (header.msg_iov as u64) + .checked_add(u64::try_from(offset).unwrap_or(u64::MAX)) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?, + )?; + let start = data.len(); + let end = start + .checked_add(iov.iov_len) + .filter(|length| u16::try_from(*length).is_ok()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EMSGSIZE))?; + data.resize(end, 0); + task_memory::read_exact(tid, iov.iov_base as u64, &mut data[start..end])?; + } + Ok(SendMessage { + data, + destination, + flags, + result_length_address, + }) +} + +fn read_sendmmsg_messages(notification: Notification) -> io::Result> { + let count = usize::try_from(notification.args[2]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if count == 0 || count > 32 { + return Err(io::Error::from_raw_os_error(libc::EMSGSIZE)); + } + let flags = i32::try_from(notification.args[3]) + .map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + (0..count) + .map(|index| { + let offset = index + .checked_mul(size_of::()) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + let base = notification.args[1] + .checked_add(u64::try_from(offset).unwrap_or(u64::MAX)) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?; + read_sendmsg_message( + notification.tid, + base, + flags, + Some( + base.checked_add( + u64::try_from(std::mem::offset_of!(libc::mmsghdr, msg_len)) + .unwrap_or(u64::MAX), + ) + .ok_or_else(|| io::Error::from_raw_os_error(libc::EOVERFLOW))?, + ), + ) + }) + .collect() +} + +fn read_task_value(tid: u32, address: u64) -> io::Result { + let mut bytes = vec![0_u8; size_of::()]; + task_memory::read_exact(tid, address, &mut bytes)?; + // SAFETY: `bytes` contains exactly one copied native value; unaligned read + // avoids imposing alignment on the task-memory scratch allocation. + Ok(unsafe { std::ptr::read_unaligned(bytes.as_ptr().cast::()) }) +} + +fn send_dns_message(fd: RawFd, message: &SendMessage) -> io::Result<()> { + // SAFETY: `fd` is the retained exact UDP socket and the buffer remains + // valid for the duration of the syscall. + let sent = unsafe { + libc::send( + fd, + message.data.as_ptr().cast(), + message.data.len(), + message.flags, + ) + }; + if sent < 0 { + return Err(io::Error::last_os_error()); + } + if usize::try_from(sent).ok() == Some(message.data.len()) { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(libc::EIO)) + } +} + +fn get_peer_name( + registry: &Mutex, + listener: &NotificationListener, + notification: Notification, +) -> io::Result<()> { + let fd = raw_fd(notification.args[0])?; + let registry = lock(registry); + let Ok(entry) = registry.resolve(notification.tid, fd) else { + return listener.respond_continue(notification.id); + }; + let peer = match entry.state() { + SocketState::Connected { original_peer } => *original_peer, + SocketState::Local { peer } | SocketState::AcceptedLocal { peer } => *peer, + _ => return Err(io::Error::from_raw_os_error(libc::ENOTCONN)), + }; + write_socket_addr( + listener, + notification.id, + notification.tid, + notification.args[1], + notification.args[2], + peer, + )?; + listener.respond_value(notification.id, 0) +} + +fn connect_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { + // Never let a blocking connect pin the single notification dispatcher. + // O_NONBLOCK is an OFD flag, so restore the workload's original setting + // after the bounded connect attempt completes. + // SAFETY: F_GETFL/F_SETFL operate on the live retained socket descriptor. + let original_flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if original_flags < 0 { + return Err(io::Error::last_os_error()); + } + let changed_flags = original_flags & libc::O_NONBLOCK == 0; + if changed_flags + && unsafe { libc::fcntl(fd, libc::F_SETFL, original_flags | libc::O_NONBLOCK) } < 0 + { + return Err(io::Error::last_os_error()); + } + let result = with_sockaddr(address, |pointer, length| { + // SAFETY: pointer/length describe a live native sockaddr and `fd` is + // the retained exact socket OFD. + let result = unsafe { libc::connect(fd, pointer, length) }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::EINPROGRESS) { + return Err(error); + } + let mut poll = libc::pollfd { + fd, + events: libc::POLLOUT, + revents: 0, + }; + // SAFETY: poll points to one live pollfd. + let timeout = i32::try_from(RELAY_CONNECT_TIMEOUT.as_millis()) + .expect("relay timeout fits poll milliseconds"); + if unsafe { libc::poll(&raw mut poll, 1, timeout) } <= 0 { + return Err(io::Error::from_raw_os_error(libc::ETIMEDOUT)); + } + let mut socket_error = 0_i32; + let mut size = libc::socklen_t::try_from(size_of::()).expect("SO_ERROR size fits"); + // SAFETY: getsockopt writes one i32 into live storage. + if unsafe { + libc::getsockopt( + fd, + libc::SOL_SOCKET, + libc::SO_ERROR, + (&raw mut socket_error).cast(), + &raw mut size, + ) + } < 0 + { + return Err(io::Error::last_os_error()); + } + if socket_error == 0 { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(socket_error)) + } + }); + let restore = if changed_flags && unsafe { libc::fcntl(fd, libc::F_SETFL, original_flags) } < 0 + { + Err(io::Error::last_os_error()) + } else { + Ok(()) + }; + result.and(restore) +} + +fn bind_exact(fd: RawFd, address: SocketAddr) -> io::Result<()> { + with_sockaddr(address, |pointer, length| { + // SAFETY: pointer/length describe a live native sockaddr. + if unsafe { libc::bind(fd, pointer, length) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } + }) +} + +fn socket_local_addr(fd: RawFd) -> io::Result { + let mut storage = std::mem::MaybeUninit::::zeroed(); + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr storage size fits"); + // SAFETY: storage and length are live output buffers. + if unsafe { libc::getsockname(fd, storage.as_mut_ptr().cast(), &raw mut length) } < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: getsockname initialized `length` bytes, including the family. + decode_sockaddr( + unsafe { storage.assume_init() }, + usize::try_from(length).unwrap_or(0), + ) +} + +fn read_socket_addr(tid: u32, address: u64, length: u64) -> io::Result { + let length = usize::try_from(length).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if length < size_of::() || length > size_of::() { + return Err(io::Error::from_raw_os_error(libc::EINVAL)); + } + let mut bytes = vec![0_u8; length]; + task_memory::read_exact(tid, address, &mut bytes)?; + let mut storage = std::mem::MaybeUninit::::zeroed(); + // SAFETY: destination spans sockaddr_storage and `length` was bounded. + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), storage.as_mut_ptr().cast(), length); + decode_sockaddr(storage.assume_init(), length) + } +} + +fn socket_address_is_inet(tid: u32, address: u64, length: u64) -> io::Result { + let length = usize::try_from(length).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; + if address == 0 || length < size_of::() { + return Err(io::Error::from_raw_os_error(libc::EFAULT)); + } + let mut family = [0_u8; size_of::()]; + task_memory::read_exact(tid, address, &mut family)?; + Ok(matches!( + i32::from(libc::sa_family_t::from_ne_bytes(family)), + libc::AF_INET | libc::AF_INET6 + )) +} + +fn decode_sockaddr(storage: libc::sockaddr_storage, length: usize) -> io::Result { + match i32::from(storage.ss_family) { + libc::AF_INET if length >= size_of::() => { + // SAFETY: family and length establish sockaddr_in layout. + let address = unsafe { *(&raw const storage).cast::() }; + Ok(SocketAddr::new( + IpAddr::V4(Ipv4Addr::from(address.sin_addr.s_addr.to_ne_bytes())), + u16::from_be(address.sin_port), + )) + } + libc::AF_INET6 if length >= size_of::() => { + // SAFETY: family and length establish sockaddr_in6 layout. + let address = unsafe { *(&raw const storage).cast::() }; + Ok(SocketAddr::new( + IpAddr::V6(Ipv6Addr::from(address.sin6_addr.s6_addr)), + u16::from_be(address.sin6_port), + )) + } + _ => Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)), + } +} + +fn write_socket_addr( + listener: &NotificationListener, + notification_id: u64, + tid: u32, + address: u64, + length_address: u64, + value: SocketAddr, +) -> io::Result<()> { + let mut supplied_length = [0_u8; size_of::()]; + task_memory::read_exact(tid, length_address, &mut supplied_length)?; + let supplied_length = libc::socklen_t::from_ne_bytes(supplied_length); + let (bytes, actual_length) = sockaddr_bytes(value); + let copied = usize::try_from(supplied_length) + .unwrap_or(0) + .min(bytes.len()); + listener.validate_id(notification_id)?; + if copied != 0 { + task_memory::write_exact(tid, address, &bytes[..copied])?; + } + listener.validate_id(notification_id)?; + task_memory::write_exact(tid, length_address, &actual_length.to_ne_bytes()) +} + +fn sockaddr_bytes(address: SocketAddr) -> (Vec, libc::socklen_t) { + match address { + SocketAddr::V4(address) => { + let native = libc::sockaddr_in { + sin_family: libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t"), + sin_port: address.port().to_be(), + sin_addr: libc::in_addr { + s_addr: u32::from_ne_bytes(address.ip().octets()), + }, + sin_zero: [0; 8], + }; + // SAFETY: native is plain initialized storage. + let bytes = unsafe { + std::slice::from_raw_parts( + (&raw const native).cast::(), + size_of::(), + ) + }; + ( + bytes.to_vec(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in size fits socklen_t"), + ) + } + SocketAddr::V6(address) => { + let native = libc::sockaddr_in6 { + sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) + .expect("AF_INET6 fits sa_family_t"), + sin6_port: address.port().to_be(), + sin6_flowinfo: address.flowinfo(), + sin6_addr: libc::in6_addr { + s6_addr: address.ip().octets(), + }, + sin6_scope_id: address.scope_id(), + }; + // SAFETY: native is plain initialized storage. + let bytes = unsafe { + std::slice::from_raw_parts( + (&raw const native).cast::(), + size_of::(), + ) + }; + ( + bytes.to_vec(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in6 size fits socklen_t"), + ) + } + } +} + +fn with_sockaddr( + address: SocketAddr, + operation: impl FnOnce(*const libc::sockaddr, libc::socklen_t) -> io::Result, +) -> io::Result { + match address { + SocketAddr::V4(address) => { + let native = libc::sockaddr_in { + sin_family: libc::sa_family_t::try_from(libc::AF_INET) + .expect("AF_INET fits sa_family_t"), + sin_port: address.port().to_be(), + sin_addr: libc::in_addr { + s_addr: u32::from_ne_bytes(address.ip().octets()), + }, + sin_zero: [0; 8], + }; + operation( + (&raw const native).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in size fits socklen_t"), + ) + } + SocketAddr::V6(address) => { + let native = libc::sockaddr_in6 { + sin6_family: libc::sa_family_t::try_from(libc::AF_INET6) + .expect("AF_INET6 fits sa_family_t"), + sin6_port: address.port().to_be(), + sin6_flowinfo: address.flowinfo(), + sin6_addr: libc::in6_addr { + s6_addr: address.ip().octets(), + }, + sin6_scope_id: address.scope_id(), + }; + operation( + (&raw const native).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in6 size fits socklen_t"), + ) + } + } +} + +fn raw_fd(value: u64) -> io::Result { + RawFd::try_from(value).map_err(|_| io::Error::from_raw_os_error(libc::EBADF)) +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn error_to_errno(error: &io::Error) -> i32 { + error.raw_os_error().unwrap_or(libc::EACCES).max(1) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read as _, Write as _}; + use std::os::unix::net::{UnixListener, UnixStream}; + + #[test] + fn pending_external_open_slots_are_bounded_and_reusable() { + let active = Arc::new(AtomicUsize::new(OPEN_QUEUE_CAPACITY - 1)); + let last = acquire_pending_open_slot(&active).expect("last available slot"); + assert_eq!( + acquire_pending_open_slot(&active) + .expect_err("open limit must fail closed") + .raw_os_error(), + Some(libc::EAGAIN) + ); + drop(last); + let reused = acquire_pending_open_slot(&active).expect("released slot"); + drop(reused); + assert_eq!(active.load(Ordering::Acquire), OPEN_QUEUE_CAPACITY - 1); + } + + #[test] + fn dns_worker_slots_are_bounded_and_reusable() { + let active = Arc::new(AtomicUsize::new(DNS_WORKER_CAPACITY - 1)); + let last = acquire_pending_dns_slot(&active).expect("last available slot"); + assert_eq!( + acquire_pending_dns_slot(&active) + .expect_err("DNS worker limit must fail closed") + .raw_os_error(), + Some(libc::EAGAIN) + ); + drop(last); + let reused = acquire_pending_dns_slot(&active).expect("released slot"); + drop(reused); + assert_eq!(active.load(Ordering::Acquire), DNS_WORKER_CAPACITY - 1); + } + + #[test] + fn unix_connect_remains_kernel_driven() { + let directory = tempfile::tempdir().expect("temporary Unix socket directory"); + let path = directory.path().join("service.sock"); + let service = UnixListener::bind(&path).expect("bind Unix service"); + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result<()> { + let mut stream = UnixStream::connect(path)?; + stream.write_all(b"unix") + }) + .expect("launcher result") + }); + let (mut stream, _) = service.accept().expect("accept Unix client"); + let mut payload = [0_u8; 4]; + stream.read_exact(&mut payload).expect("read Unix payload"); + assert_eq!(&payload, b"unix"); + client.join().expect("join client").expect("Unix client"); + } + + #[test] + fn accepted_loopback_stream_is_registered_for_notified_operations() { + let reservation = TcpListener::bind("127.0.0.1:0").expect("reserve loopback port"); + let address = reservation.local_addr().expect("reserved address"); + drop(reservation); + + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let workload = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result { + let listener = TcpListener::bind(address)?; + ready_tx + .send(()) + .map_err(|_| io::Error::other("test client disappeared"))?; + let (stream, _) = listener.accept()?; + let peer = stream.peer_addr()?; + let payload = b"accepted"; + let iov = libc::iovec { + iov_base: payload.as_ptr().cast_mut().cast(), + iov_len: payload.len(), + }; + let message = libc::msghdr { + msg_name: std::ptr::null_mut(), + msg_namelen: 0, + msg_iov: (&raw const iov).cast_mut(), + msg_iovlen: 1, + msg_control: std::ptr::null_mut(), + msg_controllen: 0, + msg_flags: 0, + }; + // SAFETY: message references one live immutable payload; + // the accepted stream remains open for the call. + let sent = unsafe { libc::sendmsg(stream.as_raw_fd(), &raw const message, 0) }; + if sent != isize::try_from(payload.len()).expect("payload fits isize") { + return Err(io::Error::last_os_error()); + } + Ok(peer) + }) + .expect("launcher result") + }); + + ready_rx.recv().expect("listener ready"); + let mut client = TcpStream::connect(address).expect("connect loopback client"); + let mut payload = [0_u8; 8]; + client + .read_exact(&mut payload) + .expect("read accepted stream"); + assert_eq!(&payload, b"accepted"); + assert!( + workload + .join() + .expect("join workload") + .expect("accepted workload") + .ip() + .is_loopback() + ); + } + + #[test] + fn external_connect_waits_for_explicit_relay_decision() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(|| -> io::Result<()> { + let mut stream = TcpStream::connect("203.0.113.7:443")?; + stream.write_all(b"request")?; + let mut response = [0_u8; 8]; + stream.read_exact(&mut response)?; + if &response != b"response" { + return Err(io::Error::other("relay returned wrong response")); + } + Ok(()) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let pending = runtime.block_on(broker.accept()).expect("pending TCP open"); + assert_eq!(pending.destination, "203.0.113.7:443".parse().unwrap()); + assert!(pending.socket.socket_cookie != 0); + let mut relay = runtime + .block_on(pending.complete(NetworkOpenResult::RelayReady)) + .expect("complete relay") + .expect("authorized relay stream"); + let mut request = [0_u8; 7]; + relay + .read_exact(&mut request) + .expect("read relayed request"); + assert_eq!(&request, b"request"); + relay.write_all(b"response").expect("write relay response"); + client.join().expect("join client").expect("client relay"); + } + + #[test] + fn denied_external_connect_keeps_socket_unconnected() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let client = std::thread::spawn(move || { + launcher + .execute(|| TcpStream::connect("198.51.100.9:80")) + .expect("launcher result") + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let pending = runtime.block_on(broker.accept()).expect("pending TCP open"); + assert!( + runtime + .block_on(pending.complete(NetworkOpenResult::Denied { + errno: libc::EACCES, + })) + .expect("complete denial") + .is_none() + ); + assert_eq!( + client + .join() + .expect("join client") + .expect_err("connect must be denied") + .raw_os_error(), + Some(libc::EACCES) + ); + } + + #[test] + fn udp_dns_uses_exact_local_relay_source() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result { + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.set_read_timeout(Some(Duration::from_secs(5)))?; + socket.send_to(b"dns-query", dns_address)?; + let mut response = [0_u8; 32]; + let (length, source) = socket.recv_from(&mut response)?; + if &response[..length] != b"dns-response" { + return Err(io::Error::other("wrong DNS response")); + } + Ok(source) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + assert_eq!(query.transport, DnsTransport::Udp); + assert_eq!(query.request, b"dns-query"); + query.complete(Ok(b"dns-response".to_vec())).unwrap(); + assert_eq!( + client.join().expect("join client").expect("DNS client"), + dns_address + ); + } + + #[test] + fn tcp_dns_preserves_length_framing() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result> { + let mut stream = TcpStream::connect(dns_address)?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.write_all(&[0, 3, 1, 2, 3])?; + let mut response = vec![0_u8; 5]; + stream.read_exact(&mut response)?; + Ok(response) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + assert_eq!(query.transport, DnsTransport::Tcp); + assert_eq!(query.request, [0, 3, 1, 2, 3]); + query.complete(Ok(vec![0, 3, 4, 5, 6])).unwrap(); + assert_eq!( + client.join().expect("join client").expect("DNS client"), + [0, 3, 4, 5, 6] + ); + } +} diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-sandbox/src/process.rs similarity index 88% rename from crates/openshell-supervisor-process/src/process.rs rename to crates/openshell-sandbox/src/process.rs index f06d94d989..18e6d9dfe8 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -6,33 +6,27 @@ use crate::child_env; #[cfg(target_os = "linux")] use crate::managed_children; -#[cfg(target_os = "linux")] -use crate::netns::NetworkNamespace; use crate::sandbox; #[cfg(target_os = "linux")] use miette::WrapErr; use miette::{IntoDiagnostic, Result}; use nix::sys::signal::{self, Signal}; use nix::unistd::{Gid, Group, Pid, Uid, User}; -use openshell_core::policy::{NetworkMode, SandboxPolicy}; +use openshell_core::policy::SandboxPolicy; use std::collections::HashMap; use std::ffi::CString; #[cfg(unix)] use std::os::fd::AsRawFd; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; -#[cfg(target_os = "linux")] -use std::os::unix::ffi::OsStrExt; #[cfg(unix)] use std::os::unix::fs::{MetadataExt, PermissionsExt}; #[cfg(any(test, unix))] use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; #[cfg(target_os = "linux")] use std::sync::OnceLock; -#[cfg(target_os = "linux")] -use std::sync::mpsc; +use std::sync::atomic::{AtomicBool, Ordering}; use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; use tracing::{debug, info}; @@ -48,18 +42,6 @@ fn set_controlling_tty(fd: libc::c_int) -> std::io::Result<()> { Ok(()) } -/// Process/filesystem enforcement performed by the process supervisor. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ProcessEnforcementMode { - /// Preserve the existing supervisor behavior: prepare filesystem policy, - /// drop privileges, and apply Landlock/seccomp to workload processes. - Full, - /// Preserve process launch and SSH/session behavior, but skip controls - /// that require root or extra Linux capabilities. Kubernetes sidecar mode - /// uses this when network policy is enforced by the network sidecar. - NetworkOnly, -} - /// Numeric identity components resolved once from driver-owned metadata. /// /// A component is `None` when the corresponding policy field was explicit and @@ -128,34 +110,43 @@ impl ResolvedWorkspace { } } -impl ProcessEnforcementMode { - #[must_use] - pub const fn uses_privileged_process_setup(self) -> bool { - matches!(self, Self::Full) - } - - #[must_use] - pub const fn enforces_child_sandbox(self) -> bool { - matches!(self, Self::Full | Self::NetworkOnly) - } -} - #[cfg(target_os = "linux")] pub(crate) fn prepare_child_sandbox( policy: &SandboxPolicy, workdir: Option<&str>, - enforcement_mode: ProcessEnforcementMode, + runtime_read_only: &[PathBuf], ) -> Result> { - if !enforcement_mode.enforces_child_sandbox() { - return Ok(None); + let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); + let prepared = sandbox::linux::prepare_capability_free(&effective_policy, workdir)?; + Ok(Some(prepared)) +} + +#[cfg(target_os = "linux")] +fn policy_with_runtime_read_only( + policy: &SandboxPolicy, + runtime_read_only: &[PathBuf], +) -> SandboxPolicy { + let mut effective_policy = policy.clone(); + for path in runtime_read_only { + if !effective_policy.filesystem.read_only.contains(path) { + effective_policy.filesystem.read_only.push(path.clone()); + } } + effective_policy +} - let prepared = if enforcement_mode.uses_privileged_process_setup() { - sandbox::linux::prepare(policy, workdir) - } else { - sandbox::linux::prepare_current_user(policy, workdir) - }?; - Ok(Some(prepared)) +#[cfg(target_os = "linux")] +pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { + let Some((certificate, bundle)) = ca_paths else { + return Vec::new(); + }; + let mut paths = Vec::with_capacity(3); + if let Some(directory) = certificate.parent() { + paths.push(directory.to_path_buf()); + } + paths.push(certificate.clone()); + paths.push(bundle.clone()); + paths } const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ @@ -172,6 +163,19 @@ const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, ]; +const PROXY_ENV_VARS: &[&str] = &[ + "ALL_PROXY", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "all_proxy", + "http_proxy", + "https_proxy", + "no_proxy", + "grpc_proxy", + "NODE_USE_ENV_PROXY", +]; + pub fn is_supervisor_only_env_var(key: &str) -> bool { SUPERVISOR_ONLY_ENV_VARS.contains(&key) } @@ -182,6 +186,27 @@ fn strip_supervisor_only_env(cmd: &mut Command) { } } +/// Remove ambient proxy routing from a transparently mediated child. +pub fn strip_proxy_env(cmd: &mut Command) { + for key in PROXY_ENV_VARS { + cmd.env_remove(key); + } +} + +/// [`strip_proxy_env`] for synchronous exec commands. +pub fn strip_proxy_env_std(cmd: &mut std::process::Command) { + for key in PROXY_ENV_VARS { + cmd.env_remove(key); + } +} + +/// Whether an environment key can redirect a child around transparent +/// network mediation. +#[must_use] +pub fn is_proxy_env_var(key: &str) -> bool { + PROXY_ENV_VARS.contains(&key) +} + fn inject_provider_env(cmd: &mut Command, provider_env: &HashMap) { for (key, value) in provider_env { if is_supervisor_only_env_var(key) { @@ -396,262 +421,52 @@ fn validate_capability_bounding_set_clear( } } -// Pins the pre-seccomp child mount namespace where supervisor identity sockets -// are shadowed. Children enter it with setns before dropping privileges. -#[cfg(target_os = "linux")] -static SUPERVISOR_IDENTITY_MOUNT_NS: OnceLock> = - OnceLock::new(); - -#[cfg(target_os = "linux")] -pub struct SupervisorIdentityMountNamespace { - spawn_tx: mpsc::Sender, -} - -#[cfg(target_os = "linux")] -type SupervisorIdentityNsRef = &'static SupervisorIdentityMountNamespace; -#[cfg(target_os = "linux")] -type SupervisorIdentitySpawnJob = Box; - -#[cfg(target_os = "linux")] -impl SupervisorIdentityMountNamespace { - fn from_socket_path(socket_path: &str) -> Result> { - let Some(target) = supervisor_identity_mount_target(socket_path)? else { - return Ok(None); - }; - Ok(Some(Self { - spawn_tx: start_supervisor_identity_spawn_worker(target)?, - })) - } -} - #[cfg(target_os = "linux")] -pub fn prepare_supervisor_identity_mount_namespace_from_env() -> Result<()> { - if SUPERVISOR_IDENTITY_MOUNT_NS.get().is_some() { - return Ok(()); - } - - let Some((_env_name, socket_path)) = supervisor_identity_socket_path_from_env() else { - let _ = SUPERVISOR_IDENTITY_MOUNT_NS.set(None); - return Ok(()); - }; - let namespace = SupervisorIdentityMountNamespace::from_socket_path(&socket_path)?; - let _ = SUPERVISOR_IDENTITY_MOUNT_NS.set(namespace); - Ok(()) -} +static WORKLOAD_LAUNCHER: OnceLock< + openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, +> = OnceLock::new(); +/// Install the sandbox-owned launcher that every later workload spawn must +/// traverse. A second launcher would create a second listener generation and +/// is therefore rejected. #[cfg(target_os = "linux")] -pub fn supervisor_identity_mount_from_env() -> Result> { - let Some(namespace) = SUPERVISOR_IDENTITY_MOUNT_NS.get() else { - if supervisor_identity_socket_path_from_env().is_some() { - return Err(miette::miette!( - "supervisor identity mount namespace was not prepared before startup hardening" - )); - } - return Ok(None); - }; - Ok(namespace.as_ref()) +pub fn configure_workload_launcher( + launcher: openshell_isolation_interface::linux::workload_launcher::WorkloadLauncher, +) -> std::io::Result<()> { + WORKLOAD_LAUNCHER.set(launcher).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "workload launcher was already configured", + ) + }) } #[cfg(target_os = "linux")] -pub fn spawn_command_with_supervisor_identity_namespace( - mut cmd: Command, -) -> std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_tokio_command(cmd) +pub fn spawn_command_with_workload_launcher(mut cmd: Command) -> std::io::Result { + let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "sandbox workload launcher is not configured", + ) + })?; + let runtime = tokio::runtime::Handle::current(); + launcher.execute(move || { + let _guard = runtime.enter(); + cmd.spawn() + })? } #[cfg(target_os = "linux")] -pub fn spawn_std_command_with_supervisor_identity_namespace( +pub fn spawn_std_command_with_workload_launcher( mut cmd: std::process::Command, ) -> std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_std_command(cmd) -} - -#[cfg(target_os = "linux")] -impl SupervisorIdentityMountNamespace { - fn spawn_tokio_command(&self, mut cmd: Command) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - let handle = tokio::runtime::Handle::current(); - self.spawn_tx - .send(Box::new(move || { - let _guard = handle.enter(); - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } - - fn spawn_std_command( - &self, - mut cmd: std::process::Command, - ) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - self.spawn_tx - .send(Box::new(move || { - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } -} - -#[cfg(target_os = "linux")] -fn start_supervisor_identity_spawn_worker( - target: PathBuf, -) -> Result> { - let (spawn_tx, spawn_rx) = mpsc::channel::(); - let (ready_tx, ready_rx) = mpsc::channel::>(); - std::thread::Builder::new() - .name("openshell-identity-spawn".into()) - .spawn(move || { - let setup = (|| -> std::io::Result<()> { - private_mount_namespace()?; - let target = - cstring_path(&target).map_err(|err| std::io::Error::other(err.to_string()))?; - mount_empty_tmpfs(&target) - })(); - let ready = match &setup { - Ok(()) => Ok(()), - Err(err) => Err(std::io::Error::new( - err.kind(), - format!("supervisor identity setup failed: {err}"), - )), - }; - let _ = ready_tx.send(ready); - if setup.is_err() { - return; - } - while let Ok(job) = spawn_rx.recv() { - job(); - } - }) - .map_err(|err| miette::miette!("failed to spawn supervisor identity worker: {err}"))?; - ready_rx - .recv() - .map_err(|err| miette::miette!("supervisor identity worker did not start: {err}"))? - .map_err(|err| miette::miette!("{err}"))?; - Ok(spawn_tx) -} - -#[cfg(target_os = "linux")] -fn supervisor_identity_socket_path_from_env() -> Option<(&'static str, String)> { - std::env::var(openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) - .ok() - .filter(|socket_path| !socket_path.trim().is_empty()) - .map(|socket_path| { - ( - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - socket_path, - ) - }) -} - -#[cfg(any(test, target_os = "linux"))] -fn supervisor_identity_mount_target(socket_path: &str) -> Result> { - let trimmed = socket_path.trim(); - if trimmed.is_empty() { - return Ok(None); - } - if trimmed.starts_with("tcp:") { - return Ok(None); - } - let path = trimmed.strip_prefix("unix:").unwrap_or(trimmed); - let path = Path::new(path); - if !path.is_absolute() { - return Err(miette::miette!( - "{} must be an absolute UNIX socket path", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - } - let Some(parent) = path.parent() else { - return Err(miette::miette!( - "{} has no parent directory", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - }; - if parent == Path::new("/") { - return Err(miette::miette!( - "{} must live below a dedicated directory, not directly under /", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - )); - } - if is_shared_root_mount_shadow(parent) { - return Err(miette::miette!( - "{} must live below a dedicated subdirectory; refusing to hide shared directory {}", - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - parent.display() - )); - } - Ok(Some(parent.to_path_buf())) -} - -#[cfg(any(test, target_os = "linux"))] -fn is_shared_root_mount_shadow(parent: &Path) -> bool { - matches!(parent.to_str(), Some("/run" | "/var" | "/tmp" | "/etc")) -} - -#[cfg(target_os = "linux")] -fn cstring_path(path: &Path) -> Result { - CString::new(path.as_os_str().as_bytes()) - .map_err(|_| miette::miette!("path contains an interior NUL byte: {}", path.display())) -} - -#[cfg(target_os = "linux")] -fn private_mount_namespace() -> std::io::Result<()> { - #[allow(unsafe_code)] - let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - - #[allow(unsafe_code)] - let rc = unsafe { - let flags: libc::c_ulong = libc::MS_REC | libc::MS_PRIVATE; - libc::mount( - std::ptr::null(), - c"/".as_ptr(), - std::ptr::null(), - flags, - std::ptr::null(), - ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - -#[cfg(target_os = "linux")] -fn mount_empty_tmpfs(target: &CString) -> std::io::Result<()> { - #[allow(unsafe_code)] - let rc = unsafe { - let flags: libc::c_ulong = - libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_RDONLY; - libc::mount( - c"tmpfs".as_ptr(), - target.as_ptr(), - c"tmpfs".as_ptr(), - flags, - c"mode=0555,size=4k".as_ptr().cast(), + let launcher = WORKLOAD_LAUNCHER.get().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotConnected, + "sandbox workload launcher is not configured", ) - }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) + })?; + launcher.execute(move || cmd.spawn())? } /// Handle to a running process. @@ -659,6 +474,8 @@ pub struct ProcessHandle { child: Child, pid: u32, io: Option, + terminal: Arc, + signal_lock: Arc>, #[cfg(target_os = "linux")] managed_child: Option, } @@ -688,9 +505,6 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - netns: Option<&NetworkNamespace>, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -700,9 +514,6 @@ impl ProcessHandle { workspace, interactive, policy, - resolved_identity, - enforcement_mode, - netns.and_then(NetworkNamespace::ns_fd), ca_paths, provider_env, ) @@ -721,8 +532,6 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -732,8 +541,6 @@ impl ProcessHandle { workspace, interactive, policy, - resolved_identity, - enforcement_mode, ca_paths, provider_env, ) @@ -747,9 +554,6 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - netns_fd: Option, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -800,29 +604,7 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - // When using network namespace, set proxy URL to the veth host IP - if netns_fd.is_some() { - // The proxy is on 10.200.0.1:3128 (or configured port) - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - let proxy_url = format!("http://10.200.0.1:{port}"); - // Both uppercase and lowercase variants: curl/wget use uppercase, - // gRPC C-core (libgrpc) checks lowercase http_proxy/https_proxy. - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } else if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } - } + strip_proxy_env(&mut cmd); // Set TLS trust store env vars so sandbox processes trust the ephemeral CA if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { @@ -835,25 +617,26 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } + sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - // Phase 1: Prepare Landlock ruleset by opening PathFds. - // In full mode this runs before drop_privileges() so root-only paths - // can be opened. In sidecar network-only mode the container already - // runs as the sandbox UID, so inaccessible paths are unavailable to - // the workload and best-effort compatibility skips them. + // Prepare the Landlock ruleset as the workload UID. Inaccessible paths + // are already unavailable to the child and remain omitted. #[cfg(target_os = "linux")] - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + let runtime_read_only = ca_runtime_read_only_paths(ca_paths); + let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; + #[cfg(target_os = "linux")] + let mut child_hardening = + openshell_isolation_interface::linux::child_seccomp::prepare(std::process::id()) + .map_err(|error| { + miette::miette!("prepare child self-protection filter: {error}") + })?; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain // proper terminal control for shells and interactive programs. // SAFETY: pre_exec runs after fork but before exec in the child process. // setpgid and setns are async-signal-safe and safe to call in this context. { - let policy = policy.clone(); // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] @@ -870,25 +653,6 @@ impl ProcessHandle { return Err(std::io::Error::last_os_error()); } - // Enter network namespace before applying other restrictions. - if let Some(fd) = netns_fd { - let result = libc::setns(fd, libc::CLONE_NEWNET); - if result != 0 { - return Err(std::io::Error::other(format!( - "failed to enter network namespace: {}", - std::io::Error::last_os_error() - ))); - } - } - - // Drop privileges. initgroups/setgid/setuid need access to - // /etc/group and /etc/passwd which would be blocked if - // Landlock were already enforced. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(&policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; // Phase 2 (as unprivileged user): Enforce the prepared @@ -896,7 +660,7 @@ impl ProcessHandle { // restrict_self() does not require root. #[cfg(target_os = "linux")] if let Some(prepared) = prepared_sandbox.take() { - sandbox::linux::enforce(prepared) + sandbox::linux::enforce_capability_free(prepared, &mut child_hardening) .map_err(|err| std::io::Error::other(err.to_string()))?; } @@ -910,7 +674,7 @@ impl ProcessHandle { // or interpreter, and is a common failure on images that lack the // requested shell/binary (e.g. bash on Alpine). #[cfg(target_os = "linux")] - let mut child = spawn_command_with_supervisor_identity_namespace(cmd) + let mut child = spawn_command_with_workload_launcher(cmd) .into_diagnostic() .wrap_err_with(|| format!("failed to spawn sandbox entrypoint process '{program}'"))?; #[cfg(not(target_os = "linux"))] @@ -937,6 +701,8 @@ impl ProcessHandle { child, pid, io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), #[cfg(target_os = "linux")] managed_child, }) @@ -950,8 +716,6 @@ impl ProcessHandle { workspace: &ResolvedWorkspace, interactive: bool, policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -1002,19 +766,7 @@ impl ProcessHandle { cmd.current_dir(dir); } - if matches!(policy.network.mode, NetworkMode::Proxy) { - let proxy = policy.network.proxy.as_ref().ok_or_else(|| { - miette::miette!( - "Network mode is set to proxy but no proxy configuration was provided" - ) - })?; - if let Some(http_addr) = proxy.http_addr { - let proxy_url = format!("http://{http_addr}"); - for (key, value) in child_env::proxy_env_vars(&proxy_url) { - cmd.env(key, value); - } - } - } + strip_proxy_env(&mut cmd); // Set TLS trust store env vars so sandbox processes trust the ephemeral CA if let Some((ca_cert_path, combined_bundle_path)) = ca_paths { @@ -1044,20 +796,9 @@ impl ProcessHandle { return Err(std::io::Error::last_os_error()); } - // Drop privileges before applying sandbox restrictions. - // initgroups/setgid/setuid need access to /etc/group and /etc/passwd - // which may be blocked by Landlock. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(&policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - - if enforcement_mode.enforces_child_sandbox() { - sandbox::apply(&policy, workdir.as_deref()) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } + sandbox::apply(&policy, workdir.as_deref()) + .map_err(|err| std::io::Error::other(err.to_string()))?; Ok(()) }); @@ -1085,6 +826,8 @@ impl ProcessHandle { child, pid, io: Some(io), + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -1099,6 +842,12 @@ impl ProcessHandle { self.io.take().expect("canonical process I/O already taken") } + /// Shared state used by an independent boundary signal handle. + #[must_use] + pub fn signaling_state(&self) -> (Arc, Arc>) { + (self.terminal.clone(), self.signal_lock.clone()) + } + /// Wait for the process to exit. /// /// # Errors @@ -1106,6 +855,11 @@ impl ProcessHandle { /// Returns an error if waiting fails. pub async fn wait(&mut self) -> std::io::Result { let status = self.child.wait().await; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1118,6 +872,11 @@ impl ProcessHandle { pub fn try_wait(&mut self) -> std::io::Result> { let status = self.child.try_wait()?; if status.is_some() { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); #[cfg(target_os = "linux")] if let Some(child) = self.managed_child.take() { managed_children::unregister(child); @@ -1132,6 +891,13 @@ impl ProcessHandle { /// /// Returns an error if the signal cannot be sent. pub fn signal(&self, sig: Signal) -> Result<()> { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("process has exited")); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); signal::kill(Pid::from_raw(pid), sig).into_diagnostic() } @@ -2577,18 +2343,6 @@ mod tests { assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); } - #[test] - fn full_enforcement_uses_privileged_setup_and_child_sandbox() { - assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); - assert!(ProcessEnforcementMode::Full.enforces_child_sandbox()); - } - - #[test] - fn network_only_enforcement_keeps_child_sandbox_without_privileged_setup() { - assert!(!ProcessEnforcementMode::NetworkOnly.uses_privileged_process_setup()); - assert!(ProcessEnforcementMode::NetworkOnly.enforces_child_sandbox()); - } - #[cfg(target_os = "linux")] fn capability_bounding_set_clear_available() -> bool { capctl::caps::CapState::get_current() @@ -3439,6 +3193,79 @@ mod tests { } } + #[cfg(target_os = "linux")] + #[test] + fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.filesystem.read_only = vec![PathBuf::from("/usr")]; + let certificate = PathBuf::from("/run/openshell-proxy-ca/ca.crt"); + let bundle = PathBuf::from("/run/openshell-proxy-ca/ca-bundle.crt"); + + let effective = policy_with_runtime_read_only( + &policy, + &[certificate.clone(), bundle.clone(), certificate.clone()], + ); + + assert_eq!(policy.filesystem.read_only, vec![PathBuf::from("/usr")]); + assert_eq!( + effective.filesystem.read_only, + vec![PathBuf::from("/usr"), certificate, bundle] + ); + } + + #[cfg(target_os = "linux")] + #[test] + #[allow(unsafe_code)] + fn runtime_ca_material_remains_readable_after_landlock_for_non_root_workload() { + let root = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let ca_directory = root.path().join("openshell-proxy-ca"); + std::fs::create_dir(&ca_directory).unwrap(); + std::fs::set_permissions(&ca_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); + let certificate = ca_directory.join("ca.crt"); + let bundle = ca_directory.join("ca-bundle.crt"); + let denied = root.path().join("not-authorized"); + for path in [&certificate, &bundle, &denied] { + std::fs::write(path, b"public certificate material").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + } + + let mut policy = policy_with_process(ProcessPolicy::default()); + policy.landlock = LandlockPolicy { + compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, + }; + let runtime_paths = + ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); + let Ok(Some(prepared)) = prepare_child_sandbox(&policy, None, &runtime_paths) else { + return; + }; + + match unsafe { fork() }.expect("fork should succeed") { + ForkResult::Child => { + let dropped = if nix::unistd::geteuid().is_root() { + unsafe { + libc::setgroups(0, std::ptr::null()) == 0 + && libc::setgid(42_235) == 0 + && libc::setuid(42_234) == 0 + } + } else { + true + }; + let valid = dropped + && sandbox::linux::enforce(prepared).is_ok() + && std::fs::read(&certificate).is_ok() + && std::fs::read(&bundle).is_ok() + && std::fs::read(&denied).is_err(); + unsafe { libc::_exit(i32::from(!valid)) }; + } + ForkResult::Parent { child } => assert_eq!( + waitpid(child, None).expect("waitpid should succeed"), + WaitStatus::Exited(child, 0), + "Landlock must preserve non-root access only to admitted public CA material" + ), + } + } + #[cfg(unix)] #[test] fn validate_oci_workspace_rejects_restrictive_parent() { @@ -3900,57 +3727,32 @@ mod tests { assert!(stdout.contains("OPENSHELL_ENDPOINT=https://gateway.example.test")); } - #[test] - fn supervisor_identity_mount_target_uses_socket_parent() { - assert_eq!( - supervisor_identity_mount_target("/spiffe-workload-api/spire-agent.sock") - .expect("plain path should parse"), - Some(PathBuf::from("/spiffe-workload-api")) - ); - assert_eq!( - supervisor_identity_mount_target("unix:/spiffe-workload-api/spire-agent.sock") - .expect("unix path should parse"), - Some(PathBuf::from("/spiffe-workload-api")) - ); - } - - #[test] - fn supervisor_identity_mount_target_ignores_empty_socket_path() { - assert_eq!( - supervisor_identity_mount_target(" ").expect("empty path should be ignored"), - None - ); - } + #[tokio::test] + async fn transparent_mediation_removes_ambient_proxy_routing() { + let mut cmd = Command::new("/usr/bin/env"); + cmd.env_clear() + .stdin(StdStdio::null()) + .stdout(StdStdio::piped()) + .stderr(StdStdio::null()) + .env("PATH", "/usr/bin:/bin"); + for key in PROXY_ENV_VARS { + cmd.env(key, "http://ambient-proxy.invalid:3128"); + } - #[test] - fn supervisor_identity_mount_target_rejects_unhideable_endpoints() { - assert_eq!( - supervisor_identity_mount_target("tcp:127.0.0.1:8081") - .expect("tcp endpoint should not require mount hiding"), - None - ); - assert!(supervisor_identity_mount_target("spiffe-workload-api/spire-agent.sock").is_err()); - assert!(supervisor_identity_mount_target("/spire-agent.sock").is_err()); - } + strip_proxy_env(&mut cmd); - #[test] - fn supervisor_identity_mount_target_rejects_shared_root_shadowing() { - for socket_path in [ - "/run/spire-agent.sock", - "/var/spire-agent.sock", - "/tmp/spire-agent.sock", - "/etc/spire-agent.sock", - ] { - let err = supervisor_identity_mount_target(socket_path) - .expect_err("shared root shadowing should be rejected"); - assert!(err.to_string().contains("dedicated subdirectory")); + let output = cmd.output().await.expect("spawn env"); + assert!(output.status.success()); + let stdout = String::from_utf8(output.stdout).expect("utf8"); + for key in PROXY_ENV_VARS { + assert!( + !stdout + .lines() + .any(|line| line.starts_with(&format!("{key}="))), + "{key} must not redirect a transparently mediated process" + ); } - - assert_eq!( - supervisor_identity_mount_target("/run/spire/spire-agent.sock") - .expect("dedicated subdirectory should be accepted"), - Some(PathBuf::from("/run/spire")) - ); + assert!(stdout.contains("PATH=/usr/bin:/bin")); } // ---- Numeric UID tests (Phase 2) ---- diff --git a/crates/openshell-sandbox/src/pty.rs b/crates/openshell-sandbox/src/pty.rs new file mode 100644 index 0000000000..d887342306 --- /dev/null +++ b/crates/openshell-sandbox/src/pty.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Workload-side PTY and audited pre-exec setup. + +use std::os::fd::RawFd; +use std::process::Command; + +use nix::pty::Winsize; +use nix::unistd::setsid; +use openshell_core::policy::SandboxPolicy; +#[cfg(unix)] +use std::os::unix::process::CommandExt as _; + +#[allow(unsafe_code)] +pub fn set_winsize(fd: RawFd, winsize: Winsize) -> std::io::Result<()> { + // SAFETY: fd is the owned PTY master and winsize is initialized. + let rc = unsafe { libc::ioctl(fd, libc::TIOCSWINSZ, &winsize) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Install a pre-exec hook that gives the child a dedicated process group. +#[allow(unsafe_code)] +pub fn install_dedicated_process_group(command: &mut Command) { + // SAFETY: the hook invokes only the async-signal-safe setpgid syscall. + unsafe { + command.pre_exec(|| { + if libc::setpgid(0, 0) < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } +} + +#[allow(unsafe_code, clippy::useless_conversion)] +fn set_controlling_tty(fd: RawFd) -> std::io::Result<()> { + // SAFETY: fd is the slave PTY inherited by this pre-exec child. + let rc = unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[allow( + unsafe_code, + clippy::unnecessary_wraps, + reason = "pre-exec installation remains fallible as the prepared policy evolves" +)] +pub fn install_pre_exec( + command: &mut Command, + policy: SandboxPolicy, + _workdir: Option, + slave_fd: RawFd, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + let mut prepared = prepared; + #[cfg(target_os = "linux")] + let mut child_hardening = child_hardening; + // SAFETY: all allocations and policy compilation happened before spawn; + // the hook performs only the audited child transition. + unsafe { + command.pre_exec(move || { + setsid().map_err(|error| std::io::Error::other(error.to_string()))?; + set_controlling_tty(slave_fd)?; + enter_sandbox( + &policy, + #[cfg(target_os = "linux")] + prepared.take(), + #[cfg(target_os = "linux")] + &mut child_hardening, + ) + }); + } + Ok(()) +} + +#[allow( + unsafe_code, + clippy::unnecessary_wraps, + reason = "pre-exec installation remains fallible as the prepared policy evolves" +)] +pub fn install_pre_exec_no_pty( + command: &mut Command, + policy: SandboxPolicy, + _workdir: Option, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> anyhow::Result<()> { + #[cfg(target_os = "linux")] + let mut prepared = prepared; + #[cfg(target_os = "linux")] + let mut child_hardening = child_hardening; + // SAFETY: all allocations and policy compilation happened before spawn; + // the hook performs only the audited child transition. + unsafe { + command.pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } + enter_sandbox( + &policy, + #[cfg(target_os = "linux")] + prepared.take(), + #[cfg(target_os = "linux")] + &mut child_hardening, + ) + }); + } + Ok(()) +} + +fn enter_sandbox( + policy: &SandboxPolicy, + #[cfg(target_os = "linux")] prepared: Option, + #[cfg(target_os = "linux")] + child_hardening: &mut openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> std::io::Result<()> { + crate::process::harden_child_process() + .map_err(|error| std::io::Error::other(error.to_string()))?; + + #[cfg(target_os = "linux")] + if let Some(prepared) = prepared { + crate::sandbox::linux::enforce_capability_free(prepared, child_hardening) + .map_err(|error| std::io::Error::other(error.to_string()))?; + } + + #[cfg(not(target_os = "linux"))] + crate::sandbox::apply(policy, None) + .map_err(|error| std::io::Error::other(error.to_string()))?; + + #[cfg(target_os = "linux")] + let _ = policy; + + Ok(()) +} diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs similarity index 91% rename from crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs rename to crates/openshell-sandbox/src/sandbox/linux/landlock.rs index bf42faede8..9d4502dc5f 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/landlock.rs @@ -8,7 +8,10 @@ use landlock::{ Ruleset, RulesetAttr, RulesetCreatedAttr, }; use miette::{IntoDiagnostic, Result}; -use openshell_core::policy::{LandlockCompatibility, SandboxPolicy}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkPolicy, ProcessPolicy, + SandboxPolicy, +}; use std::os::fd::AsFd; use std::path::{Path, PathBuf}; use tracing::debug; @@ -115,8 +118,8 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result Result { + let read_write = capability_free_baseline_paths(Path::new("/"))?; + if read_write.is_empty() { + return Err(miette::miette!( + "capability-free Landlock baseline found no usable root entries" + )); + } + + let policy = SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy { + read_only: Vec::new(), + read_write, + include_workdir: false, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy { + compatibility: LandlockCompatibility::HardRequirement, + }, + process: ProcessPolicy::default(), + }; + prepare_with_path_open_mode(&policy, None, PathOpenMode::CurrentUser)?.ok_or_else(|| { + miette::miette!("capability-free Landlock baseline unexpectedly produced no ruleset") + }) +} + +fn capability_free_baseline_paths(root: &Path) -> Result> { + const PRIVATE_ROOT: &str = ".openshell"; + + let mut paths = Vec::new(); + for entry in std::fs::read_dir(root).into_diagnostic()? { + let entry = entry.into_diagnostic()?; + if entry.file_name() != PRIVATE_ROOT { + paths.push(entry.path()); + } + } + paths.sort(); + Ok(paths) +} + fn prepare_with_path_open_mode( policy: &SandboxPolicy, workdir: Option<&str>, @@ -488,7 +537,7 @@ fn compat_level(level: &LandlockCompatibility) -> CompatLevel { #[cfg(test)] mod tests { use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy}; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy}; fn hard_requirement_policy(read_only: Vec, read_write: Vec) -> SandboxPolicy { SandboxPolicy { @@ -522,6 +571,22 @@ mod tests { panic!("hard_requirement should accept mixed directory and device paths: {err}"); } } + + #[test] + fn capability_free_baseline_omits_only_private_root() { + let root = tempfile::tempdir().unwrap(); + for name in ["bin", "etc", "sandbox", ".openshell"] { + std::fs::create_dir(root.path().join(name)).unwrap(); + } + + let paths = capability_free_baseline_paths(root.path()).unwrap(); + assert_eq!( + paths, + ["bin", "etc", "sandbox"] + .map(|name| root.path().join(name)) + .to_vec() + ); + } fn tailored_access(path: &Path, requested_access: BitFlags) -> BitFlags { let path_fd = PathFd::new(path).unwrap(); access_for_path_fd(&path_fd, requested_access, ABI::V2).unwrap() diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs b/crates/openshell-sandbox/src/sandbox/linux/mod.rs similarity index 78% rename from crates/openshell-supervisor-process/src/sandbox/linux/mod.rs rename to crates/openshell-sandbox/src/sandbox/linux/mod.rs index 107a50e370..523d33bd0c 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/mod.rs @@ -14,7 +14,7 @@ use std::sync::Once; /// Opaque handle to a prepared-but-not-yet-enforced sandbox. /// Holds the Landlock ruleset with `PathFds` opened before child exec. pub struct PreparedSandbox { - landlock: Option, + landlock: Vec, policy: SandboxPolicy, } @@ -25,20 +25,41 @@ pub struct PreparedSandbox { pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result { let landlock = landlock::prepare(policy, workdir)?; Ok(PreparedSandbox { - landlock, + landlock: landlock.into_iter().collect(), policy: policy.clone(), }) } /// Phase 1 for already-unprivileged workloads. /// -/// Opens Landlock `PathFds` as the current UID. This is used by Kubernetes -/// sidecar mode, where the agent container already runs as the sandbox user. +/// Opens Landlock `PathFds` as the current workload UID. pub fn prepare_current_user( policy: &SandboxPolicy, workdir: Option<&str>, ) -> Result { let landlock = landlock::prepare_current_user(policy, workdir)?; + Ok(PreparedSandbox { + landlock: landlock.into_iter().collect(), + policy: policy.clone(), + }) +} + +/// Prepare the mandatory capability-free filesystem baseline plus the +/// optional user policy. +/// +/// The baseline is always a hard requirement. It grants access to each +/// top-level filesystem entry independently while deliberately omitting the +/// driver-owned `/.openshell` hierarchy. Applying the user ruleset after the +/// baseline intersects the two policies; it can narrow the baseline but can +/// never make the private hierarchy visible. +pub fn prepare_capability_free( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result { + let baseline = landlock::prepare_capability_free_baseline()?; + let user = landlock::prepare_current_user(policy, workdir)?; + let mut landlock = vec![baseline]; + landlock.extend(user); Ok(PreparedSandbox { landlock, policy: policy.clone(), @@ -50,9 +71,28 @@ pub fn prepare_current_user( /// Calls `restrict_self()` for Landlock and applies seccomp filters. /// Neither operation requires root privileges. pub fn enforce(prepared: PreparedSandbox) -> Result<()> { - if let Some(ruleset) = prepared.landlock { + for ruleset in prepared.landlock { + landlock::enforce(ruleset)?; + } + seccomp::apply(&prepared.policy)?; + Ok(()) +} + +/// Enforce the capability-free child filter stack. +/// +/// Landlock precedes sandbox-TGID self-protection. The ordinary workload +/// filter is installed last. The final filter blocks any later seccomp +/// installation, so this order is mandatory for capability-free children. +pub fn enforce_capability_free( + prepared: PreparedSandbox, + child_hardening: &mut openshell_isolation_interface::linux::child_seccomp::ChildHardeningProgram, +) -> Result<()> { + for ruleset in prepared.landlock { landlock::enforce(ruleset)?; } + child_hardening + .install() + .map_err(|error| miette::miette!("install child self-protection filter: {error}"))?; seccomp::apply(&prepared.policy)?; Ok(()) } diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs similarity index 99% rename from crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs rename to crates/openshell-sandbox/src/sandbox/linux/seccomp.rs index ddd37a502d..43fc59df51 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/seccomp.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs @@ -25,10 +25,9 @@ //! The risk is contained by existing sandbox layers: //! - **Privilege drop**: `CAP_NET_ADMIN` is not granted, so all write operations //! (add/delete routes, addresses, interfaces) fail with `EPERM` regardless. -//! - **Network namespace**: the sandboxed process sees only `lo` and one veth; -//! no host interfaces are visible. -//! - **nftables bypass rules**: all non-proxy traffic is rejected at the -//! netfilter level regardless of what the sandbox learns about its interfaces. +//! - **Driver outer fence**: direct workload egress is rejected outside this +//! process by Docker network-none, Kubernetes `NetworkPolicy`, or a NIC-less +//! VM. //! //! Every other netlink protocol (`NETLINK_SOCK_DIAG`, `NETLINK_NETFILTER`, //! `NETLINK_AUDIT`, `NETLINK_XFRM`, `NETLINK_GENERIC`, etc.) remains blocked. diff --git a/crates/openshell-supervisor-process/src/sandbox/mod.rs b/crates/openshell-sandbox/src/sandbox/mod.rs similarity index 100% rename from crates/openshell-supervisor-process/src/sandbox/mod.rs rename to crates/openshell-sandbox/src/sandbox/mod.rs diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs deleted file mode 100644 index 11f3e68e23..0000000000 --- a/crates/openshell-sandbox/src/sidecar_control.rs +++ /dev/null @@ -1,1210 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Local control channel for Kubernetes sidecar topology. -//! -//! The network sidecar owns gateway credentials. The process supervisor in the -//! agent container connects over this Unix socket to receive policy/provider -//! state without mounting gateway credentials into the agent container. - -use miette::{IntoDiagnostic, Result, WrapErr}; -use prost::Message; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, RwLock}; -use std::time::Duration; -use tokio::io::{AsyncBufReadExt, AsyncWrite, AsyncWriteExt, BufReader}; -use tokio::net::UnixListener; -use tokio::net::unix::OwnedWriteHalf; -use tokio::sync::{Mutex, broadcast, mpsc}; -use tracing::{debug, info, warn}; - -#[derive(Debug, Clone)] -pub struct BootstrapData { - pub policy_proto: openshell_core::proto::SandboxPolicy, - pub provider_env_revision: u64, - pub provider_env_generation: u64, - pub provider_child_env: HashMap, - pub agent_proposals_enabled: bool, - pub proxy_ca_cert_path: Option, - pub proxy_ca_bundle_path: Option, -} - -#[derive(Debug, Clone)] -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -pub struct EntrypointStarted { - pub pid: u32, - pub start_session: bool, - pub instance_id: String, - pub exit_code: Option, - pub finalized: bool, -} - -#[derive(Debug, Clone, Copy)] -pub struct ExpectedPeer { - pub uid: u32, - pub gid: u32, -} - -#[derive(Debug, Clone)] -pub enum ControlUpdate { - ProviderEnv { - revision: u64, - generation: u64, - provider_child_env: HashMap, - }, - Policy { - policy_proto: Box, - policy_hash: String, - config_revision: u64, - }, - AgentProposals { - enabled: bool, - config_revision: u64, - }, - MainProcessExitAck { - instance_id: String, - }, -} - -#[derive(Clone)] -pub struct Publisher { - state: Arc>, - updates: broadcast::Sender, -} - -impl Publisher { - pub fn publish_provider_env(&self, revision: u64, provider_child_env: HashMap) { - let mut state = self.state.write().expect("sidecar control state poisoned"); - if revision == state.provider_env_revision { - return; - } - state.provider_env_revision = revision; - state.provider_env_generation = state - .provider_env_generation - .checked_add(1) - .expect("sidecar provider environment generation overflow"); - state.provider_child_env.clone_from(&provider_child_env); - - // Keep generation assignment, bootstrap state, and publication under - // one lock so cloned publishers cannot emit generations out of order. - let _ = self.updates.send(WireServerMessage::ProviderEnvUpdated { - revision, - generation: state.provider_env_generation, - provider_child_env, - }); - } - - pub fn publish_policy( - &self, - policy_proto: openshell_core::proto::SandboxPolicy, - policy_hash: String, - config_revision: u64, - ) { - { - let mut state = self.state.write().expect("sidecar control state poisoned"); - state.policy_proto = policy_proto.clone(); - } - - let _ = self.updates.send(WireServerMessage::PolicyUpdated { - policy_proto: policy_proto.encode_to_vec(), - policy_hash, - config_revision, - }); - } - - pub fn publish_agent_proposals(&self, enabled: bool, config_revision: u64) { - { - let mut state = self.state.write().expect("sidecar control state poisoned"); - if state.agent_proposals_enabled == enabled { - return; - } - state.agent_proposals_enabled = enabled; - } - - let _ = self.updates.send(WireServerMessage::AgentProposalsUpdated { - enabled, - config_revision, - }); - } - - #[cfg(any(target_os = "linux", test))] - pub fn publish_main_process_exit_ack(&self, instance_id: String) { - let _ = self - .updates - .send(WireServerMessage::MainProcessExitAck { instance_id }); - } -} - -pub struct ServerHandle { - publisher: Publisher, - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - entrypoint_rx: mpsc::Receiver, - connection_task: tokio::task::JoinHandle<()>, -} - -impl ServerHandle { - pub fn publisher(&self) -> Publisher { - self.publisher.clone() - } - - #[cfg(test)] - pub fn into_entrypoint_receiver(self) -> mpsc::Receiver { - self.entrypoint_rx - } - - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - pub fn into_runtime_parts( - self, - ) -> ( - mpsc::Receiver, - tokio::task::JoinHandle<()>, - ) { - (self.entrypoint_rx, self.connection_task) - } -} - -pub struct ProcessConnection { - pub writer: Arc>, - pub updates: mpsc::UnboundedReceiver, - pub closed: tokio::sync::oneshot::Receiver<()>, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum WireClientMessage { - BootstrapRequest { supervisor_pid: u32 }, - EntrypointStarted { pid: u32, instance_id: String }, - MainProcessExited { instance_id: String, exit_code: i32 }, - MainProcessFinalized { instance_id: String }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum WireServerMessage { - BootstrapResponse { - policy_proto: Vec, - provider_env_revision: u64, - provider_env_generation: u64, - provider_child_env: HashMap, - agent_proposals_enabled: bool, - proxy_ca_cert_path: Option, - proxy_ca_bundle_path: Option, - }, - ProviderEnvUpdated { - revision: u64, - generation: u64, - provider_child_env: HashMap, - }, - PolicyUpdated { - policy_proto: Vec, - policy_hash: String, - config_revision: u64, - }, - AgentProposalsUpdated { - enabled: bool, - config_revision: u64, - }, - MainProcessExitAck { - instance_id: String, - }, -} - -impl BootstrapData { - #[cfg_attr(not(target_os = "linux"), allow(dead_code))] - fn to_wire(&self) -> WireServerMessage { - WireServerMessage::BootstrapResponse { - policy_proto: self.policy_proto.encode_to_vec(), - provider_env_revision: self.provider_env_revision, - provider_env_generation: self.provider_env_generation, - provider_child_env: self.provider_child_env.clone(), - agent_proposals_enabled: self.agent_proposals_enabled, - proxy_ca_cert_path: self - .proxy_ca_cert_path - .as_ref() - .map(|path| path.display().to_string()), - proxy_ca_bundle_path: self - .proxy_ca_bundle_path - .as_ref() - .map(|path| path.display().to_string()), - } - } -} - -impl TryFrom for BootstrapData { - type Error = miette::Report; - - fn try_from(message: WireServerMessage) -> Result { - let WireServerMessage::BootstrapResponse { - policy_proto, - provider_env_revision, - provider_env_generation, - provider_child_env, - agent_proposals_enabled, - proxy_ca_cert_path, - proxy_ca_bundle_path, - } = message - else { - return Err(miette::miette!( - "expected sidecar bootstrap response, received update message" - )); - }; - - let policy_proto = openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) - .into_diagnostic() - .wrap_err("failed to decode sidecar bootstrap policy")?; - let policy_proto = canonicalize_sidecar_policy( - policy_proto, - "sidecar bootstrap policy failed validation", - )?; - - Ok(Self { - policy_proto, - provider_env_revision, - provider_env_generation, - provider_child_env, - agent_proposals_enabled, - proxy_ca_cert_path: proxy_ca_cert_path.map(PathBuf::from), - proxy_ca_bundle_path: proxy_ca_bundle_path.map(PathBuf::from), - }) - } -} - -impl TryFrom for ControlUpdate { - type Error = miette::Report; - - fn try_from(message: WireServerMessage) -> Result { - match message { - WireServerMessage::ProviderEnvUpdated { - revision, - generation, - provider_child_env, - } => Ok(Self::ProviderEnv { - revision, - generation, - provider_child_env, - }), - WireServerMessage::PolicyUpdated { - policy_proto, - policy_hash, - config_revision, - } => { - let policy_proto = - openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) - .into_diagnostic() - .wrap_err("failed to decode sidecar policy update")?; - let policy_proto = canonicalize_sidecar_policy( - policy_proto, - "sidecar policy update failed validation", - )?; - Ok(Self::Policy { - policy_proto: Box::new(policy_proto), - policy_hash, - config_revision, - }) - } - WireServerMessage::AgentProposalsUpdated { - enabled, - config_revision, - } => Ok(Self::AgentProposals { - enabled, - config_revision, - }), - WireServerMessage::MainProcessExitAck { instance_id } => { - Ok(Self::MainProcessExitAck { instance_id }) - } - WireServerMessage::BootstrapResponse { .. } => Err(miette::miette!( - "unexpected sidecar bootstrap response after initial handshake" - )), - } - } -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -pub fn spawn_server( - path: &Path, - bootstrap: BootstrapData, - expected_peer: ExpectedPeer, -) -> Result { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to create sidecar control socket dir {}", - parent.display() - ) - })?; - } - match std::fs::remove_file(path) { - Ok(()) => {} - Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} - Err(err) => { - return Err(err).into_diagnostic().wrap_err_with(|| { - format!( - "failed to remove stale sidecar control socket {}", - path.display() - ) - }); - } - } - - let listener = UnixListener::bind(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to bind sidecar control socket {}", path.display()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to set permissions on sidecar control socket {}", - path.display() - ) - })?; - } - - let state = Arc::new(RwLock::new(bootstrap)); - let (updates, _) = broadcast::channel(32); - let (entrypoint_tx, entrypoint_rx) = mpsc::channel(8); - let publisher = Publisher { - state: state.clone(), - updates: updates.clone(), - }; - - let connection_task = tokio::spawn(accept_authoritative_connection( - listener, - path.to_path_buf(), - expected_peer, - state, - updates, - entrypoint_tx, - )); - info!(path = %path.display(), "Sidecar control socket listening"); - - Ok(ServerHandle { - publisher, - entrypoint_rx, - connection_task, - }) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -async fn accept_authoritative_connection( - listener: UnixListener, - socket_path: PathBuf, - expected_peer: ExpectedPeer, - state: Arc>, - updates: broadcast::Sender, - entrypoint_tx: mpsc::Sender, -) { - let stream = match listener.accept().await { - Ok((stream, _addr)) => stream, - Err(err) => { - warn!(error = %err, "Failed to accept authoritative sidecar control connection"); - return; - } - }; - - // The process supervisor connects before it launches the workload. Drop - // the listener and unlink its pathname after that first accept so workload - // processes can neither open a second control channel nor impersonate a - // restarted server at the trusted path. - drop(listener); - if let Err(err) = std::fs::remove_file(&socket_path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - path = %socket_path.display(), - error = %err, - "Failed to unlink accepted sidecar control socket" - ); - } - - if let Err(err) = handle_connection(stream, expected_peer, state, updates, entrypoint_tx).await - { - warn!(error = %err, "Authoritative sidecar control connection closed"); - } -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -async fn handle_connection( - stream: tokio::net::UnixStream, - expected_peer: ExpectedPeer, - state: Arc>, - updates: broadcast::Sender, - entrypoint_tx: mpsc::Sender, -) -> Result<()> { - let credentials = stream - .peer_cred() - .into_diagnostic() - .wrap_err("failed to read sidecar control peer credentials")?; - if credentials.uid() != expected_peer.uid || credentials.gid() != expected_peer.gid { - return Err(miette::miette!( - "sidecar control peer identity mismatch: expected uid:gid {}:{}, got {}:{}", - expected_peer.uid, - expected_peer.gid, - credentials.uid(), - credentials.gid(), - )); - } - let peer_pid = credentials - .pid() - .and_then(|pid| u32::try_from(pid).ok()) - .ok_or_else(|| miette::miette!("sidecar control peer PID is unavailable"))?; - - let (reader, mut writer) = stream.into_split(); - let mut lines = BufReader::new(reader).lines(); - - let first_line = - lines.next_line().await.into_diagnostic()?.ok_or_else(|| { - miette::miette!("sidecar control client disconnected before bootstrap") - })?; - match decode_client_message(&first_line)? { - WireClientMessage::BootstrapRequest { supervisor_pid } => { - if supervisor_pid == 0 || supervisor_pid != peer_pid { - return Err(miette::miette!( - "sidecar bootstrap PID mismatch: peer PID {peer_pid}, claimed PID {supervisor_pid}" - )); - } - entrypoint_tx - .send(EntrypointStarted { - pid: supervisor_pid, - start_session: false, - instance_id: String::new(), - exit_code: None, - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::EntrypointStarted { .. } - | WireClientMessage::MainProcessExited { .. } - | WireClientMessage::MainProcessFinalized { .. } => { - return Err(miette::miette!( - "sidecar control client sent entrypoint event before bootstrap" - )); - } - } - - // Subscribe before taking the bootstrap snapshot so an update can neither - // be missed between the snapshot and the live update stream nor omitted - // from the snapshot itself. - let mut update_rx = updates.subscribe(); - let bootstrap = { - let state = state.read().expect("sidecar control state poisoned"); - state.to_wire() - }; - write_json_line(&mut writer, &bootstrap).await?; - - loop { - tokio::select! { - line = lines.next_line() => { - let Some(line) = line.into_diagnostic()? else { - return Ok(()); - }; - match decode_client_message(&line)? { - WireClientMessage::BootstrapRequest { .. } => { - debug!("Ignoring duplicate sidecar bootstrap request"); - } - WireClientMessage::EntrypointStarted { pid, instance_id } => { - if pid == 0 { - warn!("Ignoring sidecar entrypoint event with pid=0"); - continue; - } - entrypoint_tx - .send(EntrypointStarted { - pid, - start_session: true, - instance_id, - exit_code: None, - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::MainProcessExited { - instance_id, - exit_code, - } => { - entrypoint_tx - .send(EntrypointStarted { - pid: 0, - start_session: false, - instance_id, - exit_code: Some(exit_code), - finalized: false, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - WireClientMessage::MainProcessFinalized { instance_id } => { - entrypoint_tx - .send(EntrypointStarted { - pid: 0, - start_session: false, - instance_id, - exit_code: None, - finalized: true, - }) - .await - .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; - } - } - } - update = update_rx.recv() => { - match update { - Ok(message) => write_json_line(&mut writer, &message).await?, - Err(broadcast::error::RecvError::Lagged(skipped)) => { - warn!(skipped, "Sidecar control client lagged behind updates"); - } - Err(broadcast::error::RecvError::Closed) => return Ok(()), - } - } - } - } -} - -pub async fn connect_process_client( - path: &Path, - timeout: Duration, -) -> Result<(BootstrapData, ProcessConnection)> { - let stream = connect_with_retry(path, timeout).await?; - let (reader, mut writer) = stream.into_split(); - write_json_line( - &mut writer, - &WireClientMessage::BootstrapRequest { - supervisor_pid: std::process::id(), - }, - ) - .await?; - - let mut lines = BufReader::new(reader).lines(); - let first_line = lines - .next_line() - .await - .into_diagnostic()? - .ok_or_else(|| miette::miette!("sidecar control closed before bootstrap response"))?; - let bootstrap = BootstrapData::try_from(decode_server_message(&first_line)?)?; - - let (update_tx, updates) = mpsc::unbounded_channel(); - let (closed_tx, closed) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - while let Ok(Some(line)) = lines.next_line().await { - match decode_server_message(&line).and_then(ControlUpdate::try_from) { - Ok(update) => { - if update_tx.send(update).is_err() { - break; - } - } - Err(err) => { - warn!(error = %err, "Ignoring invalid sidecar control update"); - } - } - } - let _ = closed_tx.send(()); - }); - - Ok(( - bootstrap, - ProcessConnection { - writer: Arc::new(Mutex::new(writer)), - updates, - closed, - }, - )) -} - -async fn connect_with_retry(path: &Path, timeout: Duration) -> Result { - let deadline = tokio::time::Instant::now() + timeout; - loop { - match tokio::net::UnixStream::connect(path).await { - Ok(stream) => return Ok(stream), - Err(err) if tokio::time::Instant::now() < deadline => { - debug!( - path = %path.display(), - error = %err, - "Waiting for sidecar control socket" - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - Err(err) => { - return Err(err).into_diagnostic().wrap_err_with(|| { - format!( - "timed out waiting for sidecar control socket {}", - path.display() - ) - }); - } - } - } -} - -pub async fn send_entrypoint_started( - writer: &Arc>, - pid: u32, - instance_id: String, -) -> Result<()> { - let message = WireClientMessage::EntrypointStarted { pid, instance_id }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -pub async fn send_main_process_exited( - writer: &Arc>, - instance_id: String, - exit_code: i32, -) -> Result<()> { - let message = WireClientMessage::MainProcessExited { - instance_id, - exit_code, - }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -pub async fn send_main_process_finalized( - writer: &Arc>, - instance_id: String, -) -> Result<()> { - let message = WireClientMessage::MainProcessFinalized { instance_id }; - let mut writer = writer.lock().await; - write_json_line(&mut *writer, &message).await -} - -async fn write_json_line(writer: &mut W, value: &T) -> Result<()> -where - W: AsyncWrite + Unpin + Send, - T: Serialize + Sync, -{ - let bytes = serde_json::to_vec(value).into_diagnostic()?; - writer.write_all(&bytes).await.into_diagnostic()?; - writer.write_all(b"\n").await.into_diagnostic()?; - writer.flush().await.into_diagnostic()?; - Ok(()) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn decode_client_message(line: &str) -> Result { - serde_json::from_str(line) - .into_diagnostic() - .wrap_err("failed to decode sidecar client message") -} - -fn decode_server_message(line: &str) -> Result { - serde_json::from_str(line) - .into_diagnostic() - .wrap_err("failed to decode sidecar server message") -} - -fn canonicalize_sidecar_policy( - policy: openshell_core::proto::SandboxPolicy, - error_message: &'static str, -) -> Result { - // Bootstrap and update messages must expose the same canonical typed - // policy to every process-supervisor consumer. Keep validation details - // out of this channel error because they can contain authored values. - openshell_policy::validate_and_canonicalize_sandbox_policy(policy) - .map_err(|_| miette::miette!(error_message)) -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::proto::{McpOptions, NetworkEndpoint, NetworkPolicyRule, SandboxPolicy}; - - fn defaultable_mcp_policy(mcp: Option) -> SandboxPolicy { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "mcp".to_string(), - NetworkPolicyRule { - name: "mcp".to_string(), - endpoints: vec![NetworkEndpoint { - host: "mcp.example.com".to_string(), - port: 443, - protocol: "mcp".to_string(), - mcp, - rules: vec![openshell_core::proto::L7Rule { - allow: Some(openshell_core::proto::L7Allow { - method: "tools/list".to_string(), - ..Default::default() - }), - }], - ..Default::default() - }], - ..Default::default() - }, - ); - policy - } - - fn mcp_versions(policy: &SandboxPolicy) -> &[String] { - policy.network_policies["mcp"].endpoints[0] - .mcp - .as_ref() - .expect("canonical MCP options") - .versions - .as_slice() - } - - fn bootstrap_message(policy: &SandboxPolicy) -> WireServerMessage { - WireServerMessage::BootstrapResponse { - policy_proto: policy.encode_to_vec(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - } - } - - fn policy_update_message(policy: &SandboxPolicy) -> WireServerMessage { - WireServerMessage::PolicyUpdated { - policy_proto: policy.encode_to_vec(), - policy_hash: "hash".to_string(), - config_revision: 1, - } - } - - fn current_peer() -> ExpectedPeer { - ExpectedPeer { - uid: nix::unistd::Uid::current().as_raw(), - gid: nix::unistd::Gid::current().as_raw(), - } - } - - #[test] - fn policy_messages_canonicalize_defaultable_mcp_versions() { - for raw in [ - defaultable_mcp_policy(None), - defaultable_mcp_policy(Some(McpOptions::default())), - ] { - let bootstrap = BootstrapData::try_from(bootstrap_message(&raw)) - .expect("defaultable MCP policy must pass bootstrap ingress"); - assert_eq!(mcp_versions(&bootstrap.policy_proto), ["2025-11-25"]); - - let update = ControlUpdate::try_from(policy_update_message(&raw)) - .expect("defaultable MCP policy must pass update ingress"); - let ControlUpdate::Policy { policy_proto, .. } = update else { - panic!("expected policy update"); - }; - assert_eq!(mcp_versions(&policy_proto), ["2025-11-25"]); - } - } - - #[test] - fn policy_messages_reject_invalid_mcp_versions_without_echoing_values() { - let invalid = defaultable_mcp_policy(Some(McpOptions { - versions: vec!["latest".to_string()], - ..Default::default() - })); - - let bootstrap_error = BootstrapData::try_from(bootstrap_message(&invalid)) - .expect_err("invalid MCP policy must not pass bootstrap ingress") - .to_string(); - assert_eq!( - bootstrap_error, - "sidecar bootstrap policy failed validation" - ); - assert!(!bootstrap_error.contains("latest")); - - let update_error = ControlUpdate::try_from(policy_update_message(&invalid)) - .expect_err("invalid MCP policy must not pass update ingress") - .to_string(); - assert_eq!(update_error, "sidecar policy update failed validation"); - assert!(!update_error.contains("latest")); - } - - #[tokio::test] - async fn bootstrap_round_trips_policy_and_provider_env() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let mut env = HashMap::new(); - env.insert("GITHUB_TOKEN".to_string(), "secret".to_string()); - let bootstrap = BootstrapData { - policy_proto: SandboxPolicy { - version: 7, - ..SandboxPolicy::default() - }, - provider_env_revision: 3, - provider_env_generation: 0, - provider_child_env: env.clone(), - agent_proposals_enabled: true, - proxy_ca_cert_path: Some(PathBuf::from("/tmp/ca.pem")), - proxy_ca_bundle_path: Some(PathBuf::from("/tmp/bundle.pem")), - }; - - let _server = spawn_server(&socket, bootstrap, current_peer()).unwrap(); - let (received, _connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - assert_eq!(received.policy_proto.version, 7); - assert_eq!(received.provider_env_revision, 3); - assert_eq!(received.provider_env_generation, 0); - assert_eq!(received.provider_child_env, env); - assert!(received.agent_proposals_enabled); - assert_eq!( - received.proxy_ca_cert_path, - Some(PathBuf::from("/tmp/ca.pem")) - ); - assert_eq!( - received.proxy_ca_bundle_path, - Some(PathBuf::from("/tmp/bundle.pem")) - ); - } - - #[tokio::test] - async fn provider_env_updates_use_generation_not_fingerprint_order() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: u64::MAX, - provider_env_generation: 7, - provider_child_env: HashMap::from([("TOKEN".to_string(), "first".to_string())]), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - publisher.publish_provider_env( - 1, - HashMap::from([("TOKEN".to_string(), "second".to_string())]), - ); - - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - assert_eq!(revision, 1); - assert_eq!(generation, 8); - assert_eq!( - provider_child_env.get("TOKEN").map(String::as_str), - Some("second") - ); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - - publisher.publish_provider_env( - 1, - HashMap::from([("TOKEN".to_string(), "duplicate".to_string())]), - ); - assert!( - tokio::time::timeout(Duration::from_millis(50), connection.updates.recv()) - .await - .is_err(), - "an identical fingerprint must remain a no-op" - ); - - publisher.publish_provider_env( - u64::MAX, - HashMap::from([("TOKEN".to_string(), "third".to_string())]), - ); - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::ProviderEnv { - revision, - generation, - provider_child_env, - } => { - assert_eq!(revision, u64::MAX); - assert_eq!(generation, 9); - assert_eq!( - provider_child_env.get("TOKEN").map(String::as_str), - Some("third") - ); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - } - - #[tokio::test] - async fn agent_proposals_update_is_delivered_to_process_client() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - publisher.publish_agent_proposals(true, 9); - - let update = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - match update { - ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - assert!(enabled); - assert_eq!(config_revision, 9); - } - other => panic!("unexpected sidecar update: {other:?}"), - } - } - - #[tokio::test] - async fn entrypoint_started_is_delivered_to_server() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let publisher = server.publisher(); - let mut entrypoint_rx = server.into_entrypoint_receiver(); - let (_bootstrap, mut connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - let anchor = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(anchor.pid, std::process::id()); - assert!(!anchor.start_session); - - send_entrypoint_started(&connection.writer, 4242, "instance-1".to_string()) - .await - .unwrap(); - - let started = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(started.pid, 4242); - assert!(started.start_session); - assert_eq!(started.instance_id, "instance-1"); - assert!(started.exit_code.is_none()); - - send_main_process_exited(&connection.writer, "instance-1".to_string(), 0) - .await - .unwrap(); - let terminal = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(terminal.exit_code, Some(0)); - assert!(!terminal.finalized); - - assert!( - tokio::time::timeout(Duration::from_millis(20), connection.updates.recv()) - .await - .is_err(), - "process side must not observe a durable ACK before gateway persistence" - ); - publisher.publish_main_process_exit_ack("instance-1".to_string()); - let ack = tokio::time::timeout(Duration::from_secs(1), connection.updates.recv()) - .await - .unwrap() - .unwrap(); - assert!(matches!( - ack, - ControlUpdate::MainProcessExitAck { instance_id } if instance_id == "instance-1" - )); - - send_main_process_finalized(&connection.writer, "instance-1".to_string()) - .await - .unwrap(); - let delivered = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .unwrap(); - assert!(delivered.exit_code.is_none()); - assert!(delivered.finalized); - } - - #[tokio::test] - async fn second_control_client_is_rejected_after_authoritative_bootstrap() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let _server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - - let (_bootstrap, _connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - let err = tokio::net::UnixStream::connect(&socket) - .await - .expect_err("control listener must be removed after the first bootstrap"); - assert!( - matches!( - err.kind(), - std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused - ), - "unexpected second-client error: {err}" - ); - } - - #[tokio::test] - async fn authoritative_connection_task_ends_when_process_supervisor_disconnects() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); - let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - drop(connection); - tokio::time::timeout(Duration::from_secs(1), connection_task) - .await - .expect("server must observe authoritative client disconnect") - .expect("control task must not panic"); - } - - #[tokio::test] - async fn process_client_reports_network_sidecar_restart() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); - let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) - .await - .unwrap(); - - connection_task.abort(); - let _ = connection_task.await; - tokio::time::timeout(Duration::from_secs(1), connection.closed) - .await - .expect("process supervisor must observe network sidecar disconnect") - .expect("disconnect notifier must remain live"); - } - - #[tokio::test] - async fn bootstrap_rejects_claimed_pid_that_does_not_match_peer_credentials() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("control.sock"); - let server = spawn_server( - &socket, - BootstrapData { - policy_proto: SandboxPolicy::default(), - provider_env_revision: 0, - provider_env_generation: 0, - provider_child_env: HashMap::new(), - agent_proposals_enabled: false, - proxy_ca_cert_path: None, - proxy_ca_bundle_path: None, - }, - current_peer(), - ) - .unwrap(); - let mut entrypoint_rx = server.into_entrypoint_receiver(); - - let mut stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); - write_json_line( - &mut stream, - &WireClientMessage::BootstrapRequest { - supervisor_pid: std::process::id().saturating_add(1), - }, - ) - .await - .unwrap(); - - assert!( - tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) - .await - .unwrap() - .is_none(), - "mismatched bootstrap must not publish a process anchor" - ); - } - - #[test] - fn malformed_client_message_is_rejected() { - let err = decode_client_message("not-json").unwrap_err(); - assert!( - err.to_string() - .contains("failed to decode sidecar client message") - ); - } -} diff --git a/crates/openshell-sandbox/tests/stdout_logging.rs b/crates/openshell-sandbox/tests/stdout_logging.rs index c4f5213de8..403ed31cb3 100644 --- a/crates/openshell-sandbox/tests/stdout_logging.rs +++ b/crates/openshell-sandbox/tests/stdout_logging.rs @@ -6,21 +6,16 @@ use std::process::Command; #[test] fn startup_logs_go_to_stderr_not_stdout() { let output = Command::new(env!("CARGO_BIN_EXE_openshell-sandbox")) - .arg("--") - .arg("/usr/bin/printf") - .arg("hello") + .arg("--bootstrap") + .arg("/does/not/exist/openshell-boundary.json") .env("OPENSHELL_LOG_LEVEL", "info") .env_remove("RUST_LOG") - .env_remove("OPENSHELL_POLICY_RULES") - .env_remove("OPENSHELL_POLICY_DATA") - .env_remove("OPENSHELL_SANDBOX_ID") - .env_remove("OPENSHELL_ENDPOINT") .output() .expect("spawn openshell-sandbox"); assert!( !output.status.success(), - "expected sandbox startup to fail without a policy source" + "expected sandbox startup to fail without bootstrap material" ); let stdout = String::from_utf8_lossy(&output.stdout); @@ -31,11 +26,7 @@ fn startup_logs_go_to_stderr_not_stdout() { "expected startup logs on stderr only, got stdout: {stdout}" ); assert!( - stderr.contains("Starting sandbox"), - "expected startup log on stderr, got: {stderr}" - ); - assert!( - stderr.contains("Sandbox policy required"), - "expected missing-policy error on stderr, got: {stderr}" + stderr.contains("capability-free sandbox probe") || stderr.contains("read boundary config"), + "expected startup qualification or bootstrap error on stderr, got: {stderr}" ); } diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 64fd40cee2..252c7bea63 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -2207,10 +2207,7 @@ fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool { let phase = SandboxPhase::try_from(sandbox.phase()).ok(); matches!(phase, Some(SandboxPhase::Ready)) || (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error)) - && state.supervisor_sessions.has_session(sandbox.object_id()) - && !state - .supervisor_sessions - .terminal_delivery_finalized(sandbox.object_id())) + && state.supervisor_sessions.has_session(sandbox.object_id())) } pub(super) async fn handle_create_ssh_session( @@ -5814,6 +5811,9 @@ mod tests { .supervisor_sessions .finalize_main_process_exit("sandbox-work") ); + assert!(sandbox_relay_reachable(&state, &sandbox)); + + assert!(state.supervisor_sessions.disconnect("sandbox-work")); assert!(!sandbox_relay_reachable(&state, &sandbox)); } @@ -6577,7 +6577,16 @@ mod tests { let mut sandbox = test_sandbox("cross-ws", Vec::new()); sandbox.metadata.as_mut().unwrap().workspace = "other-workspace".to_string(); + sandbox.set_phase(SandboxPhase::Completed as i32); state.store.put_message(&sandbox).await.unwrap(); + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + let _ = state.supervisor_sessions.register( + sandbox.object_id().to_string(), + "retained-terminal-session".to_string(), + tx, + shutdown_tx, + ); // --- handle_watch_sandbox --- let err = handle_watch_sandbox( diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index e3fa6d36b4..bb6001f023 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -900,8 +900,8 @@ _matching_endpoint_configs := [cfg | # Full matched endpoint records are kept separate from the legacy # endpoint-config list, which intentionally contains only connection/L7 # metadata. The policy name and array index identify the endpoint within this -# policy generation while the complete endpoint preserves explicit protocol -# markers needed by later policy-DNS correlation. +# policy generation while the complete endpoint preserves protocol markers +# needed by later policy-DNS correlation. _policy_endpoint_records(policy_name, policy) := [record | some endpoint_index, ep in policy.endpoints @@ -922,12 +922,15 @@ _matching_endpoint_records := [record | # Endpoints eligible for policy DNS are a policy-data snapshot, not an # authorization decision. In particular, they do not depend on input.exec or -# grant access to any process. Only endpoints that explicitly opt into raw TCP -# and provide a resolvable host plus concrete ports are materialized. +# grant access to any process. Every supported endpoint protocol is carried by +# TCP, and an omitted protocol is the default L4 TCP form. Endpoints with a +# resolvable host plus concrete ports are therefore materialized regardless of +# whether later stream handling is L4, HTTP, WebSocket, or another L7 adapter. policy_dns_eligible_endpoint_records := [record | some policy_name, policy in data.network_policies some endpoint_index, ep in policy.endpoints - lower(object.get(ep, "protocol", "")) == "tcp" + protocol := lower(object.get(ep, "protocol", "tcp")) + protocol in {"tcp", "rest", "websocket", "graphql", "sql", "json-rpc", "mcp"} object.get(ep, "host", "") != "" ports := object.get(ep, "ports", []) count(ports) > 0 diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 93315a671a..fb5db6b94a 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -3301,16 +3301,30 @@ fn parse_status_code(headers: &str) -> Option { code_str.parse().ok() } -/// Check if the response headers contain `Connection: close`. +/// Check whether the response is delimited by closing the connection. +/// +/// HTTP/1.0 closes by default unless the server explicitly negotiates +/// keep-alive. HTTP/1.1 keeps connections alive by default unless the server +/// sends `Connection: close`. fn parse_connection_close(headers: &str) -> bool { + let http_1_0 = headers + .lines() + .next() + .is_some_and(|line| line.starts_with("HTTP/1.0 ")); for line in headers.lines().skip(1) { let lower = line.to_ascii_lowercase(); if lower.starts_with("connection:") { let val = lower.split_once(':').map_or("", |(_, v)| v.trim()); - return val.contains("close"); + return if http_1_0 { + !val.split(',') + .any(|token| token.trim().eq_ignore_ascii_case("keep-alive")) + } else { + val.split(',') + .any(|token| token.trim().eq_ignore_ascii_case("close")) + }; } } - false + http_1_0 } fn response_is_event_stream(headers: &str) -> bool { @@ -5477,6 +5491,10 @@ mod tests { assert!(!parse_connection_close( "HTTP/1.1 200 OK\r\nHost: x\r\n\r\n" )); + assert!(parse_connection_close("HTTP/1.0 200 OK\r\nHost: x\r\n\r\n")); + assert!(!parse_connection_close( + "HTTP/1.0 200 OK\r\nConnection: keep-alive\r\n\r\n" + )); } #[test] @@ -5549,6 +5567,41 @@ mod tests { ); } + #[tokio::test] + async fn relay_response_http_1_0_defaults_to_connection_close() { + let response = b"HTTP/1.0 200 OK\r\nServer: test\r\n\r\nhello world"; + let (mut upstream_read, mut upstream_write) = tokio::io::duplex(4096); + let (mut client_read, mut client_write) = tokio::io::duplex(4096); + + tokio::spawn(async move { + upstream_write.write_all(response).await.unwrap(); + upstream_write.shutdown().await.unwrap(); + }); + + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(2), + relay_response( + "POST", + &mut upstream_read, + &mut client_write, + RelayResponseOptions::default(), + ), + ) + .await + .expect("HTTP/1.0 close-delimited response should not deadlock") + .expect("HTTP/1.0 response should relay"); + assert!(matches!(outcome, RelayOutcome::Consumed)); + + client_write.shutdown().await.unwrap(); + let mut received = Vec::new(); + client_read.read_to_end(&mut received).await.unwrap(); + assert!( + received + .windows(b"hello world".len()) + .any(|value| value == b"hello world") + ); + } + #[tokio::test] async fn relay_response_no_framing_event_stream_reads_until_eof() { let response = diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index d3def44743..4b8be4736e 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -8,7 +8,7 @@ //! store, terminates TLS from the client (presenting dynamic certs per hostname), //! inspects the plaintext HTTP, then re-encrypts to upstream using real root CAs. -use miette::{IntoDiagnostic, Result, miette}; +use miette::{IntoDiagnostic, Result, WrapErr, miette}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ServerConfig}; @@ -66,6 +66,76 @@ impl SandboxCa { pub fn cert_pem(&self) -> &str { &self.ca_cert_pem } + + /// Returns the CA private key in PKCS#8 PEM format. + pub fn private_key_pem(&self) -> String { + self.ca_key.serialize_pem() + } + + /// Load a durable CA certificate and matching private key from absolute paths. + pub fn load_from_paths(certificate_path: &Path, private_key_path: &Path) -> Result { + if !certificate_path.is_absolute() || !private_key_path.is_absolute() { + return Err(miette!( + "proxy CA certificate and key paths must be absolute" + )); + } + if certificate_path == private_key_path { + return Err(miette!( + "proxy CA certificate and private key must use different paths" + )); + } + let certificate_pem = std::fs::read_to_string(certificate_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA certificate {}", certificate_path.display()) + })?; + let private_key_pem = std::fs::read_to_string(private_key_path) + .into_diagnostic() + .wrap_err_with(|| { + format!("read proxy CA private key {}", private_key_path.display()) + })?; + Self::from_pem(&certificate_pem, &private_key_pem) + } + + /// Load a durable CA while preserving the exact certificate bytes supplied + /// by the provisioner for boundary launch replay. + pub fn from_pem(certificate_pem: &str, private_key_pem: &str) -> Result { + let _ = rustls::crypto::ring::default_provider().install_default(); + let ca_key = KeyPair::from_pem(private_key_pem) + .into_diagnostic() + .wrap_err("parse proxy CA private key")?; + let certificates = rustls_pemfile::certs(&mut certificate_pem.as_bytes()) + .collect::, _>>() + .into_diagnostic() + .wrap_err("parse proxy CA certificate")?; + if certificates.len() != 1 { + return Err(miette!( + "proxy CA certificate file must contain exactly one certificate" + )); + } + let private_key = rustls_pemfile::private_key(&mut private_key_pem.as_bytes()) + .into_diagnostic() + .wrap_err("parse proxy CA private key")? + .ok_or_else(|| miette!("proxy CA private key file contains no private key"))?; + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .into_diagnostic() + .wrap_err("proxy CA certificate and private key do not match")?; + + let params = CertificateParams::from_ca_cert_pem(certificate_pem) + .into_diagnostic() + .wrap_err("parse proxy CA signing certificate")?; + let ca_cert = params + .self_signed(&ca_key) + .into_diagnostic() + .wrap_err("initialize proxy CA signer")?; + Ok(Self { + ca_cert, + ca_key, + ca_cert_pem: certificate_pem.to_string(), + }) + } } /// A leaf certificate chain and private key for a specific hostname. @@ -561,4 +631,23 @@ mod tests { "bundle should contain at least one cert", ); } + + #[test] + fn durable_ca_round_trip_preserves_certificate_bytes() { + let generated = SandboxCa::generate().unwrap(); + let certificate = generated.cert_pem().to_string(); + let private_key = generated.private_key_pem(); + let loaded = SandboxCa::from_pem(&certificate, &private_key).unwrap(); + + assert_eq!(loaded.cert_pem(), certificate); + assert_eq!(loaded.private_key_pem(), private_key); + } + + #[test] + fn durable_ca_rejects_mismatched_key_and_relative_paths() { + let certificate = SandboxCa::generate().unwrap(); + let other_key = SandboxCa::generate().unwrap(); + assert!(SandboxCa::from_pem(certificate.cert_pem(), &other_key.private_key_pem()).is_err()); + assert!(SandboxCa::load_from_paths(Path::new("ca.pem"), Path::new("ca.key")).is_err()); + } } diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index a828f75fba..f5537d28f3 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -19,6 +19,7 @@ pub mod procfs; pub mod proxy; pub mod run; pub mod sigv4; +mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 8d77da76e9..e9e03e7a03 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -92,15 +92,6 @@ pub struct NetworkInput { pub cmdline_paths: Vec, } -pub(crate) fn network_binary_identity_required() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY).map_or(true, |value| { - !matches!( - value.as_str(), - "relaxed" | "disabled" | "endpoint-only" | "false" | "0" - ) - }) -} - fn inject_runtime_policy_data(data: &mut serde_json::Value, require_binary_identity: bool) { let Some(obj) = data.as_object_mut() else { return; @@ -318,7 +309,7 @@ impl OpaEngine { engine .add_policy_from_file(policy_path) .map_err(|e| miette::miette!("{e}"))?; - let require_binary_identity = network_binary_identity_required(); + let require_binary_identity = true; emit_binary_identity_mode(require_binary_identity, "files"); let data_json = preprocess_yaml_data( &yaml_str, @@ -335,7 +326,7 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_strings(policy: &str, data_yaml: &str) -> Result { - Self::from_strings_with_options(policy, data_yaml, network_binary_identity_required(), None) + Self::from_strings_with_options(policy, data_yaml, true, None) } pub fn from_strings_with_middleware_config( @@ -343,12 +334,7 @@ impl OpaEngine { data_yaml: &str, validate_middleware_config: Option<&MiddlewareConfigValidator>, ) -> Result { - Self::from_strings_with_options( - policy, - data_yaml, - network_binary_identity_required(), - validate_middleware_config, - ) + Self::from_strings_with_options(policy, data_yaml, true, validate_middleware_config) } #[cfg(test)] @@ -401,11 +387,7 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { - Self::from_proto_with_pid_and_binary_identity_required( - proto, - entrypoint_pid, - network_binary_identity_required(), - ) + Self::from_proto_with_pid_and_binary_identity_required(proto, entrypoint_pid, true) } fn from_proto_with_pid_and_binary_identity_required( @@ -815,6 +797,7 @@ impl OpaEngine { /// generation comparison and callback linearizes state derived from an OPA /// snapshot with every policy reload and fail-closed transition. Callers /// must not perform I/O or other long-running work in `operation`. + #[allow(dead_code)] pub(crate) fn with_current_generation( &self, expected_generation: u64, @@ -2318,13 +2301,13 @@ process: "#; #[test] - fn policy_dns_snapshot_is_tcp_only_stable_and_generation_consistent() { + fn policy_dns_snapshot_includes_every_tcp_carried_endpoint() { let engine = OpaEngine::from_strings(TEST_POLICY, POLICY_DNS_SNAPSHOT_DATA).unwrap(); let snapshot = engine.policy_dns_eligibility_snapshot().unwrap(); assert_eq!(snapshot.generation, engine.current_generation()); - assert_eq!(snapshot.endpoints.len(), 2); + assert_eq!(snapshot.endpoints.len(), 4); assert_eq!(snapshot.endpoints[0].policy_name, "dns_transport"); assert_eq!(snapshot.endpoints[0].endpoint_index, 0); assert_eq!( @@ -2337,15 +2320,17 @@ process: panic!("eligible endpoint must retain concrete ports"); }; assert_eq!(ports.as_ref(), &[53.into(), 853.into()]); - assert_eq!(snapshot.endpoints[1].endpoint_index, 4); + assert_eq!(snapshot.endpoints[1].endpoint_index, 1); + assert_eq!(snapshot.endpoints[2].endpoint_index, 2); + assert_eq!(snapshot.endpoints[3].endpoint_index, 4); engine .reload(TEST_POLICY, POLICY_DNS_SNAPSHOT_DATA) .unwrap(); let reloaded = engine.policy_dns_eligibility_snapshot().unwrap(); assert_eq!(reloaded.generation, snapshot.generation + 1); - assert_eq!(reloaded.endpoints.len(), 2); - assert_eq!(reloaded.endpoints[1].endpoint_index, 4); + assert_eq!(reloaded.endpoints.len(), 4); + assert_eq!(reloaded.endpoints[3].endpoint_index, 4); } #[test] @@ -2363,7 +2348,13 @@ process: fn policy_dns_snapshot_accepts_the_default_multi_policy_shape() { let engine = OpaEngine::from_strings(TEST_POLICY, TEST_DATA_YAML).unwrap(); let snapshot = engine.policy_dns_eligibility_snapshot().unwrap(); - assert!(snapshot.endpoints.is_empty()); + assert_eq!(snapshot.endpoints.len(), 15); + assert!( + snapshot + .endpoints + .iter() + .any(|endpoint| endpoint.policy_name == "claude_code") + ); } #[test] diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 60cea680a5..8b260294c9 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -34,7 +34,7 @@ pub(crate) use store::{ use crate::opa::OpaEngine; use crate::proxy::destination::{build_validation_plan, filter_resolved_addresses}; -use crate::proxy::is_host_gateway_alias; +use crate::proxy::{INFERENCE_LOCAL_HOST, INFERENCE_LOCAL_PORT, is_host_gateway_alias}; use openshell_core::host_pattern::HostSelector; use openshell_ocsf::{ ActionId, ActivityId, ConfigStateChangeBuilder, DispositionId, Endpoint, @@ -122,11 +122,16 @@ impl PolicyDnsService { .policy .policy_dns_eligibility_snapshot() .map_err(|error| PolicyDnsError::Policy(error.to_string()))?; - let eligible = eligible_endpoints( - &snapshot.endpoints, - &normalized_name, - self.trusted_host_gateway, - )?; + let system_inference = normalized_name.as_str() == INFERENCE_LOCAL_HOST; + let eligible = if system_inference { + vec![system_inference_endpoint(family)?] + } else { + eligible_endpoints( + &snapshot.endpoints, + &normalized_name, + self.trusted_host_gateway, + )? + }; if eligible.is_empty() { emit_dns_denial( &normalized_name, @@ -139,18 +144,34 @@ impl PolicyDnsService { // The trusted resolver is invoked only after the immutable snapshot // proved policy eligibility. It never consults sandbox resolver state. let endpoint_context = eligible_endpoint_context(&eligible); - let trusted_answer = match self.resolver.resolve(&normalized_name, family).await { - Ok(answer) => answer, - Err(error) => { - emit_dns_failure( - &normalized_name, - family, - &endpoint_context, - snapshot.generation, - resolver_failure_detail(&error), - "Policy DNS trusted resolver query failed", - ); - return Err(PolicyDnsError::Resolver(error)); + let trusted_answer = if system_inference { + TrustedAnswer { + addresses: vec![family_loopback(family)], + ttl: MAX_MAPPING_TTL, + } + } else if is_host_gateway_alias(normalized_name.as_str()) { + let address = self + .trusted_host_gateway + .filter(|address| family.accepts(*address)) + .ok_or(PolicyDnsError::NoValidAddress)?; + TrustedAnswer { + addresses: vec![address], + ttl: MAX_MAPPING_TTL, + } + } else { + match self.resolver.resolve(&normalized_name, family).await { + Ok(answer) => answer, + Err(error) => { + emit_dns_failure( + &normalized_name, + family, + &endpoint_context, + snapshot.generation, + resolver_failure_detail(&error), + "Policy DNS trusted resolver query failed", + ); + return Err(PolicyDnsError::Resolver(error)); + } } }; let ttl = clamp_mapping_ttl(trusted_answer.ttl); @@ -253,6 +274,28 @@ impl PolicyDnsService { } } +fn family_loopback(family: AddressFamily) -> std::net::IpAddr { + match family { + AddressFamily::Ipv4 => std::net::Ipv4Addr::LOCALHOST.into(), + AddressFamily::Ipv6 => std::net::Ipv6Addr::LOCALHOST.into(), + } +} + +fn system_inference_endpoint(family: AddressFamily) -> Result { + let address = family_loopback(family); + let destination_plan = crate::proxy::destination::build_pinned_validation_plan(vec![address]) + .map_err(|error| PolicyDnsError::Policy(error.reason))?; + Ok(EligibleEndpoint { + endpoint_id: PolicyEndpointId { + policy_name: "openshell-system-inference".to_string(), + endpoint_index: 0, + }, + ports: vec![INFERENCE_LOCAL_PORT], + destination_plan, + contract_fingerprint: "openshell-system-inference-local".to_string(), + }) +} + struct EligibleEndpoint { endpoint_id: PolicyEndpointId, ports: Vec, @@ -285,8 +328,8 @@ fn eligible_endpoints( let destination_plan = build_validation_plan( name.as_str(), name.as_str(), - None, trusted_host_gateway, + None, &raw_allowed_ips, exact_declared_host, ) @@ -613,6 +656,36 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn publishes_system_inference_without_upstream_resolution_or_user_policy() { + let service = service(BASE_POLICY, vec!["8.8.8.8".parse().unwrap()]); + let now = Instant::now(); + + let answer = service + .answer_query(INFERENCE_LOCAL_HOST, AddressFamily::Ipv4, now) + .await + .unwrap(); + + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); + let mapping = service + .store + .lookup( + answer.address, + INFERENCE_LOCAL_PORT, + answer.policy_generation, + now, + ) + .unwrap(); + assert_eq!( + mapping.record.normalized_name.as_str(), + INFERENCE_LOCAL_HOST + ); + assert_eq!( + mapping.pinned_addresses(), + [IpAddr::V4(Ipv4Addr::LOCALHOST)] + ); + } + #[tokio::test] async fn eligible_nxdomain_fails_without_publishing_a_mapping() { let policy = Arc::new( @@ -806,26 +879,42 @@ process: { run_as_user: sandbox, run_as_group: sandbox } .unwrap(); assert_eq!(mapping.record.contracts[0].pinned_addresses, [trusted]); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } #[tokio::test] - async fn reserved_gateway_alias_rejects_mismatch_metadata_private_and_wrong_family_answers() { + async fn reserved_gateway_alias_accepts_an_exact_private_backend_gateway() { + let trusted: IpAddr = "172.23.0.1".parse().unwrap(); + let service = gateway_service(Vec::new(), Some(trusted)); + let now = Instant::now(); + + let answer = service + .answer_query("host.openshell.internal", AddressFamily::Ipv4, now) + .await + .unwrap(); + let mapping = service + .store + .lookup(answer.address, 8080, answer.policy_generation, now) + .unwrap(); + + assert_eq!(mapping.record.contracts[0].pinned_addresses, [trusted]); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn reserved_gateway_alias_rejects_wrong_address_family_without_resolver_fallback() { let trusted: IpAddr = "169.254.1.2".parse().unwrap(); - for (family, address) in [ - (AddressFamily::Ipv4, "169.254.1.3"), - (AddressFamily::Ipv4, "169.254.169.254"), - (AddressFamily::Ipv4, "10.2.3.4"), - (AddressFamily::Ipv6, "fe80::2"), - ] { - let service = gateway_service(vec![address.parse().unwrap()], Some(trusted)); - let result = service - .answer_query("host.openshell.internal", family, Instant::now()) - .await; - assert!( - matches!(result, Err(PolicyDnsError::NoValidAddress)), - "{address} must not satisfy the trusted gateway contract" - ); - } + let service = gateway_service(vec!["fe80::2".parse().unwrap()], Some(trusted)); + let result = service + .answer_query( + "host.openshell.internal", + AddressFamily::Ipv6, + Instant::now(), + ) + .await; + + assert!(matches!(result, Err(PolicyDnsError::NoValidAddress))); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } struct BlockingResolver { diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index ad6095efa9..8f721749d5 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -9,6 +9,7 @@ use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_isolation_interface::contract::{DnsMediationSource, DnsTransport}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -63,6 +64,77 @@ pub(crate) struct PolicyDnsRuntime { } impl PolicyDnsRuntime { + /// Start policy DNS over an isolation-backend exchange source. No UDP or + /// TCP listener is bound in the supervisor namespace. + pub(crate) fn start_mediated( + policy: Arc, + source: Arc, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + mut engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let task = tokio::spawn(async move { + if engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + loop { + let Ok(query) = source.accept().await else { + return; + }; + let service = service.clone(); + tokio::spawn(async move { + let response = match query.transport { + DnsTransport::Udp => { + wire::handle_udp_query_with_ipv6(&service, &query.request, false).await + } + DnsTransport::Tcp => { + wire::handle_tcp_query_with_ipv6(&service, &query.request, false).await + } + } + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process(format!( + "policy DNS response failed: {error}" + )) + }); + if query.response.send(response).is_err() { + tracing::warn!("sandbox DNS response channel closed before delivery"); + } + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message("Policy DNS connected to isolation boundary") + .build() + ); + Ok(Self { + store, + tasks: vec![task, expiry_task], + }) + } + pub(crate) fn start( policy: Arc, udp: tokio::net::UdpSocket, diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index ea2cd7c0f8..439cdcfa36 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -73,6 +73,17 @@ pub(crate) struct MappingLookup { } impl MappingLookup { + pub(crate) fn pinned_addresses(&self) -> Vec { + let mut seen = HashSet::new(); + self.record + .contracts + .iter() + .filter(|contract| contract.port == self.port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .filter(|address| seen.insert(*address)) + .collect() + } + pub(crate) fn endpoint_ids(&self) -> impl Iterator { self.record .contracts diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index d7a6c697a3..d89ffd61db 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,8 +10,9 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; +use crate::policy_dns::ResolvedEndpointStore; #[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; +use crate::policy_dns::{MappingLookupError, PolicyEndpointId}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; @@ -24,6 +25,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation_interface::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + NetworkOpenResult, PendingNetworkOpen, ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,16 +41,32 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; -use tokio::net::{TcpListener, TcpStream}; +use tokio::net::TcpListener; +#[cfg(any(target_os = "linux", test))] +use tokio::net::TcpStream; use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +type ProxyClient = tokio::io::BufReader; +type AcceptedProxyConnection = ( + BoundaryDuplexStream, + Option>, + Option<(SocketAddr, SocketAddr)>, + Option, +); + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation_interface::contract::BackendError), +} + use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, - validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan, + build_validation_plan, validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -62,12 +83,10 @@ const TUNNEL_PROTOCOL_PEEK_TIMEOUT: std::time::Duration = std::time::Duration::f const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(5); #[cfg(test)] const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(1); -const INFERENCE_LOCAL_HOST: &str = "inference.local"; -const INFERENCE_LOCAL_PORT: u16 = 443; +pub(crate) const INFERENCE_LOCAL_HOST: &str = "inference.local"; +pub(crate) const INFERENCE_LOCAL_PORT: u16 = 443; const FORWARD_ENCODED_SLASH_REJECTION_DETAIL: &str = "request-target contains an encoded '/' (%2F) which is not allowed on this endpoint"; -#[cfg(target_os = "linux")] -const SIDECAR_SUPERVISOR_TOPOLOGY: &str = "sidecar"; fn emit_credential_endpoint_mismatch(host: &str, port: u16, policy_name: &str) { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -256,6 +275,8 @@ impl ProxyHandle { engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, backend_host_gateway: Option, + network_mediation_source: Option>, + policy_dns_store: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -271,15 +292,27 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let source_backed = network_mediation_source.is_some(); + let listener = if source_backed { + None + } else { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) .severity(SeverityId::Informational) .status(StatusId::Success) .dst_endpoint(Endpoint::from_ip(local_addr.ip(), local_addr.port())) - .message(format!("Proxy listening on {local_addr}")) + .message(if source_backed { + "Proxy consuming isolation-boundary streams".to_string() + } else { + format!("Proxy listening on {local_addr}") + }) .build(); ocsf_emit!(event); } @@ -373,11 +406,44 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + match source.accept().await { + Ok(connection) => { + let Some(connection) = preauthorize_transparent_open( + connection, + policy_dns_store.as_ref(), + &opa_engine, + *backend_host_gateway, + *trusted_host_gateway, + ) + .await + else { + continue; + }; + Ok(connection) + } + Err(error) => Err(ProxyAcceptError::Source(error)), + } + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr), None) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs, transparent_destination)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -389,6 +455,7 @@ impl ProxyHandle { let backend_gw = backend_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); let credentials = provider_credentials.clone(); + let dns_store = policy_dns_store.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -401,8 +468,12 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, + transparent_destination, + dns_store, opa, cache, spid, @@ -431,7 +502,19 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!( + "Network-mediation source failed; proxy accept loop exiting: {err}" + )) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -469,7 +552,7 @@ impl ProxyHandle { }); Ok(Self { - http_addr: Some(local_addr), + http_addr: (!source_backed).then_some(local_addr), join, exited_rx: Some(exited_rx), }) @@ -485,6 +568,190 @@ impl ProxyHandle { } } +async fn preauthorize_transparent_open( + connection: PendingNetworkOpen, + policy_dns_store: Option<&Arc>, + opa_engine: &OpaEngine, + backend_host_gateway: Option, + trusted_host_gateway: Option, +) -> Option { + let PendingNetworkOpen { + stream, + binary_identity, + destination, + socket: _, + policy_generation: _, + result, + } = connection; + let host = match transparent_destination_host(destination, policy_dns_store, opa_engine) { + Ok(host) => host, + Err(error) => { + warn!(%destination, %error, "Denied staged transparent connection"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &error.to_string(), + "transparent_tcp_mapping_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + }; + if host != INFERENCE_LOCAL_HOST || destination.port() != INFERENCE_LOCAL_PORT { + let mut decision = authorize_supplied_identity( + opa_engine, + EgressIntent::connect(host.clone(), destination.port()), + &binary_identity, + ); + if let NetworkAction::Deny { reason } = &decision.action { + warn!(%destination, %reason, "Denied staged transparent connection"); + emit_staged_transparent_denial( + destination, + &binary_identity, + reason, + "transparent_tcp_policy_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + if let Err(denial) = + hydrate_destination_plan(&mut decision, backend_host_gateway, trusted_host_gateway) + { + warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &denial.reason, + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + if let Some(mapping) = policy_dns_store.and_then(|store| { + store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .ok() + }) { + let Ok(plan) = build_pinned_validation_plan(mapping.pinned_addresses()) else { + emit_staged_transparent_denial( + destination, + &binary_identity, + "policy DNS produced an invalid pinned destination", + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + }; + decision.endpoint.destination = Some(plan); + } + let plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); + if let Err(denial) = validate_destination(DestinationRequest { + host: &host, + port: destination.port(), + sandbox_entrypoint_pid: 0, + plan, + }) + .await + { + warn!(%destination, reason = %denial.reason, "Denied staged transparent destination"); + emit_staged_transparent_denial( + destination, + &binary_identity, + &denial.reason, + "transparent_tcp_destination_denied", + ); + let _ = result.send(NetworkOpenResult::Denied { + errno: libc::EACCES, + }); + return None; + } + } + if result.send(NetworkOpenResult::RelayReady).is_err() { + return None; + } + Some((stream, Some(binary_identity), None, Some(destination))) +} + +fn emit_staged_transparent_denial( + destination: SocketAddr, + identity: &Result, + reason: &str, + status_detail: &'static str, +) { + let (binary, ancestors, cmdline) = identity.as_ref().map_or_else( + |_| ("-".to_string(), "-".to_string(), "-".to_string()), + |identity| { + ( + identity.binary_path.display().to_string(), + identity + .ancestors + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(" -> "), + identity + .cmdline_paths + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "), + ) + }, + ); + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_ip(destination.ip(), destination.port())) + .actor_process(Process::from_bypass(&binary, "-", &ancestors).with_cmd_line(&cmdline)) + .message(format!("Transparent TCP denied before relay: {reason}")) + .status_detail(status_detail) + .build() + ); +} + +fn transparent_destination_host( + destination: SocketAddr, + policy_dns_store: Option<&Arc>, + opa_engine: &OpaEngine, +) -> Result { + let Some(store) = policy_dns_store else { + return Ok(destination.ip().to_string()); + }; + match store.lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) { + Ok(mapping) => Ok(mapping.record.normalized_name.as_str().to_string()), + Err(MappingLookupError::Missing) => Ok(destination.ip().to_string()), + Err(error) => Err(miette::miette!( + "transparent destination mapping is unavailable: {error}" + )), + } +} + impl Drop for ProxyHandle { fn drop(&mut self) { self.join.abort(); @@ -1196,21 +1463,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1588,8 +1858,8 @@ fn build_forward_destination_deny_ocsf_event( } #[allow(clippy::too_many_arguments)] -async fn deny_connect_destination( - client: &mut TcpStream, +async fn deny_connect_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1601,7 +1871,10 @@ async fn deny_connect_destination( decision: &EgressDecision, denial_tx: &Option>, activity_tx: &Option, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_connect_destination_deny_ocsf_event( denial, peer_addr, host, port, binary, pid, ancestors, cmdline, @@ -1634,8 +1907,8 @@ async fn deny_connect_destination( } #[allow(clippy::too_many_arguments)] -async fn deny_forward_destination( - client: &mut TcpStream, +async fn deny_forward_destination( + client: &mut C, denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1650,7 +1923,10 @@ async fn deny_forward_destination( decision: &EgressDecision, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ let detail = destination_denial_detail(denial.kind); ocsf_emit!(build_forward_destination_deny_ocsf_event( denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, @@ -1685,9 +1961,10 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1710,6 +1987,121 @@ async fn handle_tcp_connection( denial_tx: Option>, activity_tx: Option, ) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + None, + None, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + backend_host_gateway, + trusted_host_gateway, + upstream_proxy, + provider_credentials, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +/// Adapt a transparent application stream to the existing CONNECT pipeline. +/// The synthetic CONNECT request is supervisor-owned and its successful 200 +/// response is consumed before bytes are returned to the workload. +fn virtual_connect_stream( + workload: BoundaryDuplexStream, + authority: String, +) -> BoundaryDuplexStream { + let (handler, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + let (mut workload_read, mut workload_write) = tokio::io::split(workload); + tokio::spawn(async move { + let request = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); + if bridge_write.write_all(request.as_bytes()).await.is_ok() { + let _ = tokio::io::copy(&mut workload_read, &mut bridge_write).await; + } + let _ = bridge_write.shutdown().await; + }); + tokio::spawn(async move { + let mut header = Vec::with_capacity(256); + let mut byte = [0_u8; 1]; + while header.len() < MAX_HEADER_BYTES { + match bridge_read.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => header.push(byte[0]), + } + if header.ends_with(b"\r\n\r\n") { + break; + } + } + if !header.starts_with(b"HTTP/1.1 200 ") && !header.starts_with(b"HTTP/1.0 200 ") { + let _ = workload_write.shutdown().await; + return; + } + let _ = tokio::io::copy(&mut bridge_read, &mut workload_write).await; + let _ = workload_write.shutdown().await; + }); + Box::new(handler) +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, + transparent_destination: Option, + policy_dns_store: Option>, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + backend_host_gateway: Arc>, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + provider_credentials: Option, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let transparent_mapping = if let Some(destination) = transparent_destination { + let host = + transparent_destination_host(destination, policy_dns_store.as_ref(), &opa_engine)?; + let mapping = policy_dns_store.as_ref().and_then(|store| { + store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .ok() + }); + let authority = format!("{host}:{}", destination.port()); + client = tokio::io::BufReader::new(virtual_connect_stream(client.into_inner(), authority)); + Some(mapping) + } else { + None + } + .flatten(); let mut buf = vec![0u8; MAX_HEADER_BYTES]; let mut used = 0usize; @@ -1762,6 +2154,8 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, @@ -1809,22 +2203,31 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); // Evaluate OPA policy with process-identity binding. // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -1962,6 +2365,13 @@ async fn handle_tcp_connection( return Ok(()); } } + if let Some(mapping) = transparent_mapping.as_ref() { + decision.endpoint.destination = Some( + build_pinned_validation_plan(mapping.pinned_addresses()).map_err(|denial| { + miette::miette!("transparent destination mapping denied: {}", denial.reason) + })?, + ); + } let destination_plan = decision .endpoint .destination @@ -2211,7 +2621,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2643,18 +3053,6 @@ fn authorize_egress_intent( } }; - if !crate::opa::network_binary_identity_required() { - let result = evaluate_endpoint_only_opa(engine, intent); - debug!( - "authorize_egress_intent endpoint-only: host={} port={} transport={:?} action={:?}", - result.intent.destination.host, - result.intent.destination.port, - result.intent.transport, - result.action - ); - return result; - } - let entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( @@ -2732,18 +3130,10 @@ fn authorize_egress_intent( #[cfg(target_os = "linux")] fn proc_net_anchor_pid(entrypoint_pid: u32) -> Option { - if entrypoint_pid != 0 { - return Some(entrypoint_pid); - } - sidecar_topology_enabled().then(std::process::id) -} - -#[cfg(target_os = "linux")] -fn sidecar_topology_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) + (entrypoint_pid != 0).then_some(entrypoint_pid) } +#[cfg(test)] fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> EgressDecision { let input = crate::opa::NetworkInput { host: intent.destination.host.clone(), @@ -2786,6 +3176,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.authorize_egress(&input) { + Ok(authorization) => EgressDecision { + intent, + action: authorization.action.clone(), + policy_generation: authorization.generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::from_authorization(&authorization), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2795,10 +3256,6 @@ fn authorize_egress_intent( _entrypoint_pid: &AtomicU32, intent: EgressIntent, ) -> EgressDecision { - if !crate::opa::network_binary_identity_required() { - return evaluate_endpoint_only_opa(engine, intent); - } - EgressDecision { intent, action: NetworkAction::Deny { @@ -2830,13 +3287,16 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. -async fn handle_inference_interception( - client: TcpStream, +async fn handle_inference_interception( + client: S, host: &str, port: u16, tls_state: Option<&Arc>, inference_ctx: Option<&Arc>, -) -> Result { +) -> Result +where + S: TokioAsyncRead + TokioAsyncWrite + Unpin + Send, +{ let Some(ctx) = inference_ctx else { return Ok(InferenceOutcome::Denied { reason: "cluster inference context not configured".to_string(), @@ -3425,13 +3885,16 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -async fn reject_stale_connect_policy( - client: &mut TcpStream, +async fn reject_stale_connect_policy( + client: &mut C, host: &str, port: u16, activity_tx: Option<&ActivitySender>, error: miette::Report, -) -> Result<()> { +) -> Result<()> +where + C: TokioAsyncWrite + Unpin, +{ warn!( host, port, @@ -4826,7 +5289,9 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -4931,19 +5396,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -5893,6 +6366,17 @@ async fn handle_forward_proxy( ), ) .await?; + client.shutdown().await.into_diagnostic()?; + let mut discard = [0_u8; 1024]; + let _ = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match client.read(&mut discard).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; } else { respond( client, @@ -6169,8 +6653,9 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; + client.flush().await.into_diagnostic()?; Ok(()) } @@ -6309,11 +6794,14 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// HTTP status (the flaw this replaces). Returns `true` when the connection was /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. -async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, +async fn refuse_connect_when_tls_unavailable( + client: &mut C, tls_state_present: bool, effective_tls_skip: bool, -) -> Result { +) -> Result +where + C: TokioAsyncWrite + Unpin, +{ if tls_state_present || effective_tls_skip { return Ok(false); } @@ -6364,6 +6852,188 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn supplied_identity_preserves_authorized_endpoint_metadata() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + inspected: + name: inspected + endpoints: + - host: api.example.com + port: 443 + protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + allowed_ips: ["192.0.2.0/24"] + rules: + - allow: { method: GET, path: /allowed } + binaries: + - path: /usr/bin/python3 +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .expect("load policy"); + let identity = Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/python3"), + binary_digest: Some("00".repeat(32).parse().expect("digest")), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }); + + let mut decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &identity, + ); + + assert_eq!(query_allowed_ips(&decision), ["192.0.2.0/24"]); + hydrate_l7_route(&mut decision); + let route = decision + .endpoint + .l7_route + .expect("supplied identity must retain L7 metadata"); + assert_eq!(route.configs.len(), 1); + assert!(route.configs[0].config.request_body_credential_rewrite); + } + + #[tokio::test] + async fn staged_transparent_open_waits_for_l4_policy() { + let engine = OpaEngine::from_strings( + include_str!("../data/sandbox-policy.rego"), + r#" +network_policies: + allowed: + name: allowed + endpoints: + - host: 203.0.113.7 + port: 443 + - host: 169.254.169.254 + port: 80 + binaries: + - path: /usr/bin/curl +filesystem_policy: + include_workdir: true + read_only: [] + read_write: [] +landlock: + compatibility: best_effort +process: + run_as_user: sandbox + run_as_group: sandbox +"#, + ) + .unwrap(); + let identity = || { + Ok(ContractBinaryIdentity { + binary_path: PathBuf::from("/usr/bin/curl"), + binary_digest: Some("00".repeat(32).parse().unwrap()), + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + }) + }; + let pending = |destination: &str| { + let (stream, _peer) = tokio::io::duplex(64); + let (result, completion) = tokio::sync::oneshot::channel(); + ( + PendingNetworkOpen { + stream: Box::new(stream), + binary_identity: identity(), + destination: destination.parse().unwrap(), + socket: openshell_isolation_interface::contract::NetworkSocketMetadata { + socket_cookie: 7, + nonblocking: false, + process_generation: 1, + }, + policy_generation: engine.current_generation(), + result, + }, + completion, + ) + }; + + let (allowed, allowed_result) = pending("203.0.113.7:443"); + assert!( + preauthorize_transparent_open(allowed, None, &engine, None, None) + .await + .is_some() + ); + assert_eq!(allowed_result.await.unwrap(), NetworkOpenResult::RelayReady); + + let (unsafe_destination, unsafe_result) = pending("169.254.169.254:80"); + assert!( + preauthorize_transparent_open(unsafe_destination, None, &engine, None, None) + .await + .is_none() + ); + assert_eq!( + unsafe_result.await.unwrap(), + NetworkOpenResult::Denied { + errno: libc::EACCES + } + ); + + let (denied, denied_result) = pending("203.0.113.8:443"); + assert!( + preauthorize_transparent_open(denied, None, &engine, None, None) + .await + .is_none() + ); + assert_eq!( + denied_result.await.unwrap(), + NetworkOpenResult::Denied { + errno: libc::EACCES + } + ); + } + + struct FailedMediationSource; + + #[tokio::test] + async fn virtual_connect_is_portless_and_hides_the_synthetic_handshake() { + let (workload, mut workload_peer) = tokio::io::duplex(1024); + let mut handler = virtual_connect_stream(Box::new(workload), "api.example.com:443".into()); + + workload_peer.write_all(b"client-tls").await.unwrap(); + let mut request = vec![0_u8; 128]; + let length = handler.read(&mut request).await.unwrap(); + let request = &request[..length]; + assert!(request.starts_with(b"CONNECT api.example.com:443 HTTP/1.1\r\n")); + + handler + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\nserver-tls") + .await + .unwrap(); + let mut response = [0_u8; 10]; + workload_peer.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"server-tls"); + } + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + PendingNetworkOpen, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + ), + ) + } + } + struct DenyWebSocketPreflight; #[tonic::async_trait] @@ -6551,6 +7221,49 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_proxy() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let mut handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + None, + Some(Arc::new(FailedMediationSource)), + None, + ) + .await + .expect("proxy starts before source accept"); + let exited = handle + .take_exit_receiver() + .expect("proxy exposes its exit receiver"); + + tokio::time::timeout(std::time::Duration::from_secs(1), exited) + .await + .expect("source failure must stop the proxy") + .expect_err("proxy task drops the exit sender"); + assert!(handle.join.is_finished()); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -6636,7 +7349,13 @@ network_policies: .expect("read proxy response"); response }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); tokio::time::timeout( std::time::Duration::from_secs(30), @@ -6646,6 +7365,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -6771,7 +7492,13 @@ network_policies: .await .unwrap(); }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let (proxy_connection, _) = proxy_listener.accept().await.unwrap(); + let socket_addrs = proxy_connection + .peer_addr() + .ok() + .zip(proxy_connection.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(proxy_connection); + let mut proxy_connection = tokio::io::BufReader::new(stream); let handler = tokio::spawn(async move { handle_forward_proxy( @@ -6780,6 +7507,8 @@ network_policies: request.as_bytes(), request.len(), &mut proxy_connection, + None, + socket_addrs, engine, Arc::new(BinaryIdentityCache::new()), Arc::new(AtomicU32::new(std::process::id())), @@ -7365,14 +8094,14 @@ network_policies: let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); - let (server, _) = listener.accept().await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut tokio::io::BufReader::new(&mut server)) .await .expect("peek should succeed") .expect("client sent bytes"); diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 314596b048..55c4dd9099 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Transport-neutral egress inputs and authorization results. //! //! Explicit proxy adapters normalize their protocol-specific request into an diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index e89c08225f..5ed4fd1c68 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -490,26 +490,20 @@ async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, #[test] #[ignore = "manual proxy allocation/query/latency baseline"] fn proxy_performance_baseline() { - temp_env::with_vars( - [( - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - Some("endpoint-only"), - )], - || { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap() - .block_on(async { - // Benchmark the full fail-closed path using a declared loopback - // destination. This is deterministic and never opens a listener - // outside the local process, so it does not trigger host firewall - // prompts during manual baseline collection. - let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); - - let policy = format!( - r#" + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + // Benchmark the full fail-closed path using a declared loopback + // destination. This is deterministic and never opens a listener + // outside the local process, so it does not trigger host firewall + // prompts during manual baseline collection. + let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + + let policy = format!( + r#" network_policies: proxy_compatibility: name: proxy_compatibility @@ -518,94 +512,92 @@ network_policies: port: {port} tls: skip binaries: - - path: "/**" + - path: "/no-such-benchmark-binary" "#, - host = target.ip(), - port = target.port(), - ); - let engine = Arc::new( - OpaEngine::from_strings_with_binary_identity_required( - include_str!("../../../data/sandbox-policy.rego"), - &policy, - false, - ) - .unwrap(), - ); - let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy_addr = proxy_listener.local_addr().unwrap(); - let proxy_engine = engine.clone(); - let proxy_task = tokio::spawn(async move { - while let Ok((stream, _)) = proxy_listener.accept().await { - let engine = proxy_engine.clone(); - tokio::spawn(async move { - Box::pin(handle_tcp_connection( - stream, - engine, - Arc::new(BinaryIdentityCache::new()), - Arc::new(AtomicU32::new(0)), - None, - None, - None, - AgentProposals::default(), - Arc::new(None), - Arc::new(None), - Arc::new(None), - None, - None, - None, - None, - None, - )) - .await - .unwrap(); - }); - } + host = target.ip(), + port = target.port(), + ); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + include_str!("../../../data/sandbox-policy.rego"), + &policy, + true, + ) + .unwrap(), + ); + let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = proxy_listener.local_addr().unwrap(); + let proxy_engine = engine.clone(); + let proxy_task = tokio::spawn(async move { + while let Ok((stream, _)) = proxy_listener.accept().await { + let engine = proxy_engine.clone(); + tokio::spawn(async move { + Box::pin(handle_tcp_connection( + stream, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(std::process::id())), + None, + None, + None, + AgentProposals::default(), + Arc::new(None), + Arc::new(None), + Arc::new(None), + None, + None, + None, + None, + None, + )) + .await + .unwrap(); }); - - for connect in [true, false] { - exercise_benchmark_request(proxy_addr, target, connect).await; - } - - let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|value| *value > 0) - .unwrap_or(25); - let mut results = serde_json::Map::new(); - for (name, connect) in [("connect", true), ("forward", false)] { - crate::test_alloc::reset(); - crate::opa::reset_test_opa_query_count(); - let started = std::time::Instant::now(); - for _ in 0..iterations { - exercise_benchmark_request(proxy_addr, target, connect).await; - } - let elapsed = started.elapsed(); - let queries = crate::opa::test_opa_query_count(); - let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); - let expected_queries = 4; - assert_eq!(queries, expected_queries * iterations); - results.insert( - name.to_string(), - serde_json::json!({ - "allocated_bytes_per_request": allocated_bytes / iterations, - "allocations_per_request": allocations / iterations, - "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), - "opa_queries_per_request": queries / iterations, - }), - ); - } - println!( - "{}", - serde_json::json!({ - "iterations": iterations, - "proxy_performance_baseline": results, - "scenario": "declared_loopback_destination_denied", - "schema_version": 1, - }) - ); - - proxy_task.abort(); - }); - }, - ); + } + }); + + for connect in [true, false] { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + + let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(25); + let mut results = serde_json::Map::new(); + for (name, connect) in [("connect", true), ("forward", false)] { + crate::test_alloc::reset(); + crate::opa::reset_test_opa_query_count(); + let started = std::time::Instant::now(); + for _ in 0..iterations { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + let elapsed = started.elapsed(); + let queries = crate::opa::test_opa_query_count(); + let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); + let expected_queries = 4; + assert_eq!(queries, expected_queries * iterations); + results.insert( + name.to_string(), + serde_json::json!({ + "allocated_bytes_per_request": allocated_bytes / iterations, + "allocations_per_request": allocations / iterations, + "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), + "opa_queries_per_request": queries / iterations, + }), + ); + } + println!( + "{}", + serde_json::json!({ + "iterations": iterations, + "proxy_performance_baseline": results, + "scenario": "declared_loopback_destination_denied", + "schema_version": 1, + }) + ); + + proxy_task.abort(); + }); } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 0f29e331ff..7f136492cb 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation_interface::contract::{DnsMediationSource, NetworkMediationSource}; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -155,6 +156,7 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + _mediated_policy_dns: Option, #[cfg(target_os = "linux")] _policy_dns: Option, #[cfg(target_os = "linux")] @@ -198,6 +200,8 @@ pub async fn run_networking( upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, + network_mediation_source: Option>, + dns_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -314,10 +318,29 @@ pub async fn run_networking( // the proxy, so it's owned here. let identity_cache = opa_engine.map(|_| Arc::new(BinaryIdentityCache::new())); - // Generate ephemeral CA and TLS state for HTTPS L7 inspection. + // Load a provisioned CA when the boundary lifetime outlives this control + // process; otherwise generate an ephemeral CA. // The CA cert is written to disk so sandbox processes can trust it. let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { - match SandboxCa::generate() { + let configured_ca = match ( + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_CERT), + std::env::var_os(openshell_core::sandbox_env::PROXY_CA_KEY), + ) { + (Some(certificate), Some(private_key)) => Some(SandboxCa::load_from_paths( + std::path::Path::new(&certificate), + std::path::Path::new(&private_key), + )?), + (None, None) => None, + _ => { + return Err(miette::miette!( + "{} and {} must be configured together", + openshell_core::sandbox_env::PROXY_CA_CERT, + openshell_core::sandbox_env::PROXY_CA_KEY, + )); + } + }; + let durable_ca = configured_ca.is_some(); + match configured_ca.map_or_else(SandboxCa::generate, Ok) { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); @@ -357,7 +380,11 @@ pub async fn run_networking( .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "enabled") - .message("TLS termination enabled: ephemeral CA generated") + .message(if durable_ca { + "TLS termination enabled: provisioned CA loaded" + } else { + "TLS termination enabled: ephemeral CA generated" + }) .build() ); (Some(state), Some(paths)) @@ -403,6 +430,21 @@ pub async fn run_networking( (None, None) }; + let mediated_policy_dns = if let Some(source) = dns_mediation_source { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("Mediated DNS requires an OPA engine"))?; + Some(crate::policy_dns::PolicyDnsRuntime::start_mediated( + engine, + source, + host_gateway_ip, + crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(0)?, + engine_ready_rx.clone(), + )?) + } else { + None + }; + let proxy_handle = if matches!(policy.network.mode, NetworkMode::Proxy) { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") @@ -450,6 +492,10 @@ pub async fn run_networking( engine_ready_rx, upstream_proxy_args, host_gateway_ip, + network_mediation_source, + mediated_policy_dns + .as_ref() + .map(|runtime| runtime.store.clone()), ) .await?; Some(proxy_handle) @@ -495,6 +541,7 @@ pub async fn run_networking( proxy: proxy_handle, ca_file_paths, policy_local_ctx, + _mediated_policy_dns: mediated_policy_dns, #[cfg(target_os = "linux")] _policy_dns: policy_dns, #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor-network/src/spiffe_endpoint.rs b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs new file mode 100644 index 0000000000..b3b6816f1e --- /dev/null +++ b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +#[allow(dead_code)] +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} diff --git a/crates/openshell-supervisor-network/src/upstream_proxy.rs b/crates/openshell-supervisor-network/src/upstream_proxy.rs index f95c5a3e81..490602c4d8 100644 --- a/crates/openshell-supervisor-network/src/upstream_proxy.rs +++ b/crates/openshell-supervisor-network/src/upstream_proxy.rs @@ -83,6 +83,9 @@ const CONNECT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); pub struct ProxyEndpoint { host: String, port: u16, + /// Optional driver-pinned address used only for the TCP dial. The + /// configured host remains authoritative for TLS identity and logging. + dial_ip: Option, /// Pre-computed `Basic ` header value from the proxy auth file. /// Never logged. proxy_authorization: Option, @@ -106,6 +109,7 @@ impl std::fmt::Debug for ProxyEndpoint { f.debug_struct("ProxyEndpoint") .field("host", &self.host) .field("port", &self.port) + .field("dial_ip", &self.dial_ip) .field("proxy_authorization", &self.proxy_authorization.is_some()) .field("tls", &self.tls.is_some()) .finish() @@ -334,6 +338,9 @@ pub struct UpstreamProxyArgs { /// `http://host:port` or `https://host:port` corporate proxy URL, or /// `None` for direct egress. pub https_proxy: Option, + /// Optional compute-driver-selected IP for reaching the proxy from the + /// supervisor's network namespace without changing its TLS identity. + pub proxy_dial_ip: Option, /// Comma-separated `NO_PROXY` list. pub no_proxy: Option, /// Path to the root-only credential mount (`user:pass`). @@ -354,6 +361,7 @@ pub struct UpstreamProxyArgs { // Supervisor CLI flag names for the corporate-proxy settings, used as the // dispatch keys in `from_lookup` and in operator-facing error messages. const ARG_HTTPS_PROXY: &str = "--upstream-proxy"; +const ARG_PROXY_DIAL_IP: &str = "--upstream-proxy-dial-ip"; const ARG_NO_PROXY: &str = "--upstream-no-proxy"; const ARG_PROXY_AUTH_FILE: &str = "--upstream-proxy-auth-file"; const ARG_PROXY_AUTH_ALLOW_INSECURE: &str = "--upstream-proxy-auth-allow-insecure"; @@ -391,6 +399,8 @@ impl UpstreamProxyConfig { Self::from_lookup(|name| { if name == ARG_HTTPS_PROXY { args.https_proxy.clone() + } else if name == ARG_PROXY_DIAL_IP { + args.proxy_dial_ip.map(|ip| ip.to_string()) } else if name == ARG_NO_PROXY { args.no_proxy.clone() } else if name == ARG_PROXY_AUTH_FILE { @@ -424,6 +434,12 @@ impl UpstreamProxyConfig { let https = var(ARG_HTTPS_PROXY)? .map(|url| parse_proxy_url(&url, ARG_HTTPS_PROXY)) .transpose()?; + let proxy_dial_ip = var(ARG_PROXY_DIAL_IP)? + .map(|raw| { + raw.parse::() + .map_err(|error| format!("{ARG_PROXY_DIAL_IP} is invalid: {error}")) + }) + .transpose()?; let auth_file = var(ARG_PROXY_AUTH_FILE)?; let auth_allow_insecure = var(ARG_PROXY_AUTH_ALLOW_INSECURE)?; let connect_by_hostname_raw = var(ARG_PROXY_CONNECT_BY_HOSTNAME)?; @@ -435,6 +451,7 @@ impl UpstreamProxyConfig { // silently running with direct egress. for (name, value) in [ (ARG_PROXY_AUTH_FILE, &auth_file), + (ARG_PROXY_DIAL_IP, &proxy_dial_ip.map(|ip| ip.to_string())), (ARG_PROXY_AUTH_ALLOW_INSECURE, &auth_allow_insecure), (ARG_PROXY_CONNECT_BY_HOSTNAME, &connect_by_hostname_raw), (ARG_NO_PROXY, &no_proxy_list), @@ -446,6 +463,7 @@ impl UpstreamProxyConfig { } return Ok(None); }; + https.dial_ip = proxy_dial_ip; // CONNECT-target mode. The default binds the tunnel to a validated // address; hostname CONNECT re-opens proxy-side DNS resolution and @@ -592,6 +610,7 @@ fn parse_proxy_url(raw: &str, var_name: &str) -> Result<(ProxyEndpoint, bool), S ProxyEndpoint { host: addr.host, port: addr.port, + dial_ip: None, proxy_authorization: None, tls: None, }, @@ -1000,7 +1019,10 @@ async fn connect_via_inner( port: u16, target: ConnectTarget, ) -> std::io::Result { - let tcp = TcpStream::connect((endpoint.host.as_str(), endpoint.port)).await?; + let tcp = match endpoint.dial_ip { + Some(ip) => TcpStream::connect(SocketAddr::new(ip, endpoint.port)).await?, + None => TcpStream::connect((endpoint.host.as_str(), endpoint.port)).await?, + }; set_tcp_nodelay_best_effort(&tcp); // For an `https://` proxy, wrap the connection in TLS (verifying the proxy // certificate against the configured roots) before the CONNECT handshake. @@ -1113,6 +1135,7 @@ mod tests { ARG_PROXY_AUTH_ALLOW_INSECURE as PROXY_AUTH_ALLOW_INSECURE, ARG_PROXY_AUTH_FILE as PROXY_AUTH_FILE, ARG_PROXY_CA_BUNDLE as PROXY_CA_BUNDLE, ARG_PROXY_CONNECT_BY_HOSTNAME as PROXY_CONNECT_BY_HOSTNAME, + ARG_PROXY_DIAL_IP as PROXY_DIAL_IP, }; fn config_from(pairs: &[(&str, &str)]) -> Result, String> { @@ -1834,6 +1857,7 @@ mod tests { ProxyEndpoint { host: addr.ip().to_string(), port: addr.port(), + dial_ip: None, proxy_authorization: auth.map(str::to_string), tls: None, } @@ -2241,14 +2265,16 @@ mod tests { // -- TLS (https://) proxies -- - /// A fake `https://` proxy: a TLS server with a self-signed cert for - /// 127.0.0.1 that answers CONNECT with 200. Returns the listen address, + /// A fake `https://` proxy: a TLS server with a self-signed cert for the + /// requested identity that answers CONNECT with 200. Returns the listen address, /// the server task (yielding the received CONNECT request), and the /// server certificate PEM to use as the corporate CA bundle. - async fn fake_tls_proxy() -> (SocketAddr, tokio::task::JoinHandle, String) { + async fn fake_tls_proxy( + tls_identity: &str, + ) -> (SocketAddr, tokio::task::JoinHandle, String) { install_crypto_provider(); let key = rcgen::KeyPair::generate().unwrap(); - let cert = rcgen::CertificateParams::new(vec!["127.0.0.1".to_string()]) + let cert = rcgen::CertificateParams::new(vec![tls_identity.to_string()]) .unwrap() .self_signed(&key) .unwrap(); @@ -2288,18 +2314,23 @@ mod tests { #[tokio::test] async fn connect_via_https_proxy_with_corporate_ca_bundle() { - let (addr, handle, cert_pem) = fake_tls_proxy().await; + const PROXY_IDENTITY: &str = "proxy.corp.test"; + let (addr, handle, cert_pem) = fake_tls_proxy(PROXY_IDENTITY).await; let ca_file = tempfile::NamedTempFile::new().unwrap(); std::fs::write(ca_file.path(), cert_pem).unwrap(); - let proxy_url = format!("https://{addr}"); + let proxy_url = format!("https://{PROXY_IDENTITY}:{}", addr.port()); let ca_path = ca_file.path().to_string_lossy().into_owned(); + let dial_ip = addr.ip().to_string(); let cfg = config_ok(&[ (HTTPS_PROXY, proxy_url.as_str()), + (PROXY_DIAL_IP, dial_ip.as_str()), (PROXY_CA_BUNDLE, ca_path.as_str()), ]); let endpoint = &cfg.https; assert!(endpoint.tls.is_some()); + assert_eq!(endpoint.host, PROXY_IDENTITY); + assert_eq!(endpoint.dial_ip, Some(addr.ip())); let stream = connect_via(endpoint, "api.example.com", 443, ConnectTarget::Hostname) .await @@ -2314,7 +2345,7 @@ mod tests { async fn connect_via_https_proxy_rejects_untrusted_cert() { // No corporate CA bundle: the self-signed proxy cert must not verify // against the built-in / system roots, so the handshake fails closed. - let (addr, _handle, _cert_pem) = fake_tls_proxy().await; + let (addr, _handle, _cert_pem) = fake_tls_proxy("127.0.0.1").await; let proxy_url = format!("https://{addr}"); let cfg = config_ok(&[(HTTPS_PROXY, proxy_url.as_str())]); let endpoint = &cfg.https; diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index aa80aaeb60..cfcea38555 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "openshell-supervisor-process" -description = "Process component of the OpenShell supervisor: entrypoint spawn, SSH server, supervisor session, netns, bypass monitor" +description = "Process access and gateway session runtime for the OpenShell supervisor" version.workspace = true edition.workspace = true license.workspace = true @@ -14,14 +14,12 @@ rust-version.workspace = true openshell-core = { path = "../openshell-core" } openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-ocsf = { path = "../openshell-ocsf" } -openshell-policy = { path = "../openshell-policy" } anyhow = { workspace = true } async-trait = "0.1" base64 = { workspace = true } bytes = { workspace = true } hex = "0.4" -ipnet = "2" miette = { workspace = true } nix = { workspace = true } rand = "0.10" @@ -37,14 +35,6 @@ uuid = { workspace = true } [target.'cfg(unix)'.dependencies] libc = "0.2" -rustix = { workspace = true } - -[target.'cfg(target_os = "linux")'.dependencies] -capctl = "0.2.4" -landlock = "0.4" -seccompiler = "0.5" -socket2 = { workspace = true } -tempfile = "3" [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs deleted file mode 100644 index 44847b0d13..0000000000 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ /dev/null @@ -1,651 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Bypass detection monitor — reads kernel log messages from `/dev/kmsg` to -//! detect and report direct connection attempts that bypass the HTTP CONNECT -//! proxy. -//! -//! When the sandbox network namespace has nftables log rules installed (see -//! `NetworkNamespace::install_bypass_rules`), the kernel writes a log line for -//! each dropped packet. This module reads those messages, parses the nftables -//! LOG format, and emits structured tracing events + denial aggregator entries. -//! -//! ## Graceful degradation -//! -//! If `/dev/kmsg` cannot be opened (e.g., restricted container environment), -//! the monitor logs a one-time warning and returns. The nftables reject rules -//! still provide fast-fail UX — the monitor only adds diagnostic visibility. - -mod procfs; - -use openshell_core::activity::{ActivitySender, try_record_activity}; -use openshell_core::denial::DenialEvent; -use openshell_ocsf::{ - ActionId, ActivityId, ConfidenceId, DetectionFindingBuilder, DispositionId, Endpoint, - FindingInfo, NetworkActivityBuilder, Process, SeverityId, ocsf_emit, -}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; -use tokio::sync::mpsc; -use tracing::debug; - -/// A parsed nftables log entry from `/dev/kmsg`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BypassEvent { - /// Destination IP address. - pub dst_addr: String, - /// Destination port. - pub dst_port: u16, - /// Source port (used for process identity resolution). - pub src_port: u16, - /// Protocol (TCP or UDP). - pub proto: String, - /// UID of the process that initiated the connection. - pub uid: Option, -} - -/// Parse a nftables log line from `/dev/kmsg`. -/// -/// Expected format (from the kernel LOG target): -/// ```text -/// ...,;openshell:bypass::IN= OUT=veth-s-... SRC=10.200.0.2 DST=93.184.216.34 -/// LEN=60 ... PROTO=TCP SPT=48012 DPT=443 ... UID=1000 -/// ``` -/// -/// Returns `None` if the line doesn't match the expected prefix or is malformed. -pub fn parse_kmsg_line(line: &str, namespace_prefix: &str) -> Option { - // Check that this line contains our namespace prefix. - let prefix_pos = line.find(namespace_prefix)?; - let relevant = &line[prefix_pos + namespace_prefix.len()..]; - - let dst_addr = extract_field(relevant, "DST=")?; - let dst_port = extract_field(relevant, "DPT=")?.parse::().ok()?; - let src_port = extract_field(relevant, "SPT=") - .and_then(|s| s.parse::().ok()) - .unwrap_or(0); - let proto = extract_field(relevant, "PROTO=") - .unwrap_or_else(|| "unknown".to_string()) - .to_lowercase(); - let uid = extract_field(relevant, "UID=").and_then(|s| s.parse::().ok()); - - Some(BypassEvent { - dst_addr, - dst_port, - src_port, - proto, - uid, - }) -} - -fn build_bypass_ocsf_events( - event: &BypassEvent, - binary: &str, - binary_pid: &str, - ancestors: &str, -) -> (openshell_ocsf::OcsfEvent, openshell_ocsf::OcsfEvent) { - let hint = hint_for_event(event); - let reason = "direct connection bypassed HTTP CONNECT proxy"; - let dst_port = event.dst_port.to_string(); - let dst_ep = event.dst_addr.parse::().map_or_else( - |_| Endpoint::from_domain(&event.dst_addr, event.dst_port), - |ip| Endpoint::from_ip(ip, event.dst_port), - ); - - let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Refuse) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .dst_endpoint(dst_ep) - .actor_process(Process::from_bypass(binary, binary_pid, ancestors)) - .firewall_rule("bypass-detect", "nftables") - .observation_point(3) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - - let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .is_alert(true) - .confidence(ConfidenceId::High) - .finding_info(FindingInfo::new("bypass-detect", "Proxy Bypass Detected").with_desc(reason)) - .remediation(hint) - .evidence_pairs(&[ - ("dst_addr", event.dst_addr.as_str()), - ("dst_port", dst_port.as_str()), - ("proto", event.proto.as_str()), - ("binary", binary), - ("binary_pid", binary_pid), - ("ancestors", ancestors), - ]) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - - (net_event, finding_event) -} - -/// Extract a single space-delimited field value from a nftables log line. -/// -/// Given `"DST="` and a string like `"...DST=93.184.216.34 LEN=60..."`, -/// returns `Some("93.184.216.34")`. -fn extract_field(s: &str, key: &str) -> Option { - let start = s.find(key)? + key.len(); - let rest = &s[start..]; - let end = rest.find(' ').unwrap_or(rest.len()); - let value = &rest[..end]; - if value.is_empty() { - None - } else { - Some(value.to_string()) - } -} - -/// Generate a protocol-appropriate hint for the bypass event. -fn hint_for_event(event: &BypassEvent) -> &'static str { - if event.proto == "udp" && event.dst_port == 53 { - "DNS queries should route through the sandbox proxy; check resolver configuration" - } else if event.proto == "udp" { - "UDP traffic must route through the sandbox proxy" - } else { - "ensure process honors HTTP_PROXY/HTTPS_PROXY; for Node.js set NODE_USE_ENV_PROXY=1" - } -} - -/// Spawn the bypass monitor as a background tokio task. -/// -/// Uses `dmesg --follow` to tail the kernel ring buffer for nftables log -/// entries matching the given namespace. Falls back gracefully if `dmesg` -/// is not available. -/// -/// We use `dmesg` rather than reading `/dev/kmsg` directly because the -/// container runtime's device cgroup policy blocks direct `/dev/kmsg` access -/// even with `CAP_SYSLOG`. The `dmesg` command reads via the `syslog(2)` -/// syscall which is permitted with `CAP_SYSLOG`. -/// -/// Returns a `JoinHandle` if the monitor was started, or `None` if `dmesg` -/// is not available. -pub fn spawn( - namespace_name: String, - entrypoint_pid: Arc, - denial_tx: Option>, - activity_tx: Option, -) -> Option> { - use std::io::BufRead; - use std::process::{Command, Stdio}; - - // Verify dmesg is available before spawning the monitor. - let dmesg_check = Command::new("dmesg") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - - if !dmesg_check.is_ok_and(|s| s.success()) { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message( - "dmesg not available; bypass detection monitor will not run. \ - Bypass REJECT rules still provide fast-fail behavior.", - ) - .build(); - ocsf_emit!(event); - return None; - } - - let namespace_prefix = format!("openshell:bypass:{namespace_name}:"); - debug!( - namespace = %namespace_name, - "Starting bypass detection monitor via dmesg --follow" - ); - - let handle = tokio::task::spawn_blocking(move || { - // Start dmesg in follow mode to tail new kernel messages. - let mut child = match Command::new("dmesg") - .args(["--follow", "--notime"]) - .stdout(Stdio::piped()) - .stderr(Stdio::null()) - .spawn() - { - Ok(c) => c, - Err(e) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message(format!( - "Failed to start dmesg --follow; bypass monitor will not run: {e}" - )) - .build(); - ocsf_emit!(event); - return; - } - }; - - let Some(stdout) = child.stdout.take() else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .severity(SeverityId::Low) - .message("dmesg --follow produced no stdout; bypass monitor will not run") - .build(); - ocsf_emit!(event); - return; - }; - - let reader = std::io::BufReader::new(stdout); - for line in reader.lines() { - let line = match line { - Ok(l) => l, - Err(e) => { - debug!(error = %e, "Error reading dmesg line, continuing"); - continue; - } - }; - - let Some(event) = parse_kmsg_line(&line, &namespace_prefix) else { - continue; - }; - - // Attempt process identity resolution (best-effort, TCP only). - let pid = entrypoint_pid.load(Ordering::Acquire); - let (binary, binary_pid, ancestors) = - if event.proto == "tcp" && event.src_port > 0 && pid > 0 { - resolve_process_identity(pid, event.src_port) - } else { - ("-".to_string(), "-".to_string(), "-".to_string()) - }; - - // Dual-emit: Network Activity [4001] + Detection Finding [2004] - let (net_event, finding_event) = - build_bypass_ocsf_events(&event, &binary, &binary_pid, &ancestors); - ocsf_emit!(net_event); - ocsf_emit!(finding_event); - - // Send to denial aggregator if available. - if let Some(ref tx) = denial_tx { - let ancestors_vec: Vec = if ancestors == "-" { - vec![] - } else { - ancestors.split(" -> ").map(String::from).collect() - }; - - let _ = tx.send(DenialEvent { - host: event.dst_addr.clone(), - port: event.dst_port, - binary: binary.clone(), - ancestors: ancestors_vec, - deny_reason: "direct connection bypassed HTTP CONNECT proxy".to_string(), - denial_stage: "bypass".to_string(), - l7_method: None, - l7_path: None, - }); - } - if let Some(ref tx) = activity_tx { - let _ = try_record_activity(tx, true, "bypass"); - } - } - - // Clean up the dmesg child process. - let _ = child.kill(); - let _ = child.wait(); - debug!("Bypass monitor: dmesg reader exited"); - }); - - Some(handle) -} - -/// Resolve process identity from a TCP source port. -/// -/// Returns `(binary_path, pid, ancestors)` as display strings. -/// Falls back to `("-", "-", "-")` on any failure (race condition, etc.). -fn resolve_process_identity(entrypoint_pid: u32, src_port: u16) -> (String, String, String) { - match procfs::resolve_tcp_peer_socket_owners(entrypoint_pid, src_port) { - Ok(socket_owners) => { - let mut identities = Vec::new(); - for owner in &socket_owners.owners { - let Ok(binary_path) = procfs::binary_path(owner.pid.cast_signed()) else { - continue; - }; - let ancestors = procfs::collect_ancestor_binaries(owner.pid, entrypoint_pid); - identities.push((owner.pid, binary_path, ancestors)); - } - - if identities.is_empty() { - return ("-".to_string(), "-".to_string(), "-".to_string()); - } - - identities.sort_by_key(|(pid, _, _)| *pid); - let first_identity = (identities[0].1.clone(), identities[0].2.clone()); - let ambiguous = identities - .iter() - .skip(1) - .any(|(_, binary_path, ancestors)| { - binary_path != &first_identity.0 || ancestors != &first_identity.1 - }); - - if ambiguous { - let pids = identities - .iter() - .map(|(pid, _, _)| pid.to_string()) - .collect::>() - .join(", "); - let owner_summary = identities - .iter() - .map(|(pid, binary_path, ancestors)| { - let ancestors_str = if ancestors.is_empty() { - "-".to_string() - } else { - ancestors - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(" -> ") - }; - format!( - "pid={pid} binary={} ancestors=[{ancestors_str}]", - binary_path.display() - ) - }) - .collect::>() - .join("; "); - return ("ambiguous".to_string(), pids, owner_summary); - } - - let (pid, binary_path, ancestors) = identities.remove(0); - let ancestors_str = if ancestors.is_empty() { - "-".to_string() - } else { - ancestors - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(" -> ") - }; - ( - binary_path.display().to_string(), - pid.to_string(), - ancestors_str, - ) - } - Err(_) => ("-".to_string(), "-".to_string(), "-".to_string()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_kmsg_line_tcp_bypass() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=93.184.216.34 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=12345 \ - DF PROTO=TCP SPT=48012 DPT=443 WINDOW=65535 RES=0x00 SYN URGP=0 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "93.184.216.34"); - assert_eq!(event.dst_port, 443); - assert_eq!(event.src_port, 48012); - assert_eq!(event.proto, "tcp"); - assert_eq!(event.uid, Some(1000)); - } - - #[test] - fn parse_kmsg_line_udp_dns_bypass() { - let line = "6,5678,9012,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=8.8.8.8 LEN=40 TOS=0x00 PREC=0x00 TTL=64 ID=0 \ - DF PROTO=UDP SPT=53421 DPT=53 LEN=32 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "8.8.8.8"); - assert_eq!(event.dst_port, 53); - assert_eq!(event.src_port, 53421); - assert_eq!(event.proto, "udp"); - assert_eq!(event.uid, Some(1000)); - } - - #[test] - fn parse_kmsg_line_no_uid() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=10.200.0.2 DST=10.0.0.5 LEN=60 PROTO=TCP SPT=12345 DPT=6379"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "10.0.0.5"); - assert_eq!(event.dst_port, 6379); - assert_eq!(event.proto, "tcp"); - assert_eq!(event.uid, None); - } - - #[test] - fn parse_kmsg_line_wrong_namespace_returns_none() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-other:IN= OUT=veth \ - SRC=10.200.0.2 DST=1.2.3.4 PROTO=TCP SPT=1111 DPT=80"; - - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_unrelated_message_returns_none() { - let line = "6,1234,5678,-;audit: type=1400 audit(1234567890.123:1): something else"; - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_missing_dst_returns_none() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth \ - SRC=10.200.0.2 PROTO=TCP SPT=1111 DPT=80"; - // Missing DST= field - let result = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:"); - assert!(result.is_none()); - } - - #[test] - fn parse_kmsg_line_ipv6_address() { - let line = "6,1234,5678,-;openshell:bypass:sandbox-abcd1234:IN= OUT=veth-s-abcd1234 \ - SRC=fd00::2 DST=2001:4860:4860::8888 LEN=60 PROTO=TCP SPT=55555 DPT=443 UID=1000"; - - let event = parse_kmsg_line(line, "openshell:bypass:sandbox-abcd1234:").unwrap(); - assert_eq!(event.dst_addr, "2001:4860:4860::8888"); - assert_eq!(event.dst_port, 443); - assert_eq!(event.proto, "tcp"); - } - - #[test] - fn hint_for_tcp_event() { - let event = BypassEvent { - dst_addr: "1.2.3.4".to_string(), - dst_port: 443, - src_port: 12345, - proto: "tcp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("HTTP_PROXY")); - } - - #[test] - fn hint_for_dns_bypass() { - let event = BypassEvent { - dst_addr: "8.8.8.8".to_string(), - dst_port: 53, - src_port: 12345, - proto: "udp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("DNS")); - } - - #[test] - fn hint_for_non_dns_udp() { - let event = BypassEvent { - dst_addr: "1.2.3.4".to_string(), - dst_port: 5060, - src_port: 12345, - proto: "udp".to_string(), - uid: None, - }; - assert!(hint_for_event(&event).contains("UDP")); - } - - #[test] - fn bypass_ocsf_contract_is_stable() { - let event = BypassEvent { - dst_addr: "93.184.216.34".to_string(), - dst_port: 443, - src_port: 48012, - proto: "tcp".to_string(), - uid: Some(1000), - }; - let (network, finding) = - build_bypass_ocsf_events(&event, "/usr/bin/curl", "42", "/usr/bin/sh"); - let network = serde_json::to_value(network).unwrap(); - assert_eq!(network["class_name"], "Network Activity"); - assert_eq!(network["activity_name"], "Refuse"); - assert_eq!(network["action"], "Denied"); - assert_eq!(network["disposition"], "Blocked"); - assert_eq!(network["severity"], "Medium"); - assert!(network.get("status").is_none()); - assert_eq!(network["dst_endpoint"]["ip"], "93.184.216.34"); - assert_eq!(network["dst_endpoint"]["port"], 443); - assert_eq!(network["actor"]["process"]["name"], "/usr/bin/curl"); - assert_eq!(network["firewall_rule"]["name"], "bypass-detect"); - assert_eq!(network["firewall_rule"]["type"], "nftables"); - assert_eq!(network["observation_point_id"], 3); - assert!( - network["message"] - .as_str() - .unwrap() - .contains("action=reject") - ); - - let finding = serde_json::to_value(finding).unwrap(); - assert_eq!(finding["class_name"], "Detection Finding"); - assert_eq!(finding["action"], "Denied"); - assert_eq!(finding["disposition"], "Blocked"); - assert_eq!(finding["severity"], "Medium"); - assert_eq!(finding["confidence"], "High"); - assert_eq!(finding["is_alert"], true); - assert_eq!(finding["finding_info"]["uid"], "bypass-detect"); - assert_eq!(finding["finding_info"]["title"], "Proxy Bypass Detected"); - assert_eq!(finding["evidences"][0]["data"]["dst_port"], "443"); - } - - #[test] - fn resolve_process_identity_surfaces_ambiguous_shared_socket() { - use std::ffi::CString; - use std::net::{TcpListener, TcpStream}; - use std::os::fd::AsRawFd; - use std::time::{Duration, Instant}; - - if !std::path::Path::new("/bin/sleep").exists() { - eprintln!("skipping: /bin/sleep not available"); - return; - } - - let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener"); - let listener_port = listener.local_addr().unwrap().port(); - let stream = TcpStream::connect(("127.0.0.1", listener_port)).expect("connect"); - let peer_port = stream.local_addr().unwrap().port(); - let (_accepted, _) = listener.accept().expect("accept"); - - let fd = stream.as_raw_fd(); - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - let flags = libc::fcntl(fd, libc::F_GETFD); - assert!(flags >= 0, "F_GETFD failed"); - assert_eq!( - libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC), - 0, - "F_SETFD failed" - ); - } - - let sleep_path = CString::new("/bin/sleep").unwrap(); - let arg0 = CString::new("sleep").unwrap(); - let arg1 = CString::new("30").unwrap(); - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - let child_pid = unsafe { libc::fork() }; - assert!(child_pid >= 0, "fork failed"); - if child_pid == 0 { - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - libc::execl( - sleep_path.as_ptr(), - arg0.as_ptr(), - arg1.as_ptr(), - std::ptr::null::(), - ); - libc::_exit(127); - } - } - - if std::fs::read_link(format!("/proc/{child_pid}/exe")).is_err() - || std::fs::read_dir(format!("/proc/{child_pid}/fd")).is_err() - { - #[allow(unsafe_code)] - unsafe { - libc::kill(child_pid, libc::SIGKILL); - libc::waitpid(child_pid, std::ptr::null_mut(), 0); - } - eprintln!("skipping: cannot read /proc/{child_pid} (restricted /proc)"); - return; - } - - let deadline = Instant::now() + Duration::from_secs(2); - loop { - if let Ok(link) = std::fs::read_link(format!("/proc/{child_pid}/exe")) - && link.to_string_lossy().contains("sleep") - { - break; - } - assert!( - Instant::now() < deadline, - "child pid {child_pid} did not exec into sleep within 2s" - ); - std::thread::sleep(Duration::from_millis(20)); - } - - let (binary, pid, ancestors) = resolve_process_identity(std::process::id(), peer_port); - - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - unsafe { - libc::kill(child_pid, libc::SIGKILL); - libc::waitpid(child_pid, std::ptr::null_mut(), 0); - } - - assert_eq!(binary, "ambiguous"); - assert!(pid.contains(&std::process::id().to_string())); - assert!(pid.contains(&child_pid.to_string())); - assert!(ancestors.contains("binary=")); - } - - #[test] - fn extract_field_basic() { - let s = "DST=1.2.3.4 LEN=60"; - assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); - assert_eq!(extract_field(s, "LEN="), Some("60".to_string())); - } - - #[test] - fn extract_field_missing() { - let s = "DST=1.2.3.4 LEN=60"; - assert_eq!(extract_field(s, "PROTO="), None); - } - - #[test] - fn extract_field_at_end_of_string() { - let s = "DST=1.2.3.4"; - assert_eq!(extract_field(s, "DST="), Some("1.2.3.4".to_string())); - } -} diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs b/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs deleted file mode 100644 index 98bf9634a6..0000000000 --- a/crates/openshell-supervisor-process/src/bypass_monitor/procfs.rs +++ /dev/null @@ -1,318 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Linux `/proc` filesystem reading for bypass-monitor process identity. -//! -//! Trimmed copy of `openshell-supervisor-network`'s `procfs` module: only -//! the helpers the bypass monitor calls when resolving the originating PID -//! and binary for an nftables LOG entry. The networking leaf keeps its own -//! richer copy because it also needs sha256 hashing, cmdline scraping, and -//! ambiguity-failure helpers for its proxy identity cache. - -use miette::Result; -use std::collections::HashSet; -use std::path::PathBuf; - -/// Where a socket owner was discovered while scanning `/proc`. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum SocketOwnerSource { - /// Owner was found in the entrypoint process tree at the given BFS depth. - Descendant { depth: usize }, - /// Owner was found by scanning all of `/proc` after the descendant scan. - ProcFallback, -} - -/// A process with an fd pointing at a target socket inode. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct SocketOwner { - pub pid: u32, - pub source: SocketOwnerSource, -} - -/// All process owners for a TCP peer socket. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct TcpPeerSocketOwners { - pub inode: u64, - pub owners: Vec, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct DescendantPid { - pid: u32, - depth: usize, -} - -/// Read the binary path of a process via `/proc/{pid}/exe` symlink. -/// -/// Strips the kernel-added `" (deleted)"` suffix when the raw readlink -/// target cannot be stat'd, so callers see a clean path. See the networking -/// crate's procfs documentation for the full rationale. -pub fn binary_path(pid: i32) -> Result { - use std::ffi::OsString; - use std::io::ErrorKind; - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - const DELETED_SUFFIX: &[u8] = b" (deleted)"; - - let link = format!("/proc/{pid}/exe"); - let target = std::fs::read_link(&link).map_err(|e| { - miette::miette!( - "Failed to read /proc/{pid}/exe: {e}. \ - Cannot determine binary identity — denying request. \ - Hint: the proxy may need CAP_SYS_PTRACE or to run as the same user." - ) - })?; - - let raw_target_missing = - matches!(std::fs::metadata(&target), Err(err) if err.kind() == ErrorKind::NotFound); - - let bytes = target.as_os_str().as_bytes(); - if raw_target_missing && bytes.ends_with(DELETED_SUFFIX) { - let stripped = bytes[..bytes.len() - DELETED_SUFFIX.len()].to_vec(); - return Ok(PathBuf::from(OsString::from_vec(stripped))); - } - - Ok(target) -} - -/// Resolve all process owners for the TCP peer inside a sandbox network namespace. -pub fn resolve_tcp_peer_socket_owners( - entrypoint_pid: u32, - peer_port: u16, -) -> Result { - let inode = parse_proc_net_tcp(entrypoint_pid, peer_port)?; - let owners = find_socket_inode_owners(inode, entrypoint_pid)?; - Ok(TcpPeerSocketOwners { inode, owners }) -} - -/// Read the `PPid` (parent PID) from `/proc//status`. -fn read_ppid(pid: u32) -> Option { - let status = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; - for line in status.lines() { - if let Some(rest) = line.strip_prefix("PPid:") { - return rest.trim().parse().ok(); - } - } - None -} - -/// Walk the process tree upward from `pid`, collecting binary paths. -/// -/// Stops at PID 1 (init), `stop_pid` (the entrypoint process), or after -/// 64 ancestors. The returned vec excludes `pid` itself. -#[allow(clippy::similar_names)] -pub fn collect_ancestor_binaries(pid: u32, stop_pid: u32) -> Vec { - const MAX_DEPTH: usize = 64; - let mut ancestors = Vec::new(); - let mut current = pid; - - for _ in 0..MAX_DEPTH { - let ppid = match read_ppid(current) { - Some(p) if p > 0 && p != current => p, - _ => break, - }; - - if let Ok(path) = binary_path(ppid.cast_signed()) { - ancestors.push(path); - } - - if ppid == stop_pid || ppid == 1 { - break; - } - current = ppid; - } - - ancestors -} - -fn parse_proc_net_tcp(pid: u32, peer_port: u16) -> Result { - for suffix in &["tcp", "tcp6"] { - let path = format!("/proc/{pid}/net/{suffix}"); - let Ok(content) = std::fs::read_to_string(&path) else { - continue; - }; - - for line in content.lines().skip(1) { - let fields: Vec<&str> = line.split_whitespace().collect(); - if fields.len() < 10 { - continue; - } - - let local_addr = fields[1]; - let local_port = match local_addr.rsplit_once(':') { - Some((_, port_hex)) => u16::from_str_radix(port_hex, 16).unwrap_or(0), - None => continue, - }; - - let state = fields[3]; - if state != "01" { - continue; - } - - if local_port == peer_port { - let inode: u64 = fields[9] - .parse() - .map_err(|_| miette::miette!("Failed to parse inode from {}", fields[9]))?; - if inode == 0 { - continue; - } - return Ok(inode); - } - } - } - - Err(miette::miette!( - "No ESTABLISHED TCP connection found for port {} in /proc/{}/net/tcp{{,6}}", - peer_port, - pid - )) -} - -fn find_socket_inode_owners(inode: u64, entrypoint_pid: u32) -> Result> { - let target = format!("socket:[{inode}]"); - let mut owners = Vec::new(); - let mut checked = HashSet::new(); - - let descendants = collect_descendant_pids_with_depth(entrypoint_pid); - - for descendant in &descendants { - checked.insert(descendant.pid); - if check_pid_fds(descendant.pid, &target) { - owners.push(SocketOwner { - pid: descendant.pid, - source: SocketOwnerSource::Descendant { - depth: descendant.depth, - }, - }); - } - } - - if let Ok(proc_dir) = std::fs::read_dir("/proc") { - let mut proc_pids = Vec::new(); - for entry in proc_dir.flatten() { - let name = entry.file_name(); - if let Ok(pid) = name.to_string_lossy().parse::() { - proc_pids.push(pid); - } - } - proc_pids.sort_unstable(); - - for pid in proc_pids { - if checked.contains(&pid) { - continue; - } - checked.insert(pid); - if check_pid_fds(pid, &target) { - owners.push(SocketOwner { - pid, - source: SocketOwnerSource::ProcFallback, - }); - } - } - } - - if !owners.is_empty() { - return Ok(owners); - } - - Err(miette::miette!( - "No process found owning socket inode {} \ - (scanned {} descendants of entrypoint PID {}). \ - Hint: the container may need --cap-add=SYS_PTRACE to read /proc//fd/ \ - for processes running as a different user.", - inode, - descendants.len(), - entrypoint_pid - )) -} - -fn check_pid_fds(pid: u32, target: &str) -> bool { - let fd_dir = format!("/proc/{pid}/fd"); - let Some(fds) = std::fs::read_dir(&fd_dir).ok() else { - return false; - }; - for fd_entry in fds.flatten() { - if let Ok(link) = std::fs::read_link(fd_entry.path()) - && link.to_string_lossy() == target - { - return true; - } - } - false -} - -fn collect_descendant_pids_with_depth(root_pid: u32) -> Vec { - let mut pids = vec![DescendantPid { - pid: root_pid, - depth: 0, - }]; - let mut seen = HashSet::from([root_pid]); - let mut i = 0; - while i < pids.len() { - let pid = pids[i].pid; - let child_depth = pids[i].depth + 1; - let task_dir = format!("/proc/{pid}/task"); - if let Ok(tasks) = std::fs::read_dir(&task_dir) { - for task_entry in tasks.flatten() { - let children_path = task_entry.path().join("children"); - if let Ok(children_str) = std::fs::read_to_string(&children_path) { - for child in children_str.split_whitespace() { - if let Ok(child_pid) = child.parse::() - && seen.insert(child_pid) - { - pids.push(DescendantPid { - pid: child_pid, - depth: child_depth, - }); - } - } - } - } - } - i += 1; - } - pids -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn binary_path_reads_current_process() { - let pid = std::process::id().cast_signed(); - let path = binary_path(pid).unwrap(); - assert!(path.exists()); - } - - #[test] - #[allow(clippy::similar_names)] - fn read_ppid_returns_parent() { - let pid = std::process::id(); - let ppid = read_ppid(pid); - assert!(ppid.is_some(), "Should be able to read PPid of self"); - assert!(ppid.unwrap() > 0, "PPid should be > 0"); - } - - #[test] - fn read_ppid_nonexistent_pid() { - let result = read_ppid(999_999_999); - assert!(result.is_none()); - } - - #[test] - fn collect_ancestor_binaries_returns_parents() { - let pid = std::process::id(); - let ancestors = collect_ancestor_binaries(pid, 1); - assert!( - !ancestors.is_empty(), - "Should have at least one ancestor binary" - ); - for path in &ancestors { - assert!( - !path.as_os_str().is_empty(), - "Ancestor path should not be empty" - ); - } - } -} diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs new file mode 100644 index 0000000000..ad1e8faecc --- /dev/null +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -0,0 +1,256 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-owned access-plane assembly for a remote sandbox. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use miette::Result; +use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward, BoundaryProcess}; +use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; + +fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { + openshell_ocsf::ctx::ctx() +} + +/// Supervisor-owned SSH and gateway-session tasks for a running sandbox. +pub struct BoundaryAccess { + instance_id: String, + terminating: Arc, + ssh_task: Option>, + session_task: Option>, + main_session: Option>, +} + +impl BoundaryAccess { + /// Stable supervisor instance ID used for lifecycle reporting. + #[must_use] + pub fn instance_id(&self) -> &str { + &self.instance_id + } + + /// Publish the canonical process's terminal status to attached clients. + pub async fn publish_main_exit(&self, exit_code: i32, attachment_expected: bool) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + let _ = main_session + .finish_remote(exit_code, attachment_expected) + .await; + } + + /// Release terminal delivery after the gateway acknowledges the exit, then + /// wait for attached clients to consume the terminal status. + pub async fn drain_main_terminal_delivery(&self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; + main_session.mark_terminal_reported(); + main_session.wait_for_terminal_attachments().await; + } +} + +impl Drop for BoundaryAccess { + fn drop(&mut self) { + self.terminating.store(true, Ordering::Release); + if let Some(task) = self.ssh_task.take() { + task.abort(); + } + if let Some(task) = self.session_task.take() { + task.abort(); + } + } +} + +/// Start the supervisor access plane using sandbox-supplied exec and +/// loopback-forwarding capabilities. +#[allow(clippy::too_many_arguments)] +pub async fn start_boundary_access( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option<&str>, + shared_ssh_socket: bool, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + boundary_exec: Arc, + port_forward: Arc, + agent: Arc, +) -> Result { + let instance_id = uuid::Uuid::new_v4().to_string(); + let terminating = Arc::new(AtomicBool::new(false)); + let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else { + return Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: None, + session_task: None, + main_session: None, + }); + }; + + let attachment = agent + .attach() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let main_session = crate::main_session::MainSession::from_boundary(attachment, agent); + + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + let listen_path = ssh_socket_path.clone(); + let ssh_port_forward = port_forward.clone(); + let ssh_main_session = main_session.clone(); + let ssh_task = tokio::spawn(async move { + if let Err(error) = crate::ssh::run_ssh_server( + listen_path, + ssh_ready_tx, + ca_file_paths, + shared_ssh_socket, + ssh_port_forward, + boundary_exec, + Some(ssh_main_session), + ) + .await + { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message(format!("SSH server failed: {error}")) + .build() + ); + } + }); + + match tokio::time::timeout(Duration::from_secs(10), ssh_ready_rx).await { + Ok(Ok(Ok(()))) => {} + Ok(Ok(Err(error))) => { + ssh_task.abort(); + return Err(error.context("SSH server failed during startup")); + } + Ok(Err(_)) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server task ended before signaling readiness" + )); + } + Err(_) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server did not start within 10 seconds" + )); + } + } + + let session_task = match (openshell_endpoint, sandbox_id) { + (Some(endpoint), Some(id)) => { + let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + terminating.clone(), + instance_id.clone(), + ); + match tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) + .await + { + Ok(Ok(_)) => Some(task), + Ok(Err(_)) => { + task.abort(); + return Err(miette::miette!( + "supervisor session ended before gateway acceptance" + )); + } + Err(_) => { + task.abort(); + return Err(miette::miette!( + "gateway did not accept supervisor session within 10 seconds" + )); + } + } + } + _ => None, + }; + + Ok(BoundaryAccess { + instance_id, + terminating, + ssh_task: Some(ssh_task), + session_task, + main_session: Some(main_session), + }) +} + +/// Report the canonical process exit until the gateway acknowledges it. +pub async fn report_main_process_exit( + endpoint: &str, + sandbox_id: &str, + instance_id: &str, + exit_code: i32, +) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::report_main_process_exit( + endpoint, + sandbox_id, + instance_id, + exit_code, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process exit report failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +/// Finalize canonical process terminal delivery until acknowledged. +pub async fn finalize_main_process_exit(endpoint: &str, sandbox_id: &str, instance_id: &str) { + let mut delay = Duration::from_millis(250); + loop { + match crate::supervisor_session::finalize_main_process_exit( + endpoint, + sandbox_id, + instance_id, + ) + .await + { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, "main-process finalization failed; retrying"); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn expected_post_exit_attachment_is_preserved_for_remote_main() { + let main_session = crate::main_session::MainSession::inert(); + let access = BoundaryAccess { + instance_id: "instance".to_string(), + terminating: Arc::new(AtomicBool::new(false)), + ssh_task: None, + session_task: None, + main_session: Some(main_session.clone()), + }; + + access.publish_main_exit(7, true).await; + + main_session + .begin_terminal_attachment() + .expect("declared CLI attachment must remain valid after a fast remote main exits"); + main_session.end_terminal_attachment(); + } +} diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index ee6bedeb22..023a6c8e73 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -1,32 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Process component of the `OpenShell` supervisor. +//! Access-plane component of the `OpenShell` supervisor. //! -//! Owns the entrypoint process spawn, SSH server, supervisor session, network -//! namespace, bypass monitor, child environment construction, skills install, -//! and log push. Populated by follow-up commits as modules migrate out of -//! `openshell-sandbox`. +//! Owns SSH access, retained canonical-process I/O, gateway supervisor +//! sessions, skills, and log forwarding. Workload spawning and in-sandbox +//! enforcement live exclusively in `openshell-sandbox`. -pub mod boundary_exec; -pub mod boundary_io; -pub mod child_env; pub mod debug_rpc; -#[cfg(unix)] -pub mod identity; +pub mod delegated; pub mod log_push; pub mod main_session; -pub mod managed_children; -pub mod process; -pub mod run; -pub mod sandbox; pub mod skills; pub mod ssh; pub mod supervisor_session; mod unix_socket; - -#[cfg(target_os = "linux")] -pub mod bypass_monitor; -#[cfg(target_os = "linux")] -pub mod netns; diff --git a/crates/openshell-supervisor-process/src/main_session.rs b/crates/openshell-supervisor-process/src/main_session.rs index 00dd2ea53c..3ffa24d1d0 100644 --- a/crates/openshell-supervisor-process/src/main_session.rs +++ b/crates/openshell-supervisor-process/src/main_session.rs @@ -17,10 +17,22 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::Notify; use tokio::sync::watch; -use crate::process::ProcessIo; +use openshell_isolation_interface::contract::{ + BoundaryProcess, BoundarySignal, BoundaryTerminal, ProcessAttachment, +}; const OUTPUT_BUFFER_BYTES: usize = 1024 * 1024; +/// Canonical-process I/O retained by the supervisor session multiplexer. +pub enum ProcessIo { + Pty(std::fs::File), + Pipes { + stdin: tokio::process::ChildStdin, + stdout: tokio::process::ChildStdout, + stderr: tokio::process::ChildStderr, + }, +} + #[derive(Clone, Debug)] pub enum MainOutput { Stdout(Bytes), @@ -186,6 +198,8 @@ pub struct MainSession { input_owner: Mutex>, next_owner: AtomicU64, pty_master: Option>, + boundary_process: Option>, + boundary_terminal: Option>, readers_remaining: AtomicUsize, readers_done: Notify, finished: std::sync::atomic::AtomicBool, @@ -194,6 +208,7 @@ pub struct MainSession { } impl MainSession { + const REMOTE_OUTPUT_DRAIN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); #[cfg(test)] pub fn inert() -> Arc { let (input, _input_rx) = tokio::sync::mpsc::channel(64); @@ -205,6 +220,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master: None, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(0), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -256,6 +273,8 @@ impl MainSession { input_owner: Mutex::new(None), next_owner: AtomicU64::new(1), pty_master, + boundary_process: None, + boundary_terminal: None, readers_remaining: AtomicUsize::new(if terminal { 1 } else { 2 }), readers_done: Notify::new(), finished: std::sync::atomic::AtomicBool::new(false), @@ -270,6 +289,81 @@ impl MainSession { session } + /// Build the control-side multiplexer around a boundary-owned admitted + /// process. Process lifecycle and PTY operations remain delegated to the + /// boundary process handle. + #[must_use] + pub fn from_boundary( + attachment: ProcessAttachment, + process: Arc, + ) -> Arc { + let ProcessAttachment { + stdin, + stdout, + stderr, + terminal, + } = attachment; + let terminal_mode = terminal.is_some(); + let (input, mut input_rx) = tokio::sync::mpsc::channel::>(64); + let session = Arc::new(Self { + pid: 0, + terminal: terminal_mode, + input, + output: OutputLog::new(), + input_owner: Mutex::new(None), + next_owner: AtomicU64::new(1), + pty_master: None, + boundary_process: Some(process), + boundary_terminal: terminal, + readers_remaining: AtomicUsize::new(if terminal_mode { 1 } else { 2 }), + readers_done: Notify::new(), + finished: std::sync::atomic::AtomicBool::new(false), + terminal_attachments: Mutex::new(TerminalAttachmentState { + active: 0, + process_finished: false, + expectation: AttachmentExpectation::None, + }), + terminal_attachments_done: Notify::new(), + }); + let stdout_session = Arc::clone(&session); + tokio::spawn(async move { + let mut stdout = stdout; + let mut buffer = [0u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stdout_session + .publish(MainOutput::Stdout(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stdout_session.reader_finished(); + }); + if let Some(mut stderr) = stderr { + let stderr_session = Arc::clone(&session); + tokio::spawn(async move { + let mut buffer = [0u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(read) => stderr_session + .publish(MainOutput::Stderr(Bytes::copy_from_slice(&buffer[..read]))), + } + } + stderr_session.reader_finished(); + }); + } + tokio::spawn(async move { + let mut stdin = stdin; + while let Some(data) = input_rx.recv().await { + if stdin.write_all(&data).await.is_err() { + break; + } + let _ = stdin.flush().await; + } + }); + session + } + fn start_io( this: &Arc, io: ProcessIo, @@ -380,10 +474,39 @@ impl MainSession { /// /// Returns whether terminal delivery must complete before shutdown. pub async fn finish(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.wait_for_output_readers().await; + self.complete_finish(exit_code, attachment_expected) + } + + /// Finish a remotely owned process without allowing descendants that keep + /// inherited output descriptors open to block terminal publication forever. + pub async fn finish_remote(&self, exit_code: i32, attachment_expected: bool) -> bool { + self.finish_remote_with_timeout( + exit_code, + attachment_expected, + Self::REMOTE_OUTPUT_DRAIN_TIMEOUT, + ) + .await + } + + async fn finish_remote_with_timeout( + &self, + exit_code: i32, + attachment_expected: bool, + timeout: std::time::Duration, + ) -> bool { + let _ = tokio::time::timeout(timeout, self.wait_for_output_readers()).await; + self.complete_finish(exit_code, attachment_expected) + } + + async fn wait_for_output_readers(&self) { let notified = self.readers_done.notified(); if self.readers_remaining.load(Ordering::Acquire) != 0 { notified.await; } + } + + fn complete_finish(&self, exit_code: i32, attachment_expected: bool) -> bool { let delivery_pending = { let mut state = self .terminal_attachments @@ -410,6 +533,23 @@ impl MainSession { self.output.subscribe() } + /// Return the bounded output sequence range currently retained for a + /// replacement supervisor. A nonzero first sequence is an explicit + /// truncation watermark rather than silent data loss. + #[must_use] + pub fn output_window(&self) -> (u64, u64, bool) { + let state = self + .output + .state + .lock() + .expect("main output log lock poisoned"); + let first_sequence = state + .events + .front() + .map_or(state.next_sequence, |event| event.sequence); + (first_sequence, state.next_sequence, first_sequence != 0) + } + /// Wait until the gateway durably acknowledges the main-process result. pub async fn wait_for_terminal_reported(&self) { let notified = self.output.terminal_reported_notify.notified(); @@ -499,7 +639,16 @@ impl MainSession { } } - pub fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + pub async fn resize(&self, columns: u32, rows: u32, pixel_width: u32, pixel_height: u32) { + if let Some(terminal) = self.boundary_terminal.as_ref() { + let _ = terminal + .resize( + u16::try_from(columns.max(1)).unwrap_or(u16::MAX), + u16::try_from(rows.max(1)).unwrap_or(u16::MAX), + ) + .await; + return; + } let Some(master) = self.pty_master.as_ref() else { return; }; @@ -515,9 +664,23 @@ impl MainSession { } } - pub fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), nix::errno::Errno> { + pub async fn signal_group(&self, signal: nix::sys::signal::Signal) -> Result<(), String> { + if let Some(process) = self.boundary_process.as_ref() { + let signal = match signal { + nix::sys::signal::Signal::SIGHUP => BoundarySignal::Hup, + nix::sys::signal::Signal::SIGINT => BoundarySignal::Int, + nix::sys::signal::Signal::SIGKILL => BoundarySignal::Kill, + nix::sys::signal::Signal::SIGTERM => BoundarySignal::Term, + other => return Err(format!("boundary signal {other:?} is unsupported")), + }; + return process + .signal(signal) + .await + .map_err(|error| error.to_string()); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); nix::sys::signal::kill(nix::unistd::Pid::from_raw(-pid), signal) + .map_err(|error| error.to_string()) } #[must_use] @@ -544,6 +707,83 @@ fn set_nonblocking(file: &std::fs::File) -> Result<(), nix::errno::Errno> { #[cfg(test)] mod tests { use super::*; + use openshell_isolation_interface::contract::{ + BackendError, BoundaryExitStatus, BoundaryInput, BoundaryOutput, + }; + + struct TestBoundaryProcess { + signals: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryProcess for TestBoundaryProcess { + async fn wait(&self) -> Result { + Ok(BoundaryExitStatus::Exited(0)) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.signals.lock().unwrap().push(signal); + Ok(()) + } + + async fn terminate(&self) -> Result<(), BackendError> { + Ok(()) + } + } + + struct TestBoundaryTerminal { + size: Mutex>, + } + + #[async_trait::async_trait] + impl BoundaryTerminal for TestBoundaryTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + *self.size.lock().unwrap() = Some((cols, rows)); + Ok(()) + } + } + + #[tokio::test] + async fn boundary_attachment_drives_main_io_signal_and_terminal() { + let (stdin, mut stdin_peer) = tokio::io::duplex(1024); + let (stdout, mut stdout_peer) = tokio::io::duplex(1024); + let process = Arc::new(TestBoundaryProcess { + signals: Mutex::new(Vec::new()), + }); + let terminal = Arc::new(TestBoundaryTerminal { + size: Mutex::new(None), + }); + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let attachment = ProcessAttachment { + stdin, + stdout, + stderr: None, + terminal: Some(terminal.clone()), + }; + let session = MainSession::from_boundary(attachment, process.clone()); + let mut output = session.subscribe(); + + stdout_peer.write_all(b"ready\n").await.unwrap(); + assert!(matches!( + output.recv().await.unwrap(), + MainOutput::Stdout(data) if data == b"ready\n"[..] + )); + + let (_owner, input) = session.acquire_input().unwrap(); + input.send(b"hello\n".to_vec()).await.unwrap(); + let mut received = [0_u8; 6]; + stdin_peer.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"hello\n"); + + session.resize(120, 40, 0, 0).await; + assert_eq!(*terminal.size.lock().unwrap(), Some((120, 40))); + session + .signal_group(nix::sys::signal::Signal::SIGINT) + .await + .unwrap(); + assert_eq!(*process.signals.lock().unwrap(), vec![BoundarySignal::Int]); + } #[test] fn input_lease_has_one_owner_and_can_be_reacquired() { @@ -632,6 +872,27 @@ mod tests { .expect("closing the attachment should wake the waiter"); } + #[tokio::test] + async fn remote_finish_bounds_output_drain_before_publishing_exit() { + let mut session = MainSession::inert(); + Arc::get_mut(&mut session) + .expect("sole test session reference") + .readers_remaining = AtomicUsize::new(1); + let mut output = session.subscribe(); + + session + .finish_remote_with_timeout(19, false, std::time::Duration::from_millis(10)) + .await; + + assert!(matches!( + output + .recv() + .await + .expect("terminal status after bounded drain"), + MainOutput::Exit(19) + )); + } + #[tokio::test] async fn declared_attachment_waits_for_connection_then_natural_close() { let session = MainSession::inert(); diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs deleted file mode 100644 index 2b4ea554ed..0000000000 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ /dev/null @@ -1,1239 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Network namespace isolation for sandboxed processes. -//! -//! Creates an isolated network namespace with a veth pair connecting -//! the sandbox to the host. This ensures the sandboxed process can only -//! communicate through the proxy running on the host side of the veth. - -mod nft_ruleset; - -use miette::{IntoDiagnostic, Result}; -use std::net::IpAddr; -use std::os::unix::io::RawFd; -use std::path::Path; -use std::process::Command; -use tracing::{debug, warn}; -use uuid::Uuid; - -/// Default subnet for sandbox networking. -const SUBNET_PREFIX: &str = "10.200.0"; -const HOST_IP_SUFFIX: u8 = 1; -const SANDBOX_IP_SUFFIX: u8 = 2; -/// Unprivileged port owned by the supervisor's policy DNS service. Workload -/// queries still target the standard DNS port and nftables redirects them to -/// this listener before the bypass fence runs. -pub const POLICY_DNS_PORT: u16 = 15_053; -pub const TRANSPARENT_TCP_PORT: u16 = 15_001; -const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; -const NSENTER_SEARCH_PATHS: &[&str] = &[ - "/usr/bin/nsenter", - "/bin/nsenter", - "/usr/sbin/nsenter", - "/sbin/nsenter", -]; - -/// Handle to a network namespace with veth pair. -/// -/// The namespace and veth interfaces are automatically cleaned up on drop. -#[derive(Debug)] -pub struct NetworkNamespace { - /// Namespace name (e.g., "sandbox-{uuid}") - name: String, - /// Host-side veth interface name - veth_host: String, - /// Sandbox-side veth interface name (inside namespace, used only during setup) - _veth_sandbox: String, - /// Host-side IP address (proxy binds here) - host_ip: IpAddr, - /// Sandbox-side IP address - sandbox_ip: IpAddr, - /// File descriptor for the namespace (for setns) - ns_fd: Option, -} - -impl NetworkNamespace { - /// Create a new isolated network namespace with veth pair. - /// - /// Sets up: - /// - A new network namespace named `sandbox-{uuid}` - /// - A veth pair connecting host and sandbox - /// - IP addresses on both ends (10.200.0.1/24 and 10.200.0.2/24) - /// - Default route in sandbox pointing to host - /// - /// # Errors - /// - /// Returns an error if namespace creation or network setup fails. - pub fn create() -> Result { - let id = Uuid::new_v4(); - let short_id = &id.to_string()[..8]; - let name = format!("sandbox-{short_id}"); - let veth_host = format!("veth-h-{short_id}"); - let veth_sandbox = format!("veth-s-{short_id}"); - - let host_ip: IpAddr = format!("{SUBNET_PREFIX}.{HOST_IP_SUFFIX}").parse().unwrap(); - let sandbox_ip: IpAddr = format!("{SUBNET_PREFIX}.{SANDBOX_IP_SUFFIX}") - .parse() - .unwrap(); - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "creating") - .message(format!( - "Creating network namespace [ns:{name} host_veth:{veth_host} sandbox_veth:{veth_sandbox}]" - )) - .build() - ); - - // Create the namespace - run_ip(&["netns", "add", &name])?; - - // Create veth pair - if let Err(e) = run_ip(&[ - "link", - "add", - &veth_host, - "type", - "veth", - "peer", - "name", - &veth_sandbox, - ]) { - // Cleanup namespace on failure - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Move sandbox veth into namespace - if let Err(e) = run_ip(&["link", "set", &veth_sandbox, "netns", &name]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Configure host side - let host_cidr = format!("{host_ip}/24"); - if let Err(e) = run_ip(&["addr", "add", &host_cidr, "dev", &veth_host]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - if let Err(e) = run_ip(&["link", "set", &veth_host, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Configure sandbox side (inside namespace) - let sandbox_cidr = format!("{sandbox_ip}/24"); - if let Err(e) = run_ip_netns(&name, &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - if let Err(e) = run_ip_netns(&name, &["link", "set", &veth_sandbox, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Bring up loopback in namespace - if let Err(e) = run_ip_netns(&name, &["link", "set", "lo", "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Add default route via host - let host_ip_str = host_ip.to_string(); - if let Err(e) = run_ip_netns(&name, &["route", "add", "default", "via", &host_ip_str]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); - return Err(e); - } - - // Open the namespace file descriptor for later use with setns - let ns_path = openshell_core::container_paths::netns_path(&name); - let ns_fd = match nix::fcntl::open( - ns_path.as_path(), - nix::fcntl::OFlag::O_RDONLY, - nix::sys::stat::Mode::empty(), - ) { - Ok(fd) => Some(fd), - Err(e) => { - warn!(error = %e, "Failed to open namespace fd, will use nsenter fallback"); - None - } - }; - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "created") - .message(format!( - "Network namespace created [ns:{name} host_ip:{host_ip} sandbox_ip:{sandbox_ip}]" - )) - .build() - ); - - Ok(Self { - name, - veth_host, - _veth_sandbox: veth_sandbox, - host_ip, - sandbox_ip, - ns_fd, - }) - } - - /// Get the host-side IP address (proxy should bind to this). - #[must_use] - pub const fn host_ip(&self) -> IpAddr { - self.host_ip - } - - /// Get the sandbox-side IP address. - #[must_use] - pub const fn sandbox_ip(&self) -> IpAddr { - self.sandbox_ip - } - - /// Get the namespace name. - #[must_use] - pub fn name(&self) -> &str { - &self.name - } - - /// Enter this network namespace. - /// - /// Must be called from the child process after fork, before exec. - /// Uses `setns()` to switch the calling process into the namespace. - /// - /// # Errors - /// - /// Returns an error if setns fails. - /// - /// # Safety - /// - /// This function should only be called in a `pre_exec` context after fork. - pub fn enter(&self) -> Result<()> { - if let Some(fd) = self.ns_fd { - debug!(namespace = %self.name, "Entering network namespace via setns"); - // SAFETY: setns is safe to call after fork, before exec - // libc/syscall FFI requires unsafe - #[allow(unsafe_code)] - let result = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if result != 0 { - return Err(miette::miette!( - "setns failed: {}", - std::io::Error::last_os_error() - )); - } - Ok(()) - } else { - Err(miette::miette!( - "No namespace file descriptor available for setns" - )) - } - } - - /// Get the namespace file descriptor for use with clone/unshare. - #[must_use] - pub const fn ns_fd(&self) -> Option { - self.ns_fd - } - - /// Install nftables rules for bypass detection inside the namespace. - /// - /// Sets up OUTPUT chain rules that: - /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) - /// 2. ACCEPT loopback traffic - /// 3. ACCEPT established/related connections (response packets) - /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) - /// - /// This provides two benefits: - /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of - /// a 30-second timeout when they bypass the proxy - /// - **Diagnostics**: nftables LOG entries are picked up by the bypass - /// monitor to emit structured tracing events - /// - /// Degrades gracefully if `nft` is not available — the namespace - /// still provides isolation via routing, just without fast-fail and - /// diagnostic logging. - pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { - let Some(nft_path) = find_nft() else { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "degraded") - .message(format!( - "nft not found; bypass detection rules will not be installed [ns:{}]", - self.name - )) - .build() - ); - return Ok(()); - }; - - let host_ip_str = self.host_ip.to_string(); - let log_prefix = format!("openshell:bypass:{}:", &self.name); - - // The kernel's nf_log_syslog module suppresses log output from - // non-init network namespaces by default. Enable it so the bypass - // monitor can see log entries from the sandbox namespace. - enable_nf_log_all_netns(); - - let commands = - nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "failed") - .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", - self.name - )) - .build() - ); - return Err(e); - } - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "installed") - .message(format!( - "Bypass detection rules installed [ns:{}]", - self.name - )) - .build() - ); - - Ok(()) - } - - /// Replace the ordinary bypass fence with the policy-DNS and transparent - /// TCP ruleset. This is fail-closed: callers must not release workload - /// execution unless every required rule was installed. - pub fn install_transparent_tcp_rules( - &self, - proxy_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - self.validate_synthetic_pool_routes(synthetic_ipv4_cidr, synthetic_ipv6_cidr)?; - // The inner namespace has an IPv4 default route, but not an IPv6 - // default route. Install only the active synthetic IPv6 epoch so the - // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to - // the local transparent listener. - run_ip_netns( - &self.name, - &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], - )?; - let nft_path = find_nft().ok_or_else(|| { - miette::miette!( - "trusted nft helper not found; policy DNS and transparent TCP require nftables" - ) - })?; - let host_ip = self.host_ip.to_string(); - let log_prefix = format!("openshell:bypass:{}:", self.name); - let commands = nft_ruleset::generate_transparent_tcp_commands( - &host_ip, - proxy_port, - POLICY_DNS_PORT, - TRANSPARENT_TCP_PORT, - synthetic_ipv4_cidr, - synthetic_ipv6_cidr, - Some(&log_prefix), - ); - run_nft_commands_netns(&self.name, &nft_path, &commands)?; - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Enabled, "installed") - .message(format!( - "Policy DNS and transparent TCP capture installed [ns:{}]", - self.name - )) - .build() - ); - Ok(()) - } - - fn validate_synthetic_pool_routes( - &self, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - let reserved = [ - synthetic_ipv4_cidr - .parse::() - .into_diagnostic()?, - synthetic_ipv6_cidr - .parse::() - .into_diagnostic()?, - ]; - for family in ["-4", "-6"] { - let routes = - run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; - if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { - return Err(miette::miette!( - "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" - )); - } - } - Ok(()) - } - - /// Bind IPv4 and IPv6 transparent listeners inside the workload network - /// namespace without moving an async runtime worker into that namespace. - pub async fn bind_transparent_tcp_listeners( - &self, - ) -> std::io::Result> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - let mut listeners = Vec::with_capacity(2); - for (domain, address) in [ - ( - socket2::Domain::IPV4, - format!("0.0.0.0:{TRANSPARENT_TCP_PORT}"), - ), - ( - socket2::Domain::IPV6, - format!("[::]:{TRANSPARENT_TCP_PORT}"), - ), - ] { - let socket = socket2::Socket::new( - domain, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - socket.set_reuse_address(true)?; - if domain == socket2::Domain::IPV6 { - socket.set_only_v6(true)?; - } - let address: std::net::SocketAddr = address.parse().map_err(|error| { - std::io::Error::other(format!("invalid listener address: {error}")) - })?; - socket.bind(&address.into())?; - socket.listen(128)?; - let listener: std::net::TcpListener = socket.into(); - listener.set_nonblocking(true)?; - listeners.push(listener); - } - Ok(listeners) - })(); - let _ = tx.send(result); - }); - rx.await - .map_err(|_| std::io::Error::other("netns bind thread panicked"))?? - .into_iter() - .map(tokio::net::TcpListener::from_std) - .collect() - } - - /// Bind UDP and TCP DNS listeners inside the workload network namespace. - /// The workload keeps its image-provided resolver configuration; nftables - /// redirects port 53 to these sockets before the bypass fence runs. - pub async fn bind_policy_dns_sockets( - &self, - ) -> std::io::Result<(tokio::net::UdpSocket, tokio::net::TcpListener)> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result<(std::net::UdpSocket, std::net::TcpListener)> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - // Bind the exact REDIRECT destination instead of INADDR_ANY. - // For UDP this keeps replies sourced from loopback so - // conntrack can reverse the port/address translation before - // delivering them to libc in nested rootless namespaces. - let address: std::net::SocketAddr = format!("127.0.0.1:{POLICY_DNS_PORT}") - .parse() - .map_err(|error| { - std::io::Error::other(format!("invalid DNS listener address: {error}")) - })?; - - let udp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::DGRAM, - Some(socket2::Protocol::UDP), - )?; - udp.set_reuse_address(true)?; - udp.bind(&address.into())?; - udp.set_nonblocking(true)?; - - let tcp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - tcp.set_reuse_address(true)?; - tcp.bind(&address.into())?; - tcp.listen(128)?; - tcp.set_nonblocking(true)?; - - Ok((udp.into(), tcp.into())) - })(); - let _ = tx.send(result); - }); - let (udp, tcp) = rx - .await - .map_err(|_| std::io::Error::other("netns DNS bind thread panicked"))??; - Ok(( - tokio::net::UdpSocket::from_std(udp)?, - tokio::net::TcpListener::from_std(tcp)?, - )) - } - - /// Bind a TCP listener inside this network namespace on a dedicated thread. - /// - /// Spawns a short-lived OS thread that enters the namespace via `setns`, - /// binds a `std::net::TcpListener`, then exits. The listener fd is handed - /// back as a non-blocking `tokio::net::TcpListener`. Using a dedicated - /// thread (not `spawn_blocking`) avoids contaminating the tokio thread - /// pool's namespace state. - /// - /// Returns `Err` if the namespace has no fd, `setns` fails, or bind fails. - pub async fn bind_tcp_in_netns(&self, addr: &str) -> std::io::Result { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let addr = addr.to_string(); - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - // SAFETY: setns is safe to call; this is a dedicated thread - // that exits after binding. The thread's namespace state does - // not contaminate any thread pool. - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpListener::bind(&addr) - })(); - let _ = tx.send(result); - }); - - let std_listener = rx - .await - .map_err(|_| std::io::Error::other("netns bind thread panicked"))??; - std_listener.set_nonblocking(true)?; - tokio::net::TcpListener::from_std(std_listener) - } -} - -impl Drop for NetworkNamespace { - fn drop(&mut self) { - debug!(namespace = %self.name, "Cleaning up network namespace"); - - // Close the fd if we have one - if let Some(fd) = self.ns_fd.take() { - let _ = nix::unistd::close(fd); - } - - // Delete the host-side veth (this also removes the peer) - if let Err(e) = run_ip(&["link", "delete", &self.veth_host]) { - warn!( - error = %e, - veth = %self.veth_host, - "Failed to delete veth interface" - ); - } - - // Delete the namespace - if let Err(e) = run_ip(&["netns", "delete", &self.name]) { - warn!( - error = %e, - namespace = %self.name, - "Failed to delete network namespace" - ); - } - - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Disabled, "cleaned_up") - .message(format!("Network namespace cleaned up [ns:{}]", self.name)) - .build() - ); - } -} - -/// Create the workload's network namespace and install bypass detection -/// rules. Returns `None` when the policy is not in proxy mode. -/// -/// The namespace is shared infrastructure: the proxy binds to its host-side -/// veth IP and reads /dev/kmsg from inside it for bypass detection, while -/// the workload child and SSH sessions enter it via `setns()`. -/// -/// # Errors -/// -/// Returns an error if proxy mode is requested but the namespace cannot be -/// created (e.g., missing `CAP_NET_ADMIN` / `CAP_SYS_ADMIN` or `iproute2`). -/// Failure to install nftables bypass-detection rules is non-fatal and is -/// reported via OCSF instead. -pub fn create_netns_for_proxy( - policy: &openshell_core::policy::SandboxPolicy, -) -> Result> { - use openshell_core::policy::NetworkMode; - use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; - - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Ok(None); - } - match NetworkNamespace::create() { - Ok(ns) => { - let proxy_port = policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map_or(3128, |addr| addr.port()); - if let Err(e) = ns.install_bypass_rules(proxy_port) { - ocsf_emit!( - ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "degraded") - .message(format!( - "Failed to install bypass detection rules (non-fatal): {e}" - )) - .build() - ); - } - Ok(Some(ns)) - } - Err(e) => Err(miette::miette!( - "Network namespace creation failed and proxy mode requires isolation. \ - Ensure CAP_NET_ADMIN and CAP_SYS_ADMIN are available and iproute2 is installed. \ - Error: {e}" - )), - } -} - -/// Install pod-network bypass enforcement for Kubernetes sidecar topology. -/// -/// This runs in the current network namespace, not in a per-workload netns. -/// The rules allow loopback and the sidecar proxy UID, then reject direct -/// TCP/UDP egress from other UIDs so traffic must use the sidecar's local -/// proxy. -/// -/// # Errors -/// -/// Returns an error when `nft` is unavailable or the ruleset cannot be loaded. -pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { - match install_sidecar_nft_bypass_rules(proxy_uid) { - Ok(()) => Ok(()), - Err(nft_error) => { - warn!( - error = %nft_error, - "Failed to install nftables sidecar rules; trying iptables-legacy fallback" - ); - install_sidecar_iptables_legacy_bypass_rules(proxy_uid).map_err(|iptables_error| { - miette::miette!( - "sidecar nft ruleset load failed: {nft_error}; sidecar iptables-legacy fallback failed: {iptables_error}" - ) - }) - } - } -} - -fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { - let nft_cmd = find_nft().ok_or_else(|| { - miette::miette!( - "trusted nft helper not found; sidecar network enforcement requires nftables" - ) - })?; - let log_prefix = Some("openshell:sidecar-bypass:"); - let commands = nft_ruleset::generate_sidecar_bypass_commands(proxy_uid, log_prefix); - run_nft_commands_current_namespace(&nft_cmd, &commands) -} - -const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; -const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; - -fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { - miette::miette!( - "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" - ) - })?; - - let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { - Some(find_ip6tables_legacy().ok_or_else(|| { - miette::miette!( - "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" - ) - })?) - } else { - warn!( - "Skipping IPv6 sidecar iptables-legacy fallback because the current namespace has no non-loopback IPv6 interface" - ); - None - }; - - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); - - if let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ipv4_filter_tool, - proxy_uid, - "icmp-port-unreachable", - ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - ipv6_fence_tool.as_deref(), - ); - return Err(e); - } - - if let Some(ipv6_fence_tool) = ipv6_fence_tool - && let Err(e) = install_sidecar_iptables_legacy_family_rules( - &ipv6_fence_tool, - proxy_uid, - "icmp6-port-unreachable", - ) - { - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, Some(&ipv6_fence_tool)); - return Err(e); - } - - Ok(()) -} - -fn current_namespace_has_non_loopback_ipv6() -> Result { - match std::fs::read_to_string(PROC_NET_IF_INET6_PATH) { - Ok(content) => Ok(has_non_loopback_ipv6_interface(&content)), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(e) => Err(miette::miette!( - "failed to inspect {PROC_NET_IF_INET6_PATH} before installing sidecar IPv6 fence: {e}" - )), - } -} - -fn has_non_loopback_ipv6_interface(content: &str) -> bool { - content.lines().any(|line| { - line.split_whitespace() - .nth(5) - .is_some_and(|iface| iface != "lo") - }) -} - -fn install_sidecar_iptables_legacy_family_rules( - cmd: &str, - proxy_uid: u32, - udp_reject_with: &str, -) -> Result<()> { - let proxy_uid_arg = proxy_uid.to_string(); - let commands: Vec> = vec![ - vec!["-N", SIDECAR_IPTABLES_CHAIN], - vec!["-A", SIDECAR_IPTABLES_CHAIN, "-o", "lo", "-j", "ACCEPT"], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-m", - "conntrack", - "--ctstate", - "ESTABLISHED,RELATED", - "-j", - "ACCEPT", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-m", - "owner", - "--uid-owner", - &proxy_uid_arg, - "-j", - "ACCEPT", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-p", - "tcp", - "-j", - "REJECT", - "--reject-with", - "tcp-reset", - ], - vec![ - "-A", - SIDECAR_IPTABLES_CHAIN, - "-p", - "udp", - "-j", - "REJECT", - "--reject-with", - udp_reject_with, - ], - vec!["-A", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], - ]; - - for args in commands { - if let Err(e) = run_iptables_legacy_current_namespace(cmd, &args) { - cleanup_sidecar_iptables_legacy_rules(cmd); - return Err(e); - } - } - - Ok(()) -} - -fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { - while run_iptables_legacy_current_namespace( - iptables_cmd, - &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], - ) - .is_ok() - {} - let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-F", SIDECAR_IPTABLES_CHAIN]); - let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); -} - -fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { - cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); - if let Some(ipv6_cmd) = ipv6_cmd { - cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); - } -} - -/// Run an `ip` command on the host. -fn run_ip(args: &[&str]) -> Result<()> { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - - debug!(command = %format!("{ip_path} {}", args.join(" ")), "Running ip command"); - - let output = Command::new(ip_path) - .args(args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{ip_path} {} failed: {}", - args.join(" "), - stderr.trim() - )); - } - - Ok(()) -} - -fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { - debug!( - command = %format!("{iptables_cmd} {}", args.join(" ")), - "Running iptables-legacy sidecar command" - ); - - let output = Command::new(iptables_cmd) - .args(args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{iptables_cmd} {} failed: {}", - args.join(" "), - stderr.trim() - )); - } - - Ok(()) -} - -/// Run a sequence of nft commands in the current network namespace. -/// -/// Each command is executed as a separate `nft` invocation to avoid atomic -/// batch rollback (where one unsupported expression like `ct state` or `log` -/// causes the entire transaction, including table creation, to fail). -/// -/// Commands marked as non-required are allowed to fail with a warning. -/// Required commands that fail abort the sequence immediately. -fn run_nft_commands_current_namespace( - nft_cmd: &str, - commands: &[nft_ruleset::NftCommand], -) -> Result<()> { - for cmd in commands { - let args_str = cmd.args.join(" "); - debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); - - let output = Command::new(nft_cmd) - .args(&cmd.args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - if cmd.required { - return Err(miette::miette!( - "{nft_cmd} {args_str} failed: {}", - stderr.trim() - )); - } - warn!( - command = %args_str, - error = %stderr.trim(), - "non-required nft command failed (continuing)" - ); - } - } - Ok(()) -} - -/// Run an `ip` command inside a network namespace via `nsenter --net=`. -/// -/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` -/// remounts `/sys` to reflect the target namespace's sysfs entries. That -/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, -/// which is unavailable in rootless container runtimes (e.g. rootless -/// Podman). `nsenter --net=` enters only the network namespace without -/// changing the mount namespace, avoiding the sysfs remount entirely. -/// The supervisor's operations (addr add, link set, route add) are all -/// netlink-based and do not need sysfs access. -fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - run_ip_netns_output(netns, args).map(|_| ()) -} - -fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - let mut full_args = vec![net_flag.as_str(), "--", ip_path]; - full_args.extend(args); - - debug!( - command = %format!("{nsenter_path} {}", full_args.join(" ")), - "Running ip in namespace via nsenter" - ); - - let output = Command::new(nsenter_path) - .args(&full_args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "{nsenter_path} --net={} {ip_path} {} failed: {}", - ns_path.display(), - args.join(" "), - stderr.trim() - )); - } - - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - -fn first_route_overlap( - routes: &str, - reserved: &[ipnet::IpNet], -) -> Option<(ipnet::IpNet, ipnet::IpNet)> { - routes.lines().find_map(|line| { - line.split_whitespace().find_map(|token| { - let route = token - .parse::() - .ok() - .or_else(|| token.parse::().ok().map(ipnet::IpNet::from))?; - reserved - .iter() - .copied() - .find(|pool| { - let same_family = route.addr().is_ipv4() == pool.addr().is_ipv4(); - let overlaps = - route.contains(&pool.network()) || pool.contains(&route.network()); - same_family && overlaps - }) - .map(|pool| (route, pool)) - }) - }) -} - -/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. -/// -/// Each command is executed as a separate invocation to avoid atomic batch -/// rollback. See [`run_nft_commands_current_namespace`] for rationale. -fn run_nft_commands_netns( - netns: &str, - nft_cmd: &str, - commands: &[nft_ruleset::NftCommand], -) -> Result<()> { - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - for cmd in commands { - let args_str = cmd.args.join(" "); - debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), - "Running nft command in namespace" - ); - - let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; - let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); - full_args.extend(&arg_refs); - - let output = Command::new(nsenter_path) - .args(&full_args) - .output() - .into_diagnostic()?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - if cmd.required { - return Err(miette::miette!( - "nft {args_str} failed in netns {netns}: {}", - stderr.trim() - )); - } - warn!( - command = %args_str, - error = %stderr.trim(), - netns = %netns, - "non-required nft command failed in namespace (continuing)" - ); - } - } - Ok(()) -} - -const NF_LOG_ALL_NETNS_PATH: &str = "/proc/sys/net/netfilter/nf_log_all_netns"; - -/// Enable nftables logging from non-init network namespaces. -/// -/// The kernel's `nf_log_syslog` module silently suppresses log output from -/// non-init network namespaces unless `net.netfilter.nf_log_all_netns` is -/// set to 1. Since sandbox bypass rules live in a per-sandbox network -/// namespace, the bypass monitor can't see log entries without this. -fn enable_nf_log_all_netns() { - use std::path::Path; - if !Path::new(NF_LOG_ALL_NETNS_PATH).exists() { - debug!("nf_log_all_netns sysctl not available (may already be set by init)"); - return; - } - match std::fs::write(NF_LOG_ALL_NETNS_PATH, "1") { - Ok(()) => { - debug!("Enabled nf_log_all_netns for non-init namespace logging"); - } - Err(e) => { - debug!( - error = %e, - "Could not enable nf_log_all_netns; bypass log rules may not produce output" - ); - } - } -} - -/// Well-known paths where nft may be installed. -const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; -const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/iptables-legacy", - "/sbin/iptables-legacy", - "/usr/bin/iptables-legacy", -]; -const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/ip6tables-legacy", - "/sbin/ip6tables-legacy", - "/usr/bin/ip6tables-legacy", -]; - -fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { - paths - .iter() - .copied() - .find(|path| { - let path = Path::new(path); - path.is_absolute() && path.is_file() - }) - .ok_or_else(|| { - miette::miette!( - "trusted {name} helper not found; checked {}", - paths.join(", ") - ) - }) -} - -/// Find the nft binary path, checking well-known locations. -fn find_nft() -> Option { - find_trusted_binary("nft", NFT_SEARCH_PATHS) - .ok() - .map(String::from) -} - -fn find_iptables_legacy() -> Option { - find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) -} - -fn find_ip6tables_legacy() -> Option { - find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs; - - // These tests require root and network namespace support - // Run with: sudo cargo test -- --ignored - - #[test] - fn find_trusted_binary_uses_absolute_existing_file() { - let tempdir = tempfile::tempdir().unwrap(); - let helper = tempdir.path().join("ip"); - fs::write(&helper, b"test helper").unwrap(); - let helper = helper.to_str().unwrap(); - - assert_eq!( - find_trusted_binary("ip", &["relative-ip", "/missing/ip", helper]).unwrap(), - helper - ); - } - - #[test] - fn find_trusted_binary_rejects_missing_helpers() { - let err = - find_trusted_binary("nsenter", &["relative-nsenter", "/missing/nsenter"]).unwrap_err(); - - assert!(err.to_string().contains("trusted nsenter helper not found")); - } - - #[test] - fn nft_search_paths_are_absolute() { - for path in NFT_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "NFT_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn iptables_legacy_search_paths_are_absolute() { - for path in IPTABLES_LEGACY_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn ip6tables_legacy_search_paths_are_absolute() { - for path in IP6TABLES_LEGACY_SEARCH_PATHS { - assert!( - path.starts_with('/'), - "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" - ); - } - } - - #[test] - fn non_loopback_ipv6_detector_ignores_empty_input() { - assert!(!has_non_loopback_ipv6_interface("")); - assert!(!has_non_loopback_ipv6_interface("\n\n")); - } - - #[test] - fn non_loopback_ipv6_detector_ignores_loopback() { - let content = "00000000000000000000000000000001 01 80 10 80 lo\n"; - - assert!(!has_non_loopback_ipv6_interface(content)); - } - - #[test] - fn non_loopback_ipv6_detector_detects_pod_interface() { - let content = "\ -00000000000000000000000000000001 01 80 10 80 lo -fe800000000000000000000000000001 02 40 20 80 eth0 -"; - - assert!(has_non_loopback_ipv6_interface(content)); - } - - #[test] - fn route_overlap_detects_reserved_pool_collision() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; - let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); - assert_eq!(route.to_string(), "198.18.0.0/15"); - assert_eq!(pool.to_string(), "198.18.1.0/25"); - } - - #[test] - fn route_overlap_ignores_default_and_unrelated_routes() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; - assert_eq!(first_route_overlap(routes, &reserved), None); - } - - #[test] - #[ignore = "requires root privileges"] - fn test_create_and_drop_namespace() { - let ns = NetworkNamespace::create().expect("Failed to create namespace"); - let name = ns.name().to_string(); - - // Verify namespace exists - let ns_path = openshell_core::container_paths::netns_path(&name); - assert!(ns_path.exists(), "Namespace file should exist"); - - // Verify IPs are set correctly - assert_eq!( - ns.host_ip().to_string(), - format!("{SUBNET_PREFIX}.{HOST_IP_SUFFIX}") - ); - assert_eq!( - ns.sandbox_ip().to_string(), - format!("{SUBNET_PREFIX}.{SANDBOX_IP_SUFFIX}") - ); - - // Drop should clean up - drop(ns); - - // Verify namespace is gone - assert!( - !Path::new(&ns_path).exists(), - "Namespace should be cleaned up" - ); - } -} diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs deleted file mode 100644 index 61e9b1d1dc..0000000000 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ /dev/null @@ -1,875 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! nftables ruleset generation for sandbox network bypass enforcement. -//! -//! This module provides pure functions to generate nftables rulesets that enforce -//! the sandbox network policy: all traffic must go through the proxy, with bypass -//! attempts logged and rejected. -//! -//! Rulesets are returned as a sequence of individual nft commands rather than a -//! monolithic file. Running each command as a separate `nft` invocation avoids -//! `nft -f` atomic batch semantics, where a single unsupported expression (e.g. -//! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the -//! entire transaction including table/chain creation. - -const DNS_DESTINATION_PORT: &str = "53"; - -/// A single nft command with metadata about whether it is required. -pub struct NftCommand { - /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). - pub args: Vec, - /// When false, failure of this command is non-fatal; the caller should - /// log a warning and continue with the remaining commands. - pub required: bool, -} - -/// Generate the legacy nft commands for sandbox bypass detection. -/// -/// Creates an `inet` family table (handles both IPv4 and IPv6) with rules that: -/// 1. Accept traffic to the proxy (IPv4 only) -/// 2. Accept loopback traffic -/// 3. Accept established/related connections (optional; requires `nf_conntrack`) -/// 4. Reject TCP and UDP bypass attempts (both IPv4 and IPv6) -/// -/// If `log_prefix` is provided, log rules are inserted before each reject rule -/// so that bypass attempts are recorded in the kernel ring buffer before being -/// rejected. Log rules are non-required since they need `nf_log` support. -pub fn generate_bypass_commands( - host_ip: &str, - proxy_port: u16, - log_prefix: Option<&str>, -) -> Vec { - generate_commands(host_ip, proxy_port, log_prefix, false) -} - -/// Generate the RFC 0012 default-deny egress ceiling. -/// -/// Only the exact proxy destination and loopback are accepted. TCP and UDP -/// rejects are optional fast-fail behavior; the base-chain drop policy covers -/// every address family and protocol. No blanket conntrack exception is -/// installed because pre-existing or related flows must not bypass mediation. -#[allow(dead_code, reason = "consumed when RFC 0012 backend activation lands")] -pub fn generate_egress_ceiling_commands( - host_ip: &str, - proxy_port: u16, - log_prefix: Option<&str>, -) -> Vec { - generate_commands(host_ip, proxy_port, log_prefix, true) -} - -fn generate_commands( - host_ip: &str, - proxy_port: u16, - log_prefix: Option<&str>, - default_deny: bool, -) -> Vec { - let table = "openshell_bypass"; - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", table]), - nft_cmd(true, &["flush", "table", "inet", table]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - table, - "output", - if default_deny { - "{ type filter hook output priority 0; policy drop; }" - } else { - "{ type filter hook output priority 0; policy accept; }" - }, - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "ip", - "daddr", - host_ip, - "tcp", - "dport", - &proxy_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "oifname", "lo", "accept", - ], - ), - ]; - - if !default_deny { - cmds.push(nft_cmd( - false, - &[ - "add", - "rule", - "inet", - table, - "output", - "ct", - "state", - "established,related", - "accept", - ], - )); - } - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - !default_deny, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - !default_deny, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - !default_deny, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - !default_deny, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - cmds -} - -/// Generate the combined policy-DNS, transparent-TCP, and bypass fence. -/// -/// DNS may reach only the supervisor's trusted listener. TCP addressed to the -/// reserved synthetic pools is redirected before the terminal bypass reject; -/// all other direct TCP/UDP retains the existing fast-fail behavior. -pub fn generate_transparent_tcp_commands( - host_ip: &str, - proxy_port: u16, - dns_port: u16, - transparent_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - log_prefix: Option<&str>, -) -> Vec { - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", "openshell_transparent"]), - nft_cmd(true, &["flush", "table", "inet", "openshell_transparent"]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - "openshell_transparent", - "output", - "{ type nat hook output priority dstnat; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "udp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip", - "daddr", - synthetic_ipv4_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - // Synthetic destinations must take precedence over the generic TCP - // DNS capture. A policy endpoint may legitimately use TCP port 53; - // that connection belongs to transparent TCP, not the DNS listener. - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "tcp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip6", - "daddr", - synthetic_ipv6_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - ]; - let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); - // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the - // filter hook. Some kernels retain the packet's pre-REDIRECT output - // interface for filter matching, so `oifname lo accept` alone is not - // portable. Admit only packets that the kernel records as DNATed to the - // supervisor listeners. A direct dial to either port has no DNAT status - // and still reaches the terminal bypass reject. Transparent TCP - // authorization after accept remains bound by SO_ORIGINAL_DST plus the - // synthetic-address mapping. - let insertion = bypass - .iter() - .position(|command| { - command.args.iter().any(|arg| arg == "log") - || command.args.iter().any(|arg| arg == "reject") - }) - .unwrap_or(bypass.len()); - bypass.splice( - insertion..insertion, - [ - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "udp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &transparent_port.to_string(), - "accept", - ], - ), - ], - ); - cmds.extend(bypass); - cmds -} - -/// Generate nft commands for Kubernetes sidecar enforcement. -/// -/// The network sidecar and the process supervisor share a pod network -/// namespace. The sidecar runs as `proxy_uid` and owns external egress; -/// sandbox traffic must use loopback services hosted by that sidecar -/// (gateway forward and HTTP CONNECT proxy). The generated fence rejects -/// TCP/UDP bypass attempts from non-proxy UIDs; other L4 protocols are outside -/// the sidecar policy fence. -pub fn generate_sidecar_bypass_commands( - proxy_uid: u32, - log_prefix: Option<&str>, -) -> Vec { - let table = "openshell_sidecar_bypass"; - let uid_str = proxy_uid.to_string(); - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", table]), - nft_cmd(true, &["flush", "table", "inet", table]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - table, - "output", - "{ type filter hook output priority 0; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "oifname", "lo", "accept", - ], - ), - nft_cmd( - false, - &[ - "add", - "rule", - "inet", - table, - "output", - "ct", - "state", - "established,related", - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", "rule", "inet", table, "output", "meta", "skuid", &uid_str, "accept", - ], - ), - ]; - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "tcp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - if let Some(prefix) = log_prefix { - let quoted = nft_quote(prefix); - cmds.push(nft_cmd( - false, - &[ - "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", - "5/second", "burst", "10", "packets", "log", "prefix", "ed, "flags", "skuid", - ], - )); - } - - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv4", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmp", - "type", - "port-unreachable", - ], - )); - cmds.push(nft_cmd( - true, - &[ - "add", - "rule", - "inet", - table, - "output", - "meta", - "nfproto", - "ipv6", - "meta", - "l4proto", - "udp", - "reject", - "with", - "icmpv6", - "type", - "port-unreachable", - ], - )); - - cmds -} - -fn nft_cmd(required: bool, args: &[&str]) -> NftCommand { - NftCommand { - args: args.iter().map(|s| (*s).to_string()).collect(), - required, - } -} - -fn nft_quote(s: &str) -> String { - // nft quoted strings don't support escape sequences; strip any embedded - // double-quotes that would terminate the string early. - format!("\"{}\"", s.replace('"', "")) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn cmd_str(cmd: &NftCommand) -> String { - cmd.args.join(" ") - } - - fn all_strs(cmds: &[NftCommand]) -> String { - cmds.iter().map(cmd_str).collect::>().join("\n") - } - - #[test] - fn generates_bypass_commands_with_proxy_rule() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_bypass")); - assert!(text.contains("add chain inet openshell_bypass output")); - assert!(text.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); - } - - #[test] - fn bypass_commands_have_table_and_chain() { - let cmds = generate_bypass_commands("192.168.1.1", 3128, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_bypass")); - assert!(text.contains("type filter hook output priority 0; policy accept;")); - } - - #[test] - fn in_pod_ceiling_is_default_deny_for_all_protocols() { - let text = all_strs(&generate_egress_ceiling_commands("10.0.2.2", 3128, None)); - assert!(text.contains("policy drop")); - assert!(!text.contains("policy accept")); - assert!(!text.contains("ct state")); - } - - #[test] - fn in_pod_reject_rules_are_optional_fast_fail_over_default_drop() { - let commands = generate_egress_ceiling_commands("10.0.2.2", 3128, None); - for command in commands - .iter() - .filter(|command| command.args.iter().any(|argument| argument == "reject")) - { - assert!(!command.required); - } - } - - #[test] - fn proxy_accept_rule_uses_provided_ip_and_port() { - let cmds = generate_bypass_commands("172.16.0.1", 9999, None); - let text = all_strs(&cmds); - assert!(text.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); - } - - #[test] - fn rules_are_ordered_accept_then_reject() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - let proxy_pos = text.find("ip daddr").unwrap(); - let lo_pos = text.find("oifname lo").unwrap(); - let ct_pos = text.find("ct state established").unwrap(); - let reject_pos = text.find("reject with icmp type").unwrap(); - - assert!(proxy_pos < lo_pos); - assert!(lo_pos < ct_pos); - assert!(ct_pos < reject_pos); - } - - #[test] - fn transparent_rules_precede_bypass_rejects_and_scope_dns() { - let commands = generate_transparent_tcp_commands( - "10.200.0.1", - 3128, - 15053, - 15001, - "198.18.0.0/24", - "fd23:6f70:656e::/48", - None, - ); - let text = all_strs(&commands); - assert!(text.contains("meta nfproto ipv4 udp dport 53 redirect to :15053")); - assert!(text.contains("meta nfproto ipv4 tcp dport 53 redirect to :15053")); - assert!(!text.contains("udp dport 53 accept")); - assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); - assert!( - text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") - ); - assert!(!text.contains("meta mark")); - assert!(text.contains("ct status dnat udp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15001 accept")); - for (protocol, port) in [("udp", "15053"), ("tcp", "15053"), ("tcp", "15001")] { - assert!(!commands.iter().any(|command| { - command.args.ends_with(&[ - protocol.to_string(), - "dport".to_string(), - port.to_string(), - "accept".to_string(), - ]) && !command.args.windows(3).any(|window| { - window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] - }) - })); - } - assert!(text.contains("oifname lo accept")); - assert!( - text.find("ct status dnat tcp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat udp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto udp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat tcp dport 15001 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!(!text.contains("meta nfproto ipv6 udp dport 53 redirect")); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 tcp dport 53 redirect to :15053") - .unwrap(), - "synthetic TCP:53 must reach transparent TCP before generic DNS capture" - ); - } - - #[test] - fn both_ipv4_and_ipv6_reject_types_are_present() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - let icmp_count = text - .matches("reject with icmp type port-unreachable") - .count(); - let icmpv6_count = text - .matches("reject with icmpv6 type port-unreachable") - .count(); - assert_eq!(icmp_count, 2, "need IPv4 ICMP rejects for TCP + UDP"); - assert_eq!(icmpv6_count, 2, "need IPv6 ICMPv6 rejects for TCP + UDP"); - } - - #[test] - fn no_log_commands_omit_log_rules() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let text = all_strs(&cmds); - assert!( - !text.contains("log prefix"), - "no-log commands must not contain log rules" - ); - } - - #[test] - fn log_commands_contain_prefix_for_tcp_and_udp() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let text = all_strs(&cmds); - let count = text - .matches("log prefix \"openshell:bypass:test:\"") - .count(); - assert_eq!(count, 2, "need log rules for both TCP and UDP"); - assert!(text.contains("tcp flags syn limit rate 5/second burst 10 packets")); - assert!(text.contains("meta l4proto udp limit rate 5/second burst 10 packets")); - } - - #[test] - fn log_rules_appear_before_reject_rules() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let text = all_strs(&cmds); - let tcp_log_pos = text.find("tcp flags syn").unwrap(); - let tcp_reject_pos = text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap(); - let udp_log_pos = text.find("meta l4proto udp limit rate").unwrap(); - let udp_reject_pos = text - .find("meta nfproto ipv4 meta l4proto udp reject") - .unwrap(); - - assert!( - tcp_log_pos < tcp_reject_pos, - "TCP log rule must come before TCP reject rule" - ); - assert!( - udp_log_pos < udp_reject_pos, - "UDP log rule must come before UDP reject rule" - ); - } - - #[test] - fn ct_state_rule_is_not_required() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, None); - let ct_cmd = cmds - .iter() - .find(|c| cmd_str(c).contains("ct state")) - .unwrap(); - assert!( - !ct_cmd.required, - "ct state rule should be non-required (needs nf_conntrack)" - ); - } - - #[test] - fn log_rules_are_not_required() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - for cmd in &cmds { - if cmd_str(cmd).contains("log prefix") { - assert!( - !cmd.required, - "log rules should be non-required (needs nf_log)" - ); - } - } - } - - #[test] - fn sidecar_commands_allow_supervisor_uid_and_loopback() { - let cmds = generate_sidecar_bypass_commands(1337, None); - let text = all_strs(&cmds); - assert!(text.contains("add table inet openshell_sidecar_bypass")); - assert!(text.contains("oifname lo accept")); - assert!(text.contains("meta skuid 1337 accept")); - } - - #[test] - fn sidecar_commands_reject_tcp_and_udp_egress() { - let cmds = generate_sidecar_bypass_commands(0, Some("openshell:sidecar:test:")); - let text = all_strs(&cmds); - assert!(text.contains("meta nfproto ipv4 meta l4proto tcp reject")); - assert!(text.contains("meta nfproto ipv6 meta l4proto tcp reject")); - assert!(text.contains("meta nfproto ipv4 meta l4proto udp reject")); - assert!(text.contains("meta nfproto ipv6 meta l4proto udp reject")); - assert_eq!( - text.matches("log prefix \"openshell:sidecar:test:\"") - .count(), - 2 - ); - } - - #[test] - fn log_prefix_is_quoted_as_nft_string_literal() { - let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); - for cmd in &cmds { - let s = cmd_str(cmd); - if let Some(idx) = s.find("log prefix ") { - let after_prefix = &s[idx + "log prefix ".len()..]; - assert!( - after_prefix.starts_with('"'), - "log prefix value must be an nft-quoted string, got: {after_prefix}" - ); - } - } - } - - #[test] - fn nft_quote_wraps_in_double_quotes() { - assert_eq!(nft_quote("simple"), "\"simple\""); - assert_eq!(nft_quote("has:colons:"), "\"has:colons:\""); - assert_eq!(nft_quote("has\"quote"), "\"hasquote\""); - assert_eq!(nft_quote("has\\backslash"), "\"has\\backslash\""); - } -} diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs deleted file mode 100644 index 8c47e789ba..0000000000 --- a/crates/openshell-supervisor-process/src/run.rs +++ /dev/null @@ -1,830 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Workload supervision entry point. -//! -//! Spawns the SSH server, optional supervisor session, the entrypoint child -//! process, and waits for it to exit (with optional timeout). Long-running -//! background tasks that aren't strictly tied to the workload's lifetime -//! (policy poll loop, denial aggregator, symlink resolver) live in the -//! orchestrator, not here. - -use miette::{IntoDiagnostic, Result}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::time::Duration; -use tokio::time::timeout; -use tracing::info; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, DispositionId, LaunchTypeId, Process as OcsfProcess, - ProcessActivityBuilder, SeverityId, StatusId, ocsf_emit, -}; - -#[cfg(target_os = "linux")] -use crate::netns::NetworkNamespace; -use openshell_core::policy::{NetworkMode, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; - -#[cfg(target_os = "linux")] -use openshell_core::activity::ActivitySender; -#[cfg(target_os = "linux")] -use openshell_core::denial::DenialEvent; - -#[cfg(target_os = "linux")] -use crate::managed_children; -use crate::process::{ - ProcessEnforcementMode, ProcessHandle, ProcessStatus, ResolvedProcessIdentity, - ResolvedWorkspace, -}; - -pub enum SidecarExitReport { - Exited { - instance_id: String, - exit_code: i32, - ack: tokio::sync::oneshot::Sender>, - }, - Finalized { - instance_id: String, - ack: tokio::sync::oneshot::Sender>, - }, -} - -fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { - openshell_ocsf::ctx::ctx() -} - -/// Spawn the workload entrypoint, wire up SSH and supervisor session, and -/// wait for the entrypoint child to exit. -/// -/// # Errors -/// -/// Returns an error if SSH server startup fails, if the entrypoint child -/// fails to spawn, or if waiting for the child returns an OS error. -#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] -pub async fn run_process( - program: &str, - args: &[String], - workspace: ResolvedWorkspace, - timeout_secs: u64, - interactive: bool, - await_main_process_attachment: bool, - sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, - ssh_socket_path: Option, - shared_ssh_socket: bool, - ssh_exit_tx: Option>, - policy: &SandboxPolicy, - resolved_process_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - entrypoint_pid: Arc, - entrypoint_started_tx: Option>, - sidecar_exit_tx: Option>, - provider_credentials: ProviderCredentialState, - provider_env: std::collections::HashMap, - ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, - agent_proposals: AgentProposals, - #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, - #[cfg(target_os = "linux")] bypass_denial_tx: Option< - tokio::sync::mpsc::UnboundedSender, - >, - #[cfg(target_os = "linux")] bypass_activity_tx: Option, -) -> Result { - // Platform drivers with a resolved numeric UID/GID retain the legacy - // account-file update. OCI-image identity leaves those environment values - // empty, so the image's account files remain unchanged. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::update_sandbox_passwd_entries()?; - } - - // Validate the completed process identity before exposing a child. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::validate_sandbox_user_with_identity(policy, resolved_process_identity)?; - crate::process::validate_sandbox_group_with_identity(policy, resolved_process_identity)?; - } - - // Create read_write directories and chown newly-created ones to the - // sandbox user/group. Runs as the supervisor (root) before the child - // is forked so the workload sees writable paths it owns. - #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() { - crate::process::prepare_filesystem_with_identity( - policy, - resolved_process_identity, - workspace.root(), - workspace.home().is_some(), - )?; - } - - // Eagerly fetch initial settings and install the agent skill if the - // proposals flag is on at startup, rather than waiting for the policy - // poll loop's first tick. In offline/file-mode there is no gateway, so - // the flag stays at its default (false) and no skill is installed. - install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; - - // Provider token grants may mount supervisor-only identity sockets such as - // the SPIFFE Workload API. Prepare the child mount namespace that hides - // those mounts before supervisor seccomp hardening removes the needed - // namespace syscalls. - #[cfg(target_os = "linux")] - crate::process::prepare_supervisor_identity_mount_namespace_from_env()?; - - // Install the supervisor seccomp prelude before spawning any workload-side - // tasks. By this point the orchestrator has finished privileged startup - // helpers (network namespace setup, identity mount namespace setup, - // nftables probes via run_networking), and the SSH listener and entrypoint - // child have not been exposed yet. - crate::sandbox::apply_supervisor_startup_hardening()?; - - // Spawn the bypass detection monitor. It tails dmesg for nftables LOG - // entries fired by rules installed on the workload's network namespace - // and reports direct connection attempts that would have bypassed the - // proxy. Spawn it before the entrypoint child so the first packets are - // not missed. Best-effort: returns None when dmesg is unavailable. - #[cfg(target_os = "linux")] - let _bypass_handle = netns.and_then(|ns| { - crate::bypass_monitor::spawn( - ns.name().to_string(), - entrypoint_pid.clone(), - bypass_denial_tx, - bypass_activity_tx, - ) - }); - - // Verify the runtime PID limit can accommodate the policy's pid_max. - #[cfg(target_os = "linux")] - { - let pid_limit_mode = if std::env::var_os("OPENSHELL_REQUIRE_RUNTIME_PID_LIMIT").is_some() { - crate::process::RuntimePidLimitMode::Require - } else { - crate::process::RuntimePidLimitMode::Warn - }; - crate::process::check_runtime_pid_limit(pid_limit_mode)?; - } - - // Zombie reaper — openshell-sandbox may run as PID 1 in containers and - // must reap orphaned grandchildren (e.g. background daemons started by - // coding agents) to prevent zombie accumulation. - // - // Use waitid(..., WNOWAIT) so we can inspect exited children before - // actually reaping them. This avoids racing explicit `child.wait()` calls - // for managed children (entrypoint and SSH session processes). - #[cfg(target_os = "linux")] - tokio::spawn(async { - use nix::sys::wait::{Id, WaitPidFlag, WaitStatus, waitid, waitpid}; - use tokio::signal::unix::{SignalKind, signal}; - use tokio::time::MissedTickBehavior; - - let mut sigchld = match signal(SignalKind::child()) { - Ok(s) => s, - Err(e) => { - tracing::warn!(error = %e, "Failed to register SIGCHLD handler for zombie reaping"); - return; - } - }; - let mut retry = tokio::time::interval(Duration::from_secs(5)); - retry.set_missed_tick_behavior(MissedTickBehavior::Skip); - - loop { - tokio::select! { - _ = sigchld.recv() => {} - _ = retry.tick() => {} - } - - loop { - let status = match waitid( - Id::All, - WaitPidFlag::WEXITED | WaitPidFlag::WNOHANG | WaitPidFlag::WNOWAIT, - ) { - Ok(WaitStatus::StillAlive) | Err(nix::errno::Errno::ECHILD) => break, - Ok(status) => status, - Err(nix::errno::Errno::EINTR) => continue, - Err(e) => { - tracing::debug!(error = %e, "waitid error during zombie reaping"); - break; - } - }; - - let Some(pid) = status.pid() else { - break; - }; - - if managed_children::is_managed(pid.as_raw()) { - // Let the explicit waiter own this child status. - break; - } - - match waitpid(pid, Some(WaitPidFlag::WNOHANG)) { - Ok(WaitStatus::StillAlive) - | Err(nix::errno::Errno::ECHILD | nix::errno::Errno::EINTR) => {} - Ok(reaped) => { - tracing::debug!(?reaped, "Reaped orphaned child process"); - } - Err(e) => { - tracing::debug!(error = %e, "waitpid error during orphan reap"); - break; - } - } - } - } - }); - - // Hard network policy enforcement for SSH sessions and the persistent - // supervisor session: each session's pre-exec hook calls setns(fd, - // CLONE_NEWNET) so it lands inside the workload's network namespace. - // Without this, SSH-spawned shells run in the host namespace and bypass - // the proxy entirely. - #[cfg(target_os = "linux")] - let ssh_netns_fd = netns.and_then(NetworkNamespace::ns_fd); - #[cfg(not(target_os = "linux"))] - let ssh_netns_fd: Option = None; - - #[cfg(target_os = "linux")] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - netns, - ca_file_paths.as_ref(), - &provider_env, - )?; - - #[cfg(not(target_os = "linux"))] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - ca_file_paths.as_ref(), - &provider_env, - )?; - - let main_pid = handle.pid(); - let main_session = crate::main_session::MainSession::new(handle.take_io(), main_pid); - let main_instance_id = uuid::Uuid::new_v4().to_string(); - - // SSH-spawned shells get http_proxy=http://: exported into - // their env so cooperative tools (curl, npm, Node) route through the - // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back - // to the policy-declared http_addr directly. - #[cfg(target_os = "linux")] - let ssh_proxy_url = ssh_proxy_url_for_policy(policy, netns.map(NetworkNamespace::host_ip)); - #[cfg(not(target_os = "linux"))] - let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); - - let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); - if let Some(listen_path) = ssh_socket_path.clone() { - let policy_clone = policy.clone(); - let workspace_clone = workspace.clone(); - let proxy_url = ssh_proxy_url; - let netns_fd = ssh_netns_fd; - let ca_paths = ca_file_paths.clone(); - let provider_credentials_clone = provider_credentials.clone(); - let main_session_clone = Arc::clone(&main_session); - let user_env_clone: std::collections::HashMap = - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default(); - - let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); - - tokio::spawn(async move { - let _ssh_exit_guard = ssh_exit_tx; - if let Err(err) = crate::ssh::run_ssh_server( - listen_path, - ssh_ready_tx, - policy_clone, - workspace_clone, - netns_fd, - proxy_url, - ca_paths, - provider_credentials_clone, - user_env_clone, - resolved_process_identity, - enforcement_mode, - shared_ssh_socket, - main_session_clone, - ) - .await - { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message(format!("SSH server failed: {err}")) - .build() - ); - } - }); - - // Wait for the SSH server to bind before advertising its relay. The - // main process is already supervised; MainSession retains any output - // produced while this endpoint is being prepared. - match timeout(Duration::from_secs(10), ssh_ready_rx).await { - Ok(Ok(Ok(()))) => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .message("SSH server is ready to accept connections") - .build() - ); - } - Ok(Ok(Err(err))) => { - return Err(err.context("SSH server failed during startup")); - } - Ok(Err(_)) => { - return Err(miette::miette!( - "SSH server task panicked before signaling ready" - )); - } - Err(_) => { - return Err(miette::miette!( - "SSH server did not start within 10 seconds" - )); - } - } - } - - let supervisor_terminating = Arc::new(AtomicBool::new(false)); - // A canonical process may have completed while the SSH socket was being - // prepared. Detect that exit before entering the main wait path. - let early_exit = handle.try_wait().into_diagnostic()?; - - // Spawn the persistent supervisor session if we have a gateway endpoint - // and sandbox identity. The session provides relay channels for SSH - // connect and ExecSandbox through the gateway. - let supervisor_session_task = if let (Some(endpoint), Some(id), Some(socket)) = - (openshell_endpoint, sandbox_id, ssh_socket_path.as_ref()) - { - let task = crate::supervisor_session::spawn( - endpoint.to_string(), - id.to_string(), - socket.clone(), - ssh_netns_fd, - None, - Arc::clone(&supervisor_terminating), - main_instance_id.clone(), - ); - info!("supervisor session task spawned"); - Some(task) - } else { - None - }; - - // Store the entrypoint PID so the proxy can resolve TCP peer identity - entrypoint_pid.store(handle.pid(), Ordering::Release); - if let Some(tx) = entrypoint_started_tx { - let _ = tx.send((handle.pid(), main_instance_id.clone())); - } - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .launch_type(LaunchTypeId::Spawn) - .process(OcsfProcess::new(program, i64::from(handle.pid()))) - .message(format!("Process started: pid={}", handle.pid())) - .build() - ); - - let outcome = if let Some(status) = early_exit { - ProcessWaitOutcome::Exited(status) - } else { - wait_for_process_exit_or_shutdown(&mut handle, timeout_secs, &supervisor_terminating) - .await? - }; - - let (rendered_code, drain_terminal) = match outcome { - ProcessWaitOutcome::Exited(status) => (status.code(), true), - ProcessWaitOutcome::TimedOut => { - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message("Process timed out, killing") - .build() - ); - (124, false) - } - ProcessWaitOutcome::ShutdownSignal { signal, status } => { - info!( - signal, - exit_code = status.code(), - "Entrypoint exited after supervisor shutdown signal" - ); - (status.code(), false) - } - }; - let terminal_delivery_pending = main_session - .finish( - rendered_code, - drain_terminal && await_main_process_attachment, - ) - .await; - - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .exit_code(rendered_code) - .message(format!("Process exited with code {rendered_code}")) - .build() - ); - - if outcome.should_report_main_process_exit() { - if let Some(tx) = sidecar_exit_tx.as_ref() { - report_sidecar_main_process_exit(tx, &main_instance_id, rendered_code).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code) - .await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); - } - } else { - info!( - instance_id = %main_instance_id, - "skipping main-process exit report during supervisor shutdown" - ); - } - main_session.mark_terminal_reported(); - if outcome.should_report_main_process_exit() && drain_terminal && terminal_delivery_pending { - // The peer's SSH channel-close confirms that the terminal frames sent - // above traversed russh and the relay. Detached commands have no active - // attachment and never enter this wait. - main_session.wait_for_terminal_attachments().await; - } - if outcome.should_report_main_process_exit() { - if let Some(tx) = sidecar_exit_tx.as_ref() { - finalize_sidecar_main_process_exit(tx, &main_instance_id).await?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - finalize_main_process_exit_until_ack(endpoint, id, &main_instance_id).await; - info!(instance_id = %main_instance_id, "main-process terminal delivery finalized"); - } - } - - supervisor_terminating.store(true, Ordering::Release); - if let Some(task) = supervisor_session_task { - task.abort(); - } - - Ok(rendered_code) -} - -async fn report_main_process_exit_until_ack( - endpoint: &str, - sandbox_id: &str, - instance_id: &str, - exit_code: i32, -) { - let mut retry_delay = Duration::from_millis(250); - loop { - match crate::supervisor_session::report_main_process_exit( - endpoint, - sandbox_id, - instance_id, - exit_code, - ) - .await - { - Ok(()) => return, - Err(error) => { - tracing::warn!(%error, "main-process exit report failed; retrying"); - tokio::time::sleep(retry_delay).await; - retry_delay = (retry_delay * 2).min(Duration::from_secs(2)); - } - } - } -} - -async fn finalize_main_process_exit_until_ack(endpoint: &str, sandbox_id: &str, instance_id: &str) { - let mut retry_delay = Duration::from_millis(250); - loop { - match crate::supervisor_session::finalize_main_process_exit( - endpoint, - sandbox_id, - instance_id, - ) - .await - { - Ok(()) => return, - Err(error) => { - tracing::warn!(%error, "main-process terminal finalization failed; retrying"); - tokio::time::sleep(retry_delay).await; - retry_delay = (retry_delay * 2).min(Duration::from_secs(2)); - } - } - } -} - -async fn report_sidecar_main_process_exit( - tx: &tokio::sync::mpsc::Sender, - instance_id: &str, - exit_code: i32, -) -> Result<()> { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send(SidecarExitReport::Exited { - instance_id: instance_id.to_string(), - exit_code, - ack: ack_tx, - }) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error)) -} - -async fn finalize_sidecar_main_process_exit( - tx: &tokio::sync::mpsc::Sender, - instance_id: &str, -) -> Result<()> { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send(SidecarExitReport::Finalized { - instance_id: instance_id.to_string(), - ack: ack_tx, - }) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error)) -} - -enum ProcessWaitOutcome { - Exited(ProcessStatus), - TimedOut, - ShutdownSignal { - signal: &'static str, - status: ProcessStatus, - }, -} - -impl ProcessWaitOutcome { - /// A gateway acknowledgement is required for ordinary canonical-process - /// completion, but cannot be awaited after the supervisor itself has been - /// asked to terminate. At that point the gateway may already be shutting - /// down and no longer able to acknowledge the report. - fn should_report_main_process_exit(&self) -> bool { - !matches!(self, Self::ShutdownSignal { .. }) - } -} - -async fn wait_for_process_exit_or_shutdown( - handle: &mut ProcessHandle, - timeout_secs: u64, - terminating: &AtomicBool, -) -> Result { - let pid = handle.pid(); - let wait = handle.wait(); - tokio::pin!(wait); - - if timeout_secs > 0 { - let deadline = tokio::time::sleep(Duration::from_secs(timeout_secs)); - tokio::pin!(deadline); - tokio::select! { - result = &mut wait => { - Ok(ProcessWaitOutcome::Exited(result.into_diagnostic()?)) - } - () = &mut deadline => { - terminating.store(true, Ordering::Release); - terminate_then_kill_pid(pid).await; - Ok(ProcessWaitOutcome::TimedOut) - } - signal = wait_for_supervisor_shutdown_signal() => { - terminating.store(true, Ordering::Release); - signal_entrypoint_for_shutdown(pid, signal); - let status = (&mut wait).await.into_diagnostic()?; - Ok(ProcessWaitOutcome::ShutdownSignal { signal, status }) - } - } - } else { - tokio::select! { - result = &mut wait => { - Ok(ProcessWaitOutcome::Exited(result.into_diagnostic()?)) - } - signal = wait_for_supervisor_shutdown_signal() => { - terminating.store(true, Ordering::Release); - signal_entrypoint_for_shutdown(pid, signal); - let status = (&mut wait).await.into_diagnostic()?; - Ok(ProcessWaitOutcome::ShutdownSignal { signal, status }) - } - } - } -} - -#[cfg(unix)] -async fn terminate_then_kill_pid(pid: u32) { - signal_pid(pid, nix::sys::signal::Signal::SIGTERM, "process timeout"); - tokio::time::sleep(Duration::from_millis(100)).await; - signal_pid(pid, nix::sys::signal::Signal::SIGKILL, "process timeout"); -} - -#[cfg(not(unix))] -async fn terminate_then_kill_pid(_pid: u32) {} - -#[cfg(unix)] -fn signal_entrypoint_for_shutdown(pid: u32, signal: &'static str) { - signal_pid(pid, nix::sys::signal::Signal::SIGTERM, signal); -} - -#[cfg(not(unix))] -fn signal_entrypoint_for_shutdown(_pid: u32, _signal: &'static str) {} - -#[cfg(unix)] -fn signal_pid(pid: u32, signal: nix::sys::signal::Signal, reason: &'static str) { - let raw_pid = i32::try_from(pid).unwrap_or(i32::MAX); - if let Err(error) = nix::sys::signal::kill(nix::unistd::Pid::from_raw(-raw_pid), signal) { - tracing::warn!( - pid, - signal = ?signal, - reason, - error = %error, - "failed to signal entrypoint process group" - ); - } -} - -#[cfg(unix)] -async fn wait_for_supervisor_shutdown_signal() -> &'static str { - use tokio::signal::unix::{SignalKind, signal}; - - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(signal) => signal, - Err(error) => { - tracing::warn!( - error = %error, - "Failed to install SIGTERM handler; supervisor shutdown detection disabled" - ); - return std::future::pending::<&'static str>().await; - } - }; - - let _ = sigterm.recv().await; - info!("Received SIGTERM, shutting down supervisor process"); - "SIGTERM" -} - -#[cfg(not(unix))] -async fn wait_for_supervisor_shutdown_signal() -> &'static str { - std::future::pending::<&'static str>().await -} - -fn ssh_proxy_url_for_policy( - policy: &SandboxPolicy, - netns_proxy_host: Option, -) -> Option { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return None; - } - - let proxy = policy.network.proxy.as_ref()?; - if let Some(host) = netns_proxy_host { - let port = proxy.http_addr.map_or(3128, |addr| addr.port()); - return Some(format!("http://{host}:{port}")); - } - - proxy.http_addr.map(|addr| format!("http://{addr}")) -} - -/// Eagerly fetch initial settings and install the agent-driven policy -/// proposal skill if the flag is on at startup. -/// -/// Without this, the skill would only get installed on the policy poll -/// loop's first false→true transition, which can be ~10 s after launch — -/// long enough for an agent to start running without seeing it. -/// -/// Best-effort: any failure (no gateway, RPC error, install failure) is -/// logged but does not fail sandbox startup. -async fn install_initial_agent_skill( - sandbox_id: Option<&str>, - openshell_endpoint: Option<&str>, - agent_proposals: &AgentProposals, -) { - use openshell_core::proto::setting_value; - - if let (Some(id), Some(endpoint)) = (sandbox_id, openshell_endpoint) - && let Ok(client) = - openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await - && let Ok(result) = client.poll_settings(id).await - { - let initial = result - .settings - .get(openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) - .unwrap_or(false); - agent_proposals.set_enabled(initial); - } - - if agent_proposals.enabled() { - match crate::skills::install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill" - ), - Err(error) => tracing::warn!( - error = %error, - "Failed to install sandbox agent skill" - ), - } - } else { - tracing::debug!( - "agent_policy_proposals_enabled is false at startup; skipping skill install" - ); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - - fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode, - proxy: http_addr.map(|http_addr| ProxyPolicy { - http_addr: Some(http_addr), - }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - #[test] - fn ssh_proxy_url_uses_policy_addr_without_netns() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); - - assert_eq!( - ssh_proxy_url_for_policy(&policy, None).as_deref(), - Some("http://127.0.0.1:3128") - ); - } - - #[test] - fn ssh_proxy_url_prefers_netns_host_with_policy_port() { - let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); - - assert_eq!( - ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), - Some("http://10.200.0.1:8080") - ); - } - - #[test] - fn ssh_proxy_url_skips_non_proxy_mode() { - let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); - - assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); - } - - #[cfg(unix)] - #[test] - fn supervisor_shutdown_exit_skips_gateway_acknowledgement() { - use std::os::unix::process::ExitStatusExt; - - let status = ProcessStatus::from(std::process::ExitStatus::from_raw(libc::SIGTERM)); - - assert!(ProcessWaitOutcome::Exited(status).should_report_main_process_exit()); - assert!(ProcessWaitOutcome::TimedOut.should_report_main_process_exit()); - assert!( - !ProcessWaitOutcome::ShutdownSignal { - signal: "SIGTERM", - status, - } - .should_report_main_process_exit() - ); - } -} diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 14bf287e0c..9f9aac35db 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -3,24 +3,11 @@ //! Embedded SSH server for sandbox access. -use crate::child_env; use crate::main_session::{MainOutput, MainSession}; -#[cfg(target_os = "linux")] -use crate::managed_children; -use crate::process::{ - ProcessEnforcementMode, ResolvedProcessIdentity, ResolvedWorkspace, - drop_privileges_with_identity, is_supervisor_only_env_var, session_user_and_home, -}; -use crate::sandbox; #[cfg(unix)] use libc; use miette::{IntoDiagnostic, Result}; -use nix::pty::{Winsize, openpty}; -use nix::unistd::setsid; use openshell_core::VERSION; -use openshell_core::net::set_tcp_nodelay_best_effort; -use openshell_core::policy::SandboxPolicy; -use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; @@ -29,16 +16,35 @@ use russh::server::{Auth, ChannelOpenHandle, Handle, Session}; use russh::{ChannelId, ChannelOpenFailure, Sig}; use std::borrow::Cow; use std::collections::HashMap; -use std::io::{Read, Write}; -use std::os::fd::{AsRawFd, RawFd}; use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; use std::sync::{Arc, mpsc}; use std::time::Duration; use tokio::net::UnixListener; use tracing::warn; const NO_LOGIN_SHELL_ENV: (&str, &str) = ("OPENSHELL_NO_LOGIN_SHELL", "1"); +const MAIN_DETACH_PREFIX: u8 = 0x10; +const MAIN_DETACH_KEY: u8 = 0x11; + +fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { + let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); + for &byte in data { + if *prefix_pending { + if byte == MAIN_DETACH_KEY { + *prefix_pending = false; + return (forward, true); + } + forward.push(MAIN_DETACH_PREFIX); + *prefix_pending = false; + } + if byte == MAIN_DETACH_PREFIX { + *prefix_pending = true; + } else { + forward.push(byte); + } + } + (forward, false) +} /// Perform SSH server initialization: generate a host key, build the config, /// and bind the Unix socket listener. Extracted so that startup errors can be @@ -52,13 +58,11 @@ type SshServerInit = ( fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, - enforcement_mode: ProcessEnforcementMode, shared_socket: bool, ) -> Result { let mut rng = rand::rng(); let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; - // TODO: while building the SSH config, refactor the server_id to be "SSH-2.0-OpenShell_" from `openshell_core::VERSION` let mut config = russh::server::Config { server_id: russh::SshId::Standard(Cow::Owned(format!("SSH-2.0-OpenShell_{VERSION}"))), auth_rejection_time: Duration::from_secs(1), @@ -69,17 +73,14 @@ fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - // In full enforcement mode the supervisor normally starts as root and can - // isolate the SSH socket in a root-only directory before spawning - // unprivileged children. Sidecar topology is different: the gateway relay - // runs in the network sidecar as a different UID, so the shared sidecar - // state directory must stay group-accessible. Sidecar mode uses a Linux - // abstract socket instead, so the workload cannot unlink the relay target. + // A driver may place the supervisor in another container, so an explicitly + // shared socket retains group access. Linux abstract sockets avoid a + // workload-replaceable filesystem inode. let abstract_socket = crate::unix_socket::is_abstract(listen_path); if !abstract_socket && let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - if enforcement_mode.uses_privileged_process_setup() && !shared_socket { + if !shared_socket { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -94,7 +95,7 @@ fn ssh_server_init( let listener = UnixListener::bind(runtime_path.as_ref()).into_diagnostic()?; // Tighten filesystem-socket permissions. Abstract sockets have no inode; - // sidecar relay connections authenticate the listener with SO_PEERCRED. + // local relay connections authenticate the listener with SO_PEERCRED. #[cfg(unix)] if !abstract_socket { use std::os::unix::fs::PermissionsExt; @@ -119,71 +120,44 @@ fn ssh_server_init( pub async fn run_ssh_server( listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, shared_socket: bool, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init( - &listen_path, - &ca_file_paths, - enforcement_mode, - shared_socket, - ) { - Ok(v) => { - // Signal that the SSH server has bound the socket and is ready to - // accept connections. The parent task awaits this before spawning - // the entrypoint process, ensuring exec requests won't race - // against server startup. - let _ = ready_tx.send(Ok(())); - v - } - Err(err) => { - let _ = ready_tx.send(Err(err)); - return Ok(()); - } - }; - - let mut consecutive_resource_errors: u32 = 0; - let mut consecutive_unknown_errors: u32 = 0; + let (listener, config, _ca_paths) = + match ssh_server_init(&listen_path, &ca_file_paths, shared_socket) { + Ok(v) => { + // Signal that the SSH server has bound the socket and is ready to + // accept connections. The parent task awaits this before spawning + // the entrypoint process, ensuring exec requests won't race + // against server startup. + let _ = ready_tx.send(Ok(())); + v + } + Err(err) => { + let _ = ready_tx.send(Err(err)); + return Ok(()); + } + }; + let mut consecutive_resource_errors = 0; + let mut consecutive_unknown_errors = 0; loop { match listener.accept().await { Ok((stream, _peer)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; let config = config.clone(); - let policy = policy.clone(); - let workspace = workspace.clone(); - let proxy_url = proxy_url.clone(); - let ca_paths = ca_paths.clone(); - let provider_credentials = provider_credentials.clone(); - let user_environment = user_environment.clone(); - let main_session = Arc::clone(&main_session); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); + let main_session = main_session.clone(); tokio::spawn(async move { - if let Err(err) = handle_connection( - stream, - config, - policy, - workspace, - netns_fd, - proxy_url, - ca_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ) - .await + if let Err(err) = + handle_connection(stream, config, port_forward, boundary_exec, main_session) + .await { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -196,45 +170,31 @@ pub async fn run_ssh_server( } }); } - Err(err) => { - match classify_ssh_accept_error( - &err, - &mut consecutive_resource_errors, - &mut consecutive_unknown_errors, - ) { - SshAcceptAction::Terminal => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "SSH accept loop exiting on terminal error: {err}" - )) - .build() - ); - break; - } - SshAcceptAction::Retry { backoff, severity } => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "SSH accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build() - ); - tokio::time::sleep(backoff).await; - } + Err(error) => match classify_ssh_accept_error( + &error, + &mut consecutive_resource_errors, + &mut consecutive_unknown_errors, + ) { + SshAcceptAction::Terminal => { + return Err(error).into_diagnostic(); } - } + SshAcceptAction::Retry { backoff, severity } => { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(severity) + .status(StatusId::Failure) + .message(format!( + "SSH accept error (retrying in {}ms): {error}", + backoff.as_millis() + )) + .build() + ); + tokio::time::sleep(backoff).await; + } + }, } } - - Ok(()) } const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; @@ -249,13 +209,13 @@ enum SshAcceptAction { } fn classify_ssh_accept_error( - err: &std::io::Error, + error: &std::io::Error, consecutive_resource_errors: &mut u32, consecutive_unknown_errors: &mut u32, ) -> SshAcceptAction { #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) ) { return SshAcceptAction::Terminal; @@ -263,7 +223,7 @@ fn classify_ssh_accept_error( #[cfg(unix)] if matches!( - err.raw_os_error(), + error.raw_os_error(), Some( libc::EMFILE | libc::ENFILE @@ -286,26 +246,20 @@ fn classify_ssh_accept_error( ) ) { *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), + let resource_pressure = matches!( + error.raw_os_error(), Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { + if resource_pressure { *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) + let backoff_ms = 100_u64 + .saturating_mul(1_u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) .min(5_000); return SshAcceptAction::Retry { backoff: Duration::from_millis(backoff_ms), severity: SeverityId::Medium, }; } - *consecutive_resource_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), @@ -313,24 +267,25 @@ fn classify_ssh_accept_error( }; } - #[cfg(unix)] #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; + if error.raw_os_error() == Some(libc::ENONET) { *consecutive_resource_errors = 0; + *consecutive_unknown_errors = 0; return SshAcceptAction::Retry { backoff: Duration::from_millis(100), severity: SeverityId::Low, }; } + *consecutive_resource_errors = 0; *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { - return SshAcceptAction::Terminal; - } - SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, + SshAcceptAction::Terminal + } else { + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } } } @@ -338,16 +293,9 @@ fn classify_ssh_accept_error( async fn handle_connection( stream: tokio::net::UnixStream, config: Arc, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -363,18 +311,7 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new( - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ); + let handler = SshHandler::new(port_forward, boundary_exec, main_session); russh::server::run_stream(config, stream, handler) .await .map_err(|err| miette::miette!("ssh stream error: {err}"))?; @@ -387,13 +324,12 @@ async fn handle_connection( /// sender. This allows `window_change_request` to resize the correct PTY when /// multiple channels are open simultaneously (e.g. parallel shells, shell + /// sftp, etc.). -// Several independent per-channel boolean flags (login-shell opt-out and the -// main-attachment state bits) legitimately live side by side here. #[allow(clippy::struct_excessive_bools)] #[derive(Default)] struct ChannelState { input_sender: Option, - pty_master: Option, + process: Option>, + terminal: Option>, pty_request: Option, no_login_shell: bool, main_input_owner: Option, @@ -403,37 +339,6 @@ struct ChannelState { main_output_task: Option, } -const MAIN_DETACH_PREFIX: u8 = 0x10; // Ctrl-P -const MAIN_DETACH_KEY: u8 = 0x11; // Ctrl-Q - -/// Remove the `OpenShell` detach sequence from canonical-main input. -/// -/// A trailing Ctrl-P remains pending across SSH data frames. If the following -/// byte is not Ctrl-Q, both bytes are forwarded unchanged. Bytes after a -/// completed detach sequence are discarded because the attachment is closing. -fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { - let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); - - for &byte in data { - if *prefix_pending { - if byte == MAIN_DETACH_KEY { - *prefix_pending = false; - return (forward, true); - } - forward.push(MAIN_DETACH_PREFIX); - *prefix_pending = false; - } - - if byte == MAIN_DETACH_PREFIX { - *prefix_pending = true; - } else { - forward.push(byte); - } - } - - (forward, false) -} - enum InputSender { Process(mpsc::Sender>), Main(tokio::sync::mpsc::Sender>), @@ -454,28 +359,26 @@ impl InputSender { } struct SshHandler { - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + /// Loopback port-forward, injected by the orchestrator (RFC 0012). In-pod + /// this connects from inside the workload netns; a delegated backend + /// tunnels into its guest. The handler does not know which. + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, channels: HashMap, } impl Drop for SshHandler { fn drop(&mut self) { + let Some(main_session) = self.main_session.as_ref() else { + return; + }; for state in self.channels.values_mut() { if state.main_attached { - self.main_session.end_terminal_attachment(); - state.main_attached = false; + main_session.end_terminal_attachment(); } if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + main_session.release_input(owner); } if let Some(task) = state.main_output_task.take() { task.abort(); @@ -485,29 +388,14 @@ impl Drop for SshHandler { } impl SshHandler { - #[allow(clippy::too_many_arguments)] fn new( - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, + main_session: Option>, ) -> Self { Self { - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, + port_forward, + boundary_exec, main_session, channels: HashMap::new(), } @@ -550,15 +438,23 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, _session: &mut Session, ) -> Result<(), Self::Error> { - if let Some(state) = self.channels.remove(&channel) { - if state.main_attached { - self.main_session.end_terminal_attachment(); - } - if let Some(owner) = state.main_input_owner { - self.main_session.release_input(owner); + if let Some(mut state) = self.channels.remove(&channel) { + if state.main_attached + && let Some(main_session) = self.main_session.as_ref() + { + main_session.end_terminal_attachment(); + if let Some(owner) = state.main_input_owner.take() { + main_session.release_input(owner); + } + if let Some(task) = state.main_output_task.take() { + task.abort(); + } + return Ok(()); } - if let Some(task) = state.main_output_task { - task.abort(); + if let Some(process) = state.process { + // Channel ownership defines the exec lifetime. Closing an SSH + // channel must not strand an in-boundary process. + let _ = process.terminate().await; } } Ok(()) @@ -574,12 +470,6 @@ impl russh::server::Handler for SshHandler { reply: ChannelOpenHandle, _session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - reply - .reject(ChannelOpenFailure::AdministrativelyProhibited) - .await; - return Ok(()); - } // Validate port range before truncating u32 -> u16. The SSH protocol // uses u32 for ports, but valid TCP ports are 0-65535. Without this // check, port 65537 truncates to port 1 (privileged). @@ -599,9 +489,8 @@ impl russh::server::Handler for SshHandler { return Ok(()); } - // Only allow forwarding to loopback destinations to prevent the - // sandbox SSH server from being used as a generic proxy. - if !is_loopback_host(host_to_connect) { + let target = direct_tcpip_target(host_to_connect, port_to_connect); + if target.is_none() { ocsf_emit!(SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) .action(ActionId::Denied) @@ -620,16 +509,13 @@ impl russh::server::Handler for SshHandler { let host = host_to_connect.to_string(); // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. - let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - let netns_fd = self.netns_fd; - - // Confirm the channel before spawning: the task below writes to it, and - // the peer must see the open-confirmation first. + let port = u16::try_from(port_to_connect).expect("port range checked above"); + let target = target.expect("loopback target checked above"); + let port_forward = self.port_forward.clone(); reply.accept().await; tokio::spawn(async move { - let addr = format!("{host}:{port}"); - let tcp = match connect_in_netns(&addr, netns_fd).await { + let mut tcp_stream = match port_forward.connect(target).await { Ok(stream) => stream, Err(err) => { ocsf_emit!( @@ -637,7 +523,9 @@ impl russh::server::Handler for SshHandler { .activity(ActivityId::Fail) .severity(SeverityId::Low) .status(StatusId::Failure) - .message(format!("direct-tcpip: failed to connect to {addr}: {err}")) + .message(format!( + "direct-tcpip: failed to connect to {host}:{port}: {err}" + )) .build() ); let _ = channel.close().await; @@ -646,7 +534,6 @@ impl russh::server::Handler for SshHandler { }; let mut channel_stream = channel.into_stream(); - let mut tcp_stream = tcp; let _ = tokio::io::copy_bidirectional(&mut channel_stream, &mut tcp_stream).await; }); @@ -694,18 +581,17 @@ impl russh::server::Handler for SshHandler { return Ok(()); }; if state.main_attached { - self.main_session - .resize(col_width, row_height, pixel_width, pixel_height); - } else if let Some(master) = state.pty_master.as_ref() { - let winsize = Winsize { - ws_row: to_u16(row_height.max(1)), - ws_col: to_u16(col_width.max(1)), - ws_xpixel: to_u16(pixel_width), - ws_ypixel: to_u16(pixel_height), - }; - if let Err(e) = unsafe_pty::set_winsize(master.as_raw_fd(), winsize) { - warn!("failed to resize PTY for channel {channel:?}: {e}"); + if let Some(main_session) = self.main_session.as_ref() { + main_session + .resize(col_width, row_height, pixel_width, pixel_height) + .await; } + } else if let Some(terminal) = state.terminal.as_ref() + && let Err(e) = terminal + .resize(to_u16(col_width.max(1)), to_u16(row_height.max(1))) + .await + { + warn!("failed to resize PTY for channel {channel:?}: {e}"); } Ok(()) } @@ -715,10 +601,6 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; // Only allocate a PTY when the client explicitly requested one via // pty_request. VS Code Remote-SSH sends shell_request *without* a @@ -726,7 +608,7 @@ impl russh::server::Handler for SshHandler { // endings. Forcing a PTY here caused CRLF translation which made // VS Code misdetect the platform as Windows (and then try to run // `powershell`). - self.start_shell(channel, session.handle(), None)?; + self.start_shell(channel, session.handle(), None).await?; Ok(()) } @@ -736,16 +618,13 @@ impl russh::server::Handler for SshHandler { data: &[u8], session: &mut Session, ) -> Result<(), Self::Error> { - if self.main_session.finished() { - session.channel_failure(channel)?; - return Ok(()); - } session.channel_success(channel)?; let command = String::from_utf8_lossy(data).trim().to_string(); if command.is_empty() { return Ok(()); } - self.start_shell(channel, session.handle(), Some(command))?; + self.start_shell(channel, session.handle(), Some(command)) + .await?; Ok(()) } @@ -756,12 +635,11 @@ impl russh::server::Handler for SshHandler { session: &mut Session, ) -> Result<(), Self::Error> { if name == "openshell-main" { - if !self.channels.contains_key(&channel) { - return Err(anyhow::anyhow!( - "subsystem_request on unknown channel {channel:?}" - )); - } - if self.main_session.begin_terminal_attachment().is_err() { + let Some(main_session) = self.main_session.clone() else { + session.channel_failure(channel)?; + return Ok(()); + }; + if !begin_main_attachment(&main_session, self.channels.contains_key(&channel)) { session.channel_failure(channel)?; return Ok(()); } @@ -771,34 +649,33 @@ impl russh::server::Handler for SshHandler { .expect("main channel existence checked above"); state.main_attached = true; if let Some(pty) = state.pty_request.take() { - self.main_session.resize( - pty.col_width, - pty.row_height, - pty.pixel_width, - pty.pixel_height, - ); + main_session + .resize( + pty.col_width, + pty.row_height, + pty.pixel_width, + pty.pixel_height, + ) + .await; } - let (input, input_warning) = if state.main_read_only { + let (input, warning) = if state.main_read_only { (None, None) } else { - match self.main_session.acquire_input() { + match main_session.acquire_input() { Ok((owner, input)) => { state.main_input_owner = Some(owner); (Some(InputSender::Main(input)), None) } - Err(error) => { - warn!(%error, "main process input lease unavailable; attaching read-only"); - (None, Some(error)) - } + Err(error) => (None, Some(error)), } }; - state.main_detach_prefix_pending = false; state.input_sender = input; - let mut output = self.main_session.subscribe(); - let terminal_delivery = Arc::clone(&self.main_session); + state.main_detach_prefix_pending = false; + let mut output = main_session.subscribe(); + let terminal_delivery = main_session.clone(); let handle = session.handle(); session.channel_success(channel)?; - if let Some(error) = input_warning { + if let Some(error) = warning { let _ = handle .extended_data( channel, @@ -810,13 +687,13 @@ impl russh::server::Handler for SshHandler { let output_task = tokio::spawn(async move { loop { match output.recv().await { + Ok(MainOutput::Exit(code)) => { + terminal_delivery.wait_for_terminal_reported().await; + let _ = + send_main_output(&handle, channel, MainOutput::Exit(code)).await; + break; + } Ok(event) => { - if let MainOutput::Exit(code) = event { - terminal_delivery.wait_for_terminal_reported().await; - let _ = send_main_output(&handle, channel, MainOutput::Exit(code)) - .await; - break; - } let _ = send_main_output(&handle, channel, event).await; } Err(error) => { @@ -840,31 +717,24 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { state.main_output_task = Some(output_task.abort_handle()); } - } else if name == "sftp" && !self.main_session.finished() { + } else if name == "sftp" { session.channel_success(channel)?; // sftp-server speaks the SFTP binary protocol over stdin/stdout, - // which is exactly what spawn_pipe_exec wires up. This enables + // which the boundary executor preserves as separate pipes. This enables // modern scp (SFTP-based, OpenSSH 9.0+) and SFTP clients to // transfer files into and out of the sandbox. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - Some("/usr/lib/openssh/sftp-server".to_string()), - false, - session.handle(), + self.start_exec_spec( channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - state.input_sender = Some(InputSender::Process(input_sender)); + session.handle(), + openshell_isolation_interface::contract::ExecSpec { + program: "/usr/lib/openssh/sftp-server".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }, + ) + .await?; } else { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -889,11 +759,9 @@ impl russh::server::Handler for SshHandler { ) -> Result<(), Self::Error> { // Accept the env request so the client knows we handled it, but we // don't actually propagate arbitrary variables — the sandbox - // environment is controlled via policy. We must reply so VSCode - // doesn't stall. Two exceptions carry supervisor signals the SSH - // protocol has no native field for: - // - OPENSHELL_NO_LOGIN_SHELL: gateway login-shell opt-out. - // - OPENSHELL_MAIN_READ_ONLY: read-only main attachment. + // environment is controlled via policy. The login-shell opt-out is a + // supervisor signal carried over SSH because the protocol has no + // native field for it. if variable_name == NO_LOGIN_SHELL_ENV.0 && let Some(state) = self.channels.get_mut(&channel) { @@ -919,38 +787,17 @@ impl russh::server::Handler for SshHandler { warn!("data on unknown channel {channel:?}"); return Ok(()); }; - - let main_attached = state.main_attached; - let (forward, detach) = if main_attached { + let (forward, detach) = if state.main_attached { filter_main_detach_sequence(&mut state.main_detach_prefix_pending, data) } else { (data.to_vec(), false) }; - let send_error = (!forward.is_empty()) + let error = (!forward.is_empty()) .then(|| state.input_sender.as_ref()?.send(forward).err()) .flatten(); - - if let Some(error) = send_error { - let handle = session.handle(); - if main_attached { - self.close_main_attachment(channel, handle, Some(error)) - .await; - } else { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - } - return Ok(()); - } - if detach { - self.close_main_attachment(channel, session.handle(), None) + if state.main_attached && (detach || error.is_some()) { + self.close_main_attachment(channel, session.handle(), error) .await; - return Ok(()); } Ok(()) } @@ -967,8 +814,12 @@ impl russh::server::Handler for SshHandler { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached && let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() { - self.main_session.release_input(owner); + // A canonical process outlives one SSH attachment. Release + // this channel's lease without closing process stdin so a + // replacement attachment can become the input owner. + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -984,47 +835,192 @@ impl russh::server::Handler for SshHandler { signal: Sig, _session: &mut Session, ) -> Result<(), Self::Error> { - if !self + if self .channels .get(&channel) .is_some_and(|state| state.main_attached) { + let signal = match signal { + Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), + Sig::INT => Some(nix::sys::signal::Signal::SIGINT), + Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), + Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), + Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + _ => None, + }; + if let (Some(signal), Some(main_session)) = (signal, self.main_session.as_ref()) + && let Err(error) = main_session.signal_group(signal).await + { + warn!(%error, ?signal, "failed to signal canonical main process group"); + } return Ok(()); } + let Some(process) = self + .channels + .get(&channel) + .and_then(|state| state.process.clone()) + else { + return Ok(()); + }; let signal = match signal { - Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), - Sig::INT => Some(nix::sys::signal::Signal::SIGINT), - Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), - Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), - Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), + Sig::HUP => Some(openshell_isolation_interface::contract::BoundarySignal::Hup), + Sig::INT => Some(openshell_isolation_interface::contract::BoundarySignal::Int), + Sig::KILL => Some(openshell_isolation_interface::contract::BoundarySignal::Kill), + Sig::TERM => Some(openshell_isolation_interface::contract::BoundarySignal::Term), _ => None, }; if let Some(signal) = signal - && let Err(error) = self.main_session.signal_group(signal) + && let Err(error) = process.signal(signal).await { - warn!(%error, ?signal, "failed to signal canonical main process group"); + warn!(%error, ?signal, "failed to signal boundary exec process"); } Ok(()) } } -async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { - match event { - MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), - MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), - MainOutput::Exit(code) => { - let eof_sent = handle.eof(channel).await.is_ok(); - let status_sent = handle - .exit_status_request(channel, code.max(0).unsigned_abs()) +impl SshHandler { + async fn start_shell( + &mut self, + channel: ChannelId, + handle: Handle, + command: Option, + ) -> anyhow::Result<()> { + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; + let no_login_shell = state.no_login_shell; + let pty = state.pty_request.take(); + let pty_requested = pty.is_some(); + let (program, args) = command.map_or_else( + || { + if pty_requested { + ("/bin/bash".to_string(), vec!["-i".to_string()]) + } else { + ("/bin/bash".to_string(), vec![]) + } + }, + |command| { + ( + "/bin/bash".to_string(), + vec![login_shell_flag(no_login_shell).to_string(), command], + ) + }, + ); + let env = pty + .as_ref() + .map(|request| vec![("TERM".to_string(), request.term.clone())]) + .unwrap_or_default(); + self.start_exec_spec( + channel, + handle, + openshell_isolation_interface::contract::ExecSpec { + program, + args, + env, + workdir: None, + pty: pty_requested, + }, + ) + .await?; + if let (Some(pty), Some(terminal)) = ( + pty, + self.channels + .get(&channel) + .and_then(|state| state.terminal.as_ref()), + ) { + terminal + .resize(to_u16(pty.col_width.max(1)), to_u16(pty.row_height.max(1))) .await - .is_ok(); - let close_sent = handle.close(channel).await.is_ok(); - eof_sent && status_sent && close_sent + .map_err(|error| anyhow::anyhow!(error.to_string()))?; } + Ok(()) + } + + async fn start_exec_spec( + &mut self, + channel: ChannelId, + handle: Handle, + spec: openshell_isolation_interface::contract::ExecSpec, + ) -> anyhow::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut exec = self + .boundary_exec + .exec(spec) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; + let state = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("exec on unknown channel {channel:?}"))?; + state.process = Some(exec.process.clone()); + state.terminal = exec.terminal.take(); + + if let Some(mut stdin) = exec.stdin.take() { + let (sender, receiver) = mpsc::channel::>(); + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + while let Ok(bytes) = receiver.recv() { + if runtime.block_on(stdin.write_all(&bytes)).is_err() { + break; + } + } + }); + state.input_sender = Some(InputSender::Process(sender)); + } + + let mut stdout = exec.stdout; + let stdout_handle = handle.clone(); + let stdout_task = tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stdout_handle.data(channel, buffer[..size].to_vec()).await; + } + } + } + }); + let stderr_task = exec.stderr.map(|mut stderr| { + let stderr_handle = handle.clone(); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stderr_handle + .extended_data(channel, 1, buffer[..size].to_vec()) + .await; + } + } + } + }) + }); + tokio::spawn(async move { + let status = exec.process.wait().await; + let _ = stdout_task.await; + if let Some(task) = stderr_task { + let _ = task.await; + } + let code = match status { + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code)) => { + code.max(0).cast_unsigned() + } + Ok(openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + )) => (128_i32.saturating_add(signal)).max(0).cast_unsigned(), + Err(_) => 1, + }; + let _ = handle.eof(channel).await; + let _ = handle.exit_status_request(channel, code).await; + let _ = handle.close(channel).await; + }); + Ok(()) } -} -impl SshHandler { async fn close_main_attachment( &mut self, channel: ChannelId, @@ -1033,11 +1029,15 @@ impl SshHandler { ) { if let Some(state) = self.channels.get_mut(&channel) { if state.main_attached { - self.main_session.end_terminal_attachment(); + if let Some(main_session) = self.main_session.as_ref() { + main_session.end_terminal_attachment(); + } state.main_attached = false; } - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); + if let Some(owner) = state.main_input_owner.take() + && let Some(main_session) = self.main_session.as_ref() + { + main_session.release_input(owner); } state.input_sender.take(); state.main_detach_prefix_pending = false; @@ -1058,118 +1058,33 @@ impl SshHandler { let _ = handle.exit_status_request(channel, 0).await; let _ = handle.close(channel).await; } +} - fn start_shell( - &mut self, - channel: ChannelId, - handle: Handle, - command: Option, - ) -> anyhow::Result<()> { - let provider_env = self.provider_credentials.child_env_with_gcp_resolved(); - let state = self - .channels - .get_mut(&channel) - .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; - let no_login_shell = state.no_login_shell; - if let Some(pty) = state.pty_request.take() { - // PTY was requested — allocate a real PTY (interactive shell or - // exec that explicitly asked for a terminal). - let (pty_master, input_sender) = spawn_pty_shell( - &self.policy, - &self.workspace, - command, - no_login_shell, - &pty, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.pty_master = Some(pty_master); - state.input_sender = Some(InputSender::Process(input_sender)); - } else { - // No PTY requested — use plain pipes so stdout/stderr are - // separate and output has clean LF line endings. This is the - // path VSCode Remote-SSH exec commands take. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - command, - no_login_shell, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.input_sender = Some(InputSender::Process(input_sender)); +fn begin_main_attachment(main_session: &MainSession, channel_exists: bool) -> bool { + channel_exists && main_session.begin_terminal_attachment().is_ok() +} + +async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) -> bool { + match event { + MainOutput::Stdout(data) => handle.data(channel, data).await.is_ok(), + MainOutput::Stderr(data) => handle.extended_data(channel, 1, data).await.is_ok(), + MainOutput::Exit(code) => { + let eof = handle.eof(channel).await.is_ok(); + let status = handle + .exit_status_request(channel, code.max(0).unsigned_abs()) + .await + .is_ok(); + let close = handle.close(channel).await.is_ok(); + eof && status && close } - Ok(()) } } -/// Connect a TCP stream to `addr` inside the sandbox network namespace. -/// -/// The SSH supervisor runs in the host network namespace while sandbox child -/// processes run in an isolated network namespace (with their own loopback). -/// A plain `TcpStream::connect("127.0.0.1:port")` from the supervisor would -/// hit the host loopback, not the sandbox loopback where services are listening. -/// -/// On Linux, we spawn a dedicated OS thread, call `setns` to enter the sandbox -/// namespace, create the socket there, then convert it to a tokio `TcpStream`. -/// We use `std::thread::spawn` (not `spawn_blocking`) because `setns` changes -/// the calling thread's network namespace permanently — a tokio blocking-pool -/// thread could be reused for unrelated tasks and must not be contaminated. -/// On non-Linux platforms (no network namespace support), we connect directly. -pub async fn connect_in_netns( - addr: &str, - netns_fd: Option, -) -> std::io::Result { - #[cfg(target_os = "linux")] - if let Some(fd) = netns_fd { - let addr = addr.to_string(); - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - // Enter the sandbox network namespace on this dedicated thread. - // SAFETY: setns is safe to call; this is a dedicated thread that - // will exit after the connection is established. - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect(&addr) - })(); - let _ = tx.send(result); - }); - - let std_stream = rx - .await - .map_err(|_| std::io::Error::other("netns connect thread panicked"))??; - std_stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(std_stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - #[cfg(not(target_os = "linux"))] - let _ = netns_fd; - - let stream = tokio::net::TcpStream::connect(addr).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) +const fn login_shell_flag(no_login_shell: bool) -> &'static str { + if no_login_shell { "-c" } else { "-lc" } } +#[allow(dead_code)] #[derive(Clone)] struct PtyRequest { term: String, @@ -1191,596 +1106,6 @@ impl Default for PtyRequest { } } -#[allow(clippy::too_many_arguments)] -pub(crate) fn apply_child_env( - cmd: &mut Command, - session_home: &str, - session_user: &str, - term: &str, - proxy_url: Option<&str>, - ca_file_paths: Option<&(PathBuf, PathBuf)>, - provider_env: &HashMap, - user_environment: &HashMap, -) { - let path = std::env::var("PATH").unwrap_or_else(|_| "/usr/local/bin:/usr/bin:/bin".into()); - - cmd.env_clear() - .env(openshell_core::sandbox_env::SANDBOX, "1") - .env("HOME", session_home) - .env("USER", session_user) - .env("SHELL", openshell_core::shell::detect_login_shell()) - .env("PATH", &path) - .env("TERM", term); - - for (key, value) in user_environment { - if !key.starts_with("OPENSHELL_") { - cmd.env(key, value); - } - } - - if let Some(url) = proxy_url { - for (key, value) in child_env::proxy_env_vars(url) { - cmd.env(key, value); - } - } - - if let Some((ca_cert_path, combined_bundle_path)) = ca_file_paths { - for (key, value) in child_env::tls_env_vars(ca_cert_path, combined_bundle_path) { - cmd.env(key, value); - } - } - - for (key, value) in provider_env { - if is_supervisor_only_env_var(key) { - continue; - } - cmd.env(key, value); - } -} - -const fn login_shell_flag(no_login_shell: bool) -> &'static str { - if no_login_shell { "-c" } else { "-lc" } -} - -/// Build the shell command for an SSH session using a shell that exists in the -/// sandbox image (minimal images such as Alpine ship only `/bin/sh`, not bash). -/// -/// `no_command_arg` is appended only when no explicit command is given: `-i` -/// for an interactive PTY session, or `None` for the non-PTY stdin path (a -/// bare shell already reads piped stdin line-by-line). With an explicit -/// command the login-shell flag is used per `no_login_shell`. -fn build_ssh_shell_command( - shell: &str, - command: Option, - no_login_shell: bool, - no_command_arg: Option<&str>, -) -> Command { - let mut cmd = Command::new(shell); - match command { - None => { - if let Some(arg) = no_command_arg { - cmd.arg(arg); - } - } - Some(command) => { - cmd.arg(login_shell_flag(no_login_shell)).arg(command); - } - } - cmd -} - -#[allow(clippy::too_many_arguments)] -fn spawn_pty_shell( - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - command: Option, - no_login_shell: bool, - pty: &PtyRequest, - handle: Handle, - channel: ChannelId, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_env: &HashMap, - user_environment: &HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, -) -> anyhow::Result<(std::fs::File, mpsc::Sender>)> { - let winsize = Winsize { - ws_row: to_u16(pty.row_height.max(1)), - ws_col: to_u16(pty.col_width.max(1)), - ws_xpixel: to_u16(pty.pixel_width), - ws_ypixel: to_u16(pty.pixel_height), - }; - let openpty = openpty(Some(&winsize), None)?; - let master = std::fs::File::from(openpty.master); - let slave = std::fs::File::from(openpty.slave); - let slave_fd = slave.as_raw_fd(); - - let stdin = slave.try_clone()?; - let stdout = slave.try_clone()?; - let stderr = slave; - let mut reader = master.try_clone()?; - let mut writer = master.try_clone()?; - - // Resolve a shell present in the sandbox image; interactive PTY sessions - // pass `-i` when no command is given. Runs in the supervisor, so it - // inspects the sandbox filesystem. - let shell = openshell_core::shell::detect_login_shell(); - let mut cmd = build_ssh_shell_command(&shell, command, no_login_shell, Some("-i")); - - let term = if pty.term.is_empty() { - "xterm-256color" - } else { - pty.term.as_str() - }; - - // Derive USER and HOME from the policy's run_as_user when available, - // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - apply_child_env( - &mut cmd, - &session_home, - &session_user, - term, - proxy_url.as_deref(), - ca_file_paths.as_deref(), - provider_env, - user_environment, - ); - cmd.stdin(stdin).stdout(stdout).stderr(stderr); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - // Probe Landlock availability from the parent process where tracing works. - #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } - - // Phase 1: Prepare Landlock ruleset before the child applies it. - #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; - - #[cfg(unix)] - { - unsafe_pty::install_pre_exec( - &mut cmd, - policy.clone(), - workspace.owned_root(), - slave_fd, - netns_fd, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared_sandbox, - ); - } - - #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd.spawn()?; - #[cfg(target_os = "linux")] - let child_pid = child.id(); - #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); - let master_file = master; - - let (sender, receiver) = mpsc::channel::>(); - std::thread::spawn(move || { - while let Ok(bytes) = receiver.recv() { - if writer.write_all(&bytes).is_err() { - break; - } - let _ = writer.flush(); - } - }); - - let runtime = tokio::runtime::Handle::current(); - let runtime_reader = runtime.clone(); - let handle_clone = handle.clone(); - // Signal from the reader thread to the exit thread that all output has - // been forwarded. The exit thread waits for this before sending the - // exit-status and closing the channel, ensuring the correct SSH protocol - // ordering: data → EOF → exit-status → close. - let (reader_done_tx, reader_done_rx) = mpsc::channel::<()>(); - std::thread::spawn(move || { - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let handle_clone = handle_clone.clone(); - let _ = runtime_reader - .block_on(async move { handle_clone.data(channel, data).await }); - } - } - } - // Send EOF to indicate no more data will be sent on this channel. - let eof_handle = handle_clone.clone(); - let _ = runtime_reader.block_on(async move { eof_handle.eof(channel).await }); - // Notify the exit thread that all output has been forwarded. - let _ = reader_done_tx.send(()); - }); - - let handle_exit = handle; - let runtime_exit = runtime; - std::thread::spawn(move || { - let status = child.wait().ok(); - #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); - } - let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); - // Wait for the reader thread to finish forwarding all output before - // sending exit-status and closing the channel. This prevents the - // race where close() was called before exit_status_request(). - // - // Use a timeout because a backgrounded grandchild process (e.g. - // `nohup daemon &`) may hold the PTY slave open indefinitely, - // preventing the reader from reaching EOF. Two seconds is enough - // for any remaining buffered data to drain. - let _ = reader_done_rx.recv_timeout(Duration::from_secs(2)); - drop(runtime_exit.spawn(async move { - let _ = handle_exit.exit_status_request(channel, code).await; - let _ = handle_exit.close(channel).await; - })); - }); - - Ok((master_file, sender)) -} - -/// Spawn a command using plain pipes (no PTY). -/// -/// stdout is forwarded as SSH channel data and stderr as SSH extended data -/// (type 1), preserving the separation that clients like `VSCode` Remote-SSH -/// expect. Output retains clean LF line endings (no CRLF translation). -#[allow(clippy::too_many_arguments)] -fn spawn_pipe_exec( - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - command: Option, - no_login_shell: bool, - handle: Handle, - channel: ChannelId, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_env: &HashMap, - user_environment: &HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, -) -> anyhow::Result>> { - // Resolve a shell present in the sandbox image; minimal images (e.g. Alpine) - // don't ship bash, only `/bin/sh`. Runs in the supervisor, so it inspects - // the sandbox filesystem. No command → read from stdin with no `-i`: - // interactive mode reads .bashrc, writes prompts to stderr, and can add - // just enough latency for VS Code Remote-SSH's platform detection to time - // out and fall back to "windows". A plain shell with piped stdin already - // reads commands line-by-line (script mode), which is what VS Code expects. - let shell = openshell_core::shell::detect_login_shell(); - let mut cmd = build_ssh_shell_command(&shell, command, no_login_shell, None); - - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - apply_child_env( - &mut cmd, - &session_home, - &session_user, - "dumb", - proxy_url.as_deref(), - ca_file_paths.as_deref(), - provider_env, - user_environment, - ); - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - if let Some(dir) = workspace.root() { - cmd.current_dir(dir); - } - - // Probe Landlock availability from the parent process where tracing works. - #[cfg(target_os = "linux")] - if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); - } - - // Phase 1: Prepare Landlock ruleset before the child applies it. - #[cfg(target_os = "linux")] - let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; - - #[cfg(unix)] - { - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy.clone(), - workspace.owned_root(), - netns_fd, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared_sandbox, - ); - } - - #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd.spawn()?; - #[cfg(target_os = "linux")] - let child_pid = child.id(); - #[cfg(target_os = "linux")] - let managed_child = managed_children::register(child_pid); - - let child_stdin = child.stdin.take(); - let child_stdout = child.stdout.take().expect("stdout must be piped"); - let child_stderr = child.stderr.take().expect("stderr must be piped"); - - // stdin writer thread - let (sender, receiver) = mpsc::channel::>(); - std::thread::spawn(move || { - let Some(mut stdin) = child_stdin else { - return; - }; - while let Ok(bytes) = receiver.recv() { - if stdin.write_all(&bytes).is_err() { - break; - } - let _ = stdin.flush(); - } - }); - - let runtime = tokio::runtime::Handle::current(); - - // Signal from the reader threads to the exit thread that all output has - // been forwarded. - let (reader_done_tx, reader_done_rx) = mpsc::channel::<()>(); - - // stdout reader - let stdout_handle = handle.clone(); - let stdout_runtime = runtime.clone(); - let reader_done_stdout = reader_done_tx.clone(); - std::thread::spawn(move || { - let mut reader = child_stdout; - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let h = stdout_handle.clone(); - let _ = stdout_runtime.block_on(async move { h.data(channel, data).await }); - } - } - } - let _ = reader_done_stdout.send(()); - }); - - // stderr reader — sends as extended data (type 1) - let stderr_handle = handle.clone(); - let stderr_runtime = runtime.clone(); - std::thread::spawn(move || { - let mut reader = child_stderr; - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let data = buf[..n].to_vec(); - let h = stderr_handle.clone(); - let _ = stderr_runtime - .block_on(async move { h.extended_data(channel, 1, data).await }); - } - } - } - let _ = reader_done_tx.send(()); - }); - - // Exit waiter thread - let handle_exit = handle; - let runtime_exit = runtime; - std::thread::spawn(move || { - let status = child.wait().ok(); - #[cfg(target_os = "linux")] - if let Some(child) = managed_child { - managed_children::unregister(child); - } - let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); - // Wait for both reader threads. - let _ = reader_done_rx.recv_timeout(Duration::from_secs(2)); - let _ = reader_done_rx.recv_timeout(Duration::from_secs(1)); - drop(runtime_exit.spawn(async move { - let _ = handle_exit.eof(channel).await; - let _ = handle_exit.exit_status_request(channel, code).await; - let _ = handle_exit.close(channel).await; - })); - }); - - Ok(sender) -} - -pub(crate) mod unsafe_pty { - #[cfg(not(target_os = "linux"))] - use super::sandbox; - use super::{ - Command, ProcessEnforcementMode, RawFd, ResolvedProcessIdentity, SandboxPolicy, Winsize, - drop_privileges_with_identity, setsid, - }; - #[cfg(unix)] - use std::os::unix::process::CommandExt; - - #[allow(unsafe_code)] - pub fn set_winsize(fd: RawFd, winsize: Winsize) -> std::io::Result<()> { - let rc = unsafe { libc::ioctl(fd, libc::TIOCSWINSZ, &winsize) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - } - - /// Install a pre-exec hook that gives the child a dedicated process group. - /// - /// Boundary-owned pipe execs use the child's PID as the process-group ID - /// for signal delivery and tree cleanup. Keep this separate from - /// [`install_pre_exec_no_pty`] so legacy SSH exec behavior is unchanged. - #[allow(unsafe_code)] - pub fn install_dedicated_process_group(cmd: &mut Command) { - unsafe { - cmd.pre_exec(|| { - if libc::setpgid(0, 0) < 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - }); - } - } - - #[allow(unsafe_code)] - // `libc::TIOCSCTTY` is `u32` on macOS/BSD and `u64` on Linux; allow the - // cross-platform conversion so the same expression compiles everywhere. - #[allow(clippy::useless_conversion)] - fn set_controlling_tty(fd: RawFd) -> std::io::Result<()> { - let rc = unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) - } - - #[allow(unsafe_code)] - #[allow(clippy::too_many_arguments)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] - pub fn install_pre_exec( - cmd: &mut Command, - policy: SandboxPolicy, - _workdir: Option, - slave_fd: RawFd, - netns_fd: Option, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) { - // Wrap in Option so we can .take() it out of the FnMut closure. - // pre_exec is only called once (after fork, before exec). - #[cfg(target_os = "linux")] - let mut prepared = prepared; - unsafe { - cmd.pre_exec(move || { - setsid().map_err(|err| std::io::Error::other(err.to_string()))?; - set_controlling_tty(slave_fd)?; - - enter_netns_and_sandbox( - netns_fd, - &policy, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared.take(), - ) - }); - } - } - - /// Pre-exec hook for pipe-based (non-PTY) exec. - /// - /// Skips `setsid` and `TIOCSCTTY` since there is no controlling terminal. - #[allow(unsafe_code)] - #[cfg_attr( - not(target_os = "linux"), - allow( - clippy::unnecessary_wraps, - reason = "Linux pre_exec setup can fail while non-Linux setup cannot." - ) - )] - pub fn install_pre_exec_no_pty( - cmd: &mut Command, - policy: SandboxPolicy, - _workdir: Option, - netns_fd: Option, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) { - #[cfg(target_os = "linux")] - let mut prepared = prepared; - unsafe { - cmd.pre_exec(move || { - enter_netns_and_sandbox( - netns_fd, - &policy, - resolved_identity, - enforcement_mode, - #[cfg(target_os = "linux")] - prepared.take(), - ) - }); - } - } - - fn enter_netns_and_sandbox( - netns_fd: Option, - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - #[cfg(target_os = "linux")] prepared: Option, - ) -> std::io::Result<()> { - // Enter network namespace before dropping privileges. - // This ensures SSH shell processes are isolated to the same - // network namespace as the entrypoint, forcing all traffic - // through the veth pair and CONNECT proxy. - #[cfg(target_os = "linux")] - if let Some(fd) = netns_fd { - #[allow(unsafe_code)] - let result = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if result != 0 { - return Err(std::io::Error::last_os_error()); - } - } - - #[cfg(not(target_os = "linux"))] - let _ = netns_fd; - - // Drop privileges. initgroups/setgid/setuid need /etc/group and - // /etc/passwd which would be blocked if Landlock were already enforced. - if enforcement_mode.uses_privileged_process_setup() { - drop_privileges_with_identity(policy, resolved_identity) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - crate::process::harden_child_process() - .map_err(|err| std::io::Error::other(err.to_string()))?; - - // Phase 2: Enforce the prepared Landlock ruleset + seccomp. - // restrict_self() does not require root. - #[cfg(target_os = "linux")] - if let Some(prepared) = prepared { - crate::sandbox::linux::enforce(prepared) - .map_err(|err| std::io::Error::other(err.to_string()))?; - } - - #[cfg(not(target_os = "linux"))] - if enforcement_mode.enforces_child_sandbox() { - sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; - } - - Ok(()) - } -} - fn to_u16(value: u32) -> u16 { u16::try_from(value.min(u32::from(u16::MAX))).unwrap_or(u16::MAX) } @@ -1820,60 +1145,259 @@ fn is_loopback_host(host: &str) -> bool { } } +/// Resolve a (loopback-validated) destination host string to an `IpAddr`, +/// mapping `localhost` to `127.0.0.1`. +/// +/// Returns `None` for anything that does not parse to an IP, so +/// [`LoopbackTarget::new`] never sees a hostname. +fn loopback_ip(host: &str) -> Option { + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if host.eq_ignore_ascii_case("localhost") { + return Some(std::net::Ipv4Addr::LOCALHOST.into()); + } + host.parse().ok() +} + +fn direct_tcpip_target( + host: &str, + port: u32, +) -> Option { + if !is_loopback_host(host) { + return None; + } + let port = u16::try_from(port).ok()?; + let ip = loopback_ip(host)?; + openshell_isolation_interface::contract::LoopbackTarget::new(ip, port).ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, - unsafe_code, - reason = "Test code: doc text references identifiers and uses libc::winsize zero-init." + reason = "Test documentation references protocol and API identifiers." )] mod tests { use super::*; - use std::ffi::OsStr; - use std::process::Stdio; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + struct AcceptAnyServerKey; + + impl russh::client::Handler for AcceptAnyServerKey { + type Error = russh::Error; + + async fn check_server_key( + &mut self, + _server_public_key: &russh::keys::PublicKey, + ) -> Result { + Ok(true) + } + } + + struct TestPortForward; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryPortForward for TestPortForward { + async fn connect( + &self, + target: openshell_isolation_interface::contract::LoopbackTarget, + ) -> std::result::Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + let stream = tokio::net::TcpStream::connect((target.host(), target.port())) + .await + .map_err(|error| { + openshell_isolation_interface::contract::BackendError::Process( + error.to_string(), + ) + })?; + Ok(Box::new(stream)) + } + } + + struct RejectingExec; + + #[async_trait::async_trait] + impl openshell_isolation_interface::contract::BoundaryExec for RejectingExec { + async fn exec( + &self, + _spec: openshell_isolation_interface::contract::ExecSpec, + ) -> std::result::Result< + openshell_isolation_interface::contract::ExecSession, + openshell_isolation_interface::contract::BackendError, + > { + Err( + openshell_isolation_interface::contract::BackendError::Unsupported( + "exec is not used by direct-tcpip tests".into(), + ), + ) + } + } + + async fn authenticated_test_client() -> russh::client::Handle { + let host_key = { + let mut rng = rand::rng(); + PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") + }; + let mut server_config = russh::server::Config { + auth_rejection_time: Duration::from_millis(1), + ..Default::default() + }; + server_config.keys.push(host_key); - /// Regression test: SSH sessions run the shell they are given, never a - /// hardcoded bash, so sh-only images (e.g. Alpine) work. Covers both the - /// interactive PTY path (`-i` when no command) and the non-PTY path. + let handler = SshHandler::new( + Arc::new(TestPortForward), + Arc::new(RejectingExec), + Some(MainSession::inert()), + ); + let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); + tokio::spawn(async move { + if let Ok(session) = + russh::server::run_stream(Arc::new(server_config), server_stream, handler).await + { + let _ = session.await; + } + }); + + let mut client = russh::client::connect_stream( + Arc::new(russh::client::Config::default()), + client_stream, + AcceptAnyServerKey, + ) + .await + .expect("SSH handshake should complete over the duplex"); + let auth = client + .authenticate_none("sandbox") + .await + .expect("auth_none should not error"); + assert!(matches!(auth, russh::client::AuthResult::Success)); + client + } + + #[cfg(unix)] + #[test] + fn transient_accept_errors_retry_with_bounded_backoff() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let aborted = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert_eq!( + classify_ssh_accept_error(&aborted, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Low, + } + ); + + let exhausted = std::io::Error::from_raw_os_error(libc::EMFILE); + let first = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + let second = + classify_ssh_accept_error(&exhausted, &mut resource_errors, &mut unknown_errors); + assert_eq!( + first, + SshAcceptAction::Retry { + backoff: Duration::from_millis(100), + severity: SeverityId::Medium, + } + ); + assert_eq!( + second, + SshAcceptAction::Retry { + backoff: Duration::from_millis(200), + severity: SeverityId::Medium, + } + ); + } + + #[cfg(unix)] #[test] - fn build_ssh_shell_command_uses_given_shell() { - // PTY, no command → given shell + interactive flag. - let cmd = build_ssh_shell_command("/bin/sh", None, false, Some("-i")); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); - assert_eq!(cmd.get_args().collect::>(), vec![OsStr::new("-i")]); - - // Non-PTY, no command → bare shell, no args (reads piped stdin). - let cmd = build_ssh_shell_command("/bin/sh", None, false, None); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); - assert_eq!(cmd.get_args().count(), 0); - - // Explicit command → login-shell flag + command, still on the given shell. - let cmd = build_ssh_shell_command("/bin/sh", Some("echo hi".into()), false, Some("-i")); - assert_eq!(cmd.get_program(), OsStr::new("/bin/sh")); + fn invalid_listener_accept_error_is_terminal() { + let mut resource_errors = 0; + let mut unknown_errors = 0; + let error = std::io::Error::from_raw_os_error(libc::EBADF); assert_eq!( - cmd.get_args().collect::>(), - vec![OsStr::new("-lc"), OsStr::new("echo hi")] + classify_ssh_accept_error(&error, &mut resource_errors, &mut unknown_errors), + SshAcceptAction::Terminal ); + } - // OPENSHELL_NO_LOGIN_SHELL → plain -c. - let cmd = build_ssh_shell_command("/bin/sh", Some("echo hi".into()), true, None); + #[test] + fn direct_tcpip_target_rejects_non_loopback_and_out_of_range_ports() { + assert!(direct_tcpip_target("10.0.0.1", 80).is_none()); + assert!(direct_tcpip_target("127.0.0.1", 65_537).is_none()); + } + + #[test] + fn direct_tcpip_target_accepts_loopback_destinations() { + let target = direct_tcpip_target("localhost", 8_080).expect("loopback target"); assert_eq!( - cmd.get_args().collect::>(), - vec![OsStr::new("-c"), OsStr::new("echo hi")] + target.host(), + std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) ); + assert_eq!(target.port(), 8_080); } - /// Regression test: the direct-tcpip connect path sets `TCP_NODELAY`. #[tokio::test] - async fn connect_in_netns_sets_tcp_nodelay() { + async fn direct_tcpip_handler_rejects_invalid_destinations() { + for (host, port) in [("10.0.0.1", 80), ("127.0.0.1", 65_537)] { + let client = authenticated_test_client().await; + let error = client + .channel_open_direct_tcpip(host, port, "127.0.0.1", 0) + .await + .expect_err("invalid forwarding destination must be refused"); + assert!(matches!( + error, + russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) + )); + } + } + + #[tokio::test] + async fn direct_tcpip_handler_relays_loopback_bytes() { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); + .expect("bind loopback echo listener"); + let port = listener.local_addr().expect("listener address").port(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept forwarded stream"); + let mut payload = [0_u8; 4]; + socket.read_exact(&mut payload).await.expect("read payload"); + socket.write_all(&payload).await.expect("echo payload"); + }); - let stream = connect_in_netns(&addr.to_string(), None) + let client = authenticated_test_client().await; + let channel = client + .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) + .await + .expect("loopback forwarding must be allowed"); + let mut stream = channel.into_stream(); + stream.write_all(b"ping").await.expect("write channel"); + let mut echoed = [0_u8; 4]; + tokio::time::timeout(Duration::from_secs(5), stream.read_exact(&mut echoed)) .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); + .expect("forwarded response timeout") + .expect("read channel"); + assert_eq!(&echoed, b"ping"); + } + + #[tokio::test] + async fn main_attachment_accepts_declared_session_after_process_exit() { + let main_session = MainSession::inert(); + assert!(main_session.finish(23, true).await); + assert!(main_session.finished()); + + assert!(begin_main_attachment(&main_session, true)); + let mut output = main_session.subscribe(); + assert!(matches!( + output.recv().await.expect("retained terminal status"), + MainOutput::Exit(23) + )); + main_session.end_terminal_attachment(); } #[cfg(unix)] @@ -1890,15 +1414,14 @@ mod tests { #[cfg(unix)] #[tokio::test] - async fn ssh_server_init_full_enforcement_keeps_private_socket() { + async fn ssh_server_init_keeps_private_socket() { let temp = tempfile::tempdir().unwrap(); let parent = temp.path().join("ssh"); std::fs::create_dir_all(&parent).unwrap(); set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, false).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, false).unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o700); @@ -1914,8 +1437,7 @@ mod tests { set_file_mode(&parent, 0o775); let socket = parent.join("ssh.sock"); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, true).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); drop(listener); assert_eq!(file_mode(&parent), 0o775); @@ -1926,8 +1448,7 @@ mod tests { #[tokio::test] async fn ssh_server_abstract_socket_cannot_be_replaced_while_bound() { let socket = PathBuf::from(format!("@openshell-ssh-test-{}", uuid::Uuid::new_v4())); - let (listener, _, _) = - ssh_server_init(&socket, &None, ProcessEnforcementMode::NetworkOnly, true).unwrap(); + let (listener, _, _) = ssh_server_init(&socket, &None, true).unwrap(); assert!( !socket.exists(), @@ -1987,41 +1508,9 @@ mod tests { assert!( output.status.success(), "cat exited with {:?}", - output.status - ); - assert_eq!(output.stdout, b"hello"); - } - - /// Command execution selects a login shell by default and a non-login shell - /// under `--no-login-shell`, so user startup files are sourced only in the - /// default case. - #[cfg(unix)] - #[test] - fn login_shell_flag_controls_profile_sourcing() { - let home = tempfile::tempdir().unwrap(); - std::fs::write(home.path().join(".bash_profile"), "echo LOGIN_MARKER\n").unwrap(); - - let run = |flag: &str| -> String { - let out = Command::new("bash") - .arg(flag) - .arg("true") - .env("HOME", home.path()) - .env_remove("BASH_ENV") // isolate: -c still reads BASH_ENV if set - .output() - .expect("spawn bash"); - String::from_utf8_lossy(&out.stdout).into_owned() - }; - - assert_eq!(login_shell_flag(true), "-c"); - assert_eq!(login_shell_flag(false), "-lc"); - assert!( - run("-lc").contains("LOGIN_MARKER"), - "login shell must source .bash_profile" - ); - assert!( - !run("-c").contains("LOGIN_MARKER"), - "non-login shell must not source it" + output.status ); + assert_eq!(output.stdout, b"hello"); } /// Verify that the stdin writer delivers all buffered data before exiting @@ -2126,62 +1615,6 @@ mod tests { assert!(!is_loopback_host("[]")); } - // ----------------------------------------------------------------------- - // Per-channel PTY state tests (#543) - // ----------------------------------------------------------------------- - - #[test] - fn set_winsize_applies_to_correct_pty() { - // Verify that set_winsize applies to a specific PTY master FD, - // which is the mechanism that per-channel tracking relies on. - // With the old single-pty_master design, a window_change_request - // for channel N would resize whatever PTY was stored last — - // potentially belonging to a different channel. - let pty_a = openpty(None, None).expect("openpty a"); - let pty_b = openpty(None, None).expect("openpty b"); - let master_a = std::fs::File::from(pty_a.master); - let master_b = std::fs::File::from(pty_b.master); - let fd_a = master_a.as_raw_fd(); - let fd_b = master_b.as_raw_fd(); - assert_ne!(fd_a, fd_b, "two PTYs must have distinct FDs"); - - // Close the slave ends to avoid leaking FDs in the test. - drop(std::fs::File::from(pty_a.slave)); - drop(std::fs::File::from(pty_b.slave)); - - // Resize only PTY B. - let winsize_b = Winsize { - ws_row: 50, - ws_col: 120, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe_pty::set_winsize(fd_b, winsize_b).expect("set_winsize on PTY B"); - - // Resize PTY A to a different size. - let winsize_a = Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - unsafe_pty::set_winsize(fd_a, winsize_a).expect("set_winsize on PTY A"); - - // Read back sizes via ioctl to verify independence. - let mut actual_a: libc::winsize = unsafe { std::mem::zeroed() }; - let mut actual_b: libc::winsize = unsafe { std::mem::zeroed() }; - #[allow(unsafe_code)] - unsafe { - libc::ioctl(fd_a, libc::TIOCGWINSZ, &mut actual_a); - libc::ioctl(fd_b, libc::TIOCGWINSZ, &mut actual_b); - } - - assert_eq!(actual_a.ws_row, 24, "PTY A should be 24 rows"); - assert_eq!(actual_a.ws_col, 80, "PTY A should be 80 cols"); - assert_eq!(actual_b.ws_row, 50, "PTY B should be 50 rows"); - assert_eq!(actual_b.ws_col, 120, "PTY B should be 120 cols"); - } - #[test] fn channel_state_independent_input_senders() { // Verify that each channel gets its own input sender so that @@ -2232,604 +1665,4 @@ mod tests { .unwrap(); assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } - - #[test] - fn main_detach_filter_forwards_ctrl_c_unchanged() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x03after"); - - assert_eq!(forward, b"before\x03after"); - assert!(!detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_removes_sequence_and_trailing_input() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x10\x11after"); - - assert_eq!(forward, b"before"); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_recognizes_sequence_across_frames() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"before\x10"); - assert_eq!(forward, b"before"); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x11"); - assert!(forward.is_empty()); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_forwards_unmatched_prefix() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x10"); - assert!(forward.is_empty()); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"x"); - assert_eq!(forward, b"\x10x"); - assert!(!detach); - assert!(!prefix_pending); - } - - // ----------------------------------------------------------------------- - // session_user_and_home tests (Phase 2: numeric UID support) - // ----------------------------------------------------------------------- - - #[test] - fn session_user_and_home_returns_numeric_uid_as_user() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1000".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "1000"); - // Numeric UID has no passwd entry — defaults to /sandbox. - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_uses_driver_workspace_when_supplied() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1234".into()), - run_as_group: Some("1235".into()), - }, - }; - - let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); - assert_eq!(user, "1234"); - assert_eq!(home, "/workspace/project"); - } - - #[test] - fn session_user_and_home_returns_name_from_passwd() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("sandbox".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - // Name-based — should resolve via passwd (or /home/{user}). - assert!(!home.is_empty()); - } - - #[test] - fn session_user_and_home_defaults_to_sandbox_when_empty() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some(String::new()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_defaults_to_sandbox_when_none() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "sandbox"); - assert_eq!(home, "/sandbox"); - } - - #[test] - fn session_user_and_home_handles_large_numeric_uid() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1000660000".into()), - run_as_group: None, - }, - }; - let (user, home) = session_user_and_home(&policy, None); - assert_eq!(user, "1000660000"); - assert_eq!(home, "/sandbox"); - } - - /// `install_pre_exec_no_pty` runs drop_privileges and succeeds when the - /// current user/group is already the configured one (no actual uid change). - /// - /// This exercises the pre_exec hook end-to-end without needing root: a policy - /// with no run_as_user/group is a no-op when the process is already unprivileged. - #[cfg(unix)] - #[test] - fn pre_exec_always_calls_drop_privileges() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, - }; - - // No user/group configured and not running as root → drop_privileges is - // a no-op, so spawn succeeds regardless of the effective UID. - let policy = SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }; - - // Skip if running as root: drop_privileges would try to switch to - // "sandbox" which may not exist in the test environment. - if rustix::process::geteuid().is_root() { - return; - } - - let mut cmd = Command::new("echo"); - cmd.arg("drop-privileges-ok"); - cmd.stdout(Stdio::piped()); - - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy, - None, - None, // no netns fd - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::Full, - #[cfg(target_os = "linux")] - Some( - sandbox::linux::prepare( - &SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - }, - None, - ) - .expect("prepare should succeed in test environment"), - ), - ); - - let output = cmd - .spawn() - .expect("spawn must succeed") - .wait_with_output() - .expect("wait_with_output"); - assert!(output.status.success(), "echo should exit 0"); - assert!( - String::from_utf8_lossy(&output.stdout).contains("drop-privileges-ok"), - "echo output should contain 'drop-privileges-ok'" - ); - } - - /// SSH pre-exec uses the numeric identity resolved from OCI metadata rather - /// than looking the preserved declaration up through host NSS. - #[cfg(unix)] - #[test] - fn pre_exec_uses_resolved_oci_identity() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, - }; - - if rustix::process::geteuid().is_root() { - return; - } - - let policy = SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("__oci_user_not_in_host_nss__".into()), - run_as_group: Some("__oci_group_not_in_host_nss__".into()), - }, - }; - let resolved = ResolvedProcessIdentity::new( - Some(rustix::process::geteuid().as_raw()), - Some(rustix::process::getegid().as_raw()), - ); - - let mut cmd = Command::new("echo"); - cmd.arg("resolved-identity-ok"); - cmd.stdout(Stdio::piped()); - - unsafe_pty::install_pre_exec_no_pty( - &mut cmd, - policy, - None, - None, - resolved, - ProcessEnforcementMode::Full, - #[cfg(target_os = "linux")] - None, - ); - - let output = cmd - .spawn() - .expect("spawn should use resolved numeric identity") - .wait_with_output() - .expect("wait should succeed"); - assert!(output.status.success()); - assert_eq!( - String::from_utf8_lossy(&output.stdout).trim(), - "resolved-identity-ok" - ); - } - - // ----------------------------------------------------------------------- - // direct-tcpip authorization wiring (SEC-007) - // - // The `loopback_host_*` tests above cover the predicate in isolation. - // These drive the real `russh::server::Handler` over an in-memory duplex - // so the deny path itself is covered: channel-open authorization travels - // through a reply handle rather than the handler's return value, so a - // handler that never rejects anything still type-checks and still passes - // every predicate test. - // ----------------------------------------------------------------------- - - struct AcceptAnyServerKey; - - impl russh::client::Handler for AcceptAnyServerKey { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &russh::keys::PublicKey, - ) -> Result { - Ok(true) - } - } - - fn forwarding_test_policy() -> SandboxPolicy { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - - SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - } - } - - /// Serve `SshHandler` on one end of an in-memory duplex and return an - /// authenticated client handle for the other end. - /// - /// The handler gets `netns_fd: None` so `connect_in_netns` performs a plain - /// TCP connect, making the forwarding path reachable without a network - /// namespace. - async fn authenticated_test_client_with_main( - main_session: Arc, - ) -> russh::client::Handle { - // Scoped so the `!Send` ThreadRng is dropped before the first await. - let host_key = { - let mut rng = rand::rng(); - PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") - }; - let mut server_config = russh::server::Config { - auth_rejection_time: Duration::from_millis(1), - ..Default::default() - }; - server_config.keys.push(host_key); - - let handler = SshHandler::new( - forwarding_test_policy(), - ResolvedWorkspace::default(), - None, - None, - None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), - HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, - main_session, - ); - - let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(async move { - if let Ok(session) = - russh::server::run_stream(Arc::new(server_config), server_stream, handler).await - { - let _ = session.await; - } - }); - - let mut client = russh::client::connect_stream( - Arc::new(russh::client::Config::default()), - client_stream, - AcceptAnyServerKey, - ) - .await - .expect("SSH handshake should complete over the duplex"); - - let auth = client - .authenticate_none("sandbox") - .await - .expect("auth_none should not error"); - assert!( - matches!(auth, russh::client::AuthResult::Success), - "sandbox SSH server accepts the none auth method" - ); - - client - } - - async fn authenticated_test_client() -> russh::client::Handle { - authenticated_test_client_with_main(MainSession::inert()).await - } - - #[tokio::test] - async fn abrupt_transport_drop_releases_main_input_lease() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should acquire canonical input lease"); - - drop(channel); - drop(client); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.acquire_input().is_ok() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("handler drop should release canonical input lease"); - } - - #[tokio::test] - async fn main_attachment_closes_naturally_after_terminal_delivery() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let mut channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should register its attachment"); - - assert!(main_session.finish(7, false).await); - main_session.mark_terminal_reported(); - - let exit_status = tokio::time::timeout(Duration::from_secs(1), async { - let mut exit_status = None; - loop { - match channel.wait().await { - Some(russh::ChannelMsg::ExitStatus { - exit_status: status, - }) => { - exit_status = Some(status); - } - Some(russh::ChannelMsg::Close) => break exit_status, - None => panic!("main channel ended without a close message"), - Some(_) => {} - } - } - }) - .await - .expect("main channel should deliver its exit status"); - assert_eq!(exit_status, Some(7)); - drop(channel); - drop(client); - - tokio::time::timeout( - Duration::from_secs(1), - main_session.wait_for_terminal_attachments(), - ) - .await - .expect("peer channel close should release terminal delivery"); - } - - #[tokio::test] - async fn main_subsystem_applies_initial_pty_dimensions() { - let (main_session, _slave) = MainSession::terminal_for_test(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_pty(true, "xterm-256color", 200, 60, 1600, 900, &[]) - .await - .expect("request PTY"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.terminal_size_for_test() == (200, 60) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should apply the initial PTY dimensions"); - } - - #[tokio::test] - async fn direct_tcpip_rejects_non_loopback_destination() { - let client = authenticated_test_client().await; - - let err = client - .channel_open_direct_tcpip("10.0.0.1", 80, "127.0.0.1", 0) - .await - .expect_err("forwarding to a non-loopback host must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_rejects_port_above_tcp_range() { - let client = authenticated_test_client().await; - - // 65_537 truncates to port 1 when cast to u16, so the guard has to - // reject it before the cast rather than forward to a privileged port. - let err = client - .channel_open_direct_tcpip("127.0.0.1", 65_537, "127.0.0.1", 0) - .await - .expect_err("a port outside the TCP range must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_forwards_to_loopback_listener() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind loopback echo listener"); - let port = listener.local_addr().expect("listener address").port(); - tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - let mut buf = [0u8; 64]; - if let Ok(n) = socket.read(&mut buf).await - && n > 0 - { - let _ = socket.write_all(&buf[..n]).await; - } - } - }); - - let client = authenticated_test_client().await; - let channel = client - .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) - .await - .expect("forwarding to a loopback listener must be allowed"); - - let mut stream = channel.into_stream(); - stream.write_all(b"ping").await.expect("write to channel"); - - let mut echoed = [0u8; 4]; - tokio::time::timeout(Duration::from_secs(10), stream.read_exact(&mut echoed)) - .await - .expect("relayed response should arrive before the timeout") - .expect("read from channel"); - assert_eq!(&echoed, b"ping", "bytes round-trip through the tunnel"); - } } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 98a3c0497b..f9a57783b7 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -11,8 +11,6 @@ //! selection — it has no protocol awareness of the bytes flowing through. use std::net::IpAddr; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; @@ -23,6 +21,7 @@ use openshell_core::proto::{ RelayOpenResult, ReportMainProcessExitRequest, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, TcpRelayTarget, gateway_message, relay_open, supervisor_message, }; +use openshell_isolation_interface::contract::{BoundaryPortForward, LoopbackTarget}; use openshell_ocsf::{ ActivityId, ConnectionInfo, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, ocsf_emit, @@ -33,7 +32,6 @@ use tokio_stream::StreamExt; use tracing::{debug, warn}; use openshell_core::grpc_client; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::transport_errors::is_expected_transport_close_status; const INITIAL_BACKOFF: Duration = Duration::from_secs(1); @@ -278,31 +276,59 @@ pub fn spawn( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, ) -> tokio::task::JoinHandle<()> { + spawn_with_readiness( + endpoint, + sandbox_id, + ssh_socket_path, + port_forward, + expected_ssh_peer_pid, + terminating, + instance_id, + ) + .0 +} + +/// Spawn the supervisor session and expose when the gateway has accepted it. +pub fn spawn_with_readiness( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, + port_forward: Arc, + expected_ssh_peer_pid: Option, + terminating: Arc, + instance_id: String, +) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::watch::Receiver, +) { + let (ready_tx, ready_rx) = tokio::sync::watch::channel(false); let config = SessionConfig { endpoint, sandbox_id, ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, terminating, instance_id, + ready_tx, }; - tokio::spawn(run_session_loop(config)) + (tokio::spawn(run_session_loop(config)), ready_rx) } struct SessionConfig { endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, instance_id: String, + ready_tx: tokio::sync::watch::Sender, } async fn run_session_loop(config: SessionConfig) { @@ -392,6 +418,8 @@ async fn run_single_session( heartbeat_secs, ); ocsf_emit!(event); + config.ready_tx.send_replace(true); + // Main loop: receive gateway messages + send heartbeats. let mut heartbeat_interval = tokio::time::interval(Duration::from_secs(u64::from(heartbeat_secs))); @@ -411,7 +439,7 @@ async fn run_single_session( let context = GatewayMessageContext { sandbox_id: &config.sandbox_id, ssh_socket_path: &config.ssh_socket_path, - netns_fd: config.netns_fd, + port_forward: &config.port_forward, expected_ssh_peer_pid: config.expected_ssh_peer_pid, channel: &channel, tx: &tx, @@ -479,7 +507,7 @@ pub async fn finalize_main_process_exit( struct GatewayMessageContext<'a> { sandbox_id: &'a str, ssh_socket_path: &'a std::path::Path, - netns_fd: Option, + port_forward: &'a Arc, expected_ssh_peer_pid: Option, channel: &'a grpc_client::AuthedChannel, tx: &'a mpsc::Sender, @@ -498,7 +526,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let channel = context.channel.clone(); let ssh_socket_path = context.ssh_socket_path.to_path_buf(); let tx = context.tx.clone(); - let netns_fd = context.netns_fd; + let port_forward = context.port_forward.clone(); let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); @@ -510,7 +538,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< match handle_relay_open( relay_open, &ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, channel, tx, @@ -567,7 +595,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, @@ -577,7 +605,7 @@ async fn handle_relay_open( let target = match open_target( &relay_open, ssh_socket_path, - netns_fd, + &port_forward, expected_ssh_peer_pid, ) .await @@ -722,11 +750,11 @@ async fn send_relay_open_result( async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: &Arc, expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { - Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, + Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, port_forward).await, Some(relay_open::Target::Ssh(_)) | None => { let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; @@ -747,59 +775,26 @@ async fn open_target( async fn open_tcp_target( target: &TcpRelayTarget, - netns_fd: Option, + port_forward: &Arc, ) -> Result, Box> { let host = normalize_tcp_target_host(target)?; let port = u16::try_from(target.port).map_err(|_| "tcp target port must fit in u16")?; - let stream = connect_tcp_target(host, port, netns_fd).await?; + // `normalize_tcp_target_host` returns a loopback IP string; parse it and let + // `LoopbackTarget::new` re-validate before connecting. + let ip: IpAddr = host + .parse() + .map_err(|_| "tcp target host must be a loopback IP")?; + let target = LoopbackTarget::new(ip, port) + .map_err(|e| -> Box { e.to_string().into() })?; + // Connect through the sandbox-owned loopback-forward interface. The + // supervisor session remains independent of the driver's transport. + let stream = port_forward + .connect(target) + .await + .map_err(|e| -> Box { e.to_string().into() })?; Ok(Box::new(stream)) } -#[cfg(target_os = "linux")] -async fn connect_tcp_target( - host: String, - port: u16, - netns_fd: Option, -) -> Result> { - if let Some(fd) = netns_fd { - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect((host.as_str(), port)) - })(); - let _ = tx.send(result); - }); - - let stream = rx - .await - .map_err(|_| "netns tcp connect thread panicked")??; - stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - -#[cfg(not(target_os = "linux"))] -async fn connect_tcp_target( - host: String, - port: u16, - _netns_fd: Option, -) -> Result> { - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - #[cfg(test)] fn validate_tcp_target(target: &TcpRelayTarget) -> Result<(), String> { normalize_tcp_target_host(target).map(|_| ()) @@ -839,20 +834,6 @@ mod target_tests { } } - /// Regression test: the TCP relay connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_tcp_target_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback"); @@ -895,6 +876,21 @@ mod target_tests { mod ocsf_event_tests { use super::*; + struct UnusedPortForward; + + #[async_trait::async_trait] + impl BoundaryPortForward for UnusedPortForward { + async fn connect( + &self, + _target: LoopbackTarget, + ) -> Result< + openshell_isolation_interface::contract::BoundaryDuplexStream, + openshell_isolation_interface::contract::BackendError, + > { + unreachable!("SSH relay does not use loopback port forwarding") + } + } + fn ctx() -> SandboxContext { SandboxContext { sandbox_id: "sbx-1".into(), @@ -1135,7 +1131,11 @@ mod ocsf_event_tests { }); let relay = ssh_relay_open("peer-check"); - let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + // The SSH relay path does not use the port-forward (that is the TCP + // target path); connect from the supervisor's own namespace. + let port_forward: Arc = Arc::new(UnusedPortForward); + + let trusted = open_target(&relay, &socket, &port_forward, Some(std::process::id())) .await .expect("matching peer PID should be accepted"); drop(trusted); @@ -1143,7 +1143,7 @@ mod ocsf_event_tests { let Err(err) = open_target( &relay, &socket, - None, + &port_forward, Some(std::process::id().saturating_add(1)), ) .await diff --git a/crates/openshell-supervisor/Cargo.toml b/crates/openshell-supervisor/Cargo.toml new file mode 100644 index 0000000000..f4129e4f84 --- /dev/null +++ b/crates/openshell-supervisor/Cargo.toml @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-supervisor" +description = "OpenShell policy and workload supervisor" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-supervisor" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } +openshell-ocsf = { path = "../openshell-ocsf" } +openshell-policy = { path = "../openshell-policy" } +openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } +openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } +openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } +openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +clap = { workspace = true } +miette = { workspace = true } +nix = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } +rustls = { workspace = true } +rustix = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-subscriber = { workspace = true } +uuid = { workspace = true } + +[features] +default = ["telemetry", "bundled-ca-roots"] +system-ca-roots = ["telemetry"] +defaults-without-telemetry = ["bundled-ca-roots"] +telemetry = ["openshell-core/telemetry"] +bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] + +[dev-dependencies] +futures = { workspace = true } +temp-env = "0.3" +tempfile = "3" +tokio-tungstenite = { workspace = true } + +[lints] +workspace = true diff --git a/crates/openshell-sandbox/src/activity_aggregator.rs b/crates/openshell-supervisor/src/activity_aggregator.rs similarity index 99% rename from crates/openshell-sandbox/src/activity_aggregator.rs rename to crates/openshell-supervisor/src/activity_aggregator.rs index 8bd2ffdb62..33605c1df9 100644 --- a/crates/openshell-sandbox/src/activity_aggregator.rs +++ b/crates/openshell-supervisor/src/activity_aggregator.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Anonymous sandbox network activity counter aggregation. +//! Anonymous supervised network activity counter aggregation. //! //! Producer-side types (`ActivityEvent`, `ActivitySender`, //! `ACTIVITY_EVENT_QUEUE_CAPACITY`, `try_record_activity`) live in diff --git a/crates/openshell-sandbox/src/denial_aggregator.rs b/crates/openshell-supervisor/src/denial_aggregator.rs similarity index 98% rename from crates/openshell-sandbox/src/denial_aggregator.rs rename to crates/openshell-supervisor/src/denial_aggregator.rs index adbca79b04..d80d28db50 100644 --- a/crates/openshell-sandbox/src/denial_aggregator.rs +++ b/crates/openshell-supervisor/src/denial_aggregator.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Denial aggregator — collects and deduplicates proxy deny events. +//! Supervisor denial aggregator — collects and deduplicates proxy deny events. //! //! The proxy emits a [`DenialEvent`] each time a connection or request is //! denied. The [`DenialAggregator`] receives these events via an MPSC channel, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs new file mode 100644 index 0000000000..670fe897ce --- /dev/null +++ b/crates/openshell-supervisor/src/lib.rs @@ -0,0 +1,5493 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` supervisor library. +//! +//! This crate provides process sandboxing and monitoring capabilities. + +// `defaults-without-telemetry` is an alias for the default feature set minus +// `telemetry`, not a switch that turns telemetry off. Cargo cannot subtract a +// default feature, so adding it on top of the defaults would otherwise produce +// a telemetry-on build that reads as telemetry-free. Fail the build instead. +#[cfg(all(feature = "telemetry", feature = "defaults-without-telemetry"))] +compile_error!( + "features `telemetry` and `defaults-without-telemetry` are mutually exclusive; \ + build a telemetry-free supervisor with `--no-default-features --features defaults-without-telemetry`" +); + +mod activity_aggregator; +mod denial_aggregator; +mod mechanistic_mapper; + +use miette::{IntoDiagnostic, Result, WrapErr}; +use std::future::Future; +use std::io::Write as _; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32}; +use std::time::Duration; +use tracing::{debug, info, warn}; + +use openshell_core::PolicyValidationFailureMode; + +use openshell_ocsf::{ + ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, + DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, + StateId, StatusId, ocsf_emit, +}; + +// --------------------------------------------------------------------------- +// OCSF Context +// --------------------------------------------------------------------------- +// +// The following log sites intentionally remain as plain `tracing` macros +// and are NOT migrated to OCSF builders: +// +// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) +// - Transient "about to do X" events where the result is logged separately +// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") +// - Internal SSH channel warnings (unknown channel, PTY resize failures) +// - Denial flush telemetry (the individual denials are already OCSF events) +// - Status reporting failures (sync to gateway, non-actionable) +// - Route refresh interval validation warnings +// +// These are operational plumbing that don't represent security decisions, +// policy changes, or observable sandbox behavior worth structuring. +// --------------------------------------------------------------------------- + +/// Re-export the process-wide OCSF sandbox context getter. +/// +/// The singleton lives in `openshell-ocsf` so both supervisor leaves can +/// reach it without depending on `openshell-sandbox`. Initialised once during +/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. +pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; + +async fn retain_remote_access_plane( + proxy_exited: impl Future, + shutdown_requested: impl Future, +) -> Result<()> { + tokio::pin!(proxy_exited); + tokio::pin!(shutdown_requested); + tokio::select! { + () = &mut proxy_exited => Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )), + () = &mut shutdown_requested => Ok(()), + } +} + +async fn completion_phase_or_shutdown(phase: F, mut shutdown: Pin<&mut S>) -> bool +where + F: Future, + S: Future + ?Sized, +{ + tokio::pin!(phase); + tokio::select! { + () = &mut phase => false, + () = &mut shutdown => true, + } +} + +struct ControlReadiness { + task: tokio::task::JoinHandle<()>, + path: std::path::PathBuf, +} + +impl ControlReadiness { + fn start(path: std::path::PathBuf) -> Result { + prepare_control_readiness_path(&path)?; + let listener = tokio::net::UnixListener::bind(&path) + .into_diagnostic() + .wrap_err_with(|| format!("bind supervisor readiness socket on {}", path.display()))?; + let task = tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + }); + Ok(Self { task, path }) + } +} + +#[cfg(unix)] +fn prepare_control_readiness_path(path: &std::path::Path) -> Result<()> { + use std::os::unix::fs::{FileTypeExt as _, MetadataExt as _}; + + if !path.is_absolute() { + return Err(miette::miette!( + "supervisor readiness socket path must be absolute" + )); + } + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| format!("create readiness directory {}", parent.display()))?; + } + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + if !metadata.file_type().is_socket() + || metadata.uid() != rustix::process::getuid().as_raw() + { + return Err(miette::miette!( + "refusing unsafe existing readiness path {}", + path.display() + )); + } + std::fs::remove_file(path) + .into_diagnostic() + .wrap_err_with(|| format!("remove stale readiness socket {}", path.display()))?; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(error) + .into_diagnostic() + .wrap_err_with(|| format!("inspect readiness path {}", path.display())); + } + } + Ok(()) +} + +impl Drop for ControlReadiness { + fn drop(&mut self) { + self.task.abort(); + let _ = std::fs::remove_file(&self.path); + } +} + +/// Check whether the live supervisor owns its private readiness socket. +#[cfg(unix)] +pub fn check_control_readiness(path: &std::path::Path) -> Result<()> { + if !path.is_absolute() { + return Err(miette::miette!("health socket path must be absolute")); + } + std::os::unix::net::UnixStream::connect(path) + .into_diagnostic() + .wrap_err_with(|| format!("connect supervisor readiness socket {}", path.display()))?; + Ok(()) +} + +/// Health subcommands are unsupported on non-Unix hosts. +#[cfg(not(unix))] +pub fn check_control_readiness(_path: &std::path::Path) -> Result<()> { + Err(miette::miette!( + "supervisor readiness sockets require a Unix host" + )) +} + +#[cfg(unix)] +async fn wait_for_control_shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + + let mut sigterm = signal(SignalKind::terminate()).expect("install control SIGTERM handler"); + let mut sigint = signal(SignalKind::interrupt()).expect("install control SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => {} + _ = sigint.recv() => {} + } +} + +#[cfg(not(unix))] +async fn wait_for_control_shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + +use openshell_core::denial::DenialEvent; +use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_network::proxy::ProxyHandle; +use openshell_supervisor_process::skills; +use tokio::sync::mpsc::UnboundedSender; +use tokio::time::timeout; + +fn shared_ssh_socket_from_env() -> bool { + std::env::var(openshell_core::sandbox_env::SSH_SOCKET_SHARED) + .is_ok_and(|value| shared_ssh_socket_value(&value)) +} + +fn shared_ssh_socket_value(value: &str) -> bool { + value == "1" || value.eq_ignore_ascii_case("true") +} + +/// Run a command in the sandbox. +/// +/// # Errors +/// +/// Returns an error if the command fails to start or encounters a fatal error. +#[allow( + clippy::too_many_arguments, + clippy::implicit_hasher, + clippy::similar_names, + clippy::fn_params_excessive_bools +)] +pub async fn run_sandbox( + command: Vec, + workdir: Option, + timeout_secs: u64, + interactive: bool, + await_main_process_attachment: bool, + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + ssh_socket_path: Option, + health_socket_path: Option, + inference_routes: Option, + ocsf_enabled: Arc, + upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: openshell_isolation_interface::contract::TopologyDescriptor, + admitted_isolation_backend: Option, + main_exit_marker: Option, +) -> Result { + let (program, args) = command + .split_first() + .ok_or_else(|| miette::miette!("No command specified"))?; + + // Initialize the process-wide OCSF context early so that events emitted + // during policy loading (filesystem config, validation) have a context. + // Proxy IP/port use defaults here; the boundary mediation source carries + // workload-side connection metadata. + { + let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( + |_| "openshell-sandbox".to_string(), + |s| s.trim().to_string(), + ); + + if !openshell_ocsf::ctx::set_ctx(SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), + container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), + hostname, + product_version: openshell_core::VERSION.to_string(), + proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), + proxy_port: 3128, + }) { + debug!("OCSF context already initialized, keeping existing"); + } + } + + // Extension credentials are owned by this supervisor and shared by every + // gateway connection it opens, so the middleware registry's bearer slots + // and the policy poll loop that rotates them stay the same objects. + let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); + + // Load policy and initialize OPA engine + let openshell_endpoint_for_proxy = openshell_endpoint.clone(); + let sandbox_name_for_agg = sandbox.clone(); + let ( + policy, + opa_engine, + retained_proto, + middleware_registry_status, + loaded_policy_origin, + initial_agent_proposals_enabled, + initial_extension_authentication_enabled, + ) = load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + &extension_credentials, + ) + .await?; + + // Normalize the active driver's identity contract once, while both the + // policy and launched image filesystem are available. Kubernetes and + // OpenShift retain their authoritative numeric pair; Docker fills only + // omitted policy fields from OCI Config.User. A remote boundary resolves + // identity in its own filesystem instead; control must not interpret + // guest account data against the host's /etc/passwd and /etc/group. + let workspace = workdir; + + let provider_credentials = { + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + static_credential_bindings, + non_secret_environment_keys, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + result.static_credential_bindings, + result.non_secret_environment_keys, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Failed to fetch provider environment; no provider credentials are active: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + Vec::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + Vec::new(), + ) + }; + + let dynamic_credentials_fallback = dynamic_credentials.clone(); + match ProviderCredentialState::from_bound_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + static_credential_bindings, + non_secret_environment_keys, + ) { + Ok(credentials) => credentials, + Err(error) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Rejected provider environment bindings; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" + )) + .build() + ); + ProviderCredentialState::from_environment( + provider_env_revision, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + dynamic_credentials_fallback, + ) + } + } + }; + + if credential_gating_unavailable( + &loaded_policy_origin, + provider_credentials.resolver().is_some(), + true, + ) { + report_credential_gating_unavailable(); + } + + // Canonical-process overrides are deliberately applied only to the main + // child. Keep the provider snapshot pristine for later exec/editor/SFTP + // children launched by the sandbox. + + // Shared agent-proposals feature flag. Seed from the same initial settings + // snapshot that produced the policy so networking and process setup agree + // before the poll loop starts reconciling later changes. + let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); + + // Shared PID: set after process spawn so the proxy can look up + // the entrypoint process's /proc/net/tcp for identity binding. + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + + // A separated topology uses the shared authenticated boundary protocol. + // The admitted backend name is resolved independently of the protected + // descriptor, and generic supervisor code never imports a driver crate. + let admitted_backend_name = admitted_isolation_backend.ok_or_else(|| { + miette::miette!("protected topology supplied without an admitted isolation backend") + })?; + let topology: openshell_isolation_interface::boundary_protocol::BoundaryTopology = + serde_json::from_slice(&topology_descriptor.payload) + .map_err(|error| miette::miette!("decode boundary topology: {error}"))?; + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let backend: Arc = Arc::new( + openshell_isolation_interface::remote::RemoteIsolationBackend::new( + admitted_backend_name.clone(), + ca_file_paths.clone(), + provider_credentials.clone(), + ), + ); + let mut registry = openshell_isolation_interface::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(topology_descriptor, &admitted_backend_name) + .map_err(|error| miette::miette!(error.to_string()))?; + let context = openshell_isolation_interface::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation_interface::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workspace, + timeout_secs, + interactive, + }, + identity: topology.workload_identity, + }; + let bound = backend + .attach(verified, context) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary attached"); + let remote_boundary = (bound, admitted_backend_name, ca_file_paths); + + let transparent_tcp_capable = true; + let transparent_tcp_substrate_ready = true; + // The denial channel is owned by the orchestrator: the proxy (in the + // networking leaf) and the bypass monitor (in the process leaf) both + // produce DenialEvents that the denial aggregator (orchestrator-side) + // consumes via the matching receiver. Both leaves are pure producers; + // the orchestrator owns the consumer task spawned below. + let (denial_tx, denial_rx): (Option>, _) = if sandbox_id.is_some() + { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + + // Anonymous activity channel: same orchestrator-owned pattern as the + // denial channel. The proxy and the bypass monitor both emit per-event + // activity records; the orchestrator-side aggregator drains, sanitizes, + // and flushes anonymous summaries to the gateway. + let (activity_tx, activity_rx) = if sandbox_id.is_some() { + let (tx, rx) = + tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + + // Workspace watch: the policy poll loop learns the workspace from + // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local + // API read the current value so proposals target the correct workspace. + let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); + + let remote_network_source = remote_boundary.0.network_mediation_source(); + let remote_dns_source = remote_boundary.0.dns_mediation_source(); + let remote_host_gateway_ip = remote_boundary.0.host_gateway_ip(); + + let mut networking = Some( + openshell_supervisor_network::run::run_networking( + &policy, + None, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + // The sandbox supplies already-resolved identities across the + // boundary. The host supervisor cannot inspect its mount or PID + // namespace, so waiting for a host-visible entrypoint PID would + // unnecessarily delay DNS and network readiness. + false, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + remote_host_gateway_ip, + #[cfg(target_os = "linux")] + None, + Some(remote_network_source), + remote_dns_source, + ) + .await?, + ); + + let remote_ready = { + let (bound, backend_name, ca_file_paths) = remote_boundary; + ca_file_paths + .lock() + .map_err(|_| miette::miette!("boundary CA path lock is poisoned"))? + .clone_from( + &networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + ); + let ready = bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary enforcement confirmed"); + (ready, backend_name) + }; + + // Spawn the denial-aggregator flush task. The aggregator drains proxy + // denial events, batches them, and ships summaries to the gateway via + // `SubmitPolicyAnalysis`. + if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall + // back to the ID when the name isn't set. + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + + let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); + let denial_workspace_gate = workspace_rx.clone(); + let denial_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + |summaries| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = denial_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_proposals_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summaries, + ) + .await + { + warn!(error = %e, "Failed to flush denial summaries to gateway"); + } + } + }, + move || !denial_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn the activity-aggregator flush task. The aggregator drains + // anonymous activity events from the proxy, sanitizes deny groups, + // and ships periodic summaries to the gateway. + if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( + std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") + .ok() + .as_deref(), + ); + + let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); + let activity_workspace_gate = workspace_rx.clone(); + let activity_workspace_rx = workspace_rx.clone(); + + tokio::spawn(async move { + aggregator + .run( + move |summary| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = activity_workspace_rx.borrow().clone(); + async move { + if let Err(e) = flush_activity_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summary, + ) + .await + { + warn!(error = %e, "Failed to flush activity summary to gateway"); + } + } + }, + move || !activity_workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + // Spawn background policy poll task (gRPC mode only). + if let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) { + let poll_id = id.to_string(); + let poll_endpoint = endpoint.to_string(); + let poll_engine = engine.clone(); + let poll_ocsf_enabled = ocsf_enabled.clone(); + let poll_pid = entrypoint_pid.clone(); + let poll_provider_credentials = provider_credentials.clone(); + let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); + let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let poll_ctx = PolicyPollLoopContext { + endpoint: poll_endpoint, + sandbox_id: poll_id, + opa_engine: poll_engine, + loaded_policy_origin, + entrypoint_pid: poll_pid, + interval_secs: poll_interval_secs, + ocsf_enabled: poll_ocsf_enabled, + provider_credentials: poll_provider_credentials, + policy_local_ctx: poll_policy_local, + agent_proposals: agent_proposals.clone(), + middleware_registry_status, + workspace_tx, + extension_credentials: extension_credentials.clone(), + extension_authentication_enabled: initial_extension_authentication_enabled, + middleware_connector: default_middleware_connector(), + transparent_tcp: TransparentTcpReloadState { + capable: transparent_tcp_capable, + substrate_ready: transparent_tcp_substrate_ready, + }, + }; + + tokio::spawn(async move { + if let Err(e) = run_policy_poll_loop(poll_ctx).await { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .message(format!("Policy poll loop exited with error: {e}")) + .build() + ); + } + }); + } + + let proxy_exited: Pin + Send>> = if let Some(rx) = networking + .as_mut() + .and_then(|n| n.proxy.as_mut()) + .and_then(ProxyHandle::take_exit_receiver) + { + Box::pin(async { + let _ = rx.await; + }) + } else { + Box::pin(std::future::pending()) + }; + tokio::pin!(proxy_exited); + + let (confirmed, backend_name) = remote_ready; + let exit_code = { + let running = confirmed + .into_boundary() + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %backend_name, "Isolation boundary agent started"); + let agent = running.agent(); + let boundary_access = openshell_supervisor_process::delegated::start_boundary_access( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path.as_deref(), + shared_ssh_socket_from_env(), + networking + .as_ref() + .and_then(|runtime| runtime.ca_file_paths.clone()), + running.exec(), + running.port_forward(), + agent.clone(), + ) + .await?; + info!(backend = %backend_name, "Control-mode access plane started"); + let mut control_readiness = if let Some(path) = health_socket_path { + Some(ControlReadiness::start(path)?) + } else { + None + }; + let instance_id = boundary_access.instance_id().to_string(); + let wait_agent = agent.clone(); + let shutdown_requested = wait_for_control_shutdown_signal(); + tokio::pin!(shutdown_requested); + let wait = async move { + wait_agent + .wait() + .await + .map(|status| match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => { + code + } + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled( + signal, + ) => 128_i32.saturating_add(signal), + }) + .map_err(|error| miette::miette!(error.to_string())) + }; + let (exit_code, mut retain_access) = tokio::select! { + result = wait => (result?, true), + () = &mut proxy_exited => { + let _ = agent.terminate().await; + return Err(miette::miette!( + "control-mode proxy accept loop exited unexpectedly" + )); + } + () = &mut shutdown_requested => { + let _ = agent + .signal(openshell_isolation_interface::contract::BoundarySignal::Term) + .await; + let status = if let Ok(result) = timeout(Duration::from_secs(5), agent.wait()).await { + result + } else { + let _ = agent.terminate().await; + agent.wait().await + } + .map_err(|error| miette::miette!(error.to_string()))?; + let exit_code = match status { + openshell_isolation_interface::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation_interface::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + }; + (exit_code, false) + } + }; + if !retain_access { + control_readiness.take(); + } + boundary_access + .publish_main_exit(exit_code, await_main_process_attachment) + .await; + // `shutdown_requested` has already completed when shutdown won the + // lifecycle select above and must not be polled again. + let mut completion_cancelled = !retain_access; + if retain_access && let Some(marker) = main_exit_marker.as_deref() { + persist_main_exit_marker(marker, exit_code) + .into_diagnostic() + .wrap_err("persist canonical-process completion marker")?; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let report = openshell_supervisor_process::delegated::report_main_process_exit( + endpoint, + id, + &instance_id, + exit_code, + ); + completion_cancelled = + completion_phase_or_shutdown(report, shutdown_requested.as_mut()).await; + } + if !completion_cancelled { + let drain = boundary_access.drain_main_terminal_delivery(); + completion_cancelled = + completion_phase_or_shutdown(drain, shutdown_requested.as_mut()).await; + } + if !completion_cancelled + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_deref(), sandbox_id.as_deref()) + { + let finalize = openshell_supervisor_process::delegated::finalize_main_process_exit( + endpoint, + id, + &instance_id, + ); + completion_cancelled = + completion_phase_or_shutdown(finalize, shutdown_requested.as_mut()).await; + } + if completion_cancelled { + retain_access = false; + control_readiness.take(); + } + if retain_access { + info!(backend = %backend_name, "Canonical process exited; retaining control-mode access plane"); + retain_remote_access_plane(&mut proxy_exited, &mut shutdown_requested).await?; + } + drop(control_readiness); + drop(running); + drop(boundary_access); + exit_code + }; + + // Drop networking explicitly so proxy tasks tear down before we return. + drop(networking); + + Ok(exit_code) +} + +fn persist_main_exit_marker(path: &std::path::Path, exit_code: i32) -> std::io::Result<()> { + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no parent: {}", path.display()), + ) + })?; + let name = path.file_name().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("completion marker has no file name: {}", path.display()), + ) + })?; + let temporary = parent.join(format!( + ".{}.tmp-{}", + name.to_string_lossy(), + std::process::id() + )); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut file = options.open(&temporary)?; + writeln!(file, "exit_code={exit_code}")?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all() +} + +/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. +async fn flush_proposals_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summaries: Vec, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialSummary, L7RequestSample}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summaries: Vec = summaries + .into_iter() + .map(|s| DenialSummary { + sandbox_id: String::new(), + host: s.host, + port: u32::from(s.port), + binary: s.binary, + ancestors: s.ancestors, + deny_reason: s.deny_reason, + first_seen_ms: s.first_seen_ms, + last_seen_ms: s.last_seen_ms, + count: s.count, + suppressed_count: 0, + total_count: s.count, + sample_cmdlines: s.sample_cmdlines, + binary_sha256: String::new(), + persistent: false, + denial_stage: s.denial_stage, + l7_request_samples: s + .l7_samples + .into_iter() + .map(|l| L7RequestSample { + method: l.method, + path: l.path, + decision: "deny".to_string(), + count: l.count, + }) + .collect(), + l7_inspection_active: false, + }) + .collect(); + + // Run the mechanistic mapper sandbox-side to generate proposals. + // The gateway is a thin persistence + validation layer — it never + // generates proposals itself. + let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); + + info!( + sandbox_name = %sandbox_name, + summaries = proto_summaries.len(), + proposals = proposals.len(), + "Flushed denial analysis to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + proto_summaries, + proposals, + Vec::new(), + "mechanistic", + ) + .await?; + + Ok(()) +} + +/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. +async fn flush_activity_to_gateway( + endpoint: &str, + sandbox_name: &str, + workspace: &str, + summary: activity_aggregator::FlushableActivitySummary, +) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; + use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; + + let client = CachedOpenShellClient::connect(endpoint).await?; + client.set_workspace(workspace.to_string()); + + let proto_summary = NetworkActivitySummary { + network_activity_count: summary.network_activity_count, + denied_action_count: summary.denied_action_count, + denials_by_group: summary + .denials_by_group + .into_iter() + .map(|(group, count)| DenialGroupCount { + deny_group: group, + denied_count: count, + }) + .collect(), + }; + + info!( + sandbox_name = %sandbox_name, + network_activity_count = proto_summary.network_activity_count, + denied_action_count = proto_summary.denied_action_count, + "Flushed activity summary to gateway" + ); + + client + .submit_policy_analysis( + sandbox_name, + Vec::new(), + Vec::new(), + vec![proto_summary], + "activity", + ) + .await?; + + Ok(()) +} + +// ============================================================================ +// Baseline filesystem path enrichment +// ============================================================================ + +/// Minimum read-only paths required for a proxy-mode sandbox child process to +/// function: dynamic linker, shared libraries, DNS resolution, CA certs, +/// Python venv, openshell logs, process info, and random bytes. +/// +/// `/proc` and `/dev/urandom` are included here for the same reasons they +/// appear in `restrictive_default_policy()`: virtually every process needs +/// them. Before the Landlock per-path fix (#677) these were effectively free +/// because a missing path silently disabled the entire ruleset; now they must +/// be explicit. +const PROXY_BASELINE_READ_ONLY: &[&str] = &[ + "/usr", + "/lib", + "/etc", + "/app", + "/var/log", + "/proc", + "/dev/urandom", +]; + +/// Minimum read-write paths required for a proxy-mode sandbox child process. +/// The active workspace is granted separately through `include_workdir`. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; + +/// GPU read-only paths. +/// +/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced +/// socket at init time. If the directory exists but Landlock denies traversal +/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` +/// even though the daemon is optional. Only read/traversal access is needed. +/// +/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, +/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` +/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may +/// not be covered by the parent-directory Landlock rule when the mount crosses +/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal +/// is permitted regardless of Landlock's cross-mount behaviour. +const GPU_BASELINE_READ_ONLY: &[&str] = &[ + "/run/nvidia-persistenced", + "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory +]; + +/// GPU read-write paths (static). +/// +/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, +/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native +/// Linux. Landlock restricts `open(2)` on device files even when DAC allows +/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. +/// These devices do not exist on WSL2 and will be skipped by the existence +/// check in `enrich_proto_baseline_paths()`. +/// +/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver +/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects +/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux +/// and will be skipped there by the existence check. +/// +/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` +/// to set thread names. Without write access, `cuInit()` returns error 304. +/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to +/// inodes and child processes have different procfs inodes than the parent. +/// +/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by +/// `enumerate_gpu_device_nodes()` since the count varies. +const GPU_BASELINE_READ_WRITE: &[&str] = &[ + "/dev/nvidiactl", + "/dev/nvidia-uvm", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-modeset", + "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) + "/proc", +]; + +/// Returns true if GPU devices are present in the container. +/// +/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and +/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these +/// depending on the host kernel; the other will not exist. +fn has_gpu_devices() -> bool { + std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() +} + +/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). +fn enumerate_gpu_device_nodes() -> Vec { + let mut paths = Vec::new(); + if let Ok(entries) = std::fs::read_dir("/dev") { + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(suffix) = name.strip_prefix("nvidia") { + if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { + continue; + } + paths.push(entry.path().to_string_lossy().into_owned()); + } + } + } + paths +} + +fn push_unique(paths: &mut Vec, path: String) { + if !paths.iter().any(|p| p == &path) { + paths.push(path); + } +} + +fn collect_baseline_enrichment_paths( + include_proxy: bool, + include_gpu: bool, + gpu_device_nodes: Vec, +) -> (Vec, Vec) { + let mut ro = Vec::new(); + let mut rw = Vec::new(); + + if include_proxy { + for &path in PROXY_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in PROXY_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + } + + if include_gpu { + for &path in GPU_BASELINE_READ_ONLY { + push_unique(&mut ro, path.to_string()); + } + for &path in GPU_BASELINE_READ_WRITE { + push_unique(&mut rw, path.to_string()); + } + for path in gpu_device_nodes { + push_unique(&mut rw, path); + } + } + + // A path promoted to read_write (e.g. /proc for GPU) should not also + // appear in read_only — Landlock handles the overlap correctly but the + // duplicate is confusing when inspecting the effective policy. + ro.retain(|p| !rw.contains(p)); + + (ro, rw) +} + +fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { + let include_gpu = has_gpu_devices(); + let gpu_device_nodes = if include_gpu { + enumerate_gpu_device_nodes() + } else { + Vec::new() + }; + collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) +} + +/// Collect all active baseline paths for tests and diagnostics. +/// Returns `(read_only, read_write)` as owned `String` vecs. +#[cfg(test)] +fn baseline_enrichment_paths() -> (Vec, Vec) { + active_baseline_enrichment_paths(true) +} + +fn enrich_proto_baseline_paths_with( + proto: &mut openshell_core::proto::SandboxPolicy, + ro: &[String], + rw: &[String], + path_exists: F, +) -> bool +where + F: Fn(&str) -> bool, +{ + if ro.is_empty() && rw.is_empty() { + return false; + } + + let fs = proto + .filesystem + .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { + include_workdir: true, + ..Default::default() + }); + + let mut modified = false; + for path in ro { + if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { + if !path_exists(path) { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + fs.read_only.push(path.clone()); + modified = true; + } + } + for path in rw { + if fs.read_write.iter().any(|p| p == path) { + continue; + } + if !path_exists(path) { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + if fs.read_only.iter().any(|p| p == path) { + if path == "/proc" { + info!( + path, + "Promoting /proc from read-only to read-write for GPU runtime compatibility" + ); + fs.read_only.retain(|p| p != path); + fs.read_write.push(path.clone()); + modified = true; + } + continue; + } + fs.read_write.push(path.clone()); + modified = true; + } + + modified +} + +/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths +/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if +/// missing; user-specified paths are never removed. +/// +/// Returns `true` if the policy was modified (caller may want to sync back). +fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); + + // Baseline paths are system-injected, not user-specified. Skip paths + // that do not exist in this container image to avoid noisy warnings from + // Landlock and, more critically, to prevent a single missing baseline + // path from abandoning the entire Landlock ruleset under best-effort + // mode (see issue #664). + let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { + std::path::Path::new(path).exists() + }); + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } + + modified +} + +fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + openshell_policy::strip_provider_rule_names(proto) +} + +fn proto_sync_payload_for_enriched_policy( + proto: &openshell_core::proto::SandboxPolicy, + enriched: bool, +) -> Option { + if !enriched { + return None; + } + + let mut sync_policy = proto.clone(); + strip_proto_provider_policy_entries(&mut sync_policy); + Some(sync_policy) +} + +/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem +/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the +/// local-file code path where no proto is available. +fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { + let (ro, rw) = + active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); + if ro.is_empty() && rw.is_empty() { + return; + } + + let mut modified = false; + for path in &ro { + let p = std::path::PathBuf::from(path); + if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { + if !p.exists() { + debug!( + path, + "Baseline read-only path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_only.push(p); + modified = true; + } + } + for path in &rw { + let p = std::path::PathBuf::from(path); + if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { + continue; + } + if !p.exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_write.push(p); + modified = true; + } + + if modified { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enriched") + .message("Enriched policy with baseline filesystem paths for proxy mode") + .build() + ); + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod baseline_tests { + use super::*; + use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; + use std::path::PathBuf; + + #[test] + fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { + // When GPU devices are present, /proc is promoted to read_write + // (CUDA needs to write /proc//task//comm). It should + // NOT also appear in read_only. + if !has_gpu_devices() { + // Can't test GPU dedup without GPU devices; skip silently. + return; + } + let (ro, rw) = baseline_enrichment_paths(); + assert!( + rw.contains(&"/proc".to_string()), + "/proc should be in read_write when GPU is present" + ); + assert!( + !ro.contains(&"/proc".to_string()), + "/proc should NOT be in read_only when it is already in read_write" + ); + } + + #[test] + fn proc_in_read_only_without_gpu() { + if has_gpu_devices() { + // On a GPU host we can't test the non-GPU path; skip silently. + return; + } + let (ro, _rw) = baseline_enrichment_paths(); + assert!( + ro.contains(&"/proc".to_string()), + "/proc should be in read_only when GPU is not present" + ); + } + + #[test] + fn baseline_read_write_does_not_hardcode_sandbox() { + let (_ro, rw) = baseline_enrichment_paths(); + assert!(rw.contains(&"/tmp".to_string())); + assert!(!rw.contains(&"/sandbox".to_string())); + } + + #[test] + fn enumerate_gpu_device_nodes_skips_bare_nvidia() { + // "nvidia" (without a trailing digit) is a valid /dev entry on some + // systems but is not a per-GPU device node. The enumerator must + // not match it. + let nodes = enumerate_gpu_device_nodes(); + assert!( + !nodes.contains(&"/dev/nvidia".to_string()), + "bare /dev/nvidia should not be enumerated: {nodes:?}" + ); + } + + #[test] + fn no_duplicate_paths_in_baseline() { + let (ro, rw) = baseline_enrichment_paths(); + // No path should appear in both lists. + for path in &ro { + assert!( + !rw.contains(path), + "path {path} appears in both read_only and read_write" + ); + } + } + + #[test] + fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + read_only: vec!["/tmp".to_string()], + read_write: vec![], + include_workdir: false, + }); + policy.network_policies.insert( + "test".into(), + openshell_core::proto::NetworkPolicyRule { + name: "test-rule".into(), + endpoints: vec![openshell_core::proto::NetworkEndpoint { + host: "example.com".into(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + enrich_proto_baseline_paths(&mut policy); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + filesystem.read_only.contains(&"/tmp".to_string()), + "explicit read_only baseline path should be preserved" + ); + assert!( + !filesystem.read_write.contains(&"/tmp".to_string()), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + assert!(strip_proto_provider_policy_entries(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_proto_provider_policy_entries(&mut policy)); + } + + #[test] + fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "provider-derived rules alone must not trigger sync or mutate runtime policy" + ); + } + + #[test] + fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + runtime_policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) + .expect("enrichment should create a sync payload"); + + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "runtime policy must retain provider-derived rules for OPA input" + ); + assert!( + !sync_policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(sync_policy.network_policies.contains_key("sandbox_only")); + } + + #[test] + fn proto_gpu_enrichment_promotes_proc_without_network_policy() { + let mut policy = openshell_policy::restrictive_default_policy(); + assert!( + policy.network_policies.is_empty(), + "regression setup must exercise the no-network default path" + ); + let (ro, rw) = + collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); + + let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { + matches!(path, "/proc" | "/dev/nvidia0") + }); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + enriched, + "GPU enrichment should not require network policies" + ); + assert!( + filesystem.read_write.contains(&"/dev/nvidia0".to_string()), + "GPU enrichment should add enumerated device nodes without network policies" + ); + assert!( + !filesystem.read_only.contains(&"/proc".to_string()), + "GPU enrichment should remove /proc from read_only" + ); + assert!( + filesystem.read_write.contains(&"/proc".to_string()), + "GPU enrichment should promote /proc to read_write" + ); + } + + #[test] + fn gpu_baseline_read_write_contains_dxg() { + // /dev/dxg must be present so WSL2 sandboxes get the Landlock + // read-write rule for the CDI-injected DXG device. The existence + // check in enrich_proto_baseline_paths() skips it on native Linux. + assert!( + GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), + "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" + ); + } + + #[test] + fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![PathBuf::from("/tmp")], + read_write: vec![], + include_workdir: false, + }, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }; + + enrich_sandbox_baseline_paths(&mut policy); + + assert!( + policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), + "explicit read_only baseline path should be preserved" + ); + assert!( + !policy + .filesystem + .read_write + .contains(&PathBuf::from("/tmp")), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn gpu_baseline_read_only_contains_usr_lib_wsl() { + // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library + // bind-mounts are accessible under Landlock. Skipped on native Linux. + assert!( + GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), + "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" + ); + } + + #[test] + fn has_gpu_devices_reflects_dxg_or_nvidiactl() { + // Verify the OR logic: result must match the manual disjunction of + // the two path checks. Passes in all environments. + let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); + let dxg = std::path::Path::new("/dev/dxg").exists(); + assert_eq!( + has_gpu_devices(), + nvidiactl || dxg, + "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" + ); + } +} + +/// Returns `true` if the error is transient and worth retrying. +/// +/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If +/// found, only the gRPC codes that represent transient failures are retryable. +/// If no `tonic::Status` is present (e.g. a raw connection error), assume the +/// failure is transient. +fn is_retryable_error(err: &miette::Report) -> bool { + let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); + while let Some(e) = source { + if let Some(status) = e.downcast_ref::() { + return matches!( + status.code(), + tonic::Code::Unavailable + | tonic::Code::DeadlineExceeded + | tonic::Code::ResourceExhausted + | tonic::Code::Aborted + | tonic::Code::Internal + | tonic::Code::Unknown + ); + } + source = e.source(); + } + true +} + +/// Retry a gRPC operation with exponential backoff (capped at 4 s). +/// +/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, +/// `PERMISSION_DENIED`) are returned immediately without retrying. +async fn grpc_retry(op_name: &str, f: F) -> Result +where + F: Fn() -> Fut, + Fut: Future>, +{ + let mut last_err = None; + for attempt in 1..=5u32 { + match f().await { + Ok(val) => return Ok(val), + Err(e) => { + if !is_retryable_error(&e) { + return Err(e); + } + if attempt < 5 { + warn!( + attempt, + max_attempts = 5, + error = %e, + "{op_name} failed, retrying" + ); + let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); + tokio::time::sleep(backoff).await; + } + last_err = Some(e); + } + } + } + Err(miette::miette!( + "{op_name} failed after 5 attempts: {}", + last_err.expect("loop executed at least once") + )) +} + +/// Load sandbox policy from local files or gRPC. +/// +/// Priority: +/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files +/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC +/// 3. If the server returns no policy, discover from disk or use restrictive default +/// 4. Otherwise, return an error +/// +/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto +/// policy. The proto is retained so the OPA engine can be rebuilt with symlink +/// resolution after the container entrypoint starts. +async fn load_policy( + sandbox_id: Option, + sandbox: Option, + openshell_endpoint: Option, + policy_rules: Option, + policy_data: Option, + extension_credentials: &openshell_extension_core::ExtensionCredentialStore, +) -> Result<( + SandboxPolicy, + Option>, + Option, + MiddlewareRegistryStatus, + LoadedPolicyOrigin, + bool, + bool, +)> { + // File mode: load OPA engine from rego rules + YAML data (dev override) + if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "loading") + .unmapped("policy_rules", serde_json::json!(policy_file)) + .unmapped("policy_data", serde_json::json!(data_file)) + .message(format!( + "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" + )) + .build()); + let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { + openshell_supervisor_middleware_builtins::validate_config(implementation, config) + .map_err(|error| error.to_string()) + }; + let engine = OpaEngine::from_files_with_middleware_config( + std::path::Path::new(policy_file), + std::path::Path::new(data_file), + Some(&validate_middleware_config), + )?; + let middleware_registry = + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + engine.replace_middleware_registry(middleware_registry)?; + let config = engine.query_sandbox_config()?; + let mut policy = SandboxPolicy { + version: 1, + filesystem: config.filesystem, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: config.landlock, + process: config.process, + }; + enrich_sandbox_baseline_paths(&mut policy); + // File mode has no operator-registered middleware to connect. + return Ok(( + policy, + Some(Arc::new(engine)), + None, + MiddlewareRegistryStatus::Synchronized, + LoadedPolicyOrigin::LocalOverride, + false, + false, + )); + } + + // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data + if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + info!( + sandbox_id = %id, + endpoint = %endpoint, + "Fetching sandbox policy via gRPC" + ); + let mut snapshot = grpc_retry("Policy fetch", || { + openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) + }) + .await?; + + let mut proto_policy = if let Some(p) = snapshot.policy.clone() { + p + } else { + // No policy configured on the server. Discover from disk or + // fall back to the restrictive default, then sync to the + // gateway so it becomes the authoritative baseline. + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "discovery") + .message("Server returned no policy; attempting local discovery") + .build() + ); + let mut discovered = discover_policy_from_disk_or_default(); + // Enrich before syncing so the gateway baseline includes + // baseline paths from the start. + enrich_proto_baseline_paths(&mut discovered); + strip_proto_provider_policy_entries(&mut discovered); + let sandbox = sandbox.as_deref().ok_or_else(|| { + miette::miette!( + "Cannot sync discovered policy: sandbox not available.\n\ + Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." + ) + })?; + + // Sync and re-fetch over a single connection to avoid extra + // TLS handshakes. + let ws = snapshot.workspace.clone(); + snapshot = grpc_retry("Policy discovery sync", || { + openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox, + &discovered, + &ws, + ) + }) + .await?; + snapshot.policy.clone().ok_or_else(|| { + miette::miette!("Server still returned no policy after sync — this is a bug") + })? + }; + + // True only while `snapshot` describes the exact policy that will be + // constructed below. If enrichment cannot be synced and re-fetched, + // the policy remains enforceable but cannot be acknowledged by + // inferred structural equality. + let mut policy_bound_to_snapshot = true; + + // Ensure baseline filesystem paths are present for proxy-mode + // sandboxes. If the policy was enriched, sync the updated version + // back to the gateway so users can see the effective policy. + let enriched = enrich_proto_baseline_paths(&mut proto_policy); + let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); + if let Some(sync_policy) = sync_policy { + if let Some(sandbox_name) = sandbox.as_deref() { + match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + endpoint, + id, + sandbox_name, + &sync_policy, + &snapshot.workspace, + ) + .await + { + Ok(canonical) => { + if let Some(policy) = canonical.policy.clone() { + proto_policy = policy; + snapshot = canonical; + } else { + policy_bound_to_snapshot = false; + warn!( + "Gateway returned no policy after enrichment sync; initial revision will be reconciled" + ); + } + } + Err(e) => { + policy_bound_to_snapshot = false; + warn!( + error = %e, + "Failed to sync enriched policy back to gateway; initial revision will be reconciled" + ); + } + } + } else { + policy_bound_to_snapshot = false; + } + } + + let mut loaded_policy_revision = + policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); + + // Build OPA engine from baked-in rules + typed proto data. + // In cluster mode, proxy networking is always enabled so OPA is + // always required for allow/deny decisions. + // The initial load uses pid=0 (no symlink resolution) because the + // container hasn't started yet. After the entrypoint spawns, the + // engine is rebuilt with the real PID for symlink resolution. + info!("Creating OPA engine from proto policy data"); + let mut has_last_valid_policy = true; + let engine = match OpaEngine::from_proto(&proto_policy) { + Ok(engine) => Arc::new(engine), + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + let validation_error = e.to_string(); + let candidate_version = snapshot.version; + let candidate_hash = snapshot.policy_hash.clone(); + // There is no in-memory last-known-good generation during + // startup, so both configured modes necessarily fail closed. + // Load the restrictive default atomically and keep the + // rejected revision unacknowledged for poll reconciliation. + has_last_valid_policy = false; + proto_policy = openshell_policy::restrictive_default_policy(); + let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); + let disposition = apply_policy_validation_failure( + &engine, + snapshot.policy_validation_failure_mode, + has_last_valid_policy, + candidate_version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + candidate_version, + &candidate_hash, + &validation_error, + ); + loaded_policy_revision = None; + engine + } + }; + + // Install the in-process catalog before any external connection can + // fail. A newly started sandbox must always be able to resolve built-in + // bindings, even while operator-run services are unavailable. + install_builtin_middleware_registry(&engine).await?; + + // Connect operator-registered middleware services. A connect/describe + // failure keeps the built-in registry active so each request's + // `on_error` policy governs matched traffic. The policy poll loop + // retries the install without waiting for a config change. + let middleware_services = snapshot.supervisor_middleware_services.clone(); + let middleware_registry_status = if middleware_services.is_empty() { + MiddlewareRegistryStatus::Synchronized + } else if let Err(error) = grpc_retry("Middleware connect", || { + let middleware_services = middleware_services.clone(); + let extension_credentials = extension_credentials.clone(); + let extension_authentication_enabled = snapshot.extension_authentication_enabled; + async move { + let credentials = if extension_authentication_enabled { + // Share the supervisor's store so the slots installed here + // are the ones the policy poll loop later rotates in place. + openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( + endpoint, + extension_credentials, + ) + .await? + .refresh_extension_credentials(&middleware_services) + .await? + } else { + std::collections::HashMap::new() + }; + connect_middleware_registry( + &middleware_services, + &MiddlewareAuthentication { + credentials, + enabled: extension_authentication_enabled, + }, + ) + .await + } + }) + .await + .and_then(|registry| engine.replace_middleware_registry(registry)) + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(middleware_services.len()) + ) + .message(format!( + "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" + )) + .build() + ); + MiddlewareRegistryStatus::NeedsReconciliation + } else { + MiddlewareRegistryStatus::Synchronized + }; + let opa_engine = Some(engine); + + let policy = match SandboxPolicy::try_from(proto_policy.clone()) { + Ok(policy) => policy, + Err(e) => { + report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) + .await; + return Err(e); + } + }; + return Ok(( + policy, + opa_engine, + Some(proto_policy), + middleware_registry_status, + LoadedPolicyOrigin::Gateway { + revision: loaded_policy_revision, + has_last_valid_policy, + }, + agent_proposals_enabled_from_settings(&snapshot.settings), + snapshot.extension_authentication_enabled, + )); + } + + // No policy source available + Err(miette::miette!( + "Sandbox policy required. Provide one of:\n\ + - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ + - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" + )) +} + +/// Try to discover a sandbox policy from the well-known disk path, falling +/// back to the legacy path, then to the hardcoded restrictive default. +fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { + let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); + if primary.exists() { + return discover_policy_from_path(primary); + } + let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); + if legacy.exists() { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "legacy_path", + serde_json::json!(legacy.display().to_string()) + ) + .unmapped("new_path", serde_json::json!(primary.display().to_string())) + .message(format!( + "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", + legacy.display(), + primary.display() + )) + .build() + ); + return discover_policy_from_path(legacy); + } + discover_policy_from_path(primary) +} + +/// Try to read a sandbox policy YAML from `path`, falling back to the +/// hardcoded restrictive default if the file is missing or invalid. +fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { + use openshell_policy::{ + parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, + }; + + let Ok(yaml) = std::fs::read_to_string(path) else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "default") + .message(format!( + "No policy file on disk, using restrictive default [path:{}]", + path.display() + )) + .build() + ); + return restrictive_default_policy(); + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Loaded sandbox policy from container disk [path:{}]", + path.display() + )) + .build() + ); + match parse_sandbox_policy(&yaml) { + Ok(policy) => { + // Validate the disk-loaded policy for safety. + if let Err(violations) = validate_sandbox_policy(&policy) { + let messages: Vec = violations.iter().map(ToString::to_string).collect(); + ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Medium) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .finding_info( + FindingInfo::new( + "unsafe-disk-policy", + "Unsafe Disk Policy Content", + ) + .with_desc(&format!( + "Disk policy at {} contains unsafe content: {}", + path.display(), + messages.join("; "), + )), + ) + .message(format!( + "Disk policy contains unsafe content, using restrictive default [path:{}]", + path.display() + )) + .build()); + return restrictive_default_policy(); + } + policy + } + Err(e) => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "fallback") + .message(format!( + "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", + path.display() + )) + .build()); + restrictive_default_policy() + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiddlewareRegistryStatus { + Synchronized, + NeedsReconciliation, +} + +#[derive(Debug)] +enum GatewayRuntimeReloadError { + PolicyValidation(miette::Report), + TransparentTcpPrerequisite(miette::Report), + MiddlewareRegistry(miette::Report), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GatewayRuntimeFailureClass { + PolicyValidation, + TransparentTcpPrerequisite, + MiddlewareRegistry, +} + +impl GatewayRuntimeReloadError { + fn class(&self) -> GatewayRuntimeFailureClass { + match self { + Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, + Self::TransparentTcpPrerequisite(_) => { + GatewayRuntimeFailureClass::TransparentTcpPrerequisite + } + Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct FailedRuntimeRevision { + config_revision: u64, + policy_hash: String, + failure_class: GatewayRuntimeFailureClass, +} + +impl FailedRuntimeRevision { + fn new(config_revision: u64, policy_hash: &str, failure: &GatewayRuntimeReloadError) -> Self { + Self { + config_revision, + policy_hash: policy_hash.to_string(), + failure_class: failure.class(), + } + } +} + +struct MiddlewareReloadContext<'a> { + desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], + authentication: &'a MiddlewareAuthentication, + registry_changed: bool, + connector: &'a MiddlewareConnector, +} + +async fn reload_gateway_policy_runtime( + engine: &OpaEngine, + policy: Option<&openshell_core::proto::SandboxPolicy>, + entrypoint_pid: u32, + middleware: MiddlewareReloadContext<'_>, + transparent_tcp: TransparentTcpReloadState, +) -> std::result::Result<(), GatewayRuntimeReloadError> { + if let Some(policy) = policy + && policy_contains_explicit_tcp(policy) + { + if !transparent_tcp.capable { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" + ), + )); + } + if !transparent_tcp.substrate_ready { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" + ), + )); + } + } + match policy { + Some(policy) if middleware.registry_changed => { + let registry = (middleware.connector)( + middleware.desired_services.to_vec(), + middleware.authentication.clone(), + ) + .await + .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; + engine + .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) + .map_err(GatewayRuntimeReloadError::PolicyValidation) + } + // Policy-only change: the installed registry already matches the + // delivered service set, so swap the engine alone. This must not + // require middleware reachability. + Some(policy) => engine + .reload_from_proto_with_pid(policy, entrypoint_pid) + .map_err(GatewayRuntimeReloadError::PolicyValidation), + None => Err(GatewayRuntimeReloadError::PolicyValidation( + miette::miette!("runtime reload requires a policy payload but none was returned"), + )), + } +} + +fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { + policy.network_policies.values().any(|rule| { + rule.endpoints + .iter() + .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) + }) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct TransparentTcpReloadState { + capable: bool, + substrate_ready: bool, +} + +/// True when the installed middleware registry no longer matches the desired +/// service set and must be rebuilt (reconnecting every delivered service). +/// +/// A policy-only change never requires a rebuild: middleware configs were +/// validated at gateway admission and the installed registry's manifests +/// already cover the unchanged service set, so requiring the services to be +/// reachable would only let a middleware outage block the policy update. +fn middleware_registry_needs_rebuild( + registry_status: MiddlewareRegistryStatus, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> bool { + registry_status == MiddlewareRegistryStatus::NeedsReconciliation + || current_services != desired_services +} + +fn gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy: bool, + current_policy_hash: &str, + desired_policy_hash: &str, + current_services: &[openshell_core::proto::SupervisorMiddlewareService], + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + registry_status: MiddlewareRegistryStatus, +) -> bool { + reloads_gateway_policy + && (current_policy_hash != desired_policy_hash + || middleware_registry_needs_rebuild( + registry_status, + current_services, + desired_services, + )) +} + +/// Identity returned with the exact policy snapshot used to construct OPA. +#[derive(Clone, Debug, PartialEq, Eq)] +struct LoadedPolicyRevision { + version: u32, + policy_hash: String, + config_revision: u64, + policy_source: openshell_core::proto::PolicySource, +} + +/// Identifies where the policy currently loaded into OPA came from. +/// +/// A missing gateway revision means the policy was loaded from the gateway but +/// could not be bound to an authoritative snapshot (for example, enrichment +/// sync failed). That state must reconcile on the first successful poll. A +/// local-file override is different: gateway policy revisions are observed for +/// settings/provider refreshes but must never replace the explicit local OPA +/// policy. +#[derive(Clone, Debug, PartialEq, Eq)] +enum LoadedPolicyOrigin { + LocalOverride, + Gateway { + revision: Option, + has_last_valid_policy: bool, + }, +} + +impl LoadedPolicyOrigin { + fn allows_gateway_policy_reload(&self) -> bool { + matches!(self, Self::Gateway { .. }) + } + + fn has_last_valid_policy(&self) -> bool { + match self { + Self::LocalOverride => true, + Self::Gateway { + has_last_valid_policy, + .. + } => *has_last_valid_policy, + } + } +} + +impl LoadedPolicyRevision { + fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { + Self { + version: snapshot.version, + policy_hash: snapshot.policy_hash.clone(), + config_revision: snapshot.config_revision, + policy_source: snapshot.policy_source, + } + } +} + +/// A sandbox-scoped policy revision that was constructed successfully at +/// startup and must be acknowledged to the gateway exactly once. +#[derive(Clone, Debug, PartialEq, Eq)] +struct InitialPolicyAck { + version: u32, + policy_hash: String, + config_revision: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PolicyStatusUpdate { + version: u32, + loaded: bool, + error: String, + success_event: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum PolicyStatusSuccessEvent { + InitialAcknowledgement { policy_hash: String }, + UnchangedAcknowledgement { policy_hash: String }, +} + +impl PolicyStatusUpdate { + fn initial_loaded(ack: &InitialPolicyAck) -> Self { + Self { + version: ack.version, + loaded: true, + error: String::new(), + success_event: Some(PolicyStatusSuccessEvent::InitialAcknowledgement { + policy_hash: ack.policy_hash.clone(), + }), + } + } + + fn loaded(version: u32) -> Self { + Self { + version, + loaded: true, + error: String::new(), + success_event: None, + } + } + + fn unchanged_loaded(version: u32, policy_hash: String) -> Self { + Self { + version, + loaded: true, + error: String::new(), + success_event: Some(PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash }), + } + } + + fn failed(version: u32, error: String) -> Self { + Self { + version, + loaded: false, + error, + success_event: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum InitialPollDisposition { + Acknowledge(InitialPolicyAck), + Reconcile, + TrackOnly, +} + +/// Determine whether the initially loaded policy corresponds to an +/// authoritative sandbox-scoped revision that must be acknowledged. +/// +/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose +/// captured gateway identity matches the current version and hash. Global +/// policies, local-file development policies, version zero, and changed +/// identities yield `None`, so those paths never emit a sandbox-revision +/// acknowledgement. +fn initial_policy_ack_candidate( + loaded: Option<&LoadedPolicyRevision>, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + let loaded = loaded?; + if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox + || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox + { + return None; + } + if loaded.version == 0 || canonical.version == 0 { + return None; + } + if loaded.version != canonical.version + || loaded.policy_hash != canonical.policy_hash + || canonical.config_revision < loaded.config_revision + { + return None; + } + Some(InitialPolicyAck { + version: loaded.version, + policy_hash: loaded.policy_hash.clone(), + config_revision: canonical.config_revision, + }) +} + +fn initial_poll_disposition( + origin: &LoadedPolicyOrigin, + canonical: &openshell_core::grpc_client::SettingsPollResult, +) -> InitialPollDisposition { + match origin { + LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, + LoadedPolicyOrigin::Gateway { revision, .. } => { + initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( + InitialPollDisposition::Reconcile, + InitialPollDisposition::Acknowledge, + ) + } + } +} + +fn unchanged_policy_revision_candidate( + reloads_gateway_policy: bool, + recovering_rejected_policy: bool, + current_policy_version: u32, + current_policy_hash: &str, + result: &openshell_core::grpc_client::SettingsPollResult, +) -> Option { + (reloads_gateway_policy + && !recovering_rejected_policy + && !current_policy_hash.is_empty() + && result.policy_source == openshell_core::proto::PolicySource::Sandbox + && result.version > current_policy_version + && result.policy_hash == current_policy_hash) + .then_some(result.version) +} + +fn unchanged_policy_revision_ready_to_ack( + candidate: Option, + policy_runtime_changed: bool, + policy_runtime_reconciled: bool, +) -> Option { + candidate.filter(|_| !policy_runtime_changed || policy_runtime_reconciled) +} + +/// Whether the credential-provenance gates cannot apply to the loaded policy. +/// +/// The gateway derives `provider_credentialed` and deliberately keeps it out of +/// the policy YAML schema, so a local-file policy never carries it and never +/// will: gateway revisions are observed for settings and providers but must not +/// replace the local OPA policy. Provider credentials still arrive from the +/// gateway on that path, so the raw-tunnel and WebSocket binary-frame refusals +/// have nothing to match on. The request-body backstop is unaffected because it +/// keys off the secret resolver rather than endpoint provenance. +fn credential_gating_unavailable( + origin: &LoadedPolicyOrigin, + has_resolver: bool, + network_enabled: bool, +) -> bool { + network_enabled && has_resolver && matches!(origin, LoadedPolicyOrigin::LocalOverride) +} + +/// Report that credential provenance is unavailable for the loaded policy. +/// +/// Carries no credential name, host, or value: the finding states which +/// controls are inactive, nothing about what they would have protected. +fn report_credential_gating_unavailable() { + ocsf_emit!( + DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::High) + .confidence(ConfidenceId::High) + .is_alert(true) + .finding_info( + FindingInfo::new( + "credential-gating-unavailable", + "Credential Provenance Unavailable", + ) + .with_desc( + "Provider credentials are injected, but the loaded policy comes from local \ + files and carries no gateway-derived credential provenance. Uninspected \ + credentialed tunnels and WebSocket binary frames are not refused. Load \ + policy from the gateway to enable these controls." + ), + ) + .evidence_pairs(&[ + ("policy_source", "local-override"), + ("uninspected_connect_gate", "inactive"), + ("websocket_binary_gate", "inactive"), + ("request_body_backstop", "active"), + ]) + .remediation( + "Remove the local policy override so the gateway-delivered effective policy \ + applies, or detach provider credentials from this sandbox." + ) + .message( + "Credential provenance unavailable for local-file policy; uninspected credential gates inactive" + ) + .build() + ); +} + +/// Deliver policy status updates independently from policy reconciliation. +/// +/// The channel is FIFO, so a delayed older status can never arrive after a +/// newer status and move the gateway's active version backward. Delivery uses +/// the existing bounded retry, but failures never delay policy enforcement. +#[tonic::async_trait] +trait PolicyGatewayClient: Clone + Send + Sync + 'static { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result; + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()>; + + async fn refresh_installed_extension_credentials(&self) -> Result<()> { + Ok(()) + } + + async fn extension_credentials_for( + &self, + _services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> { + Ok(std::collections::HashMap::new()) + } + + fn workspace(&self) -> String; +} + +#[tonic::async_trait] +impl PolicyGatewayClient for openshell_core::grpc_client::CachedOpenShellClient { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result { + self.poll_settings(sandbox_id).await + } + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.report_policy_status(sandbox_id, version, loaded, error) + .await + } + + async fn refresh_installed_extension_credentials(&self) -> Result<()> { + self.refresh_installed_extension_credentials().await + } + + async fn extension_credentials_for( + &self, + services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> { + self.extension_credentials_for(services).await + } + + fn workspace(&self) -> String { + self.workspace() + } +} + +async fn run_policy_status_reporter( + client: C, + sandbox_id: String, + mut updates: tokio::sync::mpsc::UnboundedReceiver, +) { + 'updates: while let Some(update) = updates.recv().await { + let operation = if matches!( + update.success_event, + Some(PolicyStatusSuccessEvent::InitialAcknowledgement { .. }) + ) { + "Initial policy acknowledgement" + } else { + "Policy status report" + }; + let mut attempt = 1_u32; + loop { + let sandbox_id = sandbox_id.clone(); + let error = update.error.clone(); + let client = client.clone(); + match client + .report_policy_status(&sandbox_id, update.version, update.loaded, &error) + .await + { + Ok(()) => break, + Err(error) if is_retryable_error(&error) => { + let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); + warn!( + %error, + attempt, + version = update.version, + loaded = update.loaded, + retry_in_secs = backoff.as_secs(), + "{operation} failed transiently; retaining ordered update" + ); + tokio::time::sleep(backoff).await; + attempt = attempt.saturating_add(1); + } + Err(error) => { + warn!( + %error, + version = update.version, + loaded = update.loaded, + "Discarding terminal policy status update" + ); + continue 'updates; + } + } + } + + if let Some(event) = update.success_event { + let (policy_hash, message) = match event { + PolicyStatusSuccessEvent::InitialAcknowledgement { policy_hash } => ( + policy_hash, + format!( + "Acknowledged initial policy revision as loaded [version:{}]", + update.version + ), + ), + PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash } => ( + policy_hash, + format!( + "Acknowledged unchanged policy revision as loaded [version:{}]", + update.version + ), + ), + }; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("version", serde_json::json!(update.version)) + .unmapped("policy_hash", serde_json::json!(policy_hash)) + .message(message) + .build() + ); + } + } +} + +fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { + let version = update.version; + if let Err(error) = sender.send(update) { + warn!( + %error, + version, + "Policy status reporter unavailable during shutdown" + ); + } +} + +/// Best-effort `FAILED` acknowledgement when initial policy construction or +/// conversion fails. +/// +/// Uses the revision identity captured with the policy that failed to build, +/// and preserves the original construction error as the reported message. A +/// delivery failure here is swallowed so it can never mask that error. +async fn report_initial_policy_failure( + endpoint: &str, + sandbox_id: &str, + revision: Option<&LoadedPolicyRevision>, + error: &miette::Report, +) { + let Some(revision) = revision.filter(|revision| { + revision.version > 0 + && revision.policy_source == openshell_core::proto::PolicySource::Sandbox + }) else { + return; + }; + let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { + Ok(client) => client, + Err(e) => { + warn!(error = %e, "Failed to connect to report initial policy failure"); + return; + } + }; + let message = error.to_string(); + if let Err(e) = grpc_retry("Initial policy failure report", || { + let client = client.clone(); + let message = message.clone(); + async move { + client + .report_policy_status(sandbox_id, revision.version, false, &message) + .await + } + }) + .await + { + warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); + } +} + +/// Background loop that polls the server for policy updates. +/// +/// When a new version is detected, attempts to reload the OPA engine via +/// `reload_from_proto_with_pid()`. Reports load success/failure back to the +/// server. On failure, the previous engine is untouched (LKG behavior). +/// +/// When the entrypoint PID is available, policy reloads include symlink +/// resolution for binary paths via the container filesystem. +struct PolicyPollLoopContext { + endpoint: String, + sandbox_id: String, + opa_engine: Arc, + /// Source of the policy currently loaded into OPA. This distinguishes an + /// explicit local-file override from an unbound gateway revision so the + /// former is never replaced by policy polling. + loaded_policy_origin: LoadedPolicyOrigin, + entrypoint_pid: Arc, + interval_secs: u64, + ocsf_enabled: Arc, + provider_credentials: ProviderCredentialState, + policy_local_ctx: Option>, + agent_proposals: AgentProposals, + middleware_registry_status: MiddlewareRegistryStatus, + workspace_tx: tokio::sync::watch::Sender, + extension_credentials: openshell_extension_core::ExtensionCredentialStore, + extension_authentication_enabled: bool, + middleware_connector: MiddlewareConnector, + /// Immutable driver capability and startup substrate state. + transparent_tcp: TransparentTcpReloadState, +} + +type MiddlewareConnector = Arc< + dyn Fn( + Vec, + MiddlewareAuthentication, + ) -> Pin< + Box< + dyn std::future::Future< + Output = Result, + > + Send, + >, + > + Send + + Sync, +>; + +#[derive(Clone, Default)] +struct MiddlewareAuthentication { + credentials: std::collections::HashMap, + enabled: bool, +} + +fn default_middleware_connector() -> MiddlewareConnector { + Arc::new(|services, authentication| { + Box::pin(async move { connect_middleware_registry(&services, &authentication).await }) + }) +} + +async fn connect_middleware_registry( + services: &[openshell_core::proto::SupervisorMiddlewareService], + authentication: &MiddlewareAuthentication, +) -> Result { + if authentication.enabled { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services_authenticated( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + &authentication.credentials, + ) + .await + } else { + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await + } +} + +async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { + let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + Vec::new(), + ) + .await?; + opa_engine.replace_middleware_registry(registry) +} + +/// Wait the configured poll interval, but never past the point at which an +/// installed extension credential must be rotated. +fn next_poll_delay( + store: &openshell_extension_core::ExtensionCredentialStore, + interval: Duration, +) -> Duration { + let now_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) + }); + store.next_refresh_delay(interval, now_ms) +} + +/// Drop credentials for services no longer in the installed registry. +/// +/// Call only after a registry swap succeeds, so a failed candidate cannot +/// invalidate the last-known-good clients. +fn retain_extension_credentials( + store: &openshell_extension_core::ExtensionCredentialStore, + installed: &[openshell_core::proto::SupervisorMiddlewareService], + extension_authentication_enabled: bool, +) { + let retained = if extension_authentication_enabled { + installed + .iter() + .map(|service| service.name.as_str()) + .collect() + } else { + std::collections::HashSet::default() + }; + store.retain(&retained); +} + +struct MiddlewareRegistryReconciliation<'a> { + desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], + authentication: MiddlewareAuthentication, + registry_changed: bool, + extension_credentials: &'a openshell_extension_core::ExtensionCredentialStore, + current_services: &'a mut Vec, + status: &'a mut MiddlewareRegistryStatus, +} + +async fn reconcile_middleware_registry( + opa_engine: &OpaEngine, + middleware_connector: &MiddlewareConnector, + reconciliation: MiddlewareRegistryReconciliation<'_>, +) { + if !reconciliation.registry_changed { + return; + } + + match middleware_connector( + reconciliation.desired_services.to_vec(), + reconciliation.authentication.clone(), + ) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + { + Ok(()) => { + retain_extension_credentials( + reconciliation.extension_credentials, + reconciliation.desired_services, + reconciliation.authentication.enabled, + ); + reconciliation.current_services.clear(); + reconciliation + .current_services + .extend_from_slice(reconciliation.desired_services); + *reconciliation.status = MiddlewareRegistryStatus::Synchronized; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(reconciliation.current_services.len()) + ) + .message(format!( + "Supervisor middleware registry reloaded [service_count:{}]", + reconciliation.current_services.len() + )) + .build() + ); + } + Err(error) => { + // Emit only on the transition into the failed state to avoid + // repeating the same finding on every poll during an outage. + if *reconciliation.status == MiddlewareRegistryStatus::Synchronized { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .message(format!( + "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" + )) + .build() + ); + } + *reconciliation.status = MiddlewareRegistryStatus::NeedsReconciliation; + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode, + mode: PolicyValidationFailureMode, + previous_policy_active: bool, + active_generation: u64, +} + +struct RejectedPolicyGeneration { + version: u32, + policy_hash: String, + validation_error: String, + configured_mode: PolicyValidationFailureMode, +} + +enum GatewayRuntimeFailureDisposition { + PolicyRejected { + error: String, + disposition: PolicyValidationFailureDisposition, + }, + MiddlewareUnavailable { + error: String, + }, + TransparentTcpExpansionRejected { + error: String, + active_generation: u64, + }, +} + +fn apply_gateway_runtime_reload_failure( + engine: &OpaEngine, + failure: GatewayRuntimeReloadError, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, +) -> Result { + match failure { + GatewayRuntimeReloadError::PolicyValidation(error) => { + let error = error.to_string(); + let disposition = apply_policy_validation_failure( + engine, + configured_mode, + has_last_valid_policy, + version, + &error, + )?; + Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) + } + GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error: error.to_string(), + active_generation: engine.current_generation(), + }, + ), + GatewayRuntimeReloadError::MiddlewareRegistry(error) => { + Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { + error: error.to_string(), + }) + } + } +} + +fn emit_transparent_tcp_expansion_rejection( + version: u32, + policy_hash: &str, + active_generation: u64, + error: &str, +) { + let message = format!( + "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" + ); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Enabled, "retained_previous_policy") + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .unmapped("active_generation", serde_json::json!(active_generation)) + .unmapped("validation_error", serde_json::json!(error)) + .message(message) + .build() + ); +} + +fn apply_policy_validation_failure( + engine: &OpaEngine, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, + error: &str, +) -> Result { + let mode = if has_last_valid_policy { + configured_mode + } else { + PolicyValidationFailureMode::FailClosed + }; + match mode { + PolicyValidationFailureMode::FailClosed => { + let reason = format!( + "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" + ); + let active_generation = engine.enter_fail_closed(reason)?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: false, + active_generation, + }) + } + PolicyValidationFailureMode::RetainLastValid => { + let active_generation = engine.exit_fail_closed()?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: true, + active_generation, + }) + } + } +} + +fn policy_validation_failure_events( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) -> [OcsfEvent; 2] { + let previous_policy_state = if disposition.previous_policy_active { + "IS active" + } else { + "IS NOT active" + }; + let state = if disposition.previous_policy_active { + (StateId::Enabled, "retained_last_valid") + } else { + (StateId::Disabled, "fail_closed") + }; + let message = format!( + "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", + disposition.configured_mode.as_str(), + disposition.mode.as_str(), + disposition.active_generation, + ); + let finding_uid = format!("policy-validation-failed-{version}"); + let version_string = version.to_string(); + let config = ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(state.0, state.1) + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped( + "validation_failure_mode", + serde_json::json!(disposition.mode.as_str()), + ) + .unmapped( + "configured_validation_failure_mode", + serde_json::json!(disposition.configured_mode.as_str()), + ) + .unmapped( + "previous_policy_active", + serde_json::json!(disposition.previous_policy_active), + ) + .unmapped( + "active_generation", + serde_json::json!(disposition.active_generation), + ) + .unmapped("validation_error", serde_json::json!(error)) + .message(message.clone()) + .build(); + let finding = DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .is_alert(true) + .finding_info( + FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), + ) + .evidence_pairs(&[ + ("candidate_version", &version_string), + ("candidate_policy_hash", policy_hash), + ("validation_failure_mode", disposition.mode.as_str()), + ( + "configured_validation_failure_mode", + disposition.configured_mode.as_str(), + ), + ( + "previous_policy_active", + if disposition.previous_policy_active { + "true" + } else { + "false" + }, + ), + ]) + .remediation("Submit a valid, unambiguous policy generation") + .message(message) + .build(); + [config, finding] +} + +fn emit_policy_validation_failure( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) { + for event in policy_validation_failure_events(disposition, version, policy_hash, error) { + ocsf_emit!(event); + } +} + +async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { + let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( + &ctx.endpoint, + ctx.extension_credentials.clone(), + ) + .await?; + run_policy_poll_loop_with_client(ctx, client).await +} + +async fn run_policy_poll_loop_with_client( + ctx: PolicyPollLoopContext, + client: C, +) -> Result<()> { + use openshell_core::proto::PolicySource; + use std::sync::atomic::Ordering; + + let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(run_policy_status_reporter( + client.clone(), + ctx.sandbox_id.clone(), + status_receiver, + )); + + let mut current_config_revision: u64 = 0; + let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; + let mut current_policy_version: u32 = 0; + let mut current_policy_hash = String::new(); + let mut current_middleware_services = Vec::new(); + let mut current_extension_authentication_enabled = ctx.extension_authentication_enabled; + let mut middleware_registry_status = ctx.middleware_registry_status; + let mut current_settings: std::collections::HashMap< + String, + openshell_core::proto::EffectiveSetting, + > = std::collections::HashMap::new(); + let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); + let mut last_failed_runtime_revision: Option = None; + let mut rejected_policy_generation: Option = None; + let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); + + // A first poll that does not match the policy already loaded into OPA must + // pass through the normal reconciliation path immediately. It must never + // seed the applied-state trackers before OPA actually loads it. + let mut pending_result = None; + + // Initialize revision from the first poll and acknowledge the initial + // policy revision the supervisor actually loaded. A mismatched result is + // reconciled below instead of being recorded as already applied. + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { + InitialPollDisposition::Acknowledge(candidate) => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(candidate.config_revision), + skills::install_static_skills, + ); + current_config_revision = candidate.config_revision; + current_policy_version = candidate.version; + current_policy_hash.clone_from(&candidate.policy_hash); + current_middleware_services = result.supervisor_middleware_services; + current_extension_authentication_enabled = + result.extension_authentication_enabled; + current_settings = result.settings; + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::initial_loaded(&candidate), + ); + debug!( + config_revision = current_config_revision, + "Settings poll: initial policy matches loaded revision" + ); + } + InitialPollDisposition::Reconcile => pending_result = Some(result), + InitialPollDisposition::TrackOnly => { + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "initial settings poll", + Some(result.config_revision), + skills::install_static_skills, + ); + current_config_revision = result.config_revision; + current_policy_hash = result.policy_hash.clone(); + current_middleware_services = result.supervisor_middleware_services; + current_extension_authentication_enabled = + result.extension_authentication_enabled; + current_settings = result.settings; + debug!( + config_revision = current_config_revision, + "Settings poll: tracking gateway config while preserving local policy override" + ); + } + } + } + Err(e) => { + warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); + } + } + + let interval = Duration::from_secs(ctx.interval_secs); + loop { + let result = if let Some(result) = pending_result.take() { + result + } else { + tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; + match client.poll_settings(&ctx.sandbox_id).await { + Ok(result) => { + let _ = ctx.workspace_tx.send(client.workspace()); + result + } + Err(e) => { + debug!(error = %e, "Settings poll: server unreachable, will retry"); + if current_extension_authentication_enabled + && let Err(refresh_error) = + client.refresh_installed_extension_credentials().await + { + warn!( + error = %refresh_error, + "Settings poll: extension credential refresh failed while configuration was unavailable" + ); + } + continue; + } + } + }; + + // Reuse installed per-service credentials, rotating only when one is + // missing or due. Rotation happens on the existing gateway channel and + // updates slots in place, so it is independent of config revision and + // registry equality. + let middleware_credentials = if result.extension_authentication_enabled { + match client + .extension_credentials_for(&result.supervisor_middleware_services) + .await + { + Ok(credentials) => credentials, + Err(error) => { + warn!(error = %error, "Settings poll: extension credential refresh failed"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; + + let config_changed = result.config_revision != current_config_revision; + let provider_env_changed = result.provider_env_revision != current_provider_env_revision; + let policy_changed = result.policy_hash != current_policy_hash; + let extension_authentication_changed = + current_extension_authentication_enabled != result.extension_authentication_enabled; + let middleware_registry_changed = extension_authentication_changed + || middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, + &result.supervisor_middleware_services, + ); + // A valid candidate may intentionally restore byte-for-byte policy + // content that was active before a rejected update. Its hash then + // equals `current_policy_hash`, but the runtime is still quarantined + // and must reload (or it would remain deny-all indefinitely). + let recovering_rejected_policy = reloads_gateway_policy + && rejected_policy_generation + .as_ref() + .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); + let policy_runtime_changed = recovering_rejected_policy + || extension_authentication_changed + || gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); + // Recovery already has its own acknowledgement path below. Giving it + // precedence here prevents a restored last-known-good policy from + // also being acknowledged as an ordinary same-hash revision. + let unchanged_policy_revision = unchanged_policy_revision_candidate( + reloads_gateway_policy, + recovering_rejected_policy, + current_policy_version, + ¤t_policy_hash, + &result, + ); + let mut policy_runtime_reconciled = false; + + // A local policy override is not coupled to the gateway policy + // snapshot, so its service registry can still be reconciled alone. + // Gateway policy snapshots, however, must install policy and registry + // as one generation below. + if !reloads_gateway_policy { + reconcile_middleware_registry( + &ctx.opa_engine, + &ctx.middleware_connector, + MiddlewareRegistryReconciliation { + desired_services: &result.supervisor_middleware_services, + authentication: MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + extension_credentials: &ctx.extension_credentials, + current_services: &mut current_middleware_services, + status: &mut middleware_registry_status, + }, + ) + .await; + if middleware_registry_status == MiddlewareRegistryStatus::Synchronized { + current_extension_authentication_enabled = result.extension_authentication_enabled; + } + } + + if !config_changed + && !provider_env_changed + && !policy_runtime_changed + && unchanged_policy_revision.is_none() + { + continue; + } + + if config_changed || provider_env_changed { + // Log which settings changed. + log_setting_changes(¤t_settings, &result.settings); + + // A posture change after a rejected update takes effect immediately. + // The compiled last-known-good engine remains available beneath a + // fail-closed quarantine, so an explicit retain_last_valid selection + // can reactivate it without accepting any part of the invalid policy. + if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { + let mode = result.policy_validation_failure_mode; + if mode != rejected.configured_mode { + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + mode, + has_last_valid_policy, + rejected.version, + &rejected.validation_error, + )?; + emit_policy_validation_failure( + &disposition, + rejected.version, + &rejected.policy_hash, + &rejected.validation_error, + ); + rejected.configured_mode = mode; + } + } + + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Other, "detected") + .unmapped("old_config_revision", serde_json::json!(current_config_revision)) + .unmapped("new_config_revision", serde_json::json!(result.config_revision)) + .unmapped("policy_changed", serde_json::json!(policy_changed)) + .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) + .message(format!( + "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", + result.config_revision + )) + .build()); + } + + if provider_env_changed { + match openshell_core::grpc_client::fetch_provider_environment( + &ctx.endpoint, + &ctx.sandbox_id, + ) + .await + { + Ok(env_result) => { + let provider_env_revision = env_result.provider_env_revision; + let install_result = ctx.provider_credentials.install_bound_environment( + provider_env_revision, + env_result.environment, + env_result.credential_expires_at_ms, + env_result.dynamic_credentials, + env_result.static_credential_bindings, + env_result.non_secret_environment_keys, + ); + if let Err(error) = install_result { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message(format!( + "Rejected provider environment refresh; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" + )) + .build() + ); + } else { + let env_count = + ctx.provider_credentials.child_env_with_gcp_resolved().len(); + current_provider_env_revision = provider_env_revision; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "provider_env_revision", + serde_json::json!(provider_env_revision) + ) + .message(format!( + "Provider environment refreshed [revision:{provider_env_revision} env_count:{env_count}]" + )) + .build() + ); + } + } + Err(e) => { + ctx.provider_credentials + .revoke_static_provider_environment(result.provider_env_revision); + warn!( + error = %e, + provider_env_revision = result.provider_env_revision, + "Settings poll: failed to refresh provider environment; static provider credentials were revoked; previous dynamic token grants remain active" + ); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "fail_closed") + .message( + "Provider environment refresh failed; static provider credentials were revoked; previous dynamic token grants remain active" + ) + .build() + ); + } + } + } + + if policy_runtime_changed { + let pid = ctx.entrypoint_pid.load(Ordering::Acquire); + let runtime_result = reload_gateway_policy_runtime( + &ctx.opa_engine, + result.policy.as_ref(), + pid, + MiddlewareReloadContext { + desired_services: &result.supervisor_middleware_services, + authentication: &MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + connector: &ctx.middleware_connector, + }, + ctx.transparent_tcp, + ) + .await; + + match runtime_result { + Ok(()) => { + policy_runtime_reconciled = true; + let policy = result + .policy + .as_ref() + .expect("successful runtime reload requires a policy payload"); + has_last_valid_policy = true; + rejected_policy_generation = None; + if policy_changed { + if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { + policy_local_ctx.set_current_policy(policy.clone()).await; + } + if result.global_policy_version > 0 { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .unmapped("global_version", serde_json::json!(result.global_policy_version)) + .message(format!( + "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", + result.policy_hash, + result.global_policy_version + )) + .build()); + } else { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + } + if result.version > 0 && result.policy_source == PolicySource::Sandbox { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + current_policy_version = result.version; + } + } else if recovering_rejected_policy + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); + current_policy_version = result.version; + } + + if middleware_registry_changed { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "supervisor_middleware_service_count", + serde_json::json!(result.supervisor_middleware_services.len()) + ) + .message(format!( + "Supervisor policy runtime reloaded atomically [service_count:{}]", + result.supervisor_middleware_services.len() + )) + .build()); + } + + current_policy_hash.clone_from(&result.policy_hash); + current_middleware_services.clone_from(&result.supervisor_middleware_services); + current_extension_authentication_enabled = + result.extension_authentication_enabled; + retain_extension_credentials( + &ctx.extension_credentials, + &result.supervisor_middleware_services, + result.extension_authentication_enabled, + ); + middleware_registry_status = MiddlewareRegistryStatus::Synchronized; + last_failed_runtime_revision = None; + } + Err(failure) => { + let failed_revision = FailedRuntimeRevision::new( + result.config_revision, + &result.policy_hash, + &failure, + ); + if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + let failure_mode = result.policy_validation_failure_mode; + match apply_gateway_runtime_reload_failure( + &ctx.opa_engine, + failure, + failure_mode, + has_last_valid_policy, + result.version, + )? { + GatewayRuntimeFailureDisposition::PolicyRejected { + error, + disposition, + } => { + emit_policy_validation_failure( + &disposition, + result.version, + &result.policy_hash, + &error, + ); + rejected_policy_generation = Some(RejectedPolicyGeneration { + version: result.version, + policy_hash: result.policy_hash.clone(), + validation_error: error.clone(), + configured_mode: failure_mode, + }); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error } => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(&error)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .message(format!( + "Supervisor middleware registry unavailable, keeping last-known-good policy runtime active [version:{} error:{error}]", + result.version + )) + .build()); + } + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error, + active_generation, + } => { + emit_transparent_tcp_expansion_rejection( + result.version, + &result.policy_hash, + active_generation, + &error, + ); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } + } + } + last_failed_runtime_revision = Some(failed_revision); + // Nothing was installed, so the registry status still + // describes the live registry. The retry is driven by the + // persisting hash/service-set mismatch (or an existing + // NeedsReconciliation), not by degrading the status here. + } + } + } + + if let Some(version) = unchanged_policy_revision_ready_to_ack( + unchanged_policy_revision, + policy_runtime_changed, + policy_runtime_reconciled, + ) { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::unchanged_loaded(version, result.policy_hash.clone()), + ); + current_policy_version = version; + } + + // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. + apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); + + // Apply the agent-proposals feature toggle. On a false→true transition + // we lazily install the skill so a sandbox that started with the flag + // off picks up the surface without a recreate. We never uninstall on + // a true→false transition: stale skill content on disk is harmless + // because route_request and agent_next_steps both gate on the live + // shared flag, so the agent that reads the skill will see 404s and an + // empty `next_steps` array regardless. + apply_agent_proposals_enabled( + &ctx.agent_proposals, + agent_proposals_enabled_from_settings(&result.settings), + "settings poll", + Some(result.config_revision), + skills::install_static_skills, + ); + + current_config_revision = result.config_revision; + if !reloads_gateway_policy { + current_policy_hash = result.policy_hash; + } + current_settings = result.settings; + } +} + +fn apply_ocsf_json_setting( + enabled: &AtomicBool, + settings: &std::collections::HashMap, +) { + use std::sync::atomic::Ordering; + + let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); + let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); + if new_ocsf != prev_ocsf { + info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); + } +} + +/// Extract a bool value from an effective setting, if present. +fn extract_bool_setting( + settings: &std::collections::HashMap, + key: &str, +) -> Option { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|v| match v { + setting_value::Value::BoolValue(b) => Some(*b), + _ => None, + }) +} + +fn agent_proposals_enabled_from_settings( + settings: &std::collections::HashMap, +) -> bool { + extract_bool_setting( + settings, + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, + ) + .unwrap_or(false) +} + +fn apply_agent_proposals_enabled( + agent_proposals: &AgentProposals, + enabled: bool, + source: &'static str, + config_revision: Option, + install_static_skills: impl FnOnce() -> Result, +) { + let previously_enabled = agent_proposals.swap_enabled(enabled); + if enabled == previously_enabled { + return; + } + + info!( + agent_policy_proposals_enabled = enabled, + source, config_revision, "agent-driven policy proposals toggled" + ); + + if enabled && !previously_enabled { + match install_static_skills() { + Ok(installed) => info!( + path = %installed.policy_advisor.display(), + "Installed sandbox agent skill on toggle-on" + ), + Err(error) => warn!( + error = %error, + "Failed to install sandbox agent skill on toggle-on" + ), + } + } +} + +/// Log individual setting changes between two snapshots. +fn log_setting_changes( + old: &std::collections::HashMap, + new: &std::collections::HashMap, +) { + for (key, new_es) in new { + let new_val = format_setting_value(new_es); + match old.get(key) { + Some(old_es) => { + let old_val = format_setting_value(old_es); + if old_val != new_val { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "updated") + .unmapped("key", serde_json::json!(key)) + .unmapped("old", serde_json::json!(old_val.clone())) + .unmapped("new", serde_json::json!(new_val.clone())) + .message(format!( + "Setting changed [key:{key} old:{old_val} new:{new_val}]" + )) + .build() + ); + } + } + None => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "enabled") + .unmapped("key", serde_json::json!(key)) + .unmapped("value", serde_json::json!(new_val.clone())) + .message(format!("Setting added [key:{key} value:{new_val}]")) + .build() + ); + } + } + } + for key in old.keys() { + if !new.contains_key(key) { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Disabled, "disabled") + .unmapped("key", serde_json::json!(key)) + .message(format!("Setting removed [key:{key}]")) + .build() + ); + } + } +} + +/// Format an `EffectiveSetting` value for log display. +fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { + use openshell_core::proto::setting_value; + match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { + None => "".to_string(), + Some(setting_value::Value::StringValue(v)) => v.clone(), + Some(setting_value::Value::BoolValue(v)) => v.to_string(), + Some(setting_value::Value::IntValue(v)) => v.to_string(), + Some(setting_value::Value::BytesValue(_)) => "".to_string(), + } +} + +#[cfg(test)] +#[allow( + clippy::needless_raw_string_hashes, + clippy::iter_on_single_items, + clippy::similar_names, + clippy::manual_string_new, + clippy::doc_markdown, + reason = "Test code: test fixtures often use idiomatic forms not flagged in production." +)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::BoolValue( + value, + )), + }), + scope: openshell_core::proto::SettingScope::Global.into(), + } + } + + #[test] + fn shared_ssh_socket_setting_is_explicit() { + assert!(shared_ssh_socket_value("1")); + assert!(shared_ssh_socket_value("true")); + assert!(shared_ssh_socket_value("TRUE")); + assert!(!shared_ssh_socket_value("0")); + assert!(!shared_ssh_socket_value("yes")); + } + + #[tokio::test] + async fn control_readiness_exists_only_while_guard_is_live() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("health.sock"); + let readiness = ControlReadiness::start(path.clone()).expect("start readiness listener"); + check_control_readiness(&path).expect("running supervisor accepts readiness probes"); + + drop(readiness); + tokio::task::yield_now().await; + assert!(check_control_readiness(&path).is_err()); + } + + #[test] + fn control_readiness_rejects_relative_path() { + let error = prepare_control_readiness_path(std::path::Path::new("health.sock")) + .expect_err("relative readiness path must be rejected"); + assert!(error.to_string().contains("must be absolute")); + } + + #[test] + fn main_exit_marker_atomically_replaces_previous_value() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("main-exited"); + std::fs::write(&marker, b"stale\n").unwrap(); + + persist_main_exit_marker(&marker, 23).unwrap(); + + assert_eq!(std::fs::read_to_string(&marker).unwrap(), "exit_code=23\n"); + assert!( + !directory + .path() + .join(format!(".main-exited.tmp-{}", std::process::id())) + .exists() + ); + } + + #[tokio::test] + async fn remote_access_plane_outlives_main_completion_until_teardown() { + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let retained = retain_remote_access_plane(std::future::pending(), async { + let _ = shutdown_rx.await; + }); + tokio::pin!(retained); + + assert!( + timeout(Duration::from_millis(10), &mut retained) + .await + .is_err(), + "access plane must remain live after canonical process completion" + ); + shutdown_tx.send(()).expect("request teardown"); + timeout(Duration::from_secs(1), &mut retained) + .await + .expect("teardown should release retained access plane") + .expect("clean teardown"); + } + + #[tokio::test] + async fn completion_retry_phase_is_cancelled_by_shutdown() { + let mut shutdown = Box::pin(std::future::ready(())); + assert!( + completion_phase_or_shutdown(std::future::pending(), shutdown.as_mut()).await, + "shutdown must cancel an indefinitely retrying completion phase" + ); + } + + #[test] + fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { + let agent_proposals = AgentProposals::default(); + let installs = AtomicUsize::new(0); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert_eq!(installs.load(Ordering::Relaxed), 1); + + apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), || { + installs.fetch_add(1, Ordering::Relaxed); + Ok(skills::InstalledSkills { + policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), + policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), + agents: None, + }) + }); + assert!(!agent_proposals.enabled()); + assert_eq!(installs.load(Ordering::Relaxed), 1); + } + + #[test] + fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { + let enabled = AtomicBool::new(false); + let mut settings = std::collections::HashMap::new(); + settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(enabled.load(Ordering::Relaxed)); + } + + #[test] + fn apply_ocsf_json_setting_disables_when_setting_is_unset() { + let enabled = AtomicBool::new(true); + let settings = std::collections::HashMap::new(); + + apply_ocsf_json_setting(&enabled, &settings); + + assert!(!enabled.load(Ordering::Relaxed)); + } + + #[test] + fn agent_proposals_setting_enables_from_initial_settings_snapshot() { + let mut settings = std::collections::HashMap::new(); + settings.insert( + openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), + effective_bool(true), + ); + + assert!(agent_proposals_enabled_from_settings(&settings)); + } + + #[test] + fn agent_proposals_setting_defaults_false_when_unset() { + let settings = std::collections::HashMap::new(); + + assert!(!agent_proposals_enabled_from_settings(&settings)); + } + + // ---- Policy disk discovery tests ---- + + #[test] + fn discover_policy_from_nonexistent_path_returns_restrictive_default() { + let path = std::path::Path::new("/nonexistent/policy.yaml"); + let policy = discover_policy_from_path(path); + // Restrictive default has no network policies. + assert!(policy.network_policies.is_empty()); + // It keeps filesystem restrictions while leaving identity to the + // active compute driver. + assert!(policy.filesystem.is_some()); + assert!(policy.process.is_none()); + } + + #[test] + fn discover_policy_from_valid_yaml_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +filesystem_policy: + include_workdir: false + read_only: + - /usr + read_write: + - /tmp +network_policies: + test: + name: test + endpoints: + - { host: example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + assert_eq!(policy.network_policies.len(), 1); + assert!(policy.network_policies.contains_key("test")); + let fs = policy.filesystem.unwrap(); + assert!(!fs.include_workdir); + } + + #[test] + fn discover_policy_from_invalid_yaml_returns_restrictive_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default. + assert!(policy.network_policies.is_empty()); + assert!(policy.filesystem.is_some()); + } + + #[test] + fn discover_policy_from_unsafe_yaml_falls_back_to_default() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("policy.yaml"); + std::fs::write( + &path, + r#" +version: 1 +process: + run_as_user: root + run_as_group: root +filesystem_policy: + include_workdir: true + read_only: + - /usr + read_write: + - /tmp +"#, + ) + .unwrap(); + + let policy = discover_policy_from_path(&path); + // Falls back to restrictive default because of root user. + assert!(policy.process.is_none()); + } + + #[test] + fn discover_policy_restrictive_default_blocks_network() { + // In cluster mode we keep proxy mode enabled so `inference.local` + // can always be routed through proxy/OPA controls. + let proto = openshell_policy::restrictive_default_policy(); + let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); + assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); + } + + // ---- Initial policy acknowledgement tests ---- + + fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::restrictive_default_policy() + } + + fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy( + r#" +version: 1 +network_policies: + redis: + name: redis + endpoints: + - host: redis.example.com + port: 6379 + protocol: tcp + binaries: + - path: /usr/bin/redis-cli +"#, + ) + .expect("parse TCP policy") + } + + fn settings_poll_result( + policy: Option, + version: u32, + source: openshell_core::proto::PolicySource, + ) -> openshell_core::grpc_client::SettingsPollResult { + openshell_core::grpc_client::SettingsPollResult { + policy, + version, + policy_hash: format!("hash-v{version}"), + config_revision: u64::from(version) * 100, + policy_source: source, + settings: std::collections::HashMap::new(), + global_policy_version: 0, + provider_env_revision: 0, + supervisor_middleware_services: Vec::new(), + workspace: String::new(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), + extension_authentication_enabled: false, + } + } + + #[derive(Clone)] + struct ScriptedPolicyGateway { + polls: Arc< + tokio::sync::Mutex< + tokio::sync::mpsc::UnboundedReceiver< + openshell_core::grpc_client::SettingsPollResult, + >, + >, + >, + reports: UnboundedSender<(u32, bool, String)>, + } + + #[tonic::async_trait] + impl PolicyGatewayClient for ScriptedPolicyGateway { + async fn poll_settings( + &self, + _sandbox_id: &str, + ) -> Result { + self.polls + .lock() + .await + .recv() + .await + .ok_or_else(|| miette::miette!("scripted policy poll channel closed")) + } + + async fn report_policy_status( + &self, + _sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.reports + .send((version, loaded, error.to_string())) + .map_err(|_| miette::miette!("scripted policy report channel closed")) + } + + fn workspace(&self) -> String { + "test-workspace".to_string() + } + } + + #[derive(Clone)] + struct CredentialRejectingPolicyGateway { + inner: ScriptedPolicyGateway, + credential_requests: Arc, + } + + #[tonic::async_trait] + impl PolicyGatewayClient for CredentialRejectingPolicyGateway { + async fn poll_settings( + &self, + sandbox_id: &str, + ) -> Result { + self.inner.poll_settings(sandbox_id).await + } + + async fn report_policy_status( + &self, + sandbox_id: &str, + version: u32, + loaded: bool, + error: &str, + ) -> Result<()> { + self.inner + .report_policy_status(sandbox_id, version, loaded, error) + .await + } + + async fn extension_credentials_for( + &self, + _services: &[openshell_core::proto::SupervisorMiddlewareService], + ) -> Result> + { + self.credential_requests.fetch_add(1, Ordering::SeqCst); + Err(miette::miette!( + "gateway extension authentication is unavailable" + )) + } + + fn workspace(&self) -> String { + self.inner.workspace() + } + } + + fn scripted_policy_gateway() -> ( + ScriptedPolicyGateway, + UnboundedSender, + tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + ) { + let (poll_tx, poll_rx) = tokio::sync::mpsc::unbounded_channel(); + let (report_tx, report_rx) = tokio::sync::mpsc::unbounded_channel(); + ( + ScriptedPolicyGateway { + polls: Arc::new(tokio::sync::Mutex::new(poll_rx)), + reports: report_tx, + }, + poll_tx, + report_rx, + ) + } + + fn policy_poll_test_context( + opa_engine: Arc, + loaded_policy_origin: LoadedPolicyOrigin, + middleware_connector: MiddlewareConnector, + ) -> PolicyPollLoopContext { + let (workspace_tx, _workspace_rx) = tokio::sync::watch::channel(String::new()); + PolicyPollLoopContext { + endpoint: String::new(), + sandbox_id: "sandbox-test".to_string(), + opa_engine, + loaded_policy_origin, + entrypoint_pid: Arc::new(AtomicU32::new(0)), + interval_secs: 0, + ocsf_enabled: Arc::new(AtomicBool::new(false)), + provider_credentials: ProviderCredentialState::from_child_env_snapshot( + 0, + std::collections::HashMap::new(), + ), + policy_local_ctx: None, + agent_proposals: AgentProposals::default(), + middleware_registry_status: MiddlewareRegistryStatus::Synchronized, + workspace_tx, + extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), + extension_authentication_enabled: false, + middleware_connector, + transparent_tcp: TransparentTcpReloadState::default(), + } + } + + async fn expect_policy_report( + reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + version: u32, + ) { + let report = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("policy report timed out") + .expect("policy reporter stopped"); + assert_eq!(report, (version, true, String::new())); + } + + async fn expect_no_policy_report( + reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, + ) { + assert!( + timeout(Duration::from_millis(50), reports.recv()) + .await + .is_err(), + "unexpected policy status report" + ); + } + + #[tokio::test] + async fn same_hash_poll_revision_is_acknowledged_once_without_opa_reload() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + + polls.send(v2.clone()).unwrap(); + expect_policy_report(&mut reports, 2).await; + polls.send(v2).unwrap(); + expect_no_policy_report(&mut reports).await; + + assert_eq!( + engine.current_generation(), + 0, + "same-hash acknowledgement must not reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { + let v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let v2 = settings_poll_result( + Some(proto_tcp_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let active_generation = engine.current_generation(); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.transparent_tcp = TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }; + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + let report = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("TCP rejection report timed out") + .expect("policy reporter stopped"); + + assert_eq!(report.0, 2); + assert!(!report.1); + assert!(report.2.contains("recreate the sandbox"), "{}", report.2); + assert!(report.2.contains("previous policy remains active")); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + handle.abort(); + } + + #[tokio::test] + async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "scripted-guard".to_string(), + grpc_endpoint: "http://scripted.invalid".to_string(), + ..Default::default() + }]; + + let connector_attempts = Arc::new(AtomicUsize::new(0)); + let (attempt_tx, mut attempt_rx) = tokio::sync::mpsc::unbounded_channel(); + let middleware_connector: MiddlewareConnector = { + let connector_attempts = connector_attempts.clone(); + Arc::new(move |_services, _authentication| { + let attempt = connector_attempts.fetch_add(1, Ordering::SeqCst) + 1; + attempt_tx.send(attempt).unwrap(); + Box::pin(async move { + if attempt == 1 { + Err(miette::miette!("scripted middleware connection failure")) + } else { + connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await + } + }) + }) + }; + + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + middleware_connector, + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + + polls.send(v2.clone()).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), attempt_rx.recv()) + .await + .unwrap(), + Some(1) + ); + expect_no_policy_report(&mut reports).await; + assert_eq!(engine.current_generation(), 0); + + polls.send(v2.clone()).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), attempt_rx.recv()) + .await + .unwrap(), + Some(2) + ); + expect_policy_report(&mut reports, 2).await; + assert_eq!(engine.current_generation(), 1); + + polls.send(v2).unwrap(); + expect_no_policy_report(&mut reports).await; + assert_eq!(connector_attempts.load(Ordering::SeqCst), 2); + handle.abort(); + } + + #[tokio::test] + async fn no_signer_capability_uses_legacy_middleware_connector_without_credentials() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "legacy-guard".to_string(), + grpc_endpoint: "http://legacy.invalid".to_string(), + ..Default::default() + }]; + assert!(!v2.extension_authentication_enabled); + + let (inner, polls, mut reports) = scripted_policy_gateway(); + let credential_requests = Arc::new(AtomicUsize::new(0)); + let client = CredentialRejectingPolicyGateway { + inner, + credential_requests: credential_requests.clone(), + }; + let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); + let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { + connector_tx + .send((authentication.credentials.len(), authentication.enabled)) + .unwrap(); + Box::pin(async move { + connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await + }) + }); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + connector, + ); + + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), connector_rx.recv()) + .await + .unwrap(), + Some((0, false)) + ); + expect_policy_report(&mut reports, 2).await; + assert_eq!(credential_requests.load(Ordering::SeqCst), 0); + handle.abort(); + } + + #[tokio::test] + async fn enabled_extension_authentication_keeps_credential_failure_fail_closed() { + let mut v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + v1.policy_hash = "same-policy".to_string(); + let mut v2 = v1.clone(); + v2.version = 2; + v2.config_revision = 200; + v2.extension_authentication_enabled = true; + v2.supervisor_middleware_services = + vec![openshell_core::proto::SupervisorMiddlewareService { + name: "authenticated-guard".to_string(), + grpc_endpoint: "https://guard.invalid".to_string(), + ..Default::default() + }]; + + let (inner, polls, mut reports) = scripted_policy_gateway(); + let credential_requests = Arc::new(AtomicUsize::new(0)); + let client = CredentialRejectingPolicyGateway { + inner, + credential_requests: credential_requests.clone(), + }; + let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); + let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { + connector_tx + .send((authentication.credentials.len(), authentication.enabled)) + .unwrap(); + Box::pin(async move { + if authentication.enabled && authentication.credentials.is_empty() { + Err(miette::miette!( + "missing authenticated middleware credential" + )) + } else { + connect_middleware_registry(&[], &authentication).await + } + }) + }); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let ctx = policy_poll_test_context( + engine, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + connector, + ); + + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + assert_eq!( + timeout(Duration::from_secs(1), connector_rx.recv()) + .await + .unwrap(), + Some((0, true)) + ); + expect_no_policy_report(&mut reports).await; + assert_eq!(credential_requests.load(Ordering::SeqCst), 1); + handle.abort(); + } + + async fn assert_poll_does_not_use_same_hash_acknowledgement( + initial: openshell_core::grpc_client::SettingsPollResult, + next: openshell_core::grpc_client::SettingsPollResult, + origin: LoadedPolicyOrigin, + initial_report: Option, + ) { + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let ctx = policy_poll_test_context(engine.clone(), origin, default_middleware_connector()); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(initial).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + + if let Some(version) = initial_report { + expect_policy_report(&mut reports, version).await; + } else { + expect_no_policy_report(&mut reports).await; + } + + polls.send(next).unwrap(); + expect_no_policy_report(&mut reports).await; + assert_eq!( + engine.current_generation(), + 0, + "negative same-hash scope must not reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn same_hash_ack_poll_loop_rejects_local_global_empty_equal_and_older_scopes() { + let mut sandbox_v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + sandbox_v1.policy_hash = "same-policy".to_string(); + let loaded_v1 = LoadedPolicyRevision::from_snapshot(&sandbox_v1); + let mut sandbox_v2 = sandbox_v1.clone(); + sandbox_v2.version = 2; + sandbox_v2.config_revision = 200; + + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + sandbox_v2.clone(), + LoadedPolicyOrigin::LocalOverride, + None, + ) + .await; + + let mut global_v2 = sandbox_v2.clone(); + global_v2.policy_source = openshell_core::proto::PolicySource::Global; + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + global_v2, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v1.clone()), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + let mut empty_v1 = sandbox_v1.clone(); + empty_v1.policy_hash.clear(); + let empty_loaded = LoadedPolicyRevision::from_snapshot(&empty_v1); + let mut empty_v2 = sandbox_v2.clone(); + empty_v2.policy_hash.clear(); + assert_poll_does_not_use_same_hash_acknowledgement( + empty_v1, + empty_v2, + LoadedPolicyOrigin::Gateway { + revision: Some(empty_loaded), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v1.clone(), + sandbox_v1.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v1.clone()), + has_last_valid_policy: true, + }, + Some(1), + ) + .await; + + let loaded_v2 = LoadedPolicyRevision::from_snapshot(&sandbox_v2); + assert_poll_does_not_use_same_hash_acknowledgement( + sandbox_v2, + sandbox_v1, + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_v2), + has_last_valid_policy: true, + }, + Some(2), + ) + .await; + } + + #[tokio::test] + async fn changed_hash_poll_uses_normal_opa_reload_and_status_path() { + let v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let v2 = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + expect_policy_report(&mut reports, 2).await; + assert_eq!( + engine.current_generation(), + 1, + "changed policy content must still reload OPA" + ); + handle.abort(); + } + + #[tokio::test] + async fn failed_external_startup_registry_build_preserves_installed_builtins() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let builtins_generation = engine.current_generation(); + assert_eq!(builtins_generation, 1); + + let invalid_external = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_payload_bytes: 1024, + ..Default::default() + }; + connect_middleware_registry(&[invalid_external], &MiddlewareAuthentication::default()) + .await + .expect_err("unavailable external service must not replace built-ins"); + + assert_eq!(engine.current_generation(), builtins_generation); + } + + #[tokio::test] + async fn unavailable_middleware_reload_keeps_last_known_good_runtime_active() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let active_generation = engine.current_generation(); + let unavailable_service = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_payload_bytes: 1024, + ..Default::default() + }; + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[unavailable_service], + authentication: &MiddlewareAuthentication::default(), + registry_changed: true, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState::default(), + ) + .await + .expect_err("unavailable middleware must fail candidate preparation"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("middleware failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[tokio::test] + async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + let active_generation = engine.current_generation(); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }, + ) + .await + .expect_err("TCP expansion must require startup substrate"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("runtime prerequisite failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + active_generation: generation, + .. + } if generation == active_generation + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[tokio::test] + async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, + TransparentTcpReloadState::default(), + ) + .await + .expect_err("unsupported runtime must reject TCP expansion"); + + assert!(matches!( + failure, + GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) + )); + assert_eq!(engine.current_generation(), 0); + } + + #[test] + fn policy_rejection_after_middleware_outage_is_not_deduplicated() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let middleware_failure = GatewayRuntimeReloadError::MiddlewareRegistry(miette::miette!( + "middleware service unavailable" + )); + let first_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &middleware_failure); + let middleware_disposition = apply_gateway_runtime_reload_failure( + &engine, + middleware_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + + assert!(matches!( + middleware_disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert!(engine.fail_closed_reason().is_none()); + + let policy_failure = GatewayRuntimeReloadError::PolicyValidation(miette::miette!( + "conflicting endpoint metadata" + )); + let second_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &policy_failure); + assert_ne!( + first_failure, second_failure, + "a changed failure class for the same candidate must be handled" + ); + + let policy_disposition = apply_gateway_runtime_reload_failure( + &engine, + policy_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + assert!(matches!( + policy_disposition, + GatewayRuntimeFailureDisposition::PolicyRejected { .. } + )); + assert!(engine.fail_closed_reason().is_some()); + } + + #[test] + fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { + let services = Vec::new(); + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + } + + #[test] + fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &no_services, + &no_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v1", + &no_services, + &desired_services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!gateway_policy_runtime_needs_reconciliation( + false, + "local-policy", + "hash-v2", + &no_services, + &desired_services, + MiddlewareRegistryStatus::NeedsReconciliation, + )); + } + + #[test] + fn policy_only_change_does_not_rebuild_middleware_registry() { + let services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + // The runtime must reconcile, but the registry (and therefore + // middleware reachability) is not part of that reconciliation. + assert!(gateway_policy_runtime_needs_reconciliation( + true, + "hash-v1", + "hash-v2", + &services, + &services, + MiddlewareRegistryStatus::Synchronized, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &services, + &services, + )); + } + + #[test] + fn registry_rebuild_requires_service_set_change_or_degraded_registry() { + let no_services = Vec::new(); + let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { + name: "guard".into(), + ..Default::default() + }]; + + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &no_services, + &desired_services, + )); + assert!(middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::NeedsReconciliation, + &desired_services, + &desired_services, + )); + assert!(!middleware_registry_needs_rebuild( + MiddlewareRegistryStatus::Synchronized, + &desired_services, + &desired_services, + )); + } + + #[test] + fn initial_ack_candidate_matches_sandbox_revision() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) + .expect("sandbox-sourced matching revision should be acknowledged"); + + assert_eq!(ack.version, 2); + assert_eq!(ack.policy_hash, "hash-v2"); + assert_eq!(ack.config_revision, 200); + } + + #[test] + fn initial_ack_candidate_ignores_global_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Global, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_version_zero() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 0, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&canonical); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_ignores_local_file_mode() { + // Local-file mode retains no proto policy, so there is nothing to + // acknowledge to the gateway. + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(None, &canonical).is_none()); + } + + #[test] + fn initial_ack_candidate_rejects_mismatched_identity() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); + } + + #[test] + fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { + let loaded_snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); + let mut newer = proto_policy_fixture(); + newer.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule::default(), + ); + let canonical = + settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); + let canonical = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "hash-provider-change".to_string(), + config_revision: loaded.config_revision + 1, + ..canonical + }; + + assert_eq!( + initial_poll_disposition( + &LoadedPolicyOrigin::Gateway { + revision: Some(loaded), + has_last_valid_policy: true, + }, + &canonical, + ), + InitialPollDisposition::Reconcile + ); + } + + #[test] + fn initial_poll_tracks_local_override_without_reconciliation() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + + assert_eq!( + initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), + InitialPollDisposition::TrackOnly + ); + assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); + } + + #[test] + fn initial_poll_reconciles_unbound_gateway_policy() { + let canonical = settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let origin = LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }; + + assert_eq!( + initial_poll_disposition(&origin, &canonical), + InitialPollDisposition::Reconcile + ); + assert!(origin.allows_gateway_policy_reload()); + } + + #[test] + fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { + let sandbox_result = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "same-policy".to_string(), + ..settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ) + }; + + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "same-policy", &sandbox_result), + Some(2) + ); + assert_eq!( + unchanged_policy_revision_candidate(true, false, 2, "same-policy", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate( + true, + false, + 1, + "different-policy", + &sandbox_result, + ), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(false, false, 1, "same-policy", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "", &sandbox_result), + None + ); + assert_eq!( + unchanged_policy_revision_candidate(true, true, 1, "same-policy", &sandbox_result), + None + ); + + let global_result = openshell_core::grpc_client::SettingsPollResult { + policy_hash: "same-policy".to_string(), + ..settings_poll_result( + Some(proto_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Global, + ) + }; + assert_eq!( + unchanged_policy_revision_candidate(true, false, 1, "same-policy", &global_result), + None + ); + } + + #[test] + fn unchanged_policy_revision_waits_for_required_runtime_reconciliation() { + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), false, false), + Some(2), + "a same-hash revision needs no OPA reload" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), true, false), + None, + "failed runtime reconciliation must keep the revision pending" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(Some(2), true, true), + Some(2), + "successful runtime reconciliation permits acknowledgement" + ); + assert_eq!( + unchanged_policy_revision_ready_to_ack(None, false, true), + None, + "runtime success cannot manufacture a revision candidate" + ); + } + + #[test] + fn credential_gating_unavailable_for_local_override_with_credentials() { + assert!(credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + true, + true + )); + } + + #[test] + fn credential_gating_available_without_local_override_or_credentials() { + // A gateway policy is stamped with provenance, so the gates apply. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }, + true, + true + )); + // No provider credentials means there is nothing to leak. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + false, + true + )); + // Without networking the proxy never evaluates endpoint provenance. + assert!(!credential_gating_unavailable( + &LoadedPolicyOrigin::LocalOverride, + true, + false + )); + } + + #[test] + fn policy_status_outbox_preserves_all_revision_order() { + let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); + for version in 1..=128 { + enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); + } + + for version in 1..=128 { + assert_eq!( + receiver.try_recv().unwrap(), + PolicyStatusUpdate::loaded(version) + ); + } + } + + #[test] + fn settings_snapshot_carries_workspace_for_policy_sync() { + let mut snapshot = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + snapshot.workspace = "beta".to_string(); + + let revision = LoadedPolicyRevision::from_snapshot(&snapshot); + assert_eq!(revision.version, 1); + assert_eq!( + snapshot.workspace, "beta", + "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" + ); + } + #[test] + fn fail_closed_validation_failure_deactivates_previous_generation() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(!disposition.previous_policy_active); + assert!(disposition.active_generation > previous_generation); + assert!( + engine + .fail_closed_reason() + .expect("quarantine reason") + .contains("candidate version 7 rejected") + ); + } + + #[test] + fn retain_validation_failure_keeps_previous_generation_active() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let quarantined = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 6, + "conflicting tls metadata", + ) + .unwrap(); + assert!(!quarantined.previous_policy_active); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(disposition.previous_policy_active); + assert!(disposition.active_generation > quarantined.active_generation); + assert!(disposition.active_generation > previous_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[test] + fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + false, + 1, + "conflicting tls metadata", + ) + .unwrap(); + + assert_eq!( + disposition.configured_mode, + PolicyValidationFailureMode::RetainLastValid + ); + assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); + assert!(!disposition.previous_policy_active); + assert!(engine.fail_closed_reason().is_some()); + + let [config, _] = policy_validation_failure_events( + &disposition, + 1, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "retain_last_valid" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + } + + #[test] + fn validation_failure_ocsf_states_whether_previous_policy_is_active() { + let fail_closed = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::FailClosed, + mode: PolicyValidationFailureMode::FailClosed, + previous_policy_active: false, + active_generation: 9, + }; + let [config, finding] = policy_validation_failure_events( + &fail_closed, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["class_uid"], 5019); + assert_eq!(config["status"], "Failure"); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "fail_closed" + ); + assert_eq!(config["unmapped"]["previous_policy_active"], false); + assert_eq!( + config["unmapped"]["validation_error"], + "conflicting tls metadata" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("error:conflicting tls metadata") + ); + + let finding = finding.to_json().unwrap(); + assert_eq!(finding["class_uid"], 2004); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + + let retained = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::RetainLastValid, + mode: PolicyValidationFailureMode::RetainLastValid, + previous_policy_active: true, + active_generation: 4, + }; + let [config, _] = policy_validation_failure_events( + &retained, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["previous_policy_active"], true); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS active") + ); + } +} diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs new file mode 100644 index 0000000000..a33e652a46 --- /dev/null +++ b/crates/openshell-supervisor/src/main.rs @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! `OpenShell` supervisor executable. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_isolation_interface::contract::TopologyDescriptor; +use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _}; + +const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; +const HEALTH_SUBCOMMAND: &str = "health"; + +#[derive(Parser, Debug)] +#[command(name = "openshell-supervisor health")] +struct HealthArgs { + /// Private supervisor readiness socket. + #[arg(long, env = "OPENSHELL_HEALTH_SOCKET_PATH")] + socket: PathBuf, +} + +#[derive(Parser, Debug)] +#[command(name = "openshell-supervisor")] +#[command(version = openshell_core::VERSION)] +#[command(about = "OpenShell policy and workload supervisor")] +#[allow(clippy::struct_excessive_bools)] +struct Args { + /// Command to execute as the canonical workload process. + #[arg(trailing_var_arg = true)] + command: Vec, + + #[arg(long, short)] + workdir: Option, + + #[arg(long, short, default_value = "0")] + timeout: u64, + + #[arg(long, short = 'i')] + interactive: bool, + + #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] + sandbox_id: Option, + + #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] + sandbox: Option, + + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: Option, + + #[arg(long, env = "OPENSHELL_POLICY_RULES")] + policy_rules: Option, + + #[arg(long, env = "OPENSHELL_POLICY_DATA")] + policy_data: Option, + + #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] + ssh_socket_path: Option, + + #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] + inference_routes: Option, + + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, + + /// Create the private readiness socket after boundary and gateway attach. + #[arg(long, env = "OPENSHELL_HEALTH_SOCKET_PATH")] + health_socket_path: Option, + + #[arg(long)] + upstream_proxy: Option, + + /// Driver-pinned TCP dial address for the configured upstream proxy. + #[arg(long)] + upstream_proxy_dial_ip: Option, + + #[arg(long)] + upstream_no_proxy: Option, + + #[arg(long)] + upstream_proxy_auth_file: Option, + + #[arg(long)] + upstream_proxy_auth_allow_insecure: bool, + + #[arg(long)] + upstream_proxy_connect_by_hostname: bool, + + #[arg(long)] + upstream_proxy_ca_bundle: Option, + + #[arg(long)] + topology_backend_name: String, + + #[arg(long)] + topology_payload_file: PathBuf, + + #[arg(long, hide = true)] + main_exit_marker: Option, +} + +fn topology(args: &Args) -> Result { + let payload = std::fs::read(&args.topology_payload_file).map_err(|error| { + miette::miette!( + "read topology payload {}: {error}", + args.topology_payload_file.display() + ) + })?; + Ok(TopologyDescriptor { + backend_name: args.topology_backend_name.clone(), + payload, + }) +} + +fn validate_main_exit_marker(marker: Option<&Path>) -> Result<()> { + if let Some(marker) = marker + && !marker.is_absolute() + { + return Err(miette::miette!( + "--main-exit-marker must be an absolute path" + )); + } + Ok(()) +} + +fn main() -> Result<()> { + let raw_args = std::env::args().collect::>(); + if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .into_diagnostic()?; + return runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; + std::process::exit(exit); + }); + } + if raw_args.get(1).map(String::as_str) == Some(HEALTH_SUBCOMMAND) { + let args = HealthArgs::parse_from(&raw_args[1..]); + return openshell_supervisor::check_control_readiness(&args.socket); + } + + let args = Args::parse(); + validate_main_exit_marker(args.main_exit_marker.as_deref())?; + let topology = topology(&args)?; + + let file_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + (writer, guard) + }); + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = + openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = + jsonl_logging.map_or((None, None), |(layer, guard)| (Some(layer), Some(guard))); + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(EnvFilter::new("info")), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with(push_layer) + .init(); + warn!("Could not open /var/log for log rotation; using stderr-only logging"); + (None, None) + }; + + let workdir = args.workdir.clone(); + let (command, interactive, await_main_process_attachment) = if !args.command.is_empty() { + (args.command, args.interactive, false) + } else if let Ok(json) = std::env::var(openshell_core::sandbox_env::MAIN_PROCESS_SPEC) { + let config = openshell_core::sandbox_env::MainProcessConfig::decode(&json) + .map_err(|error| miette::miette!("{error}"))?; + ( + config.command, + config.tty, + config.await_main_process_attachment, + ) + } else { + let config = openshell_core::sandbox_env::MainProcessConfig::scratch(); + ( + config.command, + config.tty, + config.await_main_process_attachment, + ) + }; + info!(command = ?command, "Starting sandbox supervision"); + + let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { + https_proxy: args.upstream_proxy, + proxy_dial_ip: args.upstream_proxy_dial_ip, + no_proxy: args.upstream_no_proxy, + proxy_auth_file: args.upstream_proxy_auth_file, + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, + proxy_ca_bundle: args.upstream_proxy_ca_bundle, + }; + let admitted_isolation_backend = + std::env::var(openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND).ok(); + + openshell_supervisor::run_sandbox( + command, + workdir, + args.timeout, + interactive, + await_main_process_attachment, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_socket_path, + args.health_socket_path, + args.inference_routes, + ocsf_enabled, + upstream_proxy_args, + topology, + admitted_isolation_backend, + args.main_exit_marker, + ) + .await + })?; + + std::process::exit(exit_code); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn role_specific_cli_has_no_mode_switch() { + let directory = tempfile::tempdir().expect("temporary topology directory"); + let topology_path = directory.path().join("topology.json"); + std::fs::write(&topology_path, [0]).expect("write topology payload"); + let args = Args::try_parse_from([ + "openshell-supervisor", + "--topology-backend-name", + "test", + "--topology-payload-file", + topology_path.to_str().expect("UTF-8 topology path"), + ]) + .expect("supervisor arguments"); + assert_eq!(topology(&args).expect("topology").payload, vec![0]); + } + + #[test] + fn topology_payload_is_mandatory() { + assert!( + Args::try_parse_from(["openshell-supervisor", "--topology-backend-name", "test"]) + .is_err() + ); + } + + #[test] + fn completion_marker_must_be_absolute() { + assert!(validate_main_exit_marker(Some(Path::new("relative"))).is_err()); + assert!(validate_main_exit_marker(Some(Path::new("/run/openshell/main-exit"))).is_ok()); + } +} diff --git a/crates/openshell-sandbox/src/mechanistic_mapper.rs b/crates/openshell-supervisor/src/mechanistic_mapper.rs similarity index 99% rename from crates/openshell-sandbox/src/mechanistic_mapper.rs rename to crates/openshell-supervisor/src/mechanistic_mapper.rs index 9be5f8e438..186cc68e02 100644 --- a/crates/openshell-sandbox/src/mechanistic_mapper.rs +++ b/crates/openshell-supervisor/src/mechanistic_mapper.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Mechanistic policy mapper — deterministically converts denial summaries into +//! Supervisor policy mapper — deterministically converts denial summaries into //! draft `NetworkPolicyRule` proposals. //! //! This is the "zero-LLM" baseline for policy recommendations. It inspects diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index d515fd70b1..033371e773 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -3,19 +3,29 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# The static sandbox binary is staged at: -# deploy/docker/.build/prebuilt-binaries//openshell-sandbox +# The static sandbox and supervisor binaries are staged under +# deploy/docker/.build/prebuilt-binaries//. # -# Alpine supplies nftables and iptables for pod-namespace egress enforcement. +# Alpine supplies the trusted helper runtime used by VM guest init for its +# one-shot filesystem and loopback setup. Network enforcement stays outside +# this runtime; the capability-free sandbox never receives nftables or +# iptables tooling. FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 \ + && mkdir -p /openshell-runtime \ + && cp -aL /bin /sbin /lib /usr/bin /usr/sbin /usr/lib /openshell-runtime/ \ + && mkdir -p /openshell-runtime/etc \ + && if [ -d /etc/iproute2 ]; then cp -aL /etc/iproute2 /openshell-runtime/etc/; fi \ + && test -x /openshell-runtime/bin/sh \ + && test -x /openshell-runtime/sbin/ip # Keep the binary root-owned for Podman image-volume mounts and executable by # the Kubernetes network sidecar's non-root proxy UID. COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-supervisor /openshell-supervisor -ENTRYPOINT ["/openshell-sandbox"] +ENTRYPOINT ["/openshell-supervisor"] diff --git a/e2e/rust/tests/bypass_detection.rs b/e2e/rust/tests/bypass_detection.rs index 56415a554b..569e3a60d6 100644 --- a/e2e/rust/tests/bypass_detection.rs +++ b/e2e/rust/tests/bypass_detection.rs @@ -1,13 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Verify that sandbox bypass detection provides fast-fail UX: direct TCP -//! connections that skip the HTTP CONNECT proxy are rejected with -//! ECONNREFUSED (immediate) rather than hanging until a network timeout. +//! Verify that direct TCP bypass attempts fail promptly at the sandbox +//! syscall boundary instead of reaching the runtime's external network. //! //! This test is implementation-agnostic — it validates the observable -//! behavior (fast rejection) regardless of whether the kernel rules are -//! installed via iptables or nftables. +//! behavior rather than a particular packet-filter implementation. #![cfg(feature = "e2e")] @@ -15,13 +13,12 @@ use openshell_e2e::harness::sandbox::SandboxGuard; /// Python script that attempts a raw TCP connect bypassing the proxy. /// -/// `socket.connect()` does not honor HTTP_PROXY — it goes directly through -/// the kernel, hitting the OUTPUT chain REJECT rule. The script reports the -/// outcome and wall-clock time so the test can assert on both. +/// `socket.connect()` does not honor proxy environment variables. The script +/// reports the outcome and wall-clock time so the test can assert that the +/// sandbox's seccomp mediation blocks it before the outer fence is needed. /// /// Target 198.51.100.1 is RFC 5737 TEST-NET-2 — documentation-only address -/// space that will never route. This doesn't matter because the REJECT rule -/// fires in the OUTPUT chain before the packet reaches the network. +/// space that will never route. fn bypass_attempt_script() -> &'static str { r#" import json, socket, time @@ -36,6 +33,8 @@ try: s.close() except ConnectionRefusedError: result = "refused" +except PermissionError: + result = "denied" except socket.timeout: result = "timeout" except OSError as e: @@ -46,8 +45,8 @@ print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=Tru "# } -/// A direct TCP connection bypassing the proxy should be rejected -/// immediately (ECONNREFUSED), not hang until a timeout. +/// A direct TCP connection bypassing supervision should be denied without +/// waiting for the socket's network timeout. #[tokio::test] async fn bypass_attempt_is_rejected_fast() { let guard = SandboxGuard::create(&["--", "python3", "-c", bypass_attempt_script()]) @@ -67,16 +66,14 @@ async fn bypass_attempt_is_rejected_fast() { let elapsed_ms = parsed["elapsed_ms"].as_u64().unwrap(); assert_eq!( - result, "refused", - "expected connection refused (REJECT rule), got '{result}' after {elapsed_ms}ms.\n\ - If 'timeout': REJECT rules may not be installed in the sandbox netns.\n\ + result, "denied", + "expected seccomp mediation to deny the direct connect, got '{result}' after {elapsed_ms}ms.\n\ Full output:\n{}", guard.create_output ); assert!( - elapsed_ms < 3000, - "bypass rejection took {elapsed_ms}ms — expected < 3000ms.\n\ - Fast rejection requires REJECT rules in the sandbox OUTPUT chain." + elapsed_ms < 8000, + "bypass rejection took {elapsed_ms}ms — expected < 8000ms." ); } diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index 25d5516543..9d4cf0325a 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -557,27 +557,13 @@ fn body_client_script(port: u16) -> String { r#" import os import socket -import urllib.parse host = {TEST_HOST:?} port = {port} token = os.environ[{TOKEN_ENV:?}] -proxy_url = next(os.environ[name] for name in - ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - if os.environ.get(name)) -proxy = urllib.parse.urlparse(proxy_url) -with socket.create_connection((proxy.hostname, proxy.port or 80), timeout=10) as sock: +with socket.create_connection((host, port), timeout=10) as sock: target = f"{{host}}:{{port}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode("ascii")) - response = b"" - while b"\r\n\r\n" not in response: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk - if not response.startswith(b"HTTP/1.1 200"): - raise RuntimeError("CONNECT failed") body = ("prefix-" + token + "-suffix").encode("utf-8") request = ( f"POST /token HTTP/1.1\r\nHost: {{target}}\r\n" @@ -606,14 +592,9 @@ import base64 import os import socket import struct -import urllib.parse host = {TEST_HOST:?} port = {port} -proxy_url = next(os.environ[name] for name in - ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - if os.environ.get(name)) -proxy = urllib.parse.urlparse(proxy_url) def recv_until(sock, marker): data = b"" @@ -633,11 +614,8 @@ def recv_exact(sock, size): data += chunk return data -with socket.create_connection((proxy.hostname, proxy.port or 80), timeout=10) as sock: +with socket.create_connection((host, port), timeout=10) as sock: target = f"{{host}}:{{port}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode("ascii")) - if not recv_until(sock, b"\r\n\r\n").startswith(b"HTTP/1.1 200"): - raise RuntimeError("CONNECT failed") key = base64.b64encode(os.urandom(16)).decode("ascii") request = ( f"GET /ws HTTP/1.1\r\nHost: {{target}}\r\n" diff --git a/e2e/rust/tests/forward_proxy_graphql_l7.rs b/e2e/rust/tests/forward_proxy_graphql_l7.rs index bcb2b68052..2a1b06d277 100644 --- a/e2e/rust/tests/forward_proxy_graphql_l7.rs +++ b/e2e/rust/tests/forward_proxy_graphql_l7.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! E2E tests for GraphQL L7 inspection across both proxy entry points. +//! E2E tests for GraphQL L7 inspection through transparent interception. //! //! The upstream server deliberately does not implement GraphQL. `OpenShell` //! parses and enforces GraphQL before forwarding, so any HTTP server that @@ -130,7 +130,7 @@ network_policies: #[tokio::test] #[allow(clippy::too_many_lines)] -async fn graphql_l7_enforces_allow_and_deny_rules_on_forward_and_connect_paths() { +async fn graphql_l7_enforces_high_level_and_raw_transparent_paths() { let server = start_test_server().await.expect("start test server"); let policy = write_graphql_policy(&server.host, server.port).expect("write custom policy"); let policy_path = policy @@ -142,7 +142,6 @@ async fn graphql_l7_enforces_allow_and_deny_rules_on_forward_and_connect_paths() let script = format!( r#" import json -import os import socket import time import urllib.error @@ -231,26 +230,14 @@ def retry_forward_allowed(label, request_fn): time.sleep(0.3) return last_status -def proxy_parts(*names): - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - -def forward_proxy_parts(): - return proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - -def connect_proxy_parts(): - return proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - def forward_chunked_status(query): - proxy_host, proxy_port = forward_proxy_parts() target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: + with socket.create_connection((HOST, PORT), timeout=15) as sock: request = ( - f"POST http://{{target}}/graphql HTTP/1.1\r\n" + f"POST /graphql HTTP/1.1\r\n" f"Host: {{target}}\r\n" f"Content-Type: application/json\r\n" f"Transfer-Encoding: chunked\r\n" @@ -297,22 +284,11 @@ def status_code(response, label): DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error -def connect_http_status(label, request): - proxy_host, proxy_port = connect_proxy_parts() - target = f"{{HOST}}:{{PORT}}" - +def raw_http_status(label, request): last_error = None for attempt in range(5): try: - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: - sock.sendall( - f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode() - ) - connect_response = read_until(sock, b"\r\n\r\n") - connect_code = status_code(connect_response, f"{{label}}_connect") - if connect_code != 200: - return connect_code - + with socket.create_connection((HOST, PORT), timeout=15) as sock: sock.sendall(request) sock.shutdown(socket.SHUT_WR) response = read_until(sock, b"\r\n\r\n") @@ -324,7 +300,7 @@ def connect_http_status(label, request): raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") -def connect_status(query, label): +def raw_status(query, label): target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() @@ -336,9 +312,9 @@ def connect_status(query, label): f"Connection: close\r\n" f"\r\n" ).encode() + body - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_get_status(query, label): +def raw_get_status(query, label): target = f"{{HOST}}:{{PORT}}" encoded = urllib.parse.urlencode({{"query": query}}) @@ -348,9 +324,9 @@ def connect_get_status(query, label): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_duplicate_get_status(): +def raw_duplicate_get_status(): target = f"{{HOST}}:{{PORT}}" safe = urllib.parse.quote_plus(QUERY_VIEWER) unsafe = urllib.parse.quote_plus(MUTATION_DELETE) @@ -361,9 +337,9 @@ def connect_duplicate_get_status(): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status("connect_duplicate_get_denied", request) + return raw_http_status("raw_duplicate_get_denied", request) -def connect_persisted_get_status(hash_value, label): +def raw_persisted_get_status(hash_value, label): target = f"{{HOST}}:{{PORT}}" extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) @@ -374,9 +350,9 @@ def connect_persisted_get_status(hash_value, label): f"Connection: close\r\n" f"\r\n" ).encode() - return connect_http_status(label, request) + return raw_http_status(label, request) -def connect_chunked_status(query): +def raw_chunked_status(query): target = f"{{HOST}}:{{PORT}}" body = json.dumps({{"query": query}}).encode() chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" @@ -389,7 +365,7 @@ def connect_chunked_status(query): f"Connection: close\r\n" f"\r\n" ).encode() + chunk - return connect_http_status("connect_chunked_query_allowed", request) + return raw_http_status("raw_chunked_query_allowed", request) results = {{ "forward_query_allowed": retry_forward_allowed("forward_query_allowed", lambda: forward_status(QUERY_VIEWER)), @@ -401,15 +377,15 @@ results = {{ "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), "forward_mutation_allowed": retry_forward_allowed("forward_mutation_allowed", lambda: forward_status(MUTATION_CREATE)), "forward_deny_rule_denied": forward_status(MUTATION_DELETE), - "connect_query_allowed": connect_status(QUERY_VIEWER, "connect_query_allowed"), - "connect_get_query_allowed": connect_get_status(QUERY_VIEWER, "connect_get_query_allowed"), - "connect_duplicate_get_denied": connect_duplicate_get_status(), - "connect_persisted_get_allowed": connect_persisted_get_status("abc123", "connect_persisted_get_allowed"), - "connect_unregistered_persisted_get_denied": connect_persisted_get_status("missing", "connect_unregistered_persisted_get_denied"), - "connect_chunked_query_allowed": connect_chunked_status(QUERY_VIEWER), - "connect_unlisted_field_denied": connect_status(QUERY_REPOSITORY, "connect_unlisted_field_denied"), - "connect_mutation_allowed": connect_status(MUTATION_CREATE, "connect_mutation_allowed"), - "connect_deny_rule_denied": connect_status(MUTATION_DELETE, "connect_deny_rule_denied"), + "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), + "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), + "raw_duplicate_get_denied": raw_duplicate_get_status(), + "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), + "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), + "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), + "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), + "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), + "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), }} results.update(DETAILS) print(json.dumps(results, sort_keys=True)) @@ -432,15 +408,15 @@ print(json.dumps(results, sort_keys=True)) ("forward_unlisted_field_denied", 403), ("forward_mutation_allowed", 200), ("forward_deny_rule_denied", 403), - ("connect_query_allowed", 200), - ("connect_get_query_allowed", 200), - ("connect_duplicate_get_denied", 403), - ("connect_persisted_get_allowed", 200), - ("connect_unregistered_persisted_get_denied", 403), - ("connect_chunked_query_allowed", 200), - ("connect_unlisted_field_denied", 403), - ("connect_mutation_allowed", 200), - ("connect_deny_rule_denied", 403), + ("raw_query_allowed", 200), + ("raw_get_query_allowed", 200), + ("raw_duplicate_get_denied", 403), + ("raw_persisted_get_allowed", 200), + ("raw_unregistered_persisted_get_denied", 403), + ("raw_chunked_query_allowed", 200), + ("raw_unlisted_field_denied", 403), + ("raw_mutation_allowed", 200), + ("raw_deny_rule_denied", 403), ] { let expected_fragment = format!(r#""{key}": {expected}"#); assert!( diff --git a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs index 174e3b6db9..b46dac1313 100644 --- a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs +++ b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! E2E tests for JSON-RPC L7 inspection across both proxy entry points. +//! E2E tests for JSON-RPC L7 inspection through transparent interception. //! //! The upstream server deliberately does not implement JSON-RPC. `OpenShell` //! parses and enforces JSON-RPC before forwarding, so any HTTP server that @@ -187,7 +187,7 @@ network_policies: #[tokio::test] #[allow(clippy::too_many_lines)] -async fn jsonrpc_l7_enforces_method_rules_on_forward_and_connect_paths() { +async fn jsonrpc_l7_enforces_high_level_and_raw_transparent_paths() { let server = start_test_server(RULES_TEST_SERVER_ALIAS) .await .expect("start test server"); @@ -201,25 +201,15 @@ async fn jsonrpc_l7_enforces_method_rules_on_forward_and_connect_paths() { let script = format!( r#" import json -import os import socket import time import urllib.error -import urllib.parse import urllib.request HOST = {host:?} PORT = {port} DETAILS = {{ "debug_target": {{"host": HOST, "port": PORT}}, - "debug_proxy_env": {{ - "http_proxy": os.environ.get("http_proxy"), - "https_proxy": os.environ.get("https_proxy"), - "HTTP_PROXY": os.environ.get("HTTP_PROXY"), - "HTTPS_PROXY": os.environ.get("HTTPS_PROXY"), - "NO_PROXY": os.environ.get("NO_PROXY"), - "no_proxy": os.environ.get("no_proxy"), - }}, }} def text(data): @@ -291,11 +281,6 @@ def post_invalid_json(label): except urllib.error.HTTPError as error: return record_http_error(label, error, text(encoded)) -def proxy_parts(*names): - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_until(sock, marker): data = b"" while marker not in data: @@ -339,21 +324,11 @@ def record_raw_response(label, response, body=b""): DETAILS[f"{{label}}_body"] = text(body) return code -def connect_http_status(label, request): - proxy_host, proxy_port = proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - target = f"{{HOST}}:{{PORT}}" - +def raw_http_status(label, request): last_error = None for attempt in range(5): try: - with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: - sock.sendall( - f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode() - ) - connect_response = read_until(sock, b"\r\n\r\n") - connect_code = record_raw_response(f"{{label}}_connect", connect_response) - if connect_code != 200: - return connect_code + with socket.create_connection((HOST, PORT), timeout=15) as sock: sock.sendall(request) sock.shutdown(socket.SHUT_WR) response, body = read_response(sock) @@ -365,7 +340,7 @@ def connect_http_status(label, request): raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") -def connect_jsonrpc_status(method, params, label): +def raw_jsonrpc_status(method, params, label): target = f"{{HOST}}:{{PORT}}" body = {{"jsonrpc": "2.0", "id": 1, "method": method}} if params is not None: @@ -379,7 +354,7 @@ def connect_jsonrpc_status(method, params, label): f"Connection: close\r\n" f"\r\n" ).encode() + encoded - return connect_http_status(label, request) + return raw_http_status(label, request) results = {{ # forward proxy — method-only allow rules @@ -406,12 +381,12 @@ results = {{ # forward proxy — invalid JSON body fails closed before generic rules apply "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), - # CONNECT path — representative allowed and denied cases - "connect_method_initialize_allowed": connect_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "connect_method_initialize_allowed"), - "connect_method_tools_list_allowed": connect_jsonrpc_status("tools/list", None, "connect_method_tools_list_allowed"), - "connect_method_tools_call_allowed": connect_jsonrpc_status("tools/call", {{"name": "read_status"}}, "connect_method_tools_call_allowed"), - "connect_method_tools_call_with_unmatched_params_allowed": connect_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "connect_method_tools_call_with_unmatched_params_allowed"), - "connect_method_tools_delete_denied": connect_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "connect_method_tools_delete_denied"), + # raw socket path — representative allowed and denied cases + "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "raw_method_initialize_allowed"), + "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), + "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {{"name": "read_status"}}, "raw_method_tools_call_allowed"), + "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "raw_method_tools_call_with_unmatched_params_allowed"), + "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "raw_method_tools_delete_denied"), }} results.update(DETAILS) print(json.dumps(results, sort_keys=True)) @@ -440,16 +415,13 @@ print(json.dumps(results, sort_keys=True)) ("forward_batch_one_denied", 403), // forward proxy — parse error ("forward_invalid_json_denied", 403), - // CONNECT path — allowed - ("connect_method_initialize_allowed", 200), - ("connect_method_tools_list_allowed", 200), - ("connect_method_tools_call_allowed", 200), - ( - "connect_method_tools_call_with_unmatched_params_allowed", - 200, - ), - // CONNECT path — method denied - ("connect_method_tools_delete_denied", 403), + // raw socket path — allowed + ("raw_method_initialize_allowed", 200), + ("raw_method_tools_list_allowed", 200), + ("raw_method_tools_call_allowed", 200), + ("raw_method_tools_call_with_unmatched_params_allowed", 200), + // raw socket path — method denied + ("raw_method_tools_delete_denied", 403), ] { let expected_fragment = format!(r#""{key}": {expected}"#); assert!( diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs index 7a1e12923a..ce47d3c8b7 100644 --- a/e2e/rust/tests/live_policy_update.rs +++ b/e2e/rust/tests/live_policy_update.rs @@ -32,30 +32,6 @@ use openshell_e2e::harness::output::{extract_field, strip_ansi}; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -#[cfg(feature = "e2e-docker")] -const LOCAL_OVERRIDE_REGO: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../crates/openshell-supervisor-network/data/sandbox-policy.rego" -)); - -#[cfg(feature = "e2e-docker")] -const LOCAL_OVERRIDE_DOCKERFILE: &str = r#"FROM public.ecr.aws/docker/library/python:3.13-slim - -RUN apt-get update && apt-get install -y --no-install-recommends iproute2 \ - && rm -rf /var/lib/apt/lists/* -RUN groupadd -g 1000660000 sandbox && \ - useradd -m -u 1000660000 -g sandbox sandbox - -COPY local-policy.rego /etc/openshell/local-policy.rego -COPY local-policy.yaml /etc/openshell/local-policy.yaml - -ENV OPENSHELL_POLICY_RULES=/etc/openshell/local-policy.rego -ENV OPENSHELL_POLICY_DATA=/etc/openshell/local-policy.yaml -ENV OPENSHELL_POLICY_POLL_INTERVAL_SECS=1 - -CMD ["sleep", "infinity"] -"#; - // --------------------------------------------------------------------------- // Policy YAML builders // --------------------------------------------------------------------------- @@ -146,44 +122,6 @@ landlock: Ok(file) } -#[cfg(feature = "e2e-docker")] -fn write_local_override_image() -> Result { - let dir = tempfile::tempdir().map_err(|e| format!("create image context: {e}"))?; - std::fs::write(dir.path().join("Dockerfile"), LOCAL_OVERRIDE_DOCKERFILE) - .map_err(|e| format!("write local override Dockerfile: {e}"))?; - std::fs::write(dir.path().join("local-policy.rego"), LOCAL_OVERRIDE_REGO) - .map_err(|e| format!("write local override Rego policy: {e}"))?; - std::fs::write( - dir.path().join("local-policy.yaml"), - r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /etc - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: {} -", - ) - .map_err(|e| format!("write local override policy data: {e}"))?; - Ok(dir) -} - // --------------------------------------------------------------------------- // CLI helpers // --------------------------------------------------------------------------- @@ -580,119 +518,3 @@ async fn initial_sparse_policy_is_acknowledged_as_loaded() { guard.cleanup().await; } - -/// An explicit local Rego/data override remains authoritative even when the -/// sandbox has a gateway policy and that policy changes while it is running. -/// Gateway polling must continue for settings and providers without replacing -/// the locally loaded OPA engine. -#[cfg(feature = "e2e-docker")] -#[tokio::test] -async fn local_policy_override_survives_gateway_policy_polls() { - let image_context = write_local_override_image().expect("write local override image"); - let dockerfile = image_context.path().join("Dockerfile"); - let dockerfile = dockerfile - .to_str() - .expect("Dockerfile path should be utf-8"); - - let gateway_policy_a_file = write_policy(&["example.com"]).expect("write gateway policy A"); - let gateway_policy_a_path = gateway_policy_a_file - .path() - .to_str() - .expect("gateway policy A path should be utf-8") - .to_string(); - let gateway_policy_b_file = - write_policy(&["example.com", "api.anthropic.com"]).expect("write gateway policy B"); - let gateway_policy_b_path = gateway_policy_b_file - .path() - .to_str() - .expect("gateway policy B path should be utf-8") - .to_string(); - - let mut guard = SandboxGuard::create_keep_with_args( - &[ - "--name", - "e2e-lcl-pol-ovrd", - "--from", - dockerfile, - "--policy", - &gateway_policy_a_path, - "--no-tty", - ], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create sandbox with local policy override"); - - // Allow several one-second poll intervals. Before the fix, the first poll - // immediately reloaded gateway policy A over the local override. - tokio::time::sleep(std::time::Duration::from_secs(4)).await; - let initial_logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "1m", - "--source", - "sandbox", - ]) - .await; - assert!( - initial_logs.success, - "fetch initial sandbox logs:\n{}", - initial_logs.output - ); - assert!( - initial_logs - .output - .contains("Loading OPA policy engine from local files"), - "sandbox should load the explicit local policy:\n{}", - initial_logs.output - ); - assert!( - !initial_logs.output.contains("Policy reloaded successfully"), - "the first gateway poll must not replace the local policy:\n{}", - initial_logs.output - ); - - let update = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &gateway_policy_b_path, - ]) - .await; - assert!( - update.success, - "publish gateway policy B:\n{}", - update.output - ); - - // A later gateway revision must also remain observational in local mode. - tokio::time::sleep(std::time::Duration::from_secs(4)).await; - let updated_logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "1m", - "--source", - "sandbox", - ]) - .await; - assert!( - updated_logs.success, - "fetch updated sandbox logs:\n{}", - updated_logs.output - ); - assert!( - !updated_logs.output.contains("Policy reloaded successfully"), - "gateway policy updates must not replace the local override:\n{}", - updated_logs.output - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/no_proxy.rs b/e2e/rust/tests/no_proxy.rs index ced4d02d5f..447c408f8b 100644 --- a/e2e/rust/tests/no_proxy.rs +++ b/e2e/rust/tests/no_proxy.rs @@ -5,7 +5,7 @@ use openshell_e2e::harness::sandbox::SandboxGuard; -fn localhost_bypass_script() -> &'static str { +fn localhost_transparent_script() -> &'static str { r#" import json import os @@ -13,11 +13,8 @@ import threading import urllib.request from http.server import BaseHTTPRequestHandler, HTTPServer -expected_no_proxy = '127.0.0.1,localhost,::1' -assert os.environ['HTTP_PROXY'].startswith('http://') -assert os.environ['HTTPS_PROXY'].startswith('http://') -assert os.environ['NO_PROXY'] == expected_no_proxy -assert os.environ['no_proxy'] == expected_no_proxy +for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): + assert name not in os.environ, f'unexpected proxy environment variable: {name}' class Handler(BaseHTTPRequestHandler): def log_message(self, format, *args): @@ -36,7 +33,7 @@ thread.start() try: with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: print(json.dumps({ - 'no_proxy': os.environ['NO_PROXY'], + 'proxy_env_absent': True, 'payload': json.loads(response.read().decode()), }), flush=True) finally: @@ -47,16 +44,16 @@ finally: } #[tokio::test] -async fn sandbox_bypasses_proxy_for_localhost_http() { - let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_bypass_script()]) +async fn sandbox_reaches_localhost_without_proxy_environment() { + let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_transparent_script()]) .await - .expect("sandbox create with localhost proxy bypass check"); + .expect("sandbox create with transparent localhost check"); assert!( - guard.create_output.contains( - r#"{"no_proxy": "127.0.0.1,localhost,::1", "payload": {"message": "hello"}}"# - ), - "expected localhost HTTP request to bypass proxy and succeed:\n{}", + guard + .create_output + .contains(r#"{"proxy_env_absent": true, "payload": {"message": "hello"}}"#), + "expected localhost HTTP request to stay local and succeed:\n{}", guard.create_output ); } diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs index 4ba4dbd046..d95841d07f 100644 --- a/e2e/rust/tests/websocket_conformance.rs +++ b/e2e/rust/tests/websocket_conformance.rs @@ -364,7 +364,6 @@ import os import socket import struct import time -import urllib.parse HOST = {host:?} PORT = {port} @@ -413,34 +412,15 @@ def read_frame(sock): payload = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) return first, payload -def proxy_parts(): - names = ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy") - proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) - if not proxy_url: - raise RuntimeError("proxy environment is not configured") - parsed = urllib.parse.urlparse(proxy_url) - if not parsed.hostname: - raise RuntimeError(f"invalid proxy URL: {{proxy_url!r}}") - return parsed.hostname, parsed.port or 80 - -def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): - proxy_host, proxy_port = proxy_parts() - target = f"{{host}}:{{port}}" +def transparent_socket_with_retry(host, port, timeout_seconds=20): deadline = time.monotonic() + timeout_seconds last_error = None while time.monotonic() < deadline: sock = None try: - sock = socket.create_connection((proxy_host, proxy_port), timeout=5) - if mode == "connect": - request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not (response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200")): - first_line = response.splitlines()[0] if response else "" - raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + sock = socket.create_connection((host, port), timeout=5) return sock - except (OSError, RuntimeError) as error: + except OSError as error: if sock is not None: sock.close() last_error = error @@ -449,27 +429,25 @@ def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): token = os.environ[TOKEN_ENV] payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -results = {{}} -for mode in ("connect", "forward"): - key = base64.b64encode(os.urandom(16)).decode("ascii") - with proxy_socket_with_retry(HOST, PORT, mode) as sock: - request_target = "/ws" if mode == "connect" else f"http://{{HOST}}:{{PORT}}/ws" - request = ( - f"GET {{request_target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError(f"{{mode}} websocket upgrade failed: {{response!r}}") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - results[mode] = json.loads(response_payload.decode("utf-8")) +key = base64.b64encode(os.urandom(16)).decode("ascii") +with transparent_socket_with_retry(HOST, PORT) as sock: + request = ( + "GET /ws HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {{key}}\r\n" + "Sec-WebSocket-Version: 13\r\n" + "\r\n" + ) + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not response.startswith("HTTP/1.1 101"): + raise RuntimeError(f"websocket upgrade failed: {{response!r}}") + sock.sendall(masked_text_frame(payload)) + _, response_payload = read_frame(sock) + result = json.loads(response_payload.decode("utf-8")) +results = {{"transparent": result}} print(json.dumps(results, sort_keys=True)) "#, host = host, @@ -479,7 +457,7 @@ print(json.dumps(results, sort_keys=True)) } #[tokio::test] -async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { +async fn websocket_text_placeholder_is_rewritten_transparently() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -522,14 +500,7 @@ async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { assert!( guard .create_output - .contains(r#""connect": {"saw_placeholder": false, "saw_secret": true}"#), - "expected CONNECT upstream to see only the resolved secret marker:\n{}", - guard.create_output - ); - assert!( - guard - .create_output - .contains(r#""forward": {"saw_placeholder": false, "saw_secret": true}"#), + .contains(r#""transparent": {"saw_placeholder": false, "saw_secret": true}"#), "expected upstream to see only the resolved secret marker:\n{}", guard.create_output ); diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 2be6b3f6d5..48e2ae3da1 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -19,7 +19,7 @@ links: Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. -The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. +The compute driver prepares the workload topology and trusted inputs. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. OpenShell packages that role as `openshell-sandbox --mode=control`. When the workload is separated by a container, pod, userspace kernel, or VM boundary, the same binary runs a small trusted counterpart as `openshell-sandbox --mode=boundary`. The boundary mode owns process observation and operations that cannot be implemented portably from outside the boundary; it has no gateway credentials and no policy authority. ## Motivation @@ -33,56 +33,78 @@ All three come from coupling boundary construction to boundary operation. A comm ## Non-goals -- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Standardizing a topology's resource API.** Docker, Kubernetes, VM, and other resource mechanisms remain backend-specific. - **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. -- **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. +- **Standardizing resource-specific provisioning or transport setup.** A driver still decides how to place the binary, create a private Unix socket, authenticated TCP endpoint, or vsock endpoint, and establish kernel-specific egress capture. This RFC standardizes the authenticated control-to-boundary messages carried over that endpoint. - **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. ## Proposal The mental model has three roles: -- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **compute driver** prepares placement and trusted topology inputs and owns durable resource provisioning, deletion, and reconciliation. That placement is the **topology**. - The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. - The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. Together, network policy, filesystem isolation, syscall filtering, and sandbox identity form the workload's isolation boundary. The roles above enforce that boundary and may run in one process or across several trusted components. Their placement does not change the contract. -Each active boundary has at most one logical supervisor, which may span multiple coupled processes. The backend routes all workload egress through a per-boundary source, and the supervisor consumes that source. Internal delegation and transport remain topology-private. +Each active boundary has exactly one control role and at most one boundary role. Together they implement one logical supervisor; boundary mode is not an independently authorized supervisor. The control role owns the gateway session, admitted policy, network-policy decisions, and RFC 0012 lifecycle. Boundary mode owns boundary-local process groups, `exec`, signal, wait, PTY, loopback forwarding, binary observation, and egress capture. The transport and physical placement remain driver-owned. [RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. -Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. +Admission selects the sandbox's topology and trusted context. The compute driver gives the logical supervisor a `TopologyDescriptor` for the matching backend. Its opaque payload can identify an existing resource or carry trusted prepared inputs from which `attach` establishes the boundary. The backend prepares the required controls before the agent starts. ```mermaid flowchart TB Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] - subgraph Topology["Driver-provisioned topology (placement varies)"] - Supervisor["Supervisor"] - Backend["Isolation Backend (may coordinate components)"] + subgraph Topology["Admitted topology (placement varies)"] + Control["openshell-sandbox
--mode=control"] + Backend["Remote Isolation Backend"] subgraph Boundary["Isolation boundary"] - Mediator["Network mediation"] + BoundaryAgent["openshell-sandbox
--mode=boundary"] subgraph Execution["Workload execution environment"] Workload["Workload"] end end - - Supervisor -->|"drives contract"| Backend - Backend -->|"establishes and confirms"| Boundary - Backend -.->|"routes all workload egress to"| Mediator - Supervisor -.->|"applies network policy through"| Mediator - Backend -->|"after Ready: makes admitted agent runnable"| Workload - Workload ==>|"only egress"| Mediator + Mediator["Network mediation"] + + Control -->|"drives RFC 0012"| Backend + Backend <-->|"authenticated, versioned
boundary protocol"| BoundaryAgent + BoundaryAgent -->|"start / exec / signal / wait"| Workload + Workload ==>|"captured egress"| BoundaryAgent + BoundaryAgent ==>|"attributed streams"| Mediator + Control -.->|"policy decisions"| Mediator end - Driver -->|"resources + TopologyDescriptor"| Supervisor + Driver -->|"resources + protected configs"| BoundaryAgent + Driver -->|"trusted TopologyDescriptor"| Control + Gateway <-->|"authorized session"| Control Mediator -->|"allowed egress"| Egress["Egress"] ``` -In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. +Co-located deployments may keep the existing in-process backend and omit boundary mode. Separated deployments use the shared remote backend and boundary protocol, so adding a driver changes provisioning and transport selection without adding a topology branch to the control role. + +### Supervisor modes and boundary protocol + +`openshell-sandbox --mode=control` runs outside the untrusted execution environment. It receives the admitted backend name and a protected `TopologyDescriptor`, resolves the backend without fallback, and owns the gateway-facing access plane. It is the only mode that possesses gateway credentials or applies approved network policy. + +Orchestrators may run control mode with `--health-check --health-port `. +The listener defaults to `0.0.0.0`; `--health-bind-ip ` (or +`OPENSHELL_HEALTH_BIND_IP`) selects an explicit IPv4 or IPv6 address. Kubernetes +drivers should populate the environment variable from the Downward API +`status.podIP`, which keeps the probe address family aligned with the pod. +The TCP listener becomes reachable only after `start_agent` has returned a +`RunningBoundary` and the gateway-facing access plane is established. Control +mode drops the listener when that boundary/access-plane lifetime ends, so a +Kubernetes `tcpSocket` readiness probe observes semantic control readiness +rather than mere process liveness. The listener carries no application data. + +`openshell-sandbox --mode=boundary --boundary-config ` runs inside, or immediately adjacent to, the execution environment. The driver supplies the config over a protected filesystem channel. It names one boundary, one private listener, one per-boundary bootstrap credential, and the workload identity. The identity is either a platform-resolved numeric UID/GID pair or an OCI `Config.User` declaration that boundary mode resolves against the workload filesystem. Boundary mode authenticates and scopes every request to that boundary. It never accepts policy or identity claims from workload code and it cannot authorize an operation independently of control mode. -A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. +The shared protocol is versioned independently of a driver's resource descriptor. It carries `attach`, `confirm`, `start_agent`, agent and exec `wait`/`signal`/`terminate`, PTY resize, loopback-only forwarding, and attributed egress streams. Drivers may use a private Unix socket, TLS-authenticated TCP, a Unix endpoint mapped to guest vsock, or host `AF_VSOCK`; adding a transport does not change lifecycle or operation semantics. A TCP transport that crosses a shared or operator-managed network must authenticate the server name against a driver-provisioned trust root and encrypt every control and stream request. Network isolation alone is not a confidentiality boundary. Secrets are delivered in protected files, redacted from debug output, and never placed in workload-visible environment or process arguments. When a bind-mounted file's host owner may match the workload UID, the driver requires boundary mode to re-own it as root-only before the first workload instruction. Unix socket inodes permit cross-UID control within their private driver-owned directory; the per-boundary bootstrap credential authenticates every request. + +A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver or external orchestrator owns the durable resource lifecycle even when the backend establishes a resource as part of `attach`. ### Contract invariants @@ -91,7 +113,7 @@ Six invariants hold for every boundary: 1. Workload egress is denied except through network mediation for the boundary's lifetime. 2. No untrusted instruction executes until every admitted control applicable to that process is in force. 3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. -4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the admitted execution environment. 5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. 6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. @@ -99,21 +121,22 @@ Each backend states its termination bound in its implementation documentation. L ### Provisioning -Provisioning runs on the control plane, and three rules hold in every topology: +Provisioning is selected by trusted control-plane configuration, and four rules hold in every topology: -1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. -2. **The compute driver provisions the topology** and anything the selected backend needs. -3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. +1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` must name that backend, and resolution never falls back to another backend. +2. **The descriptor supports prepared and existing resources.** A compute driver or orchestrator may identify an existing resource, or it may supply trusted prepared inputs that the backend uses to establish the resource during `attach`. +3. **The compute driver owns the durable resource lifecycle.** It provisions or prepares, deletes, and reconciles the topology independently of logical-supervisor availability. +4. **The backend establishes standing enforcement before untrusted code runs**, during `attach` or `confirm`, depending on the backend. If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. Every topology provides a trusted cleanup path that does not depend on logical-supervisor availability. -A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. +A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. It may instead prepare immutable image or disk identities, normalized runtime settings, placement results, or protected references to artifacts and encode those inputs in the descriptor. After assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend atomically establishes or locates the resource, binds that context, and returns `Bound`, or rejects it as incompatible. Attach-time establishment is idempotent on trusted sandbox identity and launch generation. Partial resources are removed or remain labeled for compute-driver reconciliation. Pool creation, claim, reset, release, and recycling remain outside this contract. ### The topology descriptor -The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. +The compute driver supplies a descriptor for every topology admitted to this contract. The common envelope names the backend and carries an opaque payload. ```rust struct TopologyDescriptor { @@ -125,7 +148,7 @@ struct TopologyDescriptor { `version` is the Isolation Backend interface version. Backend name and version match exactly; this contract does not negotiate compatibility ranges. The descriptor is transport-neutral. Provisioning supplies it to the supervisor before `attach`; how it is transported is topology-specific and outside this contract, and every transport preserves one property: workload-controlled input cannot select or modify the descriptor. -The opaque payload identifies, or gives the backend enough information to resolve, the exact driver-provisioned resource. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. +The opaque payload identifies an existing resource or gives the backend trusted prepared inputs with which to establish the exact resource during `attach`. It may also carry topology-specific endpoint or helper-role information; there are no common topology or role fields. Common verification requires: @@ -133,13 +156,13 @@ Common verification requires: - the descriptor's version is one the supervisor supports, and the resolved backend reports that same version; and - `SandboxContext` is constructed after the control plane assigns the resource to the admitted sandbox, using authenticated control-plane and trusted supervisor state. -The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically binds the provisioned resource to the trusted `SandboxContext` during `attach`. Any failure rejects the sandbox. +The supervisor validates the descriptor's common fields and produces a `VerifiedTopologyDescriptor`, then resolves its `backend_name` and version without fallback. Verification does not imply that the opaque payload is valid; the selected backend validates it and atomically establishes or locates the resource and binds it to the trusted `SandboxContext` during `attach`. Separated topologies also carry an opaque map of driver-owned immutable resource claims, such as a container ID, pod UID, or VM generation. Control mode presents those claims during authenticated attachment, and boundary mode compares them with its protected configuration before accepting the policy. Any failure rejects the sandbox. ### The lifecycle The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. -A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. +A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through one fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. ```text attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running @@ -167,12 +190,23 @@ struct SandboxContext { #[async_trait] trait BoundBoundary: Send { fn network_mediation_source(&self) -> Arc; + fn dns_mediation_source(&self) -> Option>; + fn host_gateway_ip(&self) -> Option; async fn confirm( self: Box, ) -> Result, BackendError>; } +``` +`host_gateway_ip` is the backend's trusted host-side dial target for the +well-known host-gateway aliases. A backend returns it when the mediation +service runs outside the workload boundary and therefore cannot use the +boundary's resolver view; the supervisor preserves the original hostname for +policy, HTTP, and TLS while dialing the backend-provided address. `None` +leaves host-gateway discovery to the supervisor's local environment. + +```rust #[async_trait] trait ReadyBoundary: Send { async fn start_agent( @@ -190,7 +224,7 @@ trait RunningBoundary: Send + Sync { `AgentSpec` carries the complete admitted agent launch specification, including command, arguments, working directory, timeout, and interactive mode. -`SandboxContext` carries the admitted create-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. +`SandboxContext` carries the admitted launch-time policy. [RFC 0002](../0002-agent-driven-policy-management/README.md) defines how network-policy revisions are proposed and approved. Approved revisions reach the supervisor through the existing [`GetSandboxConfig`](../../proto/sandbox.proto) gateway-supervisor contract, described in the [gateway](../../architecture/gateway.md) and [sandbox](../../architecture/sandbox.md#policy-revision-acknowledgement) architecture. The supervisor makes approved network-policy revisions effective through network mediation. If an approved network-policy revision cannot be loaded, it never becomes effective; the configured rejection posture retains the last valid generation or denies network access until a valid generation is loaded. The states have normative meanings: @@ -200,13 +234,15 @@ The states have normative meanings: `confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. -`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. +`attach` rejects a resource already bound to an active boundary or a conflicting launch generation. An idempotent retry resolves the same compatible inactive resource. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. **Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. `start_agent` is the sole operation that may make the admitted agent runnable. The backend may create or release the process, but workload-controlled code cannot run before `start_agent` applies the required controls. -`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Any exit of the admitted agent ends `Running`; the backend then terminates every remaining workload process within that environment and rejects further runtime operations, except `wait` as defined below. +A separated boundary retains the complete accepted `attach` and `start_agent` inputs for the lifetime of `Running`. If its control process restarts, the replacement replays `attach`, `confirm`, and `start_agent` with the same authenticated boundary identity, resource claims, policy, and launch inputs. The boundary returns the existing process handle without starting a second workload. Any changed input is denied. A main-process stream has one active control owner; transport closure releases that attachment so the replacement control can attach. Compute drivers fence control replacement so old and new control processes do not overlap (for example, a Kubernetes control Deployment uses `Recreate`). + +`RunningBoundary::agent()` returns a handle for the admitted agent process. Processes started through `BoundaryExec` run in the same boundary and have their own process handles. Every workload process remains within the provisioned execution environment. Exit of the admitted agent produces a stable `wait` result and ends that process generation, but it does not implicitly tear down the boundary. The control role may continue serving terminal output, `exec`, and loopback forwarding within the same confirmed boundary until the compute driver or control role explicitly tears the boundary down. Explicit teardown terminates every remaining workload process and rejects new runtime operations. ### Runtime operations @@ -257,11 +293,32 @@ trait NetworkMediationSource: Send + Sync { struct MediatedConnection { stream: BoundaryDuplexStream, binary_identity: Result, + destination: Option, +} + +#[async_trait] +trait DnsMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedDnsQuery { + request: Vec, + transport: DnsTransport, + binary_identity: Result, + response: oneshot::Sender, BackendError>>, } ``` `NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. +Explicit-proxy transports leave `destination` absent. Transparent transports +capture the original socket destination and supply it before the supervisor +consumes workload bytes. `DnsMediationSource` carries portless DNS exchanges to +the supervisor-owned policy DNS service. It is optional because explicit-proxy +topologies resolve destinations in the supervisor and do not expose workload +DNS. A backend that advertises transparent networking supplies both sources; +DNS or connection-source failure closes that boundary's egress. + Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. @@ -289,13 +346,13 @@ Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound ### The supervisor sequence -The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. +The logical supervisor resolves `backend_name` and version through a trusted implementation registry. A separated topology selects the reusable remote backend and supplies its standardized endpoint in the opaque descriptor. Adding a driver adds provisioning and endpoint construction in that driver's crate, not branches in lifecycle, proxy, SSH, session, or control-mode code. The supervisor runs the same sequence for every backend: -1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. +1. Obtain the trusted `TopologyDescriptor` and `SandboxContext` selected by admission. 2. Verify the descriptor and resolve its `backend_name` and version without fallback. -3. Call `attach` to obtain `Bound`. +3. Call `attach` to establish or locate the resource, bind it, and obtain `Bound`. 4. Connect the boundary's `NetworkMediationSource` to network mediation. 5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. 6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. @@ -312,18 +369,20 @@ enum BackendErrorKind { Invalid, Denied, Unavailable, Unsupported, Failed, Termi `Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Unsupported` identifies an optional operation the selected backend does not implement; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. -A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. +An `Unsupported` error variant reports that the selected backend does not implement an optional contract operation; it maps to the `Unavailable` kind for status purposes and never weakens a mandatory conformance requirement. + +A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per orchestration attempt. Attach-time establishment uses sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, the compute driver reclaims the topology rather than reusing an ambiguous resource. Failures resolve as follows: -- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; -- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the compute driver to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the compute driver reclaims the topology; - if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; - after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and - retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. -Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. Normal agent exit alone does not end the boundary: `BoundaryProcess::wait` returns the stable exit status while the confirmed access plane remains available until explicit teardown. A retained `wait` result may outlive teardown. ### Topologies @@ -331,11 +390,12 @@ The contract fixes the roles; a topology fixes their placement. Components may b ## Implementation plan -This RFC defines the contract; implementation lands in three phases: +This RFC defines the contract; implementation lands in four phases: 1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. -2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. -3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. +2. **Shared supervisor modes.** Add the versioned boundary protocol, reusable remote backend, and the `control` and `boundary` entrypoints to `openshell-sandbox`. +3. **Driver adoption.** Have VM, Docker, and Kubernetes provision protected boundary configs and topology descriptors in their existing driver crates. No adoption may require a shared-supervisor change; that constraint is the abstraction test. +4. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, compute-driver-owned cleanup, and failure semantics. Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. @@ -343,7 +403,8 @@ Existing placements remain outside this contract until their backend is implemen | Risk | Mitigation | |---|---| -| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep placement, preparation, durable deletion, reconciliation, and endpoint selection in the compute driver; keep enforcement sequencing in control mode and boundary-local observation in boundary mode. Generic supervisor code never imports a concrete driver. | +| Attach-time establishment could leave a partial resource after supervisor failure. | Make establishment idempotent by sandbox launch generation, label resources for compute-driver reconciliation, and do not start untrusted code before `attach` and `confirm` complete. | | Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | | Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | | The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | @@ -358,17 +419,17 @@ OpenShell could keep the current in-pod design and add topology-specific supervi Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. -### Extend the compute-driver contract +### Require every resource to exist before attach -The compute driver could own both provisioning and active-boundary operation. +The compute driver could always create the concrete resource before the supervisor calls `attach`. -This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. +This remains natural for controller-driven systems such as Kubernetes. Requiring it everywhere prevents a local backend from establishing host listeners and enforcement state before a runtime creates the workload. Allowing a trusted topology descriptor to carry prepared inputs preserves the single `attach` contract while allowing security-sensitive establishment to remain atomic with binding. The compute driver still owns deletion and reconciliation. -### Start with a remote backend service +### Give each driver its own remote control service -The contract could be expressed as a gRPC service or plugin ABI rather than an in-process Rust contract. [RFC 0001](../0001-core-architecture/README.md) chose gRPC for its gateway-facing drivers, so the question applies here. +Each separated driver could define a private gRPC service, guest agent, or runtime-specific plugin ABI behind its `IsolationBackend` implementation. -The callers differ. A gateway driver is a control-plane peer with its own release cycle, while the Isolation Backend is driven by the supervisor that operates the boundary, and the co-located topology needs no transport at all. Starting in-process serves that case directly and lets delegated implementations carry their own transport behind the same interface. A transport-bearing surface is not precluded: it is versioned contract surface, added when a concrete delegated backend requires it. +That would hide transport differences from the Rust trait, but it would duplicate lifecycle, authentication, process streaming, signaling, forwarding, and binary-identity semantics across Docker, Kubernetes, and VM implementations. The proposed versioned boundary protocol standardizes those semantics once. Drivers still choose and provision Unix socket, authenticated TCP, vsock, or adapter transport and bind their own immutable resource claims. ### Standardize topology and capabilities @@ -381,10 +442,11 @@ That would make known deployments explicit, but it would also encode current top - **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. - **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. - **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. +- **[OCI seccomp listener handoff](https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#seccomp).** The runtime specification lets a runtime send a seccomp notification FD and process state to a host Unix listener. It demonstrates why some local enforcement must exist before workload creation and informs attach-time boundary establishment. ## Open questions -None. +None for this revision. New common fields require evidence from a concrete backend and a protocol-version change when compatibility cannot be preserved. ## Appendix: codebase grounding diff --git a/rfc/0012-isolation-backend/topology-matrix.md b/rfc/0012-isolation-backend/topology-matrix.md index 8dc14d84ba..85335be545 100644 --- a/rfc/0012-isolation-backend/topology-matrix.md +++ b/rfc/0012-isolation-backend/topology-matrix.md @@ -6,18 +6,29 @@ kernel; it does not select a deployment or establish conformance. ## Representative placements -| Pattern | Logical supervisor and network-mediation placement | Backend placement | Workload-kernel relationship | Topology status | +| Pattern | Control and network-mediation placement | Boundary-mode placement | Workload-kernel relationship | Topology status | |---|---|---|---|---| -| **Co-located/in-pod** | With the workload | In the supervisor process | Trusted components share the workload's host, guest, or application kernel, depending on the runtime | Placement implemented (original topology) | -| **Same-pod composite** | Spans the workload-local supervisor process and, when used, a network-mediation sidecar | In the workload-local supervisor process | Components share the workload's kernel | Placement implemented (#2076) | -| **Delegated backend components** | With the workload and any delegated mediation component | A node or remote helper establishes some controls behind a workload-local backend | Depends on which trusted components remain with the workload | Placement proposed (#2606) | -| **Driver-hosted/shared service** | With the compute driver or another trusted service; no in-sandbox supervisor process is required | May be co-located with the logical supervisor; one host may operate many isolated boundaries | Depends on the workload runtime | Placement proposed | +| **Co-located/in-pod** | With the workload; the legacy in-process backend may omit boundary mode | Same process when used | Trusted components share the workload's host kernel | Placement implemented (original topology) | +| **Kubernetes proxy pod** | Trusted control pod | Boundary-mode workload entrypoint owning the workload PID and network namespaces | Shared cluster-node kernel; pod security boundaries separate control from workload | Implemented in #3144; requires a conforming NetworkPolicy CNI and trusted namespace | +| **Docker** | Gateway host | Trusted container entrypoint sharing the workload container's PID and network namespaces | Shared host kernel | Implemented in #2965 | +| **MicroVM** | Gateway host | Guest PID 1 | Boundary mode shares the guest kernel; control is kernel-separated | Implemented in #2945 | + +The Kubernetes proxy-pod topology uses the same boundary protocol as Docker and +VM, with per-boundary TLS because the connection traverses the pod network. +Kubernetes-specific code provisions the workload fence, pair labels, boundary +Service, control Deployment, immutable bootstrap Secret, and stable +namespace/Sandbox/Deployment/NetworkPolicy claims. The workload pod has no +direct egress; attributed proxy streams cross the TLS channel and hostname +resolution occurs on the control side. Admission requires an explicitly acknowledged conforming CNI and a +namespace in which untrusted principals cannot create pods, mutate pair labels, +or read bootstrap Secrets. Pod readiness or the existence of a `NetworkPolicy` +object alone does not prove enforcement. ## Durable rules - Every active boundary has one verified descriptor, one trusted - `SandboxContext`, and at most one logical supervisor, which may span multiple - coupled processes. + `SandboxContext`, one control role, and at most one boundary role. Those + processes form one logical supervisor. - Physical processes and listeners may be shared, but lifecycle state, policy, binary identity, enforcement, and cleanup remain isolated per boundary. - Moving a privileged component does not itself provide kernel separation. diff --git a/tasks/rust.toml b/tasks/rust.toml index e62e22b3cf..50eda4118c 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -61,8 +61,8 @@ run = [ # operators to produce these artifacts. "cargo build -p openshell-gateway --bin openshell-gateway --no-default-features --features defaults-without-telemetry", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway", - "cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features defaults-without-telemetry", - "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox", + "cargo build -p openshell-supervisor --bin openshell-supervisor --no-default-features --features defaults-without-telemetry", + "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-supervisor", ] ["rust:verify:defaults-without-telemetry"] @@ -72,11 +72,11 @@ run = "tasks/scripts/verify-defaults-without-telemetry.sh" ["rust:verify:system-ca-roots"] description = "Verify system CA roots build mode compiles and excludes bundled Mozilla root crates" run = [ - # Check that the sandbox compiles cleanly in system CA roots mode (all + # Check that the supervisor compiles cleanly in system CA roots mode (all # defaults except bundled-ca-roots). - "cargo check -p openshell-sandbox --all-targets --no-default-features --features system-ca-roots", + "cargo check -p openshell-supervisor --all-targets --no-default-features --features system-ca-roots", # Guard: webpki-roots must not appear in the dependency graph. - "bash -c 'if cargo tree -p openshell-sandbox -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo \"ERROR: webpki-roots found in system CA roots build\" >&2; exit 1; fi'", + "bash -c 'if cargo tree -p openshell-supervisor -i webpki-roots --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-roots; then echo \"ERROR: webpki-roots found in system CA roots build\" >&2; exit 1; fi'", # Guard: webpki-root-certs must not appear either (webpki-roots re-exports it). - "bash -c 'if cargo tree -p openshell-sandbox -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo \"ERROR: webpki-root-certs found in system CA roots build\" >&2; exit 1; fi'", + "bash -c 'if cargo tree -p openshell-supervisor -i webpki-root-certs --no-default-features --features system-ca-roots 2>/dev/null | grep -q webpki-root-certs; then echo \"ERROR: webpki-root-certs found in system CA roots build\" >&2; exit 1; fi'", ] diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 08ba00e066..5f055bf397 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -44,7 +44,7 @@ required_prebuilt_binaries() { echo "openshell-gateway" ;; supervisor|supervisor-sideload|supervisor-output) - echo "openshell-sandbox" + echo "openshell-sandbox openshell-supervisor" ;; esac } diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index fe4913439a..211864d007 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -109,10 +109,10 @@ components_for_target() { echo "gateway" ;; sandbox|supervisor|supervisor-output) - echo "supervisor" + echo "sandbox supervisor" ;; all) - echo "gateway supervisor" + echo "gateway sandbox supervisor" ;; *) usage @@ -128,11 +128,16 @@ resolve_component() { binary=openshell-gateway target_libc=gnu ;; - supervisor) + sandbox) crate=openshell-sandbox binary=openshell-sandbox target_libc=$(supervisor_libc) ;; + supervisor) + crate=openshell-supervisor + binary=openshell-supervisor + target_libc=$(supervisor_libc) + ;; *) echo "unsupported binary component: $1" >&2 exit 1 @@ -260,7 +265,7 @@ build_component_for_arch() { binary_path="${ROOT}/target/${target}/release/${binary}" if [[ "$component" == "gateway" ]]; then "$SCRIPT_DIR/verify-glibc-symbols.sh" 2.28 "$binary_path" - elif [[ "$component" == "supervisor" ]]; then + else "$SCRIPT_DIR/verify-static-binary.sh" "$binary_path" fi diff --git a/tasks/scripts/verify-defaults-without-telemetry.sh b/tasks/scripts/verify-defaults-without-telemetry.sh index 1fd7e67dff..104523b04b 100755 --- a/tasks/scripts/verify-defaults-without-telemetry.sh +++ b/tasks/scripts/verify-defaults-without-telemetry.sh @@ -23,7 +23,7 @@ set -euo pipefail # `defaults-without-telemetry`. CRATES=( openshell-gateway - openshell-sandbox + openshell-supervisor openshell-driver-vm ) From cc48db8d63e54155a58645c29c066bd0fef0e386 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:15:10 -0700 Subject: [PATCH 02/22] fix(network): import portable DNS mapping errors Signed-off-by: Drew Newberry --- crates/openshell-supervisor-network/src/proxy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index d89ffd61db..2ad1387c0d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,9 +10,9 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; -use crate::policy_dns::ResolvedEndpointStore; #[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId}; +use crate::policy_dns::PolicyEndpointId; +use crate::policy_dns::{MappingLookupError, ResolvedEndpointStore}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; From 26acae6fd37eacafaf81f26b18faed35fd9e4735 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:22:07 -0700 Subject: [PATCH 03/22] fix(sandbox): keep non-Linux workspace checks portable Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/delegated.rs | 4 +++- crates/openshell-sandbox/src/lib.rs | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/delegated.rs b/crates/openshell-sandbox/src/delegated.rs index a9d09a8ec5..d8c15535f0 100644 --- a/crates/openshell-sandbox/src/delegated.rs +++ b/crates/openshell-sandbox/src/delegated.rs @@ -7,7 +7,9 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Duration; -use miette::{IntoDiagnostic as _, Result, WrapErr as _}; +#[cfg(target_os = "linux")] +use miette::WrapErr as _; +use miette::{IntoDiagnostic as _, Result}; use openshell_core::policy::SandboxPolicy; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_isolation_interface::contract::{BoundaryExec, BoundaryPortForward}; diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index ed98161d78..6600099255 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -36,6 +36,15 @@ pub struct RuntimeQualification { pub tcp_deny_round_trip: bool, } +/// Placeholder used when compiling the package on a non-Linux host. +/// +/// The sandbox binary rejects execution on those hosts before constructing a +/// qualification, but retaining the type keeps the library API portable for +/// workspace-wide checks. +#[cfg(not(target_os = "linux"))] +#[derive(Clone, Copy, Debug)] +pub struct RuntimeQualification; + /// Run the authenticated boundary-local sandbox. /// /// # Errors From b4deb7b7af6600974f7bcf4071ca15573270de02 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:27:58 -0700 Subject: [PATCH 04/22] fix(sandbox): gate Linux boundary internals Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 6600099255..2bd68bd536 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -7,9 +7,11 @@ pub mod boundary_exec; pub mod boundary_io; mod boundary_server; pub mod child_env; +#[cfg(target_os = "linux")] pub(crate) mod delegated; #[cfg(unix)] pub mod identity; +#[cfg(target_os = "linux")] pub mod main_session; pub mod managed_children; #[cfg(target_os = "linux")] From a5a051613328e9b723dde6225de720434041d6b2 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:33:20 -0700 Subject: [PATCH 05/22] fix(sandbox): keep helper commands portable Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/main.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 4145d6cd00..c564575a89 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -3,13 +3,17 @@ //! `OpenShell` capability-free in-workload sandbox boundary. +#[cfg(target_os = "linux")] use std::mem::size_of; use std::path::Path; use clap::Parser; use miette::{IntoDiagnostic, Result}; +#[cfg(target_os = "linux")] use openshell_ocsf::OcsfShorthandLayer; +#[cfg(target_os = "linux")] use tracing_subscriber::EnvFilter; +#[cfg(target_os = "linux")] use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; /// Subcommand name used to self-copy the sandbox binary into a shared volume. @@ -1089,6 +1093,13 @@ fn run_capability_socket_child(args: &[String]) -> Result<()> { Ok(()) } +#[cfg(not(target_os = "linux"))] +fn run_capability_socket_child(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "socket qualification is supported only on Linux" + )) +} + #[cfg(target_os = "linux")] #[allow(unsafe_code)] fn probe_dns_socket_round_trip() -> Result<()> { From 492b17198c4a4271f97cd56c07b625444ecd42e3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:38:55 -0700 Subject: [PATCH 06/22] fix(sandbox): gate Linux bootstrap helpers Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index c564575a89..192c9dec6e 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -24,8 +24,11 @@ use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; const COPY_SELF_SUBCOMMAND: &str = "copy-self"; const BOOTSTRAP_SUBCOMMAND: &str = "bootstrap"; const SEED_WORKSPACE_SUBCOMMAND: &str = "seed-workspace"; +#[cfg(target_os = "linux")] const BOOTSTRAP_INPUT_ROOT: &str = "/.openshell/bootstrap-input"; +#[cfg(target_os = "linux")] const SANDBOX_RUNTIME_ROOT: &str = "/.openshell/runtime"; +#[cfg(target_os = "linux")] const SANDBOX_STATE_ROOT: &str = "/.openshell/state"; const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; @@ -1506,6 +1509,7 @@ fn run_capability_probe() -> Result<()> { )) } +#[cfg(target_os = "linux")] fn proc_status_hex(status: &str, field: &str) -> Result { let value = status .lines() @@ -1560,6 +1564,7 @@ fn copy_self(dest: &str) -> Result<()> { /// memory-backed volumes. The projected Secret remains mounted only in this /// trusted init container; the long-lived sandbox consumes and unlinks the /// staged configuration before it starts workload code. +#[cfg(target_os = "linux")] fn stage_kubernetes_bootstrap() -> Result<()> { stage_kubernetes_bootstrap_at( Path::new(BOOTSTRAP_INPUT_ROOT), @@ -1583,6 +1588,7 @@ fn run_kubernetes_bootstrap() -> Result<()> { )) } +#[cfg(any(target_os = "linux", test))] fn stage_kubernetes_bootstrap_at(source: &Path, runtime: &Path, state: &Path) -> Result<()> { use std::fs::{self, OpenOptions}; use std::os::unix::fs::PermissionsExt as _; @@ -1639,6 +1645,7 @@ fn stage_kubernetes_bootstrap_at(source: &Path, runtime: &Path, state: &Path) -> Ok(()) } +#[cfg(any(target_os = "linux", test))] fn copy_projected_secret_file( source_root: &Path, name: &str, @@ -1656,6 +1663,7 @@ fn copy_projected_secret_file( copy_regular_file(&canonical_source, destination, mode) } +#[cfg(any(target_os = "linux", test))] fn copy_regular_file(source: &Path, destination: &Path, mode: u32) -> Result<()> { use std::fs::{self, OpenOptions}; use std::io::{Read as _, Write as _}; From 81e48a290396b94f1a133cc6685a57500abcde93 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:41:54 -0700 Subject: [PATCH 07/22] test(sandbox): gate Linux exec coverage Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/boundary_exec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index db69d52092..5fb9ea5ab7 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -492,7 +492,7 @@ impl BoundaryProcess for LocalExecProcess { } } -#[cfg(test)] +#[cfg(all(test, target_os = "linux"))] mod tests { use super::*; use std::sync::Once; From a5409764a5b4a1e3ee61384b0ce665d372e5e7e8 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:48:43 -0700 Subject: [PATCH 08/22] test(process): gate Linux relay fixture Signed-off-by: Drew Newberry --- crates/openshell-supervisor-process/src/supervisor_session.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index f9a57783b7..aa9dc22267 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -876,8 +876,10 @@ mod target_tests { mod ocsf_event_tests { use super::*; + #[cfg(target_os = "linux")] struct UnusedPortForward; + #[cfg(target_os = "linux")] #[async_trait::async_trait] impl BoundaryPortForward for UnusedPortForward { async fn connect( From 9658ccf469a159fa0ca99061246bf4737fdcd940 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:55:32 -0700 Subject: [PATCH 09/22] ci(supervisor): stage both runtime binaries Signed-off-by: Drew Newberry --- .github/actions/build-docker-image/action.yml | 19 +++++++++++++++++++ .github/workflows/branch-e2e.yml | 1 + .github/workflows/build-sandbox-binaries.yml | 18 ++++++++++++++++-- .github/workflows/docker-build.yml | 5 +++++ .github/workflows/release-dev.yml | 1 + .github/workflows/release-tag.yml | 1 + 6 files changed, 43 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-docker-image/action.yml b/.github/actions/build-docker-image/action.yml index 08557fbfd4..578ee6091d 100644 --- a/.github/actions/build-docker-image/action.yml +++ b/.github/actions/build-docker-image/action.yml @@ -11,6 +11,10 @@ inputs: binary: description: Binary staged in the Docker build context required: true + additional-binary: + description: Optional second binary staged in the Docker build context + required: false + default: "" triple: description: Binary artifact target triple required: true @@ -53,6 +57,21 @@ runs: INPUTS_BINARY: ${{ inputs.binary }} INPUTS_ARCH: ${{ inputs.arch }} + - name: Download ${{ inputs.additional-binary }} + if: inputs.additional-binary != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.additional-binary }}-${{ inputs.triple }} + path: additional-artifact + + - name: Stage ${{ inputs.additional-binary }} + if: inputs.additional-binary != '' + shell: bash + run: install -Dm0755 additional-artifact/${INPUTS_ADDITIONAL_BINARY} deploy/docker/.build/prebuilt-binaries/${INPUTS_ARCH}/${INPUTS_ADDITIONAL_BINARY} + env: + INPUTS_ADDITIONAL_BINARY: ${{ inputs.additional-binary }} + INPUTS_ARCH: ${{ inputs.arch }} + - name: Build ${{ inputs.component }} image shell: bash env: diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2e316f88bd..4c894e3ea3 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -262,6 +262,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl secrets: inherit diff --git a/.github/workflows/build-sandbox-binaries.yml b/.github/workflows/build-sandbox-binaries.yml index 53f81bec40..bdfccc345b 100644 --- a/.github/workflows/build-sandbox-binaries.yml +++ b/.github/workflows/build-sandbox-binaries.yml @@ -32,13 +32,27 @@ jobs: - triple: x86_64-unknown-linux-musl runner: linux-amd64-cpu8 dev_shell: .#devShells.x86_64-linux.musl + package: openshell-sandbox + binary: openshell-sandbox + - triple: x86_64-unknown-linux-musl + runner: linux-amd64-cpu8 + dev_shell: .#devShells.x86_64-linux.musl + package: openshell-supervisor + binary: openshell-supervisor + - triple: aarch64-unknown-linux-musl + runner: linux-arm64-cpu8 + dev_shell: .#devShells.aarch64-linux.musl + package: openshell-sandbox + binary: openshell-sandbox - triple: aarch64-unknown-linux-musl runner: linux-arm64-cpu8 dev_shell: .#devShells.aarch64-linux.musl + package: openshell-supervisor + binary: openshell-supervisor uses: ./.github/workflows/build-binaries.yml with: - package: openshell-sandbox - binary: openshell-sandbox + package: ${{ matrix.package }} + binary: ${{ matrix.binary }} triple: ${{ matrix.triple }} runner: ${{ matrix.runner }} dev-shell: ${{ matrix.dev_shell }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d27ba1e984..c89bf6fe0e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -12,6 +12,10 @@ on: binary: required: true type: string + additional-binary: + required: false + type: string + default: "" target-suffix: required: true type: string @@ -54,6 +58,7 @@ jobs: with: component: ${{ inputs.component }} binary: ${{ inputs.binary }} + additional-binary: ${{ inputs['additional-binary'] }} triple: ${{ matrix.rust_arch }}-${{ inputs['target-suffix'] }} arch: ${{ matrix.arch }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 18b5f33a2e..14eee4e0b3 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -145,6 +145,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl secrets: inherit diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index b3c45c97fb..80c053eb54 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -184,6 +184,7 @@ jobs: with: component: supervisor binary: openshell-sandbox + additional-binary: openshell-supervisor target-suffix: unknown-linux-musl image-tag: ${{ needs.compute-versions.outputs.source_sha }} checkout-ref: ${{ inputs.tag || github.ref }} From 821805a2cb546c6719bc59b5da0d8284c6641841 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 10:06:37 -0700 Subject: [PATCH 10/22] test(sandbox): accept non-Linux startup rejection Signed-off-by: Drew Newberry --- crates/openshell-sandbox/tests/stdout_logging.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/tests/stdout_logging.rs b/crates/openshell-sandbox/tests/stdout_logging.rs index 403ed31cb3..ed3a2d5523 100644 --- a/crates/openshell-sandbox/tests/stdout_logging.rs +++ b/crates/openshell-sandbox/tests/stdout_logging.rs @@ -26,7 +26,9 @@ fn startup_logs_go_to_stderr_not_stdout() { "expected startup logs on stderr only, got stdout: {stdout}" ); assert!( - stderr.contains("capability-free sandbox probe") || stderr.contains("read boundary config"), + stderr.contains("capability-free sandbox probe") + || stderr.contains("read boundary config") + || stderr.contains("openshell-sandbox requires Linux"), "expected startup qualification or bootstrap error on stderr, got: {stderr}" ); } From 666dcaac352a41694994b5ca2b53f6367ad6c220 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 10:22:00 -0700 Subject: [PATCH 11/22] fix(images): preserve sandbox image entrypoint Signed-off-by: Drew Newberry --- deploy/docker/Dockerfile.supervisor | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index 033371e773..1a0df27353 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -28,4 +28,6 @@ RUN apk add --no-cache iproute2 \ COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-supervisor /openshell-supervisor -ENTRYPOINT ["/openshell-supervisor"] +# Keep the image default usable by drivers that run the sandbox boundary in +# this image. Split-topology drivers select /openshell-supervisor explicitly. +ENTRYPOINT ["/openshell-sandbox"] From 89a0b0896ce5f620708aca31ab3b7608a8605642 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 10:37:13 -0700 Subject: [PATCH 12/22] fix(sandbox): retry ephemeral DNS relay binding Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/network_broker.rs | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 209ab879cd..815e157805 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -281,9 +281,7 @@ fn start_dns_relay( address: SocketAddr, pending: mpsc::Sender, ) -> io::Result { - let udp = UdpSocket::bind(address)?; - let address = udp.local_addr()?; - let tcp = TcpListener::bind(address)?; + let (udp, tcp, address) = bind_dns_relay_sockets(address)?; let udp_attribution = Arc::new(Mutex::new(HashMap::new())); let tcp_attribution = Arc::new(Mutex::new(HashMap::new())); let active_workers = Arc::new(AtomicUsize::new(0)); @@ -366,6 +364,36 @@ fn start_dns_relay( Ok(relay) } +fn bind_dns_relay_sockets(address: SocketAddr) -> io::Result<(UdpSocket, TcpListener, SocketAddr)> { + const EPHEMERAL_BIND_ATTEMPTS: usize = 32; + + if address.port() != 0 { + let udp = UdpSocket::bind(address)?; + let tcp = TcpListener::bind(address)?; + return Ok((udp, tcp, address)); + } + + // TCP and UDP have independent ephemeral-port allocators. The port picked + // by the first bind can therefore already be occupied by the other + // protocol, especially while the test suite starts several brokers in + // parallel. Retry the pair rather than treating that collision as an + // unavailable network broker. + for _ in 0..EPHEMERAL_BIND_ATTEMPTS { + let udp = UdpSocket::bind(address)?; + let selected = udp.local_addr()?; + match TcpListener::bind(selected) { + Ok(tcp) => return Ok((udp, tcp, selected)), + Err(error) if error.kind() == io::ErrorKind::AddrInUse => {} + Err(error) => return Err(error), + } + } + + Err(io::Error::new( + io::ErrorKind::AddrInUse, + "could not reserve a shared ephemeral TCP/UDP DNS relay port", + )) +} + fn pending_try_send( pending: &mpsc::Sender, query: PendingDnsQuery, From 0e36df282921c42509884bfc8ec91ecec3fcf823 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 10:56:14 -0700 Subject: [PATCH 13/22] fix(isolation): resolve scratch command in sandbox Signed-off-by: Drew Newberry --- crates/openshell-core/src/sandbox_env.rs | 11 ++--- crates/openshell-core/src/shell.rs | 7 +-- .../openshell-sandbox/src/boundary_server.rs | 43 +++++++++++++++++-- crates/openshell-server/src/grpc/sandbox.rs | 2 +- crates/openshell-supervisor/src/lib.rs | 14 +++--- 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index f9cd03c539..c1c91822b6 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -39,7 +39,8 @@ const MAIN_PROCESS_SPEC_BASE64URL_PREFIX: &str = "base64url:"; pub struct MainProcessConfig { pub version: u32, /// Canonical command. Empty means "no command supplied": the supervisor - /// resolves the default login shell against the sandbox image. A non-empty + /// asks the sandbox boundary to resolve the default login shell against + /// the agent image. A non-empty /// command is the exact program+args and is run verbatim. pub command: Vec, pub tty: bool, @@ -51,8 +52,8 @@ impl MainProcessConfig { pub const VERSION: u32 = 1; /// Default config for a sandbox created without a command. The command is - /// left empty on purpose: the supervisor picks a login shell that exists in - /// the sandbox image (bash when present, otherwise `/bin/sh`). A TTY is + /// left empty on purpose: the sandbox boundary picks a login shell that + /// exists in the agent image (bash when present, otherwise `/bin/sh`). A TTY is /// requested because the default is an interactive login shell. #[must_use] pub fn scratch() -> Self { @@ -99,7 +100,7 @@ impl MainProcessConfig { )); } // An empty command is valid: it means "no command supplied", and the - // supervisor resolves the default login shell. Only a present-but-blank + // sandbox boundary resolves the default login shell. Only a present-but-blank // program is rejected. if !config.command.is_empty() && config.command[0].is_empty() { return Err(format!( @@ -316,7 +317,7 @@ mod tests { #[test] fn omitted_command_stays_empty_for_supervisor_resolution() { - // No command supplied → empty command; the supervisor resolves the + // No command supplied → empty command; the sandbox boundary resolves the // default login shell against the sandbox image. let empty = crate::proto::compute::v1::DriverSandboxSpec::default(); assert!( diff --git a/crates/openshell-core/src/shell.rs b/crates/openshell-core/src/shell.rs index 09610afe1a..d24a7bced2 100644 --- a/crates/openshell-core/src/shell.rs +++ b/crates/openshell-core/src/shell.rs @@ -3,7 +3,7 @@ //! Login-shell resolution for sandbox images. //! -//! The default sandbox command and the interactive SSH session need a shell, +//! The default sandbox command and interactive SSH sessions need a shell, //! but not every base image ships the same one. Debian-based images provide //! `bash`; minimal images such as Alpine only provide `/bin/sh` (`BusyBox` //! `ash`). Hard-coding `/bin/bash` makes sandbox startup fail on those images @@ -51,8 +51,9 @@ pub fn is_executable(path: &str) -> bool { /// Resolve a login shell that exists in the current root filesystem. /// /// Tries [`SHELL_CANDIDATES`] in order and falls back to [`POSIX_SH`]. Because -/// this inspects the filesystem, call it from the supervisor (inside the -/// sandbox), never on the gateway. +/// this inspects the filesystem, call it from the sandbox boundary or another +/// process inside the workload filesystem, never from the external supervisor +/// or gateway. /// /// `$SHELL` is intentionally not consulted: it is image/user-controlled, the /// result is later invoked with `-lc`, and an executable that is not a diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 0d793ceb92..09c05dfcec 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -1320,6 +1320,10 @@ mod linux { provider_env_revision: u64, provider_env: std::collections::HashMap, ) -> Response { + let spec = match resolve_agent_spec(spec) { + Ok(spec) => spec, + Err(error) => return guest_error("failed", error), + }; let mut state = lock(&self.state); let requested = StartedAgent { sandbox_id: sandbox_id.clone(), @@ -1669,6 +1673,24 @@ mod linux { ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, } + fn resolve_agent_spec(mut spec: AgentSpecWire) -> Result { + if !spec.program.is_empty() { + return Ok(spec); + } + if !spec.args.is_empty() { + return Err("default agent command cannot include arguments".to_string()); + } + let shell = openshell_core::shell::detect_login_shell(); + if !openshell_core::shell::is_executable(&shell) { + return Err(format!( + "sandbox image does not provide an executable login shell at {shell}" + )); + } + spec.program = shell; + spec.args = vec!["-l".to_string()]; + Ok(spec) + } + impl ManagedProcess { fn spawn( runtime: &tokio::runtime::Handle, @@ -1684,9 +1706,7 @@ mod linux { provider_env, ca_file_paths, } = launch; - if spec.program.is_empty() { - return Err("agent program must not be empty".to_string()); - } + debug_assert!(!spec.program.is_empty()); let boundary_runtime = BoundaryRuntimeState::new_exclusive_pid_namespace(); let entrypoint_pid = Arc::new(AtomicU32::new(0)); let provider_credentials = ProviderCredentialState::from_child_env_snapshot( @@ -2378,6 +2398,23 @@ mod linux { assert_eq!(ledger.entries.len(), MAX_REPLAY_LEDGER_ENTRIES); } + #[test] + fn scratch_agent_command_resolves_inside_the_workload_filesystem() { + let resolved = resolve_agent_spec(AgentSpecWire { + program: String::new(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 0, + interactive: true, + }) + .expect("resolve scratch command"); + + assert!(openshell_core::shell::is_executable(&resolved.program)); + assert_eq!(resolved.args, vec!["-l".to_string()]); + assert_eq!(resolved.workdir.as_deref(), Some("/sandbox")); + assert!(resolved.interactive); + } + fn placeholder_server_tls() -> BoundaryServerTls { BoundaryServerTls { certificate_chain_path: Path::new("/tmp/openshell-sandbox.crt").to_path_buf(), diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 252c7bea63..ad62c3a52a 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -298,7 +298,7 @@ async fn handle_create_sandbox_inner( }; // Leave an omitted command empty rather than persisting a concrete shell: - // the supervisor resolves the default login shell against the sandbox image + // the sandbox boundary resolves the default login shell against the agent image // (bash when present, otherwise /bin/sh on minimal images like Alpine), // which the gateway cannot do since it does not see the sandbox filesystem. // The default is an interactive login shell, so request a TTY. diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 670fe897ce..fdfe090220 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -246,9 +246,13 @@ pub async fn run_sandbox( admitted_isolation_backend: Option, main_exit_marker: Option, ) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; + // An empty command is the versioned scratch-sandbox sentinel. The + // external supervisor cannot inspect the workload filesystem, so preserve + // it for openshell-sandbox to resolve against the agent image. + let (program, args) = command.split_first().map_or_else( + || (String::new(), Vec::new()), + |(program, args)| (program.clone(), args.to_vec()), + ); // Initialize the process-wide OCSF context early so that events emitted // during policy loading (filesystem config, validation) have a context. @@ -453,8 +457,8 @@ pub async fn run_sandbox( sandbox_id: sandbox_id.clone().unwrap_or_default(), policy: policy.clone(), agent: openshell_isolation_interface::AgentSpec { - program: program.clone(), - args: args.to_vec(), + program, + args, workdir: workspace, timeout_secs, interactive, From 33304c36e8eaba1eb61eebfe9dbcc0b4b28cf268 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 13:07:06 -0700 Subject: [PATCH 14/22] fix(sandbox): preserve process launcher fallbacks Signed-off-by: Drew Newberry --- .../src/sandbox/linux/seccomp.rs | 75 +++++++++++-------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs index 43fc59df51..f3da87aeef 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs @@ -71,10 +71,10 @@ pub fn apply_supervisor_prelude() -> Result<()> { pub fn apply(policy: &SandboxPolicy) -> Result<()> { let allow_inet = matches!(policy.network.mode, NetworkMode::Proxy | NetworkMode::Allow); let main_filter = build_filter(allow_inet)?; - let clone3_filter = build_clone3_filter()?; + let compatibility_filter = build_compatibility_filter()?; set_no_new_privs()?; - apply_runtime_filters(&main_filter, &clone3_filter)?; + apply_runtime_filters(&main_filter, &compatibility_filter)?; Ok(()) } @@ -147,7 +147,7 @@ fn compile_filter( filter.try_into().into_diagnostic() } -/// Build a minimal BPF filter that blocks clone3 with ENOSYS. +/// Build a minimal BPF filter for unavailable process APIs. /// /// This is a separate filter from the main one because seccomp BPF cannot /// dereference the `struct clone_args *` pointer that clone3 takes as arg 0, @@ -155,27 +155,28 @@ fn compile_filter( /// unconditionally with ENOSYS so glibc falls back to the older clone /// syscall (where flags are a direct register argument and CAN be filtered). /// -/// glibc's clone3 wrapper checks for ENOSYS specifically — EPERM would be -/// treated as a hard failure and propagated to the caller instead of -/// triggering the clone fallback. -fn build_clone3_filter() -> Result { +/// glibc's clone3 wrapper and process launchers that opportunistically use +/// pidfds check for ENOSYS specifically. EPERM is treated as a hard policy +/// failure instead of triggering their portable fallback paths. +fn build_compatibility_filter() -> Result { let mut rules: BTreeMap> = BTreeMap::new(); rules.entry(libc::SYS_clone3).or_default(); + rules.entry(libc::SYS_pidfd_open).or_default(); compile_filter(rules, SeccompAction::Errno(libc::ENOSYS as u32)) } /// Install the sandbox seccomp filters in the required order. /// /// Order matters: -/// 1. Install the dedicated clone3 filter first so it can still call +/// 1. Install the compatibility filter first so it can still call /// `seccomp(SECCOMP_SET_MODE_FILTER)`. /// 2. Install the main filter second. It blocks further seccomp filter /// installation with `EPERM`, preserving the original hardening intent. fn apply_runtime_filters( main_filter: seccompiler::BpfProgramRef<'_>, - clone3_filter: seccompiler::BpfProgramRef<'_>, + compatibility_filter: seccompiler::BpfProgramRef<'_>, ) -> Result<()> { - apply_filter(clone3_filter).into_diagnostic()?; + apply_filter(compatibility_filter).into_diagnostic()?; apply_filter(main_filter).into_diagnostic()?; Ok(()) } @@ -226,8 +227,9 @@ fn build_filter_rules(allow_inet: bool) -> Result rules.entry(libc::SYS_process_vm_readv).or_default(); // Cross-process memory write (symmetric with process_vm_readv). rules.entry(libc::SYS_process_vm_writev).or_default(); - // Process handle acquisition, fd theft, and signalling via pidfd. - rules.entry(libc::SYS_pidfd_open).or_default(); + // Process fd theft and signalling via pidfd. pidfd_open is made + // unavailable with ENOSYS by the compatibility filter so runtimes can + // fall back without gaining a handle to the trusted sandbox boundary. rules.entry(libc::SYS_pidfd_getfd).or_default(); rules.entry(libc::SYS_pidfd_send_signal).or_default(); // Async I/O subsystem with extensive CVE history. @@ -278,7 +280,7 @@ fn build_filter_rules(allow_inet: bool) -> Result 0, // flags argument libc::CLONE_NEWUSER as u64, )?; - // clone3 is handled by a separate filter — see build_clone3_filter(). + // clone3 is handled by the ENOSYS compatibility filter. // seccomp(SECCOMP_SET_MODE_FILTER) would let sandboxed code replace the active filter. let condition = SeccompCondition::new( @@ -425,7 +427,6 @@ mod tests { libc::SYS_bpf, libc::SYS_process_vm_readv, libc::SYS_process_vm_writev, - libc::SYS_pidfd_open, libc::SYS_pidfd_getfd, libc::SYS_pidfd_send_signal, libc::SYS_io_uring_setup, @@ -551,19 +552,25 @@ mod tests { } #[test] - fn clone3_filter_compiles_and_blocks_clone3() { - let bpf = build_clone3_filter(); - assert!(bpf.is_ok(), "clone3 ENOSYS filter should compile"); + fn compatibility_filter_compiles() { + let bpf = build_compatibility_filter(); + assert!( + bpf.is_ok(), + "process API compatibility filter should compile" + ); } #[test] - fn clone3_not_in_main_filter() { - // clone3 must NOT be in the main filter; it has its own ENOSYS filter. + fn compatibility_syscalls_are_not_in_main_filter() { + // These APIs must NOT be in the EPERM filter; the compatibility filter + // reports them unavailable so process launchers can fall back. let filter_rules = build_filter_rules(true).unwrap(); - assert!( - !filter_rules.contains_key(&libc::SYS_clone3), - "clone3 should not be in the main filter — it uses a separate ENOSYS filter" - ); + for syscall in [libc::SYS_clone3, libc::SYS_pidfd_open] { + assert!( + !filter_rules.contains_key(&syscall), + "syscall {syscall} should use the ENOSYS compatibility filter" + ); + } } // --- Behavioral tests --- @@ -610,10 +617,10 @@ mod tests { unsafe fn install_runtime_filters_in_child( main_filter: &seccompiler::BpfProgram, - clone3_filter: &seccompiler::BpfProgram, + compatibility_filter: &seccompiler::BpfProgram, ) { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - if let Err(err) = apply_runtime_filters(main_filter, clone3_filter) { + if let Err(err) = apply_runtime_filters(main_filter, compatibility_filter) { let msg = format!("failed to install runtime seccomp filters: {err}\n"); libc::write(2, msg.as_ptr().cast(), msg.len()); libc::_exit(1); @@ -701,13 +708,13 @@ mod tests { // clone3 uses a separate filter that returns ENOSYS (not EPERM) so // glibc falls back to clone. let main_filter = build_filter(true).unwrap(); - let clone3_filter = build_clone3_filter().unwrap(); - // Apply in the same order as apply(): clone3 filter first, main filter second. + let compatibility_filter = build_compatibility_filter().unwrap(); + // Apply in the same order as apply(): compatibility filter first, main filter second. let pid = unsafe { libc::fork() }; assert!(pid >= 0, "fork failed"); if pid == 0 { unsafe { - install_runtime_filters_in_child(&main_filter, &clone3_filter); + install_runtime_filters_in_child(&main_filter, &compatibility_filter); let ret = libc::syscall(libc::SYS_clone3, 0 as libc::c_ulong, 0 as libc::c_ulong); let errno = *libc::__errno_location(); if ret == -1 && errno == libc::ENOSYS { @@ -727,17 +734,23 @@ mod tests { ); } + #[test] + fn behavioral_pidfd_open_returns_enosys() { + let filter = build_compatibility_filter().unwrap(); + unsafe { assert_blocked_in_child(&filter, libc::SYS_pidfd_open, libc::ENOSYS) }; + } + #[test] fn behavioral_third_filter_install_blocked_after_startup() { let main_filter = build_filter(true).unwrap(); - let clone3_filter = build_clone3_filter().unwrap(); - let third_filter = build_clone3_filter().unwrap(); + let compatibility_filter = build_compatibility_filter().unwrap(); + let third_filter = build_compatibility_filter().unwrap(); let pid = unsafe { libc::fork() }; assert!(pid >= 0, "fork failed"); if pid == 0 { unsafe { - install_runtime_filters_in_child(&main_filter, &clone3_filter); + install_runtime_filters_in_child(&main_filter, &compatibility_filter); match apply_filter(&third_filter) { Err(seccompiler::Error::Seccomp(e)) if e.raw_os_error() == Some(libc::EPERM) => From c9546f94b90e5d5c2e8cb546f9637b66c0066f45 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 13:56:19 -0700 Subject: [PATCH 15/22] fix(supervisor): allow null device for child launchers Signed-off-by: Drew Newberry --- crates/openshell-supervisor/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index fdfe090220..823eb4f527 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -1028,7 +1028,11 @@ const PROXY_BASELINE_READ_ONLY: &[&str] = &[ /// Minimum read-write paths required for a proxy-mode sandbox child process. /// The active workspace is granted separately through `include_workdir`. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; +// `/dev/null` is opened by common child-process launchers when they construct +// piped or discarded stdio. Without it, tools such as uv report EACCES while +// probing an otherwise executable interpreter under an explicit filesystem +// policy. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp", "/dev/null"]; /// GPU read-only paths. /// @@ -1381,6 +1385,7 @@ mod baseline_tests { fn baseline_read_write_does_not_hardcode_sandbox() { let (_ro, rw) = baseline_enrichment_paths(); assert!(rw.contains(&"/tmp".to_string())); + assert!(rw.contains(&"/dev/null".to_string())); assert!(!rw.contains(&"/sandbox".to_string())); } From 2efbd4c86f3988fe113b901049caf6390f547d7c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 14:17:29 -0700 Subject: [PATCH 16/22] fix(sandbox): attribute wildcard DNS sources Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/network_broker.rs | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 815e157805..eb075c794e 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -596,8 +596,7 @@ fn connect_socket( return Err(io::Error::from_raw_os_error(libc::EISCONN)); } let source_fd = entry.retained_preconnect()?.as_raw_fd(); - ensure_dns_source_bound(source_fd, entry.metadata().family)?; - let peer = socket_local_addr(source_fd)?; + let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; let attribution = match kind { InetKind::Tcp => &dns_relay.tcp_attribution, InetKind::DnsUdp => &dns_relay.udp_attribution, @@ -681,18 +680,26 @@ fn connect_socket( Ok(()) } -fn ensure_dns_source_bound(fd: RawFd, family: InetFamily) -> io::Result<()> { - let address = socket_local_addr(fd)?; - if address.port() != 0 { - return Ok(()); - } - bind_exact( - fd, - match family { - InetFamily::V4 => SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), - InetFamily::V6 => SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), - }, - ) +fn ensure_dns_source_bound(fd: RawFd, family: InetFamily) -> io::Result { + let mut address = socket_local_addr(fd)?; + let loopback = match family { + InetFamily::V4 => IpAddr::V4(Ipv4Addr::LOCALHOST), + InetFamily::V6 => IpAddr::V6(Ipv6Addr::LOCALHOST), + }; + if address.port() == 0 { + bind_exact(fd, SocketAddr::new(loopback, 0))?; + address = socket_local_addr(fd)?; + } + // Async resolvers commonly bind an unspecified address before sendto(2). + // A loopback destination makes the kernel select loopback as the actual + // source, so key attribution by that effective peer rather than by the + // wildcard returned before connect/send. Otherwise the relay observes + // 127.0.0.1: (or ::1:) and drops a valid query registered as + // 0.0.0.0: (or [::]:). + if address.ip().is_unspecified() { + address.set_ip(loopback); + } + Ok(address) } fn establish_relay( @@ -1074,8 +1081,7 @@ fn classify_send( let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); let entry = registry.resolve_mut(notification.tid, fd)?; let source_fd = entry.retained_preconnect()?.as_raw_fd(); - ensure_dns_source_bound(source_fd, entry.metadata().family)?; - let peer = socket_local_addr(source_fd)?; + let peer = ensure_dns_source_bound(source_fd, entry.metadata().family)?; lock(&dns_relay.udp_attribution).insert(peer, identity); if let Err(error) = connect_exact(source_fd, dns_relay.address) { lock(&dns_relay.udp_attribution).remove(&peer); @@ -1755,7 +1761,7 @@ mod tests { } #[test] - fn udp_dns_uses_exact_local_relay_source() { + fn udp_dns_normalizes_wildcard_source_for_relay_attribution() { let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() .expect("start workload launcher"); let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); @@ -1763,7 +1769,9 @@ mod tests { let client = std::thread::spawn(move || { launcher .execute(move || -> io::Result { - let socket = UdpSocket::bind("127.0.0.1:0")?; + // Tokio/Hickory-style resolvers bind a wildcard source + // before sending to the configured nameserver. + let socket = UdpSocket::bind("0.0.0.0:0")?; socket.set_read_timeout(Some(Duration::from_secs(5)))?; socket.send_to(b"dns-query", dns_address)?; let mut response = [0_u8; 32]; From 8ee7ca9faae984a82a3003e67b59fbf507f419d4 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 14:50:18 -0700 Subject: [PATCH 17/22] fix(sandbox): support UDP route probes Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/network_broker.rs | 123 +++++++++++++++++- 1 file changed, 118 insertions(+), 5 deletions(-) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index eb075c794e..e138a82c3e 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -565,7 +565,25 @@ fn connect_socket( active_opens: Arc, ) -> io::Result<()> { let fd = raw_fd(notification.args[0])?; - if !socket_address_is_inet(notification.tid, notification.args[1], notification.args[2])? { + let address_family = + read_socket_family(notification.tid, notification.args[1], notification.args[2])?; + if !matches!(address_family, libc::AF_INET | libc::AF_INET6) { + if address_family == libc::AF_UNSPEC { + let mut registry = lock(®istry); + if let Ok(entry) = registry.resolve_mut(notification.tid, fd) + && entry.metadata().kind == InetKind::DnsUdp + && matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) + { + // Address-selection implementations disconnect a temporary + // UDP route-probe socket with AF_UNSPEC before trying the next + // candidate. The probe below never connects the real OFD, so + // this is an idempotent no-op rather than a kernel CONTINUE. + return listener.respond_value(notification.id, 0); + } + } if lock(®istry).resolve(notification.tid, fd).is_ok() { // Every registered descriptor is an injected INET socket. Never // CONTINUE based on a mutable workload sockaddr for such an FD. @@ -585,6 +603,34 @@ fn connect_socket( entry.metadata().nonblocking, ) }; + if kind == InetKind::DnsUdp && destination.port() == 0 { + let mut registry = lock(®istry); + let entry = registry.resolve_mut(notification.tid, fd)?; + if !matches!( + entry.state(), + SocketState::Created | SocketState::Bound { .. } + ) { + return Err(io::Error::from_raw_os_error(libc::EISCONN)); + } + let destination_family = match destination { + SocketAddr::V4(_) => InetFamily::V4, + SocketAddr::V6(_) => InetFamily::V6, + }; + if entry.metadata().family != destination_family { + return Err(io::Error::from_raw_os_error(libc::EAFNOSUPPORT)); + } + // glibc and uv use UDP connect(..., port 0), getsockname(), and an + // AF_UNSPEC disconnect to rank resolved addresses. Bind only to the + // matching loopback family and report success; never connect the + // kernel socket to the external candidate. write(2) therefore remains + // EDESTADDRREQ and destination-bearing sends remain broker-denied. + let local = ensure_dns_source_bound( + entry.retained_preconnect()?.as_raw_fd(), + entry.metadata().family, + )?; + entry.set_state(SocketState::Bound { local }); + return listener.respond_value(notification.id, 0); + } if destination == dns_relay.address { let identity = ProcfsIdentityResolver::for_pid_namespace().resolve(notification.tid); let mut registry = lock(®istry); @@ -1401,16 +1447,20 @@ fn read_socket_addr(tid: u32, address: u64, length: u64) -> io::Result io::Result { + Ok(matches!( + read_socket_family(tid, address, length)?, + libc::AF_INET | libc::AF_INET6 + )) +} + +fn read_socket_family(tid: u32, address: u64, length: u64) -> io::Result { let length = usize::try_from(length).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL))?; if address == 0 || length < size_of::() { return Err(io::Error::from_raw_os_error(libc::EFAULT)); } let mut family = [0_u8; size_of::()]; task_memory::read_exact(tid, address, &mut family)?; - Ok(matches!( - i32::from(libc::sa_family_t::from_ne_bytes(family)), - libc::AF_INET | libc::AF_INET6 - )) + Ok(i32::from(libc::sa_family_t::from_ne_bytes(family))) } fn decode_sockaddr(storage: libc::sockaddr_storage, length: usize) -> io::Result { @@ -1798,6 +1848,69 @@ mod tests { ); } + #[test] + fn udp_port_zero_route_probes_are_local_and_reusable() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + launcher + .execute(|| -> io::Result<()> { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.connect("198.51.100.7:0")?; + let local = socket.local_addr()?; + if !local.ip().is_loopback() || local.port() == 0 { + return Err(io::Error::other(format!( + "route probe did not expose a local source: {local}" + ))); + } + + let unspecified = libc::sockaddr { + sa_family: libc::sa_family_t::try_from(libc::AF_UNSPEC) + .expect("AF_UNSPEC fits sa_family_t"), + sa_data: [0; 14], + }; + // SAFETY: unspecified is a live native sockaddr used for the + // conventional UDP disconnect operation. + let disconnected = unsafe { + libc::connect( + socket.as_raw_fd(), + (&raw const unspecified).cast(), + libc::socklen_t::try_from(size_of::()) + .expect("sockaddr size fits socklen_t"), + ) + }; + if disconnected != 0 { + return Err(io::Error::last_os_error()); + } + socket.connect("203.0.113.9:0")?; + + // The route probe never commits an external UDP peer. A + // destination-free send must therefore remain kernel-denied. + // SAFETY: payload is live for the duration of this syscall. + let sent = unsafe { + libc::send( + socket.as_raw_fd(), + b"blocked".as_ptr().cast(), + b"blocked".len(), + 0, + ) + }; + if sent >= 0 { + return Err(io::Error::other("route probe became a data path")); + } + let error = io::Error::last_os_error(); + if !matches!( + error.raw_os_error(), + Some(libc::EDESTADDRREQ | libc::ENOTCONN) + ) { + return Err(error); + } + Ok(()) + }) + .expect("launcher result") + .expect("route-probe workload"); + } + #[test] fn tcp_dns_preserves_length_framing() { let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() From eaa410a97008e2312d848c16639b78e2aa7c141b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 15:06:26 -0700 Subject: [PATCH 18/22] test(sandbox): model unbound UDP route probes Signed-off-by: Drew Newberry --- crates/openshell-sandbox/src/network_broker.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index e138a82c3e..5d1ec950dd 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -1855,7 +1855,23 @@ mod tests { let _broker = NetworkBroker::start_for_test(listener).expect("start network broker"); launcher .execute(|| -> io::Result<()> { - let socket = UdpSocket::bind("0.0.0.0:0")?; + // Address-selection probes create an unbound datagram socket; + // binding to INADDR_ANY first would intentionally preserve an + // unspecified local address and would not model that path. + // SAFETY: the return value is checked before ownership moves + // into UdpSocket. + let raw_socket = unsafe { + libc::socket( + libc::AF_INET, + libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, + libc::IPPROTO_UDP, + ) + }; + if raw_socket < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: raw_socket is a new, owned socket descriptor. + let socket = unsafe { UdpSocket::from_raw_fd(raw_socket) }; socket.connect("198.51.100.7:0")?; let local = socket.local_addr()?; if !local.ip().is_loopback() || local.port() == 0 { From b51d79850609797c061823ac0216d317e017a702 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 15:30:38 -0700 Subject: [PATCH 19/22] fix(sandbox): support repeated DNS relay sends Signed-off-by: Drew Newberry --- .../openshell-sandbox/src/network_broker.rs | 67 ++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 5d1ec950dd..55665d2007 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -1106,7 +1106,21 @@ fn classify_send( } } Ok(entry) if matches!(entry.state(), SocketState::DnsUdp { .. }) => { - if messages.iter().all(|message| message.destination.is_none()) { + let SocketState::DnsUdp { relay } = entry.state() else { + unreachable!("guard requires DNS UDP state"); + }; + // musl-based resolvers, including the statically linked `uv` + // client, send A and AAAA as separate destination-bearing + // datagrams on one socket. The first send pins the socket to the + // private relay; permit later sends only when their copied + // destination is absent or names that same relay. The mandatory + // outer network fence remains the fail-closed backstop for the + // sibling-thread pointer race inherent in seccomp CONTINUE. + if messages.iter().all(|message| { + message + .destination + .is_none_or(|destination| destination == *relay) + }) { listener.respond_continue(notification.id) } else { Err(io::Error::from_raw_os_error(libc::EACCES)) @@ -1848,6 +1862,57 @@ mod tests { ); } + #[test] + fn udp_dns_allows_repeated_destination_sends_to_the_pinned_relay() { + let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() + .expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start network broker"); + let dns_address = broker.dns_address(); + let client = std::thread::spawn(move || { + launcher + .execute(move || -> io::Result>> { + // Static musl clients send A and AAAA with two sendto(2) + // calls on the same initially-unconnected socket. + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_read_timeout(Some(Duration::from_secs(5)))?; + socket.send_to(b"dns-query-a", dns_address)?; + socket.send_to(b"dns-query-aaaa", dns_address)?; + let mut responses = Vec::new(); + for _ in 0..2 { + let mut response = [0_u8; 32]; + let (length, source) = socket.recv_from(&mut response)?; + if source != dns_address { + return Err(io::Error::other("wrong DNS response source")); + } + responses.push(response[..length].to_vec()); + } + responses.sort(); + Ok(responses) + }) + .expect("launcher result") + }); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime"); + for _ in 0..2 { + let query = runtime.block_on(broker.accept_dns()).expect("DNS query"); + let response = if query.request == b"dns-query-a" { + b"dns-response-a".to_vec() + } else if query.request == b"dns-query-aaaa" { + b"dns-response-aaaa".to_vec() + } else { + panic!("unexpected DNS query: {:?}", query.request); + }; + query.complete(Ok(response)).unwrap(); + } + assert_eq!( + client.join().expect("join client").expect("DNS client"), + vec![b"dns-response-a".to_vec(), b"dns-response-aaaa".to_vec()] + ); + } + #[test] fn udp_port_zero_route_probes_are_local_and_reusable() { let (launcher, listener) = openshell_isolation_interface::linux::workload_launcher::start() From 4ef32d6a259f26a155e39e0d3a91a03606692b7f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 16:18:37 -0700 Subject: [PATCH 20/22] fix(server): serve callbacks before sandbox restore Signed-off-by: Drew Newberry --- crates/openshell-server/src/lib.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8c8afdf08..5230d7df3a 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -698,15 +698,6 @@ pub(crate) async fn run_server( ) .await?; - if let Err(err) = state.compute.start_persisted_sandboxes().await { - warn!(error = %err, "Failed to start persisted sandboxes during startup"); - } - - state.compute.spawn_watchers(shutdown_rx.clone()); - ssh_sessions::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); - supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); - provider_refresh::spawn_refresh_worker(state.clone(), Duration::from_secs(60)); - // Create the multiplexed service let service = MultiplexService::new(state.clone()); @@ -791,6 +782,19 @@ pub(crate) async fn run_server( ))); } + // Restored supervisors need the callback listeners while the compute + // driver reconciles persisted sandboxes. Serve them before starting that + // reconciliation so policy fetch and supervisor-session registration + // cannot deadlock gateway startup. + if let Err(err) = state.compute.start_persisted_sandboxes().await { + warn!(error = %err, "Failed to start persisted sandboxes during startup"); + } + + state.compute.spawn_watchers(shutdown_rx.clone()); + ssh_sessions::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); + supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); + provider_refresh::spawn_refresh_worker(state.clone(), Duration::from_secs(60)); + shutdown_signal().await; info!("Shutdown signal received; stopping gateway"); state.gateway_shutting_down.store(true, Ordering::Release); From 5dd8d00505757463da5300f822c4cd536c7ada08 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 17:07:04 -0700 Subject: [PATCH 21/22] fix(supervisor): gate health on live gateway session Signed-off-by: Drew Newberry --- .../src/delegated.rs | 25 +++- .../src/supervisor_session.rs | 2 + crates/openshell-supervisor/src/lib.rs | 116 ++++++++++++++++-- 3 files changed, 129 insertions(+), 14 deletions(-) diff --git a/crates/openshell-supervisor-process/src/delegated.rs b/crates/openshell-supervisor-process/src/delegated.rs index ad1e8faecc..8f51bf2397 100644 --- a/crates/openshell-supervisor-process/src/delegated.rs +++ b/crates/openshell-supervisor-process/src/delegated.rs @@ -21,6 +21,7 @@ pub struct BoundaryAccess { terminating: Arc, ssh_task: Option>, session_task: Option>, + session_readiness: Option>, main_session: Option>, } @@ -31,6 +32,13 @@ impl BoundaryAccess { &self.instance_id } + /// Observe whether the gateway has accepted the current supervisor + /// session. The value returns to false while the session reconnects. + #[must_use] + pub fn session_readiness(&self) -> Option> { + self.session_readiness.clone() + } + /// Publish the canonical process's terminal status to attached clients. pub async fn publish_main_exit(&self, exit_code: i32, attachment_expected: bool) { let Some(main_session) = self.main_session.as_ref() else { @@ -85,6 +93,7 @@ pub async fn start_boundary_access( terminating, ssh_task: None, session_task: None, + session_readiness: None, main_session: None, }); }; @@ -142,7 +151,7 @@ pub async fn start_boundary_access( } } - let session_task = match (openshell_endpoint, sandbox_id) { + let (session_task, session_readiness) = match (openshell_endpoint, sandbox_id) { (Some(endpoint), Some(id)) => { let (task, mut accepted) = crate::supervisor_session::spawn_with_readiness( endpoint.to_string(), @@ -153,10 +162,12 @@ pub async fn start_boundary_access( terminating.clone(), instance_id.clone(), ); - match tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) - .await - { - Ok(Ok(_)) => Some(task), + let accepted_result = + tokio::time::timeout(Duration::from_secs(10), accepted.wait_for(|ready| *ready)) + .await + .map(|result| result.map(|_| ())); + match accepted_result { + Ok(Ok(())) => (Some(task), Some(accepted)), Ok(Err(_)) => { task.abort(); return Err(miette::miette!( @@ -171,7 +182,7 @@ pub async fn start_boundary_access( } } } - _ => None, + _ => (None, None), }; Ok(BoundaryAccess { @@ -179,6 +190,7 @@ pub async fn start_boundary_access( terminating, ssh_task: Some(ssh_task), session_task, + session_readiness, main_session: Some(main_session), }) } @@ -243,6 +255,7 @@ mod tests { terminating: Arc::new(AtomicBool::new(false)), ssh_task: None, session_task: None, + session_readiness: None, main_session: Some(main_session.clone()), }; diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index aa9dc22267..2772095532 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -340,6 +340,7 @@ async fn run_session_loop(config: SessionConfig) { match run_single_session(&config).await { Ok(()) => { + config.ready_tx.send_replace(false); let event = session_closed_event( openshell_ocsf::ctx::ctx(), &config.endpoint, @@ -349,6 +350,7 @@ async fn run_session_loop(config: SessionConfig) { break; } Err(e) => { + config.ready_tx.send_replace(false); let event = session_failed_event( openshell_ocsf::ctx::ctx(), &config.endpoint, diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 823eb4f527..e31fbf6313 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -94,21 +94,89 @@ struct ControlReadiness { } impl ControlReadiness { - fn start(path: std::path::PathBuf) -> Result { + fn start( + path: std::path::PathBuf, + mut session_readiness: Option>, + ) -> Result { + if session_readiness + .as_ref() + .is_some_and(|readiness| !*readiness.borrow()) + { + return Err(miette::miette!( + "supervisor session is not ready when starting health listener" + )); + } prepare_control_readiness_path(&path)?; let listener = tokio::net::UnixListener::bind(&path) .into_diagnostic() .wrap_err_with(|| format!("bind supervisor readiness socket on {}", path.display()))?; + let task_path = path.clone(); let task = tokio::spawn(async move { + let mut listener = Some(listener); loop { - match listener.accept().await { - Ok((stream, _)) => drop(stream), - Err(error) => { - tracing::warn!(%error, "control-mode readiness accept failed; retrying"); - tokio::time::sleep(Duration::from_millis(100)).await; + let session_unready = session_readiness + .as_ref() + .is_some_and(|readiness| !*readiness.borrow()); + if listener.is_none() || session_unready { + if session_unready { + listener.take(); + let _ = std::fs::remove_file(&task_path); + let Some(readiness) = session_readiness.as_mut() else { + break; + }; + if readiness.wait_for(|ready| *ready).await.is_err() { + break; + } + } + match prepare_control_readiness_path(&task_path).and_then(|()| { + tokio::net::UnixListener::bind(&task_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "rebind supervisor readiness socket on {}", + task_path.display() + ) + }) + }) { + Ok(rebound) => listener = Some(rebound), + Err(error) => { + tracing::warn!(%error, "control-mode readiness rebind failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + } + continue; + } + + let Some(active_listener) = listener.as_ref() else { + continue; + }; + if let Some(readiness) = session_readiness.as_mut() { + tokio::select! { + accepted = active_listener.accept() => match accepted { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + }, + changed = readiness.changed() => { + if changed.is_err() { + break; + } + } + } + } else { + match active_listener.accept().await { + Ok((stream, _)) => drop(stream), + Err(error) => { + tracing::warn!(%error, "control-mode readiness accept failed; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } } } } + let _ = std::fs::remove_file(&task_path); }); Ok(Self { task, path }) } @@ -738,7 +806,10 @@ pub async fn run_sandbox( .await?; info!(backend = %backend_name, "Control-mode access plane started"); let mut control_readiness = if let Some(path) = health_socket_path { - Some(ControlReadiness::start(path)?) + Some(ControlReadiness::start( + path, + boundary_access.session_readiness(), + )?) } else { None }; @@ -3885,7 +3956,8 @@ mod tests { async fn control_readiness_exists_only_while_guard_is_live() { let root = tempfile::tempdir().unwrap(); let path = root.path().join("health.sock"); - let readiness = ControlReadiness::start(path.clone()).expect("start readiness listener"); + let readiness = + ControlReadiness::start(path.clone(), None).expect("start readiness listener"); check_control_readiness(&path).expect("running supervisor accepts readiness probes"); drop(readiness); @@ -3893,6 +3965,34 @@ mod tests { assert!(check_control_readiness(&path).is_err()); } + #[tokio::test] + async fn control_readiness_tracks_supervisor_session() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("health.sock"); + let (session_tx, session_rx) = tokio::sync::watch::channel(true); + let _readiness = ControlReadiness::start(path.clone(), Some(session_rx)) + .expect("start readiness listener"); + check_control_readiness(&path).expect("accepted session is ready"); + + session_tx.send_replace(false); + timeout(Duration::from_secs(1), async { + while check_control_readiness(&path).is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .expect("lost session removes readiness socket"); + + session_tx.send_replace(true); + timeout(Duration::from_secs(1), async { + while check_control_readiness(&path).is_err() { + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement session restores readiness socket"); + } + #[test] fn control_readiness_rejects_relative_path() { let error = prepare_control_readiness_path(std::path::Path::new("health.sock")) From 96fbf82073fe0917faaed6efbd62cfe3b42b61b3 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 20:48:47 -0700 Subject: [PATCH 22/22] fix(server): fence sandbox restart observations Signed-off-by: Drew Newberry --- crates/openshell-server/src/compute/mod.rs | 328 +++++++++++++----- .../src/supervisor_session.rs | 28 +- 2 files changed, 258 insertions(+), 98 deletions(-) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 4e5e42bcd4..a92ba7818c 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -171,6 +171,7 @@ mod traced_driver { } const DELETE_PHASE_CAS_RETRY_LIMIT: usize = 3; +const SUPERVISOR_SESSION_CAS_RETRY_LIMIT: usize = 3; #[derive(Clone, Debug, Eq, PartialEq)] pub enum GatewayListenerRequirement { @@ -2841,18 +2842,19 @@ impl ComputeRuntime { SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown) }); - if !driver_snapshot_reports_terminal_container_exit(&incoming) - || existing_phase != SandboxPhase::Starting - { + if existing_phase != SandboxPhase::Starting { return self.apply_sandbox_update_locked(incoming, existing).await; } - // A terminal snapshot can already be queued when StartSandbox moves - // the durable phase to Starting. Release the global watch lock, wait - // for that lifecycle operation, and then reread both the driver and - // store before applying the terminal observation. Taking the - // per-sandbox gate only for this ambiguous phase avoids delaying - // unrelated watch events behind slow lifecycle operations. + // Any snapshot can already be queued when StartSandbox moves the + // durable phase to Starting. In particular, an old-generation Ready + // event followed by its terminal event can otherwise promote and then + // stop the new generation before the replacement supervisor connects. + // Release the global watch lock, wait for that lifecycle operation, + // and then reread both the driver and store before applying an + // authoritative observation. Taking the per-sandbox gate only for + // this ambiguous phase avoids delaying unrelated watch events behind + // slow lifecycle operations. let existing_name = existing_sandbox.as_ref().map_or_else( || incoming.name.clone(), |sandbox| sandbox.object_name().to_string(), @@ -2881,7 +2883,7 @@ impl ComputeRuntime { { warn!( sandbox_id = %incoming.id, - "Could not validate terminal driver snapshot; retaining current sandbox state" + "Could not validate driver snapshot during sandbox start; retaining current sandbox state" ); return Ok(()); } @@ -2993,82 +2995,129 @@ impl ComputeRuntime { instance_id: Option<&str>, terminal_delivery_finalized: bool, ) -> Result<(), String> { - let guard = self.sync_lock.lock().await; - - let Some(existing) = self + let _guard = self.sync_lock.lock().await; + let existing = self .store .get_message::(sandbox_id) .await - .map_err(|err| err.to_string())? - else { - return Ok(()); - }; - let current_phase = - SandboxPhase::try_from(existing.phase()).unwrap_or(SandboxPhase::Unknown); - if !connected - && matches!(current_phase, SandboxPhase::Error | SandboxPhase::Completed) - && terminal_delivery_finalized - { - drop(guard); - self.schedule_ephemeral_sandbox_delete(&existing); - return Ok(()); - } - if matches!( - current_phase, - SandboxPhase::Deleting - | SandboxPhase::Error - | SandboxPhase::Stopping - | SandboxPhase::Stopped - | SandboxPhase::Completed - ) { - return Ok(()); - } - if !connected && current_phase != SandboxPhase::Ready { - return Ok(()); - } - let expected_resource_version = sandbox_resource_version(&existing); - - // Use CAS to update sandbox phase based on supervisor session state - let result = self - .store - .update_message_cas::(sandbox_id, expected_resource_version, |sandbox| { - let sandbox_name = sandbox.object_name().to_string(); - if connected { - ensure_supervisor_ready_status(&mut sandbox.status, &sandbox_name); - let status = sandbox.status.get_or_insert_with(Default::default); - status.main_process_instance_id = instance_id.unwrap_or_default().to_string(); - status.exit_code = None; - sandbox.set_phase(SandboxPhase::Ready as i32); - } else { - ensure_supervisor_not_ready_status(&mut sandbox.status, &sandbox_name); - sandbox.set_phase(SandboxPhase::Provisioning as i32); - } - }) - .await; + .map_err(|err| err.to_string())?; + self.set_supervisor_session_state_from_snapshot( + sandbox_id, + connected, + instance_id, + terminal_delivery_finalized, + existing, + ) + .await + } - // Handle not found gracefully (sandbox may have been deleted) - let sandbox = match result { - Ok(s) => s, - Err(crate::persistence::PersistenceError::Database(ref msg)) - if msg.contains("not found") => - { + async fn set_supervisor_session_state_from_snapshot( + &self, + sandbox_id: &str, + connected: bool, + instance_id: Option<&str>, + terminal_delivery_finalized: bool, + mut existing: Option, + ) -> Result<(), String> { + for attempt in 1..=SUPERVISOR_SESSION_CAS_RETRY_LIMIT { + let Some(current) = existing else { return Ok(()); - } - Err(crate::persistence::PersistenceError::Conflict { - current_resource_version, - }) => { + }; + let current_phase = + SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); + if connected + && matches!( + current_phase, + SandboxPhase::Deleting | SandboxPhase::Stopping | SandboxPhase::Stopped + ) + { return Err(format!( - "concurrent modification detected (current resource_version: {})", - current_resource_version - .map_or_else(|| "unknown".to_string(), |v| v.to_string()) + "sandbox is not accepting supervisor sessions while {current_phase:?}" )); } - Err(e) => return Err(e.to_string()), - }; + if !connected + && matches!(current_phase, SandboxPhase::Error | SandboxPhase::Completed) + && terminal_delivery_finalized + { + self.schedule_ephemeral_sandbox_delete(¤t); + return Ok(()); + } + if matches!( + current_phase, + SandboxPhase::Deleting + | SandboxPhase::Error + | SandboxPhase::Stopping + | SandboxPhase::Stopped + | SandboxPhase::Completed + ) { + return Ok(()); + } + if !connected && current_phase != SandboxPhase::Ready { + return Ok(()); + } + let expected_resource_version = sandbox_resource_version(¤t); + let result = self + .store + .update_message_cas::( + sandbox_id, + expected_resource_version, + |sandbox| { + let sandbox_name = sandbox.object_name().to_string(); + if connected { + ensure_supervisor_ready_status(&mut sandbox.status, &sandbox_name); + let status = sandbox.status.get_or_insert_with(Default::default); + status.main_process_instance_id = + instance_id.unwrap_or_default().to_string(); + status.exit_code = None; + sandbox.set_phase(SandboxPhase::Ready as i32); + } else { + ensure_supervisor_not_ready_status(&mut sandbox.status, &sandbox_name); + sandbox.set_phase(SandboxPhase::Provisioning as i32); + } + }, + ) + .await; - self.sandbox_index.update_from_sandbox(&sandbox); - self.sandbox_watch_bus.notify(sandbox_id); - Ok(()) + match result { + Ok(sandbox) => { + self.sandbox_index.update_from_sandbox(&sandbox); + self.sandbox_watch_bus.notify(sandbox_id); + return Ok(()); + } + Err(crate::persistence::PersistenceError::Database(ref message)) + if message.contains("not found") => + { + return Ok(()); + } + Err(crate::persistence::PersistenceError::Conflict { + current_resource_version, + }) if attempt < SUPERVISOR_SESSION_CAS_RETRY_LIMIT => { + debug!( + sandbox_id, + attempt, + ?current_resource_version, + "Retrying supervisor session state after concurrent modification" + ); + existing = self + .store + .get_message::(sandbox_id) + .await + .map_err(|error| error.to_string())?; + } + Err(crate::persistence::PersistenceError::Conflict { + current_resource_version, + }) => { + return Err(format!( + "concurrent modification detected after {attempt} attempts (current resource_version: {})", + current_resource_version + .map_or_else(|| "unknown".to_string(), |version| version.to_string()) + )); + } + Err(error) => return Err(error.to_string()), + } + } + + unreachable!("supervisor session CAS retry loop always returns") } /// Persist a terminal canonical-process result. Successful completion is @@ -4300,18 +4349,6 @@ fn driver_snapshot_confirms_stopped(incoming: &DriverSandbox) -> bool { }) } -fn driver_snapshot_reports_terminal_container_exit(incoming: &DriverSandbox) -> bool { - incoming.status.as_ref().is_some_and(|status| { - status.conditions.iter().any(|condition| { - condition.status.eq_ignore_ascii_case("false") - && matches!( - condition.reason.to_ascii_lowercase().as_str(), - "containerexited" | "containerstopped" | "containerruntimerestart" - ) - }) - }) -} - fn driver_snapshot_reports_runtime_restart(incoming: &DriverSandbox) -> bool { incoming.status.as_ref().is_some_and(|status| { status.conditions.iter().any(|condition| { @@ -7438,6 +7475,59 @@ mod tests { } } + #[tokio::test] + async fn stale_ready_snapshot_queued_before_start_is_revalidated() { + let driver = ControlledDriver::new(); + driver.block_start(); + let sandbox = sandbox_record( + "sb-start-ready-race", + "sandbox-start-ready-race", + SandboxPhase::Stopped, + ); + let mut current = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + current.status = Some(make_driver_status(make_driver_condition( + "PodStarting", + "replacement workload is still starting", + ))); + driver.set_get_outcome(ControlledGetOutcome::Sandbox(Box::new(current))); + let mut runtime = test_runtime(driver.clone()).await; + runtime.driver_info.driver_reports_runtime_readiness = true; + runtime.store.put_message(&sandbox).await.unwrap(); + + let start_runtime = runtime.clone(); + let sandbox_name = sandbox.object_name().to_string(); + let start = + tokio::spawn( + async move { start_runtime.start_sandbox("default", &sandbox_name).await }, + ); + tokio::time::timeout(Duration::from_secs(1), driver.start_started.notified()) + .await + .expect("start did not reach the driver"); + + let stale_ready = ready_driver_sandbox(sandbox.object_id(), sandbox.object_name()); + let update_runtime = runtime.clone(); + let mut update = + tokio::spawn(async move { update_runtime.apply_sandbox_update(stale_ready).await }); + assert!( + tokio::time::timeout(Duration::from_millis(50), &mut update) + .await + .is_err(), + "queued Ready event must wait for the active start operation" + ); + + driver.release_start(); + start.await.unwrap().unwrap(); + update.await.unwrap().unwrap(); + + let stored = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Starting as i32); + } + #[tokio::test] async fn live_container_exit_during_start_still_transitions_to_error() { for reason in [ @@ -9298,6 +9388,66 @@ mod tests { ); } + #[tokio::test] + async fn supervisor_session_connected_rejects_stopped_sandbox() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + + let error = runtime + .supervisor_session_connected("sb-1", "stale-generation") + .await + .unwrap_err(); + + assert!(error.contains("Stopped")); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!(stored.phase(), SandboxPhase::Stopped as i32); + } + + #[tokio::test] + async fn supervisor_session_connected_retries_a_stale_store_snapshot() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); + runtime.store.put_message(&sandbox).await.unwrap(); + let stale = runtime.store.get_message::("sb-1").await.unwrap(); + + runtime + .store + .update_message_cas::("sb-1", 0, |sandbox| { + sandbox.set_current_policy_version(7); + }) + .await + .unwrap(); + + runtime + .set_supervisor_session_state_from_snapshot( + "sb-1", + true, + Some("test-generation"), + false, + stale, + ) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Ready + ); + assert_eq!(stored.current_policy_version(), 7); + } + #[tokio::test] async fn supervisor_session_disconnected_demotes_ready_sandbox() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index c8491dc1eb..5e69d7215f 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -785,26 +785,36 @@ pub async fn handle_connect_supervisor( return Err(Status::internal("failed to send session accepted")); } - if superseded { - state - .supervisor_sessions - .replay_pending_relays(&sandbox_id, &tx) - .await; - } - if let Err(err) = state .compute .supervisor_session_connected(&sandbox_id, &hello.instance_id) .await { + // Do not expose SessionAccepted to the supervisor when the gateway + // could not durably record the connection. Dropping the buffered + // response forces a reconnect, which gives the state transition a + // fresh chance instead of leaving a healthy-looking supervisor tied + // to a sandbox that never reaches Ready. + state + .supervisor_sessions + .remove_if_current(&sandbox_id, &session_id); warn!( sandbox_id = %sandbox_id, session_id = %session_id, error = %err, "supervisor session: failed to mark sandbox ready" ); - } else { - state.telemetry.sandbox_session_connected(&sandbox_id); + return Err(Status::aborted( + "failed to persist supervisor session state; reconnect", + )); + } + state.telemetry.sandbox_session_connected(&sandbox_id); + + if superseded { + state + .supervisor_sessions + .replay_pending_relays(&sandbox_id, &tx) + .await; } // Step 4: Spawn the session loop that reads inbound messages.