From f6d6d17f165688422c7984352b29397780d91a9a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sat, 5 Sep 2026 09:05:31 -0700 Subject: [PATCH] feat(docker): launch capability-free sandbox containers Signed-off-by: Drew Newberry --- Cargo.lock | 5 + crates/openshell-driver-docker/Cargo.toml | 9 +- crates/openshell-driver-docker/README.md | 276 +- .../openshell-driver-docker/src/isolation.rs | 152 + crates/openshell-driver-docker/src/lib.rs | 2888 +++++++++++++---- crates/openshell-driver-docker/src/tests.rs | 897 +++-- docs/reference/gateway-config.mdx | 7 +- docs/reference/sandbox-compute-drivers.mdx | 2 +- e2e/rust/tests/credential_gating.rs | 44 +- e2e/rust/tests/driver_config_volume.rs | 14 +- e2e/rust/tests/forward_proxy_l7_bypass.rs | 16 +- e2e/rust/tests/gateway_start.rs | 11 +- e2e/rust/tests/local_driver_token_restart.rs | 21 +- e2e/rust/tests/proxy_egress_pipeline.rs | 494 +-- e2e/rust/tests/transparent_tcp.rs | 7 +- rfc/0003-gateway-configuration/README.md | 3 +- 16 files changed, 3255 insertions(+), 1591 deletions(-) create mode 100644 crates/openshell-driver-docker/src/isolation.rs diff --git a/Cargo.lock b/Cargo.lock index 0322d2e96c..6b3102d2db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3916,15 +3916,20 @@ dependencies = [ "clap", "futures", "http 1.4.0", + "libc", "miette", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "opentelemetry", "opentelemetry_sdk", "prost-types", + "rand 0.9.4", + "rustix 1.1.4", "serde", "serde_json", + "sha2 0.10.9", "tar", "temp-env", "tempfile", diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 7a5fa3fe3d..38051327aa 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -38,15 +39,19 @@ miette = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } http = { workspace = true } +rand = { workspace = true } +sha2 = { workspace = true } +rustix = { workspace = true } +libc = "0.2" +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" temp-env = "0.3" -tempfile = "3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index bbd7e69b88..b8f9cd2011 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -1,148 +1,111 @@ # openshell-driver-docker -Docker-backed compute driver for local OpenShell gateways. +Docker-backed compute driver for local and remote OpenShell gateways. -When the gateway configures `[openshell.gateway.otlp]`, Docker compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-docker`. The in-process driver preserves the gateway trace -context and emits the compute-driver RPC boundary that a standalone driver -would expose. +The driver uses `bollard` to manage sandbox resources through the configured +Docker API socket. When `socket_path` is unset, it selects the first standard +local socket that responds to an API ping. An explicitly selected Docker driver +falls back to `/var/run/docker.sock` when no candidate responds. -`mise run gateway:docker` enables this export only when a local collector is -listening on `127.0.0.1:4317`. Otherwise, it omits the gateway OTLP configuration -so the development gateway does not repeatedly report export failures. - -The standalone `openshell-driver-docker` binary accepts -`OPENSHELL_OTLP_ENDPOINT`. When set, it exports Docker driver spans to that -collector, continues W3C trace context from gateway RPC metadata, and flushes -spans during graceful shutdown. - -The driver manages sandbox containers through the local Docker daemon with the -`bollard` client. It is intended for developer environments where Docker is -already available and running Kubernetes would be unnecessary. - -The driver connects to `[openshell.drivers.docker].socket_path` when configured. -Otherwise, it uses the first standard local Docker socket that responds to an -API ping, which is the same selection mechanism used by gateway auto-detection. -An explicitly selected Docker driver falls back to `/var/run/docker.sock` when -no candidate responds. +When the gateway configures `[openshell.gateway.otlp]`, the in-process driver +exports spans to the same OTLP/gRPC collector as +`openshell-driver-docker`. The standalone driver accepts +`OPENSHELL_OTLP_ENDPOINT`, continues W3C trace context from gateway RPC +metadata, and flushes spans during graceful shutdown. ## Runtime Model -The gateway runs as a host process. The Docker driver creates one container per -sandbox and starts the `openshell-sandbox` supervisor inside that container. The -supervisor then creates the nested sandbox namespace for the agent process. - -## Stop and Start - -Stop stops the managed container without removing it. Docker retains the -container writable layer, attached volumes, labels, token material, and restart -policy. Start starts that same container, so files in the resolved OCI -workspace remain available. A durably stopped sandbox is excluded from -gateway startup recovery and stays stopped across gateway restarts. Delete -continues to force-remove the container and clean up driver-owned material. -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID, raw OCI `Config.User`, and OCI -`Config.WorkingDir`. Container creation uses that image ID, preventing a -mutable tag from changing between inspection and launch. The supervisor runs as -root, resolves omitted policy identity fields from the image declaration, and -drops only agent children to the resulting identity. Named OCI components -remain names after validation; a missing group is filled with the user's -numeric primary GID. Explicit `process.run_as_user` and -`process.run_as_group` values take precedence independently. - -An absolute OCI working directory becomes the agent workspace. An empty, -root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell -creates when necessary and owns as a compatibility workspace. Any other image -workdir must already exist without symlink components. The completed identity, -including supplementary groups, must already be able to traverse every parent -and write and enter the workdir. OpenShell does not change its ownership or -mode. - -OpenShell deliberately asks the Linux kernel to make this access decision -under the completed sandbox identity instead of reproducing permission rules -from ownership and mode bits. Mode-bit inspection alone can reject authority -granted by a POSIX ACL or overlook a denial imposed by a Linux Security Module -such as SELinux or AppArmor. OpenShell does not configure or otherwise manage -ACLs or LSM policy here; the one-shot validator only observes the kernel's -effective decision. This keeps the no-authority-expansion invariant aligned -with the access the eventual workload will receive without adding a separate, -incomplete permission model to OpenShell. - -Image `VOLUME` declarations must not cover the workdir or one of its parents -because Docker would mount the volume before the supervisor could validate the -immutable image path. -Workdirs under the standard OCI runtime namespaces `/proc`, `/sys`, and `/dev` -are rejected, as are paths that overlap concrete OpenShell control resources. -The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then -reports an invalid workdir as a readiness failure. - -Docker containers join an OpenShell-managed bridge network. The driver injects -`host.openshell.internal` and `host.docker.internal` so supervisors have stable -names for reaching the gateway host. On Docker Desktop, Colima, Rancher -Desktop, OrbStack, and macOS-hosted gateways, those names use Docker's -`host-gateway` alias. The driver requests a separate IPv4 loopback callback -listener when the primary listener does not already cover it. On native Linux -Docker, the gateway also binds the bridge gateway IP so containers can call -back to the host process. +The driver creates two containers for each sandbox: + +- `openshell-sandbox` is PID 1 in the workload container. It owns the workload + process tree, seccomp notification broker, mandatory Landlock baseline, + binary identity, exec/signal/wait/PTY operations, and loopback forwarding. +- `openshell-supervisor` runs in a separate companion container. It owns the + gateway session, policy engine, credentials, interception CA, SSH relay, L7 + inspection, DNS policy, and external upstream connections. + +Both containers are non-root, request no capabilities, and set +no-new-privileges. They share only a driver-created Docker named volume. The +volume carries an authenticated Unix socket and immutable bootstrap material; +it is writable by the sandbox and read-only in the supervisor. + +The workload uses `network_mode=none`. Its seccomp user-notification broker +mediates every supported TCP and DNS operation, attributes it to the calling +binary, and sends the request across the private channel. The supervisor +authorizes the request before it opens an upstream connection. Docker's absent +workload network is the mandatory outer fence if mediation fails or is +bypassed. Only the supervisor companion joins the managed bridge network. + +The driver copies trusted runtime bytes from the configured supervisor image +through the Docker archive API. No workload launch depends on a host bind +mount or a tool supplied by the workload image, so the same path works with +local, remote, and VM-backed Docker daemons. + +## Identity and Workspace + +Before creating the workload, the driver pins the image ID and reads its +passwd/group databases through a stopped metadata container. It resolves the +admitted policy identity, or the image `Config.User` fallback, into one exact +non-root UID, primary GID, and supplementary-group set. Docker launches +`openshell-sandbox` with that identity, and the sandbox uses the same identity +for every canonical and exec process. UID or GID zero and unresolved symbolic +identities are rejected. + +An absolute OCI working directory becomes the workspace. An empty, root (`/`), +or explicit `/sandbox` declaration uses `/sandbox`. Any other workdir must +already exist without symlink components. The resolved identity must be able to +traverse every parent and write and enter the workdir; OpenShell does not +change its ownership or mode. + +Image `VOLUME` declarations and user mounts must not cover the workdir, one of +its parents, or the reserved `/.openshell` runtime/channel tree. OpenShell asks +the kernel to validate access under the final identity, so POSIX ACL and host +LSM decisions remain authoritative. ## Container Contract -The driver-controlled container settings are part of the sandbox security -contract: - | Setting | Purpose | |---|---| -| `user = "0"` | The supervisor needs root inside the container to prepare namespaces, mounts, Landlock, and seccomp. | -| `network_mode = openshell` | Places the supervisor on the managed Docker bridge network. | -| `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | -| `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | -| `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | -| CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | -| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | - -The agent child process does not retain these supervisor privileges. +| Exact non-root `user` and `group_add` | Gives sandbox and workload the same immutable UID/GID/group identity required for capability-free observation. | +| `cap_drop = ALL`, no `cap_add`, no-new-privileges | Prevents either container from acquiring Linux capabilities. | +| Docker default seccomp and AppArmor profiles | Retains runtime hardening; startup confirmation fails closed if nested seccomp notification is unavailable. | +| `network_mode = none` on the workload | Removes direct external routes. The supervisor companion alone has bridge networking. | +| `restart_policy = no` | Keeps canonical main-process exit terminal. | +| `PidsLimit` | Applies the configured sandbox PID budget. Set `sandbox_pids_limit = 0` to use the runtime default. | +| Private named volume | Carries a per-generation mutual-TLS sandbox/supervisor channel without sharing daemon-host paths. The sandbox consumes its server key at startup; only the supervisor receives the client key. | +| In-memory `/run` tmpfs | Supplies writable runtime state without changing the workload image root filesystem. | +| CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | + +## Stop, Start, and Delete + +Stop terminates the supervisor companion and stops the workload container +without removing it. Docker retains the workload writable layer and attached +volumes. Start stages a fresh sandbox bootstrap bundle, restarts that workload, +and creates a new supervisor companion. A durably stopped sandbox stays stopped +across gateway restarts. + +Delete force-removes both containers, the driver-owned channel volume, and the +host-private topology record. Missing or altered topology and channel resources +fail closed; the driver does not run an older combined-supervisor layout. ## Driver Config Mounts -The gateway forwards the `docker` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Docker -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.docker]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Docker named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. - Docker local-driver volumes created with bind options are treated as host - bind mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. - -Host bind mounts are disabled by default because they expose gateway host -paths to sandbox requests. Image mounts are not part of the Docker -driver-config schema. The driver still uses internal bind mounts for -OpenShell-owned supervisor, token, and TLS material. - -Docker `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. Docker `volume` mounts may include -`subpath`. User-supplied bind and volume mounts are read-only by default; set -`read_only: false` to make them writable. Mount `source`, `target`, and -`subpath` values must not contain surrounding whitespace. Mount targets must be -absolute container paths and must not replace or contain the resolved workspace -root. Nested workspace mounts remain valid. Mounts also must not overlap the -configured SSH socket or the reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network -namespace roots. - -Example named-volume usage: +The gateway forwards the `docker` block from `--driver-config-json`. Supported +mount types are: + +- `bind`: an absolute daemon-host path, allowed only when + `[openshell.drivers.docker].enable_bind_mounts = true`. +- `volume`: an existing named volume. The driver never creates or removes a + user-supplied volume. Bind-backed local volumes require + `enable_bind_mounts = true`. +- `tmpfs`: an in-memory filesystem with optional size and mode. + +Host bind mounts are disabled by default because they expose daemon-host paths +to sandbox requests. User bind and volume mounts are read-only by default. +Targets must be absolute, normalized paths and cannot overlap the workspace +root or OpenShell control paths. + +Example: ```shell docker volume create openshell-work @@ -152,51 +115,36 @@ openshell sandbox create \ -- claude ``` -## Supervisor Binary Resolution - -The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into -each sandbox container. Resolution order is: - -1. `supervisor_bin` in `[openshell.drivers.docker]`. -2. `supervisor_image` in `[openshell.drivers.docker]`, extracting - `/openshell-sandbox` from that image. -3. A sibling `openshell-sandbox` next to the running `openshell-gateway` binary. -4. A local Linux cargo target build for the Docker daemon architecture. -5. The release-matched default supervisor image, extracting `/openshell-sandbox`. +## Runtime Image -Release and Docker-image gateway builds bake the matching supervisor image tag -into the binary at compile time. The default Docker supervisor image is not -`:latest` unless a custom build explicitly sets that tag. +`supervisor_image` must contain `/openshell-sandbox` and +`/openshell-supervisor`. The driver extracts the sandbox binary as bytes and +stages it into the stopped workload. It starts the supervisor binary directly +in the companion container. Release and gateway image builds bake a matching +supervisor image tag into the binary. ## Callback and TLS -`OPENSHELL_ENDPOINT` is injected from the gateway's configured gRPC endpoint. -When no endpoint is configured, the driver uses -`host.openshell.internal:` with the appropriate HTTP or HTTPS -scheme. Set `host_gateway_ip` only when the host has an explicit, locally -assigned address that containers should use for callbacks; package-managed -macOS gateways should leave it unset. - -For HTTPS endpoints, the server certificate must include the endpoint host as a -subject alternative name. Docker sandboxes also need the client TLS bundle -mounted into the container and exposed with: - -- `OPENSHELL_TLS_CA` -- `OPENSHELL_TLS_CERT` -- `OPENSHELL_TLS_KEY` - -HTTP endpoints reject TLS material because the supervisor would not use it. +`OPENSHELL_ENDPOINT` and gateway authentication material are injected only into +the supervisor companion. The workload never receives the sandbox JWT, gateway +client TLS key, policy authority, or interception CA private key. -## Environment Ownership +When no endpoint is configured, the driver derives +`host.openshell.internal:`. Native Linux uses the managed bridge +gateway. Docker Desktop and compatible VM-backed daemons use Docker's +`host-gateway` route. A configured HTTPS server certificate must include the +endpoint host in its subject alternative names. -The driver merges template environment and sandbox spec environment first, then -overwrites security-critical keys: +The supervisor owns these security-critical variables: - `OPENSHELL_ENDPOINT` - `OPENSHELL_SANDBOX_ID` - `OPENSHELL_SANDBOX` +- `OPENSHELL_SANDBOX_TOKEN_FILE` - `OPENSHELL_SSH_SOCKET_PATH` - `OPENSHELL_MAIN_PROCESS_SPEC` - TLS path variables when HTTPS is enabled -Do not allow sandbox images or templates to override these values. +Template and sandbox environment is encoded in the protected bootstrap and +exposed only to workload children. Workload input cannot override +security-critical supervisor variables. diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs new file mode 100644 index 0000000000..08364d06e8 --- /dev/null +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Docker provisioning for the shared authenticated boundary protocol. +//! +//! Docker owns only the container/socket topology and immutable OCI resource +//! claims. Lifecycle, process, network, identity, and wire behavior live in +//! `openshell-isolation-interface` and `openshell-sandbox`. + +use std::collections::{BTreeMap, HashMap}; +use std::net::IpAddr; +use std::path::PathBuf; + +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +/// Driver-owned inputs that bind one Docker container to one boundary. +pub struct DockerBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub generation: String, + pub session_epoch: String, + pub container_id: String, + pub image_identity: String, + pub listener_socket: PathBuf, + pub control_socket: PathBuf, + pub sandbox_tls: BoundaryServerTls, + pub supervisor_tls: BoundaryClientTls, + pub host_gateway_ip: Option, + pub workload_identity: ResolvedWorkloadIdentity, + pub child_env: HashMap, +} + +/// Protected container config and matching host descriptor. +pub struct DockerBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl DockerBoundarySpec { + /// Produce both sides of the common protocol from the same immutable + /// Docker coordinates so attach cannot bind a different container. + #[must_use] + pub fn provision(self) -> DockerBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("docker.container_id".to_string(), self.container_id), + ("docker.image_identity".to_string(), self.image_identity), + ]); + let driver_fence = DriverFenceEvidence::Docker { + container_id: resource_claims["docker.container_id"].clone(), + network_mode: "none".to_string(), + unexpected_networks: Vec::new(), + }; + DockerBoundaryProvisioning { + boundary_config: BoundaryConfig { + boundary_id: self.boundary_id.clone(), + generation: self.generation.clone(), + session_epoch: self.session_epoch.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::Unix { + socket_path: self.listener_socket, + tls: self.sandbox_tls, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::new(), + workload_identity: self.workload_identity.clone(), + driver_fence: driver_fence.clone(), + child_env: self.child_env, + }, + topology: BoundaryTopology { + boundary_id: self.boundary_id, + generation: self.generation, + session_epoch: self.session_epoch, + workload_identity: self.workload_identity, + transport: BoundaryTransport::Unix { + socket_path: self.control_socket, + tls: self.supervisor_tls, + }, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + driver_fence, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provisioning_binds_container_and_image_claims() { + let tls = openshell_isolation_interface::boundary_protocol::generate_boundary_mutual_tls_material() + .unwrap(); + let provisioned = DockerBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + generation: "generation-1".to_string(), + session_epoch: "epoch-1".to_string(), + container_id: "sha256:container".to_string(), + image_identity: "sha256:image".to_string(), + listener_socket: PathBuf::from("/run/openshell/boundary/control.sock"), + control_socket: PathBuf::from("/host/control.sock"), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/run/openshell/boundary/server.crt"), + private_key_path: PathBuf::from("/run/openshell/boundary/server.key"), + client_ca_certificate_path: PathBuf::from("/run/openshell/boundary/client-ca.crt"), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name, + ca_certificate_pem: tls.ca_certificate_pem, + certificate_chain_pem: tls.supervisor_certificate_pem, + private_key_pem: tls.supervisor_private_key_pem, + }, + host_gateway_ip: Some(IpAddr::from([127, 0, 0, 1])), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "image".to_string(), + "sha256:image".to_string(), + ) + .unwrap(), + child_env: HashMap::new(), + } + .provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["docker.container_id"], + "sha256:container" + ); + assert_eq!( + provisioned.boundary_config.driver_fence, + provisioned.topology.driver_fence + ); + assert!( + provisioned + .topology + .driver_fence + .validate_for_backend("docker") + .is_ok() + ); + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 88624ec08f..b87da3385d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,19 +5,21 @@ #![allow(clippy::result_large_err)] +mod isolation; pub mod otel_tracing; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, - ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, EndpointSettings, HostConfig, Mount, - MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, - ProgressDetail, SystemInfo, + ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, HostConfig, Mount, + MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, ProgressDetail, + SystemInfo, VolumeCreateRequest, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, - ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, + ListContainersOptionsBuilder, ListVolumesOptionsBuilder, LogsOptionsBuilder, + RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, UploadToContainerOptionsBuilder, }; use bytes::Bytes; use futures::{Stream, StreamExt}; @@ -27,7 +29,7 @@ use openshell_core::driver_utils::{ CONDITION_EXITED, CONDITION_RUNTIME_RESTART, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, extract_first_tar_entry, supervisor_image_should_refresh, - temp_extract_container_name, validate_linux_elf_binary, write_cache_binary_atomic, + temp_extract_container_name, }; use openshell_core::gpu::{ CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, @@ -55,14 +57,21 @@ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; use openshell_core::{Error, Result as CoreResult}; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryServerTls, BoundaryTopology, generate_boundary_mutual_tls_material, +}; +use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; use opentelemetry::trace::TraceContextExt as _; +use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +#[cfg(unix)] use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio::sync::{Mutex, broadcast, mpsc, oneshot}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -74,12 +83,33 @@ const WATCH_BUFFER: usize = 128; const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; +const SANDBOX_BINARY_PATH: &str = "/.openshell/runtime/openshell-sandbox"; +const SUPERVISOR_IMAGE_CONTROL_BINARY_PATH: &str = "/openshell-supervisor"; +const SUPERVISOR_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; +const SUPERVISOR_UID: u32 = 65_534; +const SUPERVISOR_GID: u32 = 65_534; +const BOUNDARY_MOUNT_PATH: &str = "/.openshell/channel"; +const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; +const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; +const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; +const BOUNDARY_CLIENT_CA_MOUNT_PATH: &str = "/.openshell/channel/sandbox/client-ca.crt"; +const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/channel/supervisor"; +const DRIVER_ADMITTED_BACKEND: &str = "docker"; +const LABEL_ISOLATION_TOPOLOGY: &str = "openshell.ai/isolation-topology"; +const LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE: &str = "capability-free"; +const LABEL_ISOLATION_ROLE: &str = "openshell.ai/isolation-role"; +const LABEL_ISOLATION_ROLE_SANDBOX: &str = "sandbox"; +const LABEL_ISOLATION_ROLE_SUPERVISOR: &str = "supervisor"; +const LABEL_ISOLATION_ROLE_STAGING: &str = "staging"; +const LABEL_ISOLATION_ROLE_IDENTITY: &str = "identity"; +const TOPOLOGY_PAYLOAD_FILE: &str = "topology.payload"; +const MAIN_PROCESS_SPEC_FILE: &str = "main-process.json"; +const WORKSPACE_ROOT_FILE: &str = "workspace-root"; +const BOUNDARY_CONFIG_FILE: &str = "boundary-bootstrap.json"; +const BOUNDARY_CERTIFICATE_FILE: &str = "boundary-server.crt"; +const BOUNDARY_PRIVATE_KEY_FILE: &str = "boundary-server.key"; +const BOUNDARY_CLIENT_CA_FILE: &str = "boundary-client-ca.crt"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; @@ -127,12 +157,8 @@ pub struct DockerComputeConfig { /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. - pub supervisor_bin: Option, - - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. + /// Image containing the trusted `openshell-sandbox` and + /// `openshell-supervisor` binaries. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -150,9 +176,6 @@ pub struct DockerComputeConfig { /// Host gateway IP used for sandbox host aliases. pub host_gateway_ip: String, - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - /// Container cgroup PID limit for Docker-managed sandboxes. /// /// Set to `0` to leave Docker's runtime/default PID limit unchanged. @@ -172,14 +195,12 @@ impl Default for DockerComputeConfig { image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), grpc_endpoint: String::new(), - supervisor_bin: None, supervisor_image: None, guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), host_gateway_ip: String::new(), - ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, enable_bind_mounts: false, } @@ -198,14 +219,15 @@ struct DockerDriverRuntimeConfig { default_image: String, image_pull_policy: String, sandbox_namespace: String, - grpc_endpoint: String, - network_name: String, gateway_route: DockerGatewayRoute, gateway_callback_bind_address: Option, - ssh_socket_path: String, stop_timeout_secs: u32, log_level: String, - supervisor_bin: PathBuf, + sandbox_binary: Arc>, + supervisor_image_id: String, + network_name: String, + supervisor_grpc_endpoint: String, + gateway_tls_server_name: Option, guest_tls: Option, daemon_version: String, supports_gpu: bool, @@ -216,10 +238,7 @@ struct DockerDriverRuntimeConfig { #[derive(Debug, Clone, PartialEq, Eq)] enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, + Bridge { bind_address: SocketAddr }, HostGateway, } @@ -231,6 +250,30 @@ pub struct DockerComputeDriver { pending: Arc>>, gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, + control_processes: Arc>>, + runtime_failures: Arc>>, +} + +struct DockerControlProcess { + shutdown: Option>, + task: JoinHandle<()>, +} + +#[derive(Clone)] +struct DockerRuntimeFailure { + reason: &'static str, + message: String, +} + +#[derive(Clone)] +struct DockerRuntimeFailureContext { + docker: Arc, + events: broadcast::Sender, + failures: Arc>>, + sandbox: DriverSandbox, + sandbox_namespace: String, + container_id: String, + stop_timeout_secs: u32, } /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. @@ -330,6 +373,195 @@ struct DockerImageMetadata { volumes: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct DockerPasswdEntry { + name: String, + uid: u32, + gid: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DockerGroupEntry { + name: String, + gid: u32, + members: Vec, +} + +fn parse_docker_passwd(bytes: &[u8]) -> Result, Status> { + let contents = std::str::from_utf8(bytes).map_err(|error| { + Status::failed_precondition(format!("image /etc/passwd is not UTF-8: {error}")) + })?; + let mut entries = Vec::new(); + for (index, line) in contents.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = line.split(':').collect::>(); + if fields.len() < 4 || fields[0].is_empty() { + return Err(Status::failed_precondition(format!( + "image /etc/passwd line {} is malformed", + index + 1 + ))); + } + let uid = fields[2].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/passwd line {} has an invalid UID", + index + 1 + )) + })?; + let gid = fields[3].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/passwd line {} has an invalid GID", + index + 1 + )) + })?; + entries.push(DockerPasswdEntry { + name: fields[0].to_string(), + uid, + gid, + }); + } + Ok(entries) +} + +fn parse_docker_group(bytes: &[u8]) -> Result, Status> { + let contents = std::str::from_utf8(bytes).map_err(|error| { + Status::failed_precondition(format!("image /etc/group is not UTF-8: {error}")) + })?; + let mut entries = Vec::new(); + for (index, line) in contents.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = line.split(':').collect::>(); + if fields.len() < 4 || fields[0].is_empty() { + return Err(Status::failed_precondition(format!( + "image /etc/group line {} is malformed", + index + 1 + ))); + } + let gid = fields[2].parse::().map_err(|_| { + Status::failed_precondition(format!( + "image /etc/group line {} has an invalid GID", + index + 1 + )) + })?; + entries.push(DockerGroupEntry { + name: fields[0].to_string(), + gid, + members: fields[3] + .split(',') + .filter(|member| !member.is_empty()) + .map(str::to_string) + .collect(), + }); + } + Ok(entries) +} + +fn resolve_numeric_or_named_user<'a>( + selector: &str, + passwd: &'a [DockerPasswdEntry], +) -> Result<(u32, Option<&'a DockerPasswdEntry>), Status> { + if let Ok(uid) = selector.parse::() { + return Ok((uid, passwd.iter().find(|entry| entry.uid == uid))); + } + let entry = passwd + .iter() + .find(|entry| entry.name == selector) + .ok_or_else(|| { + Status::failed_precondition(format!( + "workload user '{selector}' does not exist in the pinned image" + )) + })?; + Ok((entry.uid, Some(entry))) +} + +fn resolve_numeric_or_named_group( + selector: &str, + groups: &[DockerGroupEntry], +) -> Result { + if let Ok(gid) = selector.parse::() { + return Ok(gid); + } + groups + .iter() + .find(|entry| entry.name == selector) + .map(|entry| entry.gid) + .ok_or_else(|| { + Status::failed_precondition(format!( + "workload group '{selector}' does not exist in the pinned image" + )) + }) +} + +fn resolve_docker_identity_from_accounts( + sandbox: &DriverSandbox, + image: &DockerImageMetadata, + passwd_bytes: &[u8], + group_bytes: &[u8], +) -> Result { + let passwd = parse_docker_passwd(passwd_bytes)?; + let groups = parse_docker_group(group_bytes)?; + let request = sandbox + .spec + .as_ref() + .and_then(|spec| spec.workload_identity.as_ref()); + let requested_user = request.map_or("", |request| request.user.trim()); + let requested_group = request.map_or("", |request| request.group.trim()); + let (image_user, image_group) = image.user.split_once(':').unwrap_or((&image.user, "")); + let user_selector = if requested_user.is_empty() { + image_user.trim() + } else { + requested_user + }; + if user_selector.is_empty() { + return Err(Status::failed_precondition( + "the pinned image defaults to root; configure a non-root process.run_as_user", + )); + } + let (uid, passwd_entry) = resolve_numeric_or_named_user(user_selector, &passwd)?; + let username = passwd_entry.map(|entry| entry.name.as_str()); + let group_selector = if requested_group.is_empty() { + image_group.trim() + } else { + requested_group + }; + let gid = if group_selector.is_empty() { + passwd_entry.map(|entry| entry.gid).ok_or_else(|| { + Status::failed_precondition(format!( + "numeric workload UID {uid} has no passwd entry; configure process.run_as_group" + )) + })? + } else { + resolve_numeric_or_named_group(group_selector, &groups)? + }; + let supplementary_gids = username.map_or_else(Vec::new, |username| { + groups + .iter() + .filter(|entry| { + entry.gid != gid && entry.members.iter().any(|member| member == username) + }) + .map(|entry| entry.gid) + .collect() + }); + let source = if !requested_user.is_empty() || !requested_group.is_empty() { + "policy" + } else { + "image" + }; + ResolvedWorkloadIdentity::new( + uid, + gid, + supplementary_gids, + source.to_string(), + image.id.clone(), + ) + .map_err(|error| Status::failed_precondition(error.to_string())) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] struct DockerResourceLimits { nano_cpus: Option, @@ -527,29 +759,62 @@ impl DockerComputeDriver { docker_config.grpc_endpoint = format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, + let host_grpc_endpoint = + docker_host_openshell_endpoint(&docker_config.grpc_endpoint, &gateway_route)?; + let original_gateway_url = Url::parse(&docker_config.grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid docker grpc_endpoint '{}': {error}", + docker_config.grpc_endpoint + )) + })?; + let host_gateway_url = Url::parse(&host_grpc_endpoint).map_err(|error| { + Error::config(format!( + "invalid normalized Docker host grpc_endpoint '{host_grpc_endpoint}': {error}" + )) + })?; + let gateway_tls_server_name = (original_gateway_url.scheme() == "https" + && original_gateway_url.host_str() != host_gateway_url.host_str()) + .then(|| { + original_gateway_url + .host_str() + .unwrap_or_default() + .to_string() + }); + let supervisor_grpc_endpoint = match &gateway_route { + DockerGatewayRoute::Bridge { .. } => host_grpc_endpoint, + DockerGatewayRoute::HostGateway => docker_config.grpc_endpoint.clone(), + }; + let supervisor_image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let supervisor_image_id = + ensure_supervisor_container_image(&docker, &supervisor_image).await?; + let sandbox_binary = Arc::new( + extract_supervisor_binary_bytes(&docker, &supervisor_image_id) + .await + .map_err(|error| { + Error::config(format!( + "failed to load trusted sandbox binary from Docker image '{supervisor_image}': {error}" + )) + })?, ); - let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); - let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; let guest_tls = docker_guest_tls_paths(&docker_config)?; - let driver = Self { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), sandbox_namespace: docker_config.sandbox_namespace.clone(), - grpc_endpoint, - network_name, gateway_route, gateway_callback_bind_address, - ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: gateway_log_level.to_string(), - supervisor_bin, + sandbox_binary, + supervisor_image_id, + network_name, + supervisor_grpc_endpoint, + gateway_tls_server_name, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, @@ -564,8 +829,19 @@ impl DockerComputeDriver { allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), + runtime_failures: Arc::new(Mutex::new(HashMap::new())), }; + Box::pin(driver.reconcile_runtime_resources_at_startup()) + .await + .map_err(|error| { + Error::config(format!( + "failed to reconcile Docker isolation resources: {}", + error.message() + )) + })?; + let poll_driver = driver.clone(); tokio::spawn(async move { poll_driver.poll_loop().await; @@ -757,6 +1033,74 @@ impl DockerComputeDriver { .map_err(docker_gpu_selection_status) } + async fn resolve_docker_workload_identity( + &self, + sandbox: &DriverSandbox, + image: &DockerImageMetadata, + ) -> Result { + let container_name = format!("{}-identity", temp_extract_container_name()); + self.docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(image.id.clone()), + labels: Some(docker_auxiliary_container_labels( + sandbox, + &self.config, + LABEL_ISOLATION_ROLE_IDENTITY, + )), + ..Default::default() + }, + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "create Docker identity resolver container: {error}" + )) + })?; + + let result = async { + let passwd = + download_path_from_container(&self.docker, &container_name, "/etc/passwd", true) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read immutable image /etc/passwd for workload identity: {error}" + )) + })?; + let group = + download_path_from_container(&self.docker, &container_name, "/etc/group", true) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read immutable image /etc/group for workload identity: {error}" + )) + })?; + resolve_docker_identity_from_accounts(sandbox, image, &passwd, &group) + } + .await; + + if let Err(error) = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + warn!( + container = container_name, + %error, + "Failed to remove Docker identity resolver container" + ); + } + result + } + async fn get_sandbox_snapshot( &self, sandbox_id: &str, @@ -765,9 +1109,10 @@ impl DockerComputeDriver { let container = self .find_managed_container_summary(sandbox_id, sandbox_name) .await?; - if let Some(sandbox) = + if let Some(mut sandbox) = container.and_then(|summary| sandbox_from_container_summary(&summary)) { + self.apply_runtime_failure(&mut sandbox).await; return Ok(Some(sandbox)); } @@ -781,30 +1126,39 @@ impl DockerComputeDriver { let Some(mut sandbox) = sandbox_from_container_summary(summary) else { continue; }; - // Docker's list summary carries no exit code, so an exited - // container is reported as the generic terminal `ContainerExited`. - // Inspect it to tell a machine/daemon-restart signal kill apart - // from an ordinary application exit, mirroring the Podman driver, - // so startup recovery can revive restart victims while leaving - // crashes terminal. - if summary.state == Some(ContainerSummaryStateEnum::EXITED) - && let Some(container_id) = summary.id.as_deref() - { + if let Some(container_id) = summary.id.as_deref() { match self.docker.inspect_container(container_id, None).await { - Ok(inspected) => { + Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::EXITED) => { + // Docker's list summary carries no exit code. Inspect + // exited containers so daemon-restart kills remain + // distinguishable from terminal application exits. if let Some(state) = inspected.state.as_ref() { apply_docker_exit_classification(&mut sandbox, state); } } + Ok(inspected) if summary.state == Some(ContainerSummaryStateEnum::RUNNING) => { + if let Err(status) = validate_docker_outer_fence(&inspected) { + let context = self + .control_failure_context(sandbox.clone(), container_id.to_string()); + handle_docker_runtime_failure( + context, + "OuterFenceViolation", + status.message().to_string(), + ) + .await; + } + } + Ok(_) => {} Err(err) => { debug!( container_id, error = %err, - "Could not inspect exited Docker container to classify its exit" + "Could not inspect Docker sandbox container during reconciliation" ); } } } + self.apply_runtime_failure(&mut sandbox).await; container_sandboxes.push(sandbox); } let mut by_id = self.pending_snapshot_map().await; @@ -859,7 +1213,7 @@ impl DockerComputeDriver { let provisioning_span = provisioning_span(&parent, sandbox, &image); let task = tokio::spawn( async move { - driver.provision_sandbox(sandbox_for_task).await; + Box::pin(driver.provision_sandbox(sandbox_for_task)).await; } .instrument(provisioning_span), ); @@ -875,7 +1229,7 @@ impl DockerComputeDriver { } async fn provision_sandbox(&self, sandbox: DriverSandbox) { - match self.provision_sandbox_inner(&sandbox).await { + match Box::pin(self.provision_sandbox_inner(&sandbox)).await { Ok(()) => { self.clear_pending_sandbox(&sandbox.id).await; } @@ -911,40 +1265,83 @@ impl DockerComputeDriver { image.ref = %template.image, )) .await?; - let token_file_created = write_sandbox_token_file(sandbox, &self.config) + let workload_identity = self + .resolve_docker_workload_identity(sandbox, &image) + .await + .map_err(|status| { + DockerProvisioningFailure::new("IdentityResolutionFailed", status.message()) + })?; + prepare_docker_boundary_state_dir(sandbox, &self.config).map_err(|status| { + DockerProvisioningFailure::new("BoundaryStateCreateFailed", status.message()) + })?; + create_docker_channel_volume(&self.docker, sandbox, &self.config) .await .map_err(|status| { - DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) + cleanup_docker_boundary_state(sandbox, &self.config); + DockerProvisioningFailure::new("BoundaryChannelCreateFailed", status.message()) })?; + let token_file_created = match write_sandbox_token_file(sandbox, &self.config).await { + Ok(created) => created, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + status.message(), + )); + } + }; + if !token_file_created { + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "SandboxTokenWriteFailed", + "Docker control mode requires a gateway sandbox token", + )); + } let container_name = container_name_for_sandbox(sandbox); - let gpu_devices = self + let gpu_devices = match self .resolve_gpu_cdi_devices( validated.gpu_requirements, &validated.driver_config, CdiGpuDefaultSelector::next_device_ids, ) .await - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let create_body = build_container_create_body_for_image( + { + Ok(devices) => devices, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + status.message(), + )); + } + }; + let create_body = match build_container_create_body_for_image( sandbox, &self.config, &validated.driver_config, gpu_devices.as_deref(), &image, - ) - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); + &workload_identity, + ) { + Ok(body) => body, + Err(status) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + status.message(), + )); } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - async { + }; + let create_result = async { openshell_otel::record_error_result( self.docker .create_container( @@ -955,16 +1352,7 @@ impl DockerComputeDriver { ), create_body, ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - }), + .await, ) } .instrument(tracing::info_span!( @@ -974,7 +1362,56 @@ impl DockerComputeDriver { sandbox.id = %sandbox.id, container.name = %container_name, )) - .await?; + .await; + let created = match create_result { + Ok(created) => created, + Err(error) => { + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "ContainerCreateFailed", + create_status_from_docker_error("create docker sandbox container", error), + )); + } + }; + let inspected = match self.docker.inspect_container(&created.id, None).await { + Ok(inspected) => inspected, + Err(error) => { + let _ = self + .docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "OuterFenceInspectFailed", + internal_status("inspect Docker sandbox outer fence", error), + )); + } + }; + let outer_fence_error = validate_docker_outer_fence(&inspected).err(); + drop(inspected); + if let Some(status) = outer_fence_error { + let _ = self + .docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::from_status( + "OuterFenceRejected", + status, + )); + } self.publish_docker_progress( &sandbox.id, "Created", @@ -982,6 +1419,35 @@ impl DockerComputeDriver { HashMap::from([("container_name".to_string(), container_name.clone())]), ); + let topology = match prepare_docker_boundary_files( + &self.docker, + sandbox, + &self.config, + &created.id, + &image, + &workload_identity, + ) + .await + { + Ok(topology) => topology, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "BoundaryConfigWriteFailed", + status.message(), + )); + } + }; + let start_result = async { openshell_otel::record_error_result( self.docker.start_container(&container_name, None).await, @@ -1011,14 +1477,44 @@ impl DockerComputeDriver { "Failed to clean up Docker container after start failure" ); } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } + cleanup_docker_boundary_state(sandbox, &self.config); + let _ = + remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config).await; return Err(DockerProvisioningFailure::from_status( "ContainerStartFailed", create_status_from_docker_error("start docker sandbox container", err), )); } + self.clear_runtime_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), created.id.clone()); + let control = match spawn_docker_control_process( + &self.docker, + sandbox, + &self.config, + &topology, + failure_context, + ) + .await + { + Ok(control) => control, + Err(status) => { + let _ = self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let _ = remove_docker_channel_volume_by_id(&self.docker, &sandbox.id, &self.config) + .await; + cleanup_docker_boundary_state(sandbox, &self.config); + return Err(DockerProvisioningFailure::new( + "ControlSupervisorStartFailed", + status.message(), + )); + } + }; + self.replace_control_process(&sandbox.id, control).await; self.publish_docker_progress( &sandbox.id, "Started", @@ -1039,50 +1535,380 @@ impl DockerComputeDriver { span_status.finish(Ok(())) } - async fn delete_sandbox_inner( + async fn replace_control_process(&self, sandbox_id: &str, process: DockerControlProcess) { + let previous = self + .control_processes + .lock() + .await + .insert(sandbox_id.to_string(), process); + if let Some(previous) = previous { + stop_docker_control_process(previous).await; + } + } + + async fn clear_runtime_failure(&self, sandbox_id: &str) { + self.runtime_failures.lock().await.remove(sandbox_id); + } + + fn control_failure_context( &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; - if let Some(record) = pending.as_ref() - && let Some(task) = record.task.as_ref() - { - task.abort(); + sandbox: DriverSandbox, + container_id: String, + ) -> DockerRuntimeFailureContext { + DockerRuntimeFailureContext { + docker: self.docker.clone(), + events: self.events.clone(), + failures: self.runtime_failures.clone(), + sandbox, + sandbox_namespace: self.config.sandbox_namespace.clone(), + container_id, + stop_timeout_secs: self.config.stop_timeout_secs, } + } - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = pending { - let container_name = container_name_for_sandbox(&record.sandbox); - match self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); + async fn apply_runtime_failure(&self, sandbox: &mut DriverSandbox) { + let container_is_running = sandbox.status.as_ref().is_some_and(|status| { + status.conditions.iter().any(|condition| { + condition.r#type == "Ready" + && condition.status == "True" + && condition.reason == "BackendReady" + }) + }); + if !container_is_running { + return; + } + let failure = self.runtime_failures.lock().await.get(&sandbox.id).cloned(); + if let Some(failure) = failure { + set_sandbox_ready_condition(sandbox, error_condition(failure.reason, &failure.message)); + } + } + + async fn stop_control_process(&self, sandbox_id: &str) { + let process = self.control_processes.lock().await.remove(sandbox_id); + if let Some(process) = process { + stop_docker_control_process(process).await; + } + } + + async fn remove_auxiliary_containers_for_sandbox( + &self, + sandbox_id: &str, + ) -> Result { + let filters = managed_resource_label_filters( + &self.config.sandbox_namespace, + [format!("{LABEL_SANDBOX_ID}={sandbox_id}")], + ); + let containers = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker auxiliary containers", error))?; + let mut removed = false; + for container in containers { + let role = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str); + if !matches!( + role, + Some( + LABEL_ISOLATION_ROLE_SUPERVISOR + | LABEL_ISOLATION_ROLE_STAGING + | LABEL_ISOLATION_ROLE_IDENTITY + ) + ) { + continue; + } + let Some(target) = summary_container_target(&container) else { + continue; + }; + self.docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove Docker auxiliary container", error))?; + removed = true; + } + Ok(removed) + } + + async fn reconcile_runtime_resources_at_startup(&self) -> Result<(), Status> { + let sandboxes = self.list_managed_container_summaries().await?; + let sandbox_ids = sandboxes + .iter() + .filter_map(|container| { + container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .cloned() + }) + .collect::>(); + + let filters = managed_resource_label_filters(&self.config.sandbox_namespace, []); + let auxiliary = self + .docker + .list_containers(Some( + ListContainersOptionsBuilder::default() + .all(true) + .filters(&filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker startup resources", error))?; + for container in auxiliary { + let role = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str); + if !matches!( + role, + Some( + LABEL_ISOLATION_ROLE_SUPERVISOR + | LABEL_ISOLATION_ROLE_STAGING + | LABEL_ISOLATION_ROLE_IDENTITY + ) + ) { + continue; + } + let Some(target) = summary_container_target(&container) else { + continue; + }; + self.docker + .remove_container( + &target, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove stale Docker auxiliary", error))?; + } + + let volume_filters = label_filters([ + format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), + format!( + "{LABEL_SANDBOX_NAMESPACE}={}", + self.config.sandbox_namespace + ), + format!("{LABEL_ISOLATION_TOPOLOGY}={LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE}"), + ]); + let volumes = self + .docker + .list_volumes(Some( + ListVolumesOptionsBuilder::default() + .filters(&volume_filters) + .build(), + )) + .await + .map_err(|error| internal_status("list Docker startup volumes", error))?; + for volume in volumes.volumes.unwrap_or_default() { + let sandbox_id = volume.labels.get(LABEL_SANDBOX_ID); + if sandbox_id.is_some_and(|id| sandbox_ids.contains(id)) { + continue; + } + self.docker + .remove_volume( + &volume.name, + None::, + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| internal_status("remove orphan Docker channel volume", error))?; + } + + for sandbox in &sandboxes { + if sandbox.state == Some(ContainerSummaryStateEnum::RUNNING) + && let Err(error) = self.ensure_control_process_for_container(sandbox).await + { + warn!( + sandbox_id = sandbox + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or("unknown", String::as_str), + %error, + "Failed to restore Docker supervisor during startup reconciliation" + ); + } + } + Ok(()) + } + + async fn ensure_control_process_for_container( + &self, + container: &ContainerSummary, + ) -> Result<(), Status> { + let Some(sandbox) = sandbox_from_container_summary(container) else { + return Err(Status::internal( + "managed Docker container is missing sandbox identity labels", + )); + }; + let stale = { + let mut processes = self.control_processes.lock().await; + match processes.get(&sandbox.id) { + Some(process) if !process.task.is_finished() => return Ok(()), + Some(_) => processes.remove(&sandbox.id), + None => None, + } + }; + if let Some(stale) = stale { + stop_docker_control_process(stale).await; + } + let Some(topology) = read_docker_boundary_topology(&sandbox.id, &self.config).await? else { + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let status = Status::failed_precondition( + "Docker sandbox topology is missing; refusing to leave the workload running without its supervisor", + ); + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + status.message().to_string(), + ) + .await; + return Err(status); + }; + let container_id = summary_container_target(container) + .ok_or_else(|| Status::internal("managed Docker container has no id or name"))?; + self.clear_runtime_failure(&sandbox.id).await; + let failure_context = self.control_failure_context(sandbox.clone(), container_id); + let process = match spawn_docker_control_process( + &self.docker, + &sandbox, + &self.config, + &topology, + failure_context.clone(), + ) + .await + { + Ok(process) => process, + Err(status) => { + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + format!( + "failed to start Docker control supervisor: {}", + status.message() + ), + ) + .await; + return Err(status); + } + }; + self.replace_control_process(&sandbox.id, process).await; + Ok(()) + } + + async fn delete_sandbox_inner( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; + if let Some(record) = pending.as_ref() + && let Some(task) = record.task.as_ref() + { + task.abort(); + } + if let Some(record) = pending.as_ref() { + self.stop_control_process(&record.sandbox.id).await; + self.remove_auxiliary_containers_for_sandbox(&record.sandbox.id) + .await?; + } + + let Some(container) = self + .find_managed_container_summary(sandbox_id, sandbox_name) + .await? + else { + if let Some(record) = pending { + let container_name = container_name_for_sandbox(&record.sandbox); + match self + .docker + .remove_container( + &container_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + { + Ok(()) => { + self.clear_runtime_failure(&record.sandbox.id).await; + remove_docker_channel_volume_by_id( + &self.docker, + &record.sandbox.id, + &self.config, + ) + .await?; + cleanup_docker_boundary_state(&record.sandbox, &self.config); + return Ok(true); + } + Err(err) if is_not_found_error(&err) => { + self.clear_runtime_failure(&record.sandbox.id).await; + let _ = remove_docker_channel_volume_by_id( + &self.docker, + &record.sandbox.id, + &self.config, + ) + .await; + cleanup_docker_boundary_state(&record.sandbox, &self.config); + return Ok(true); } Err(err) => { return Err(internal_status("delete docker sandbox container", err)); } } } + if !sandbox_id.is_empty() { + self.stop_control_process(sandbox_id).await; + let removed = self + .remove_auxiliary_containers_for_sandbox(sandbox_id) + .await?; + remove_docker_channel_volume_by_id(&self.docker, sandbox_id, &self.config).await?; + cleanup_docker_boundary_state_by_id(sandbox_id, &self.config); + self.clear_runtime_failure(sandbox_id).await; + return Ok(removed); + } return Ok(false); }; let Some(target) = summary_container_target(&container) else { return Ok(pending.is_some()); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; + self.remove_auxiliary_containers_for_sandbox(resolved_sandbox_id) + .await?; match self .docker @@ -1093,11 +1919,21 @@ impl DockerComputeDriver { .await { Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_runtime_failure(resolved_sandbox_id).await; + remove_docker_channel_volume_by_id(&self.docker, resolved_sandbox_id, &self.config) + .await?; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(true) } Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); + self.clear_runtime_failure(resolved_sandbox_id).await; + let _ = remove_docker_channel_volume_by_id( + &self.docker, + resolved_sandbox_id, + &self.config, + ) + .await; + cleanup_docker_boundary_state_by_id(resolved_sandbox_id, &self.config); Ok(pending.is_some()) } Err(err) => Err(internal_status("delete docker sandbox container", err)), @@ -1110,10 +1946,16 @@ impl DockerComputeDriver { .await? else { if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { + self.stop_control_process(&record.sandbox.id).await; + self.remove_auxiliary_containers_for_sandbox(&record.sandbox.id) + .await?; + self.clear_runtime_failure(&record.sandbox.id).await; if let Some(task) = record.task { task.abort(); } - cleanup_sandbox_token_file(&record.sandbox, &self.config); + remove_docker_channel_volume_by_id(&self.docker, &record.sandbox.id, &self.config) + .await?; + cleanup_docker_boundary_state(&record.sandbox, &self.config); self.publish_deleted(record.sandbox.id); return Ok(()); } @@ -1122,8 +1964,16 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Err(Status::not_found("sandbox container has no id or name")); }; + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + self.stop_control_process(resolved_sandbox_id).await; + self.remove_auxiliary_containers_for_sandbox(resolved_sandbox_id) + .await?; - match self + let result = match self .docker .stop_container( &target, @@ -1139,7 +1989,11 @@ impl DockerComputeDriver { Err(err) if is_not_modified_error(&err) => Ok(()), Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), Err(err) => Err(internal_status("stop docker sandbox container", err)), + }; + if result.is_ok() { + self.clear_runtime_failure(resolved_sandbox_id).await; } + result } /// Start a managed sandbox container that was previously stopped. Used @@ -1168,9 +2022,8 @@ impl DockerComputeDriver { let span_status = openshell_otel::ErrorStatusGuard::current(); require_sandbox_identifier(sandbox_id, sandbox_name)?; self.lifecycle_event_fences.begin_start(sandbox_id); - let result = self - .start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name) - .await; + let result = + Box::pin(self.start_sandbox_with_lifecycle_fence(sandbox_id, sandbox_name)).await; self.lifecycle_event_fences.finish_start(sandbox_id); span_status.finish(result) } @@ -1189,20 +2042,14 @@ impl DockerComputeDriver { let Some(target) = summary_container_target(&container) else { return Ok(false); }; + let inspected = self + .docker + .inspect_container(&target, None) + .await + .map_err(|error| internal_status("inspect Docker sandbox outer fence", error))?; + validate_docker_outer_fence(&inspected)?; let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if !container_state_needs_start(state) { - return Ok(true); - } - - // Fence a poll that observed this stopped run but has not published it - // yet. Use Docker's transition timestamp so a later, genuine exit from - // the restarted container remains observable. let previous_finished_at = if state == ContainerSummaryStateEnum::EXITED { - let inspected = self - .docker - .inspect_container(&target, None) - .await - .map_err(|err| internal_status("inspect docker sandbox before start", err))?; inspected .state .as_ref() @@ -1211,17 +2058,102 @@ impl DockerComputeDriver { } else { None }; + drop(inspected); + if !container_state_needs_start(state) { + self.ensure_control_process_for_container(&container) + .await?; + return Ok(true); + } + + // Fence a poll that observed this stopped run but has not published it + // yet. Use Docker's transition timestamp so a later, genuine exit from + // the restarted container remains observable. self.lifecycle_event_fences .record_previous_exit(sandbox_id, previous_finished_at.as_deref()); + let resolved_sandbox_id = container + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .map_or(sandbox_id, String::as_str); + let Some(topology) = + read_docker_boundary_topology(resolved_sandbox_id, &self.config).await? + else { + return Err(Status::failed_precondition( + "Docker sandbox topology is missing; refusing to start the workload without its supervisor", + )); + }; + let boundary_config = tokio::fs::read( + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? + .join(BOUNDARY_CONFIG_FILE), + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox bootstrap for restart: {error}" + )) + })?; + let boundary_directory = + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)?; + let boundary_certificate = + tokio::fs::read(boundary_directory.join(BOUNDARY_CERTIFICATE_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel certificate for restart: {error}" + )) + })?; + let boundary_private_key = + tokio::fs::read(boundary_directory.join(BOUNDARY_PRIVATE_KEY_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel private key for restart: {error}" + )) + })?; + let boundary_client_ca = tokio::fs::read(boundary_directory.join(BOUNDARY_CLIENT_CA_FILE)) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox channel client CA for restart: {error}" + )) + })?; + let workspace_root = tokio::fs::read_to_string( + docker_boundary_state_dir_by_id(resolved_sandbox_id, &self.config)? + .join(WORKSPACE_ROOT_FILE), + ) + .await + .map_err(|error| { + Status::failed_precondition(format!( + "read Docker sandbox workspace for restart: {error}" + )) + })?; + stage_docker_sandbox_bundle( + &self.docker, + &target, + &self.config, + &topology.workload_identity, + &boundary_config, + DockerSandboxTls { + certificate: &boundary_certificate, + private_key: &boundary_private_key, + client_ca: &boundary_client_ca, + }, + &workspace_root, + ) + .await?; + match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), + Ok(()) => {} // Already running — race with another start path or the // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), + Err(err) if is_not_modified_error(&err) => {} + Err(err) if is_not_found_error(&err) => return Ok(false), + Err(err) => return Err(internal_status("start docker sandbox container", err)), } + self.ensure_control_process_for_container(&container) + .await?; + Ok(true) } async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { @@ -1290,7 +2222,7 @@ impl DockerComputeDriver { sandbox: &DriverSandbox, failure: &DockerProvisioningFailure, ) { - cleanup_sandbox_token_file(sandbox, &self.config); + cleanup_docker_boundary_state(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, &self.config.sandbox_namespace, @@ -1327,8 +2259,9 @@ impl DockerComputeDriver { if let Some(summary) = self .find_managed_container_summary(sandbox_id, sandbox_name) .await? - && let Some(sandbox) = sandbox_from_container_summary(&summary) + && let Some(mut sandbox) = sandbox_from_container_summary(&summary) { + self.apply_runtime_failure(&mut sandbox).await; self.publish_sandbox_snapshot(sandbox); } Ok(()) @@ -1676,6 +2609,36 @@ impl DockerComputeDriver { // Standalone and in-process servers both use this wrapper. Delegating to the // driver's canonical tonic implementation keeps request validation and Docker // operation spans identical across both deployment modes. +fn validate_docker_outer_fence( + inspected: &bollard::models::ContainerInspectResponse, +) -> Result<(), Status> { + let network_mode = inspected + .host_config + .as_ref() + .and_then(|config| config.network_mode.as_deref()); + if network_mode != Some("none") { + return Err(Status::failed_precondition(format!( + "Docker sandbox outer fence requires network_mode=none, got {}", + network_mode.unwrap_or("") + ))); + } + let unexpected_networks = inspected + .network_settings + .as_ref() + .and_then(|settings| settings.networks.as_ref()) + .into_iter() + .flat_map(HashMap::keys) + .filter(|network| network.as_str() != "none") + .cloned() + .collect::>(); + if !unexpected_networks.is_empty() { + return Err(Status::failed_precondition(format!( + "Docker sandbox outer fence found attached networks: {}", + unexpected_networks.join(", ") + ))); + } + Ok(()) +} #[tonic::async_trait] impl ComputeDriver for ComputeDriverService { type WatchSandboxesStream = WatchStream; @@ -1990,7 +2953,13 @@ impl ComputeDriver for DockerComputeDriver { request: Request, ) -> Result, Status> { let request = request.into_inner(); - if !Self::start_sandbox(self, &request.sandbox_id, &request.sandbox_name).await? { + if !Box::pin(Self::start_sandbox( + self, + &request.sandbox_id, + &request.sandbox_name, + )) + .await? + { return Err(Status::not_found("sandbox not found")); } self.publish_container_snapshot(&request.sandbox_id, &request.sandbox_name) @@ -2165,6 +3134,21 @@ fn error_condition(reason: &str, message: &str) -> DriverCondition { } } +fn set_sandbox_ready_condition(sandbox: &mut DriverSandbox, condition: DriverCondition) { + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == "Ready") + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + fn platform_event( source: &str, event_type: &str, @@ -2586,79 +3570,157 @@ fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { }) } -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) +fn build_binds(_sandbox: &DriverSandbox, _config: &DockerDriverRuntimeConfig) -> Vec { + Vec::new() } -fn sandbox_token_host_path( +fn docker_boundary_state_dir( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result { - sandbox_token_host_path_by_id(&sandbox.id, config) + docker_boundary_state_dir_by_id(&sandbox.id, config) } -fn sandbox_token_host_path_by_id( +fn docker_boundary_state_dir_by_id( sandbox_id: &str, config: &DockerDriverRuntimeConfig, ) -> Result { - openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_namespace), - sandbox_id, - ) - .map_err(|err| { - Status::internal(format!( - "resolve sandbox token state directory failed: {err}" - )) + sandbox_token_host_path_by_id(sandbox_id, config).and_then(|path| { + path.parent() + .map(Path::to_path_buf) + .ok_or_else(|| Status::internal("docker boundary state path has no parent")) }) } -async fn write_sandbox_token_file( +fn docker_channel_volume_name( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, -) -> Result { - let Some(spec) = sandbox.spec.as_ref() else { - return Ok(false); - }; - if spec.sandbox_token.is_empty() { - return Ok(false); - } - let path = sandbox_token_host_path(sandbox, config)?; - if let Some(parent) = path.parent() { - openshell_core::paths::create_dir_restricted(parent).map_err(|err| { - Status::internal(format!( - "create sandbox token directory {} failed: {err}", - parent.display() - )) - })?; +) -> String { + docker_channel_volume_name_by_id(&sandbox.id, config) +} + +fn docker_channel_volume_name_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> String { + let mut hasher = Sha256::new(); + hasher.update(config.sandbox_namespace.as_bytes()); + hasher.update([0]); + hasher.update(sandbox_id.as_bytes()); + let digest = format!("{:x}", hasher.finalize()); + format!("openshell-channel-{}", &digest[..32]) +} + +async fn create_docker_channel_volume( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let name = docker_channel_volume_name(sandbox, config); + let expected_labels = HashMap::from([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + ( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + ), + ]); + docker + .create_volume(VolumeCreateRequest { + name: Some(name.clone()), + labels: Some(expected_labels.clone()), + ..Default::default() + }) + .await + .map_err(|error| { + Status::internal(format!("create Docker sandbox channel volume: {error}")) + })?; + let volume = docker.inspect_volume(&name).await.map_err(|error| { + Status::internal(format!("inspect Docker sandbox channel volume: {error}")) + })?; + if volume.driver != "local" + || !volume.options.is_empty() + || expected_labels + .iter() + .any(|(key, value)| volume.labels.get(key) != Some(value)) + { + return Err(Status::failed_precondition(format!( + "Docker sandbox channel volume '{name}' already exists without the expected local-driver ownership labels" + ))); + } + Ok(()) +} + +async fn remove_docker_channel_volume_by_id( + docker: &Docker, + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result<(), Status> { + let name = docker_channel_volume_name_by_id(sandbox_id, config); + docker + .remove_volume( + &name, + None::, + ) + .await + .or_else(|error| { + if is_not_found_error(&error) { + Ok(()) + } else { + Err(error) + } + }) + .map_err(|error| Status::internal(format!("remove Docker sandbox channel volume: {error}"))) +} + +fn sandbox_token_host_path( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + sandbox_token_host_path_by_id(&sandbox.id, config) +} + +fn sandbox_token_host_path_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result { + openshell_core::driver_utils::sandbox_token_path( + "docker-sandbox-tokens", + Some(&config.sandbox_namespace), + sandbox_id, + ) + .map_err(|err| { + Status::internal(format!( + "resolve sandbox token state directory failed: {err}" + )) + }) +} + +async fn write_sandbox_token_file( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let Some(spec) = sandbox.spec.as_ref() else { + return Ok(false); + }; + if spec.sandbox_token.is_empty() { + return Ok(false); + } + let path = sandbox_token_host_path(sandbox, config)?; + if let Some(parent) = path.parent() { + openshell_core::paths::create_dir_restricted(parent).map_err(|err| { + Status::internal(format!( + "create sandbox token directory {} failed: {err}", + parent.display() + )) + })?; } tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) .await @@ -2677,164 +3739,914 @@ async fn write_sandbox_token_file( Ok(true) } -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); +fn prepare_docker_boundary_state_dir( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + openshell_core::paths::create_dir_restricted(&directory).map_err(|error| { + Status::internal(format!( + "create Docker boundary state directory {}: {error}", + directory.display() + )) + })?; + Ok(directory) } -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, +async fn write_docker_boundary_file(path: &Path, contents: &[u8]) -> Result<(), Status> { + tokio::fs::write(path, contents).await.map_err(|error| { + Status::internal(format!( + "write Docker boundary file {}: {error}", + path.display() + )) + })?; + openshell_core::paths::set_file_owner_only(path).map_err(|error| { + Status::internal(format!( + "restrict Docker boundary file {}: {error}", + path.display() + )) + }) +} + +fn append_docker_archive_directory( + archive: &mut tar::Builder>, + path: &str, + mode: u32, + uid: u32, + gid: u32, +) -> Result<(), Status> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_mode(mode); + header.set_uid(u64::from(uid)); + header.set_gid(u64::from(gid)); + header.set_mtime(0); + header.set_size(0); + header.set_cksum(); + archive + .append_data(&mut header, path, std::io::empty()) + .map_err(|error| Status::internal(format!("build Docker sandbox archive: {error}"))) +} + +fn append_docker_archive_file( + archive: &mut tar::Builder>, + path: &str, + mode: u32, + uid: u32, + gid: u32, + contents: &[u8], +) -> Result<(), Status> { + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(mode); + header.set_uid(u64::from(uid)); + header.set_gid(u64::from(gid)); + header.set_mtime(0); + header.set_size(contents.len() as u64); + header.set_cksum(); + archive + .append_data(&mut header, path, contents) + .map_err(|error| Status::internal(format!("build Docker sandbox archive: {error}"))) +} + +#[derive(Clone, Copy)] +struct DockerSandboxTls<'a> { + certificate: &'a [u8], + private_key: &'a [u8], + client_ca: &'a [u8], +} + +fn docker_sandbox_bundle_archive( + sandbox_binary: &[u8], + boundary_config: &[u8], + boundary_tls: DockerSandboxTls<'_>, + identity: &ResolvedWorkloadIdentity, + workspace_root: &str, +) -> Result, Status> { + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory(&mut archive, ".openshell", 0o755, 0, 0)?; + append_docker_archive_directory(&mut archive, ".openshell/runtime", 0o555, 0, 0)?; + append_docker_archive_directory(&mut archive, ".openshell/channel", 0o755, 0, 0)?; + append_docker_archive_directory( + &mut archive, + ".openshell/channel/sandbox", + 0o700, + identity.uid, + identity.gid, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/runtime/openshell-sandbox", + 0o555, + 0, + 0, + sandbox_binary, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/sandbox/bootstrap.json", + 0o600, + identity.uid, + identity.gid, + boundary_config, + )?; + for (path, contents) in [ + ( + ".openshell/channel/sandbox/server.crt", + boundary_tls.certificate, + ), + ( + ".openshell/channel/sandbox/server.key", + boundary_tls.private_key, + ), + ( + ".openshell/channel/sandbox/client-ca.crt", + boundary_tls.client_ca, + ), + ] { + append_docker_archive_file( + &mut archive, + path, + 0o600, + identity.uid, + identity.gid, + contents, + )?; + } + if workspace_root == driver_mounts::DEFAULT_WORKSPACE_ROOT { + // The default workspace is driver-managed. Create it before the + // capability-free sandbox starts because that process deliberately + // has no authority to create or chown a directory beneath `/`. + append_docker_archive_directory( + &mut archive, + workspace_root.trim_start_matches('/'), + 0o700, + identity.uid, + identity.gid, + )?; + } + archive + .into_inner() + .map_err(|error| Status::internal(format!("finish Docker sandbox archive: {error}"))) +} + +async fn stage_docker_sandbox_bundle( + docker: &Docker, + container_id: &str, config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); - } + identity: &ResolvedWorkloadIdentity, + boundary_config: &[u8], + boundary_tls: DockerSandboxTls<'_>, + workspace_root: &str, +) -> Result<(), Status> { + let archive = docker_sandbox_bundle_archive( + config.sandbox_binary.as_slice(), + boundary_config, + boundary_tls, + identity, + workspace_root, + )?; + let options = UploadToContainerOptionsBuilder::default() + .path("/") + .copy_uidgid("true") + .build(); + docker + .upload_to_container( + container_id, + Some(options), + bollard::body_full(Bytes::from(archive)), + ) + .await + .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}"))) } -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { - return; - }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" - ); +async fn prepare_docker_boundary_files( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + container_id: &str, + image: &DockerImageMetadata, + workload_identity: &ResolvedWorkloadIdentity, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(Status::failed_precondition)?; + let bootstrap_token = random_boundary_token(); + let host_gateway_ip = Some(match config.gateway_route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), + }); + let tls = generate_boundary_mutual_tls_material() + .map_err(|error| Status::internal(format!("generate Docker boundary TLS: {error}")))?; + let provisioning = isolation::DockerBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + generation: random_boundary_token(), + session_epoch: random_boundary_token(), + container_id: container_id.to_string(), + image_identity: image.id.clone(), + listener_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + control_socket: PathBuf::from(BOUNDARY_SOCKET_MOUNT_PATH), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_MOUNT_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_MOUNT_PATH), + client_ca_certificate_path: PathBuf::from(BOUNDARY_CLIENT_CA_MOUNT_PATH), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name.clone(), + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem.clone(), + private_key_pem: tls.supervisor_private_key_pem.clone(), + }, + host_gateway_ip, + workload_identity: workload_identity.clone(), + child_env: docker_child_environment(sandbox), + } + .provision(); + let boundary_config = provisioning + .boundary_config + .encode() + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(BOUNDARY_CONFIG_FILE), &boundary_config).await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CERTIFICATE_FILE), + tls.sandbox_certificate_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_PRIVATE_KEY_FILE), + tls.sandbox_private_key_pem.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(BOUNDARY_CLIENT_CA_FILE), + tls.ca_certificate_pem.as_bytes(), + ) + .await?; + stage_docker_sandbox_bundle( + docker, + container_id, + config, + workload_identity, + &boundary_config, + DockerSandboxTls { + certificate: tls.sandbox_certificate_pem.as_bytes(), + private_key: tls.sandbox_private_key_pem.as_bytes(), + client_ca: tls.ca_certificate_pem.as_bytes(), + }, + &workspace_root, + ) + .await?; + let descriptor = provisioning + .topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + write_docker_boundary_file(&directory.join(TOPOLOGY_PAYLOAD_FILE), &descriptor.payload).await?; + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| Status::internal(format!("encode Docker main process spec: {error}")))?; + write_docker_boundary_file( + &directory.join(MAIN_PROCESS_SPEC_FILE), + main_process_spec.as_bytes(), + ) + .await?; + write_docker_boundary_file( + &directory.join(WORKSPACE_ROOT_FILE), + workspace_root.as_bytes(), + ) + .await?; + Ok(provisioning.topology) +} + +async fn docker_supervisor_bundle_archive( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let directory = docker_boundary_state_dir(sandbox, config)?; + let topology = tokio::fs::read(directory.join(TOPOLOGY_PAYLOAD_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker topology payload: {error}")))?; + let token = tokio::fs::read(sandbox_token_host_path(sandbox, config)?) + .await + .map_err(|error| { + Status::failed_precondition(format!("read Docker sandbox JWT: {error}")) + })?; + if token.iter().all(u8::is_ascii_whitespace) { + return Err(Status::failed_precondition( + "Docker supervisor requires a sandbox JWT", + )); } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory( + &mut archive, + ".openshell/channel/supervisor", + 0o700, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/supervisor/topology.payload", + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &topology, + )?; + append_docker_archive_file( + &mut archive, + ".openshell/channel/supervisor/sandbox.jwt", + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &token, + )?; + if let Some(tls) = &config.guest_tls { + append_docker_archive_directory( + &mut archive, + ".openshell/channel/supervisor/tls", + 0o700, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; + for (name, path) in [ + ("ca.pem", &tls.ca), + ("cert.pem", &tls.cert), + ("key.pem", &tls.key), + ] { + let contents = tokio::fs::read(path).await.map_err(|error| { + Status::internal(format!( + "read Docker supervisor TLS file {}: {error}", + path.display() + )) + })?; + append_docker_archive_file( + &mut archive, + &format!(".openshell/channel/supervisor/tls/{name}"), + 0o600, + SUPERVISOR_UID, + SUPERVISOR_GID, + &contents, + )?; + } } + archive + .into_inner() + .map_err(|error| Status::internal(format!("finish Docker supervisor archive: {error}"))) } -#[cfg(test)] -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - build_environment_for_oci_user(sandbox, config, "") +async fn read_docker_boundary_topology( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> Result, Status> { + let path = docker_boundary_state_dir_by_id(sandbox_id, config)?.join(TOPOLOGY_PAYLOAD_FILE); + let bytes = match tokio::fs::read(&path).await { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(Status::internal(format!( + "read Docker boundary topology {}: {error}", + path.display() + ))); + } + }; + serde_json::from_slice(&bytes).map(Some).map_err(|error| { + Status::internal(format!( + "decode Docker boundary topology {}: {error}", + path.display() + )) + }) } -fn build_environment_for_oci_user( +async fn stage_docker_supervisor_bundle( + docker: &Docker, sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, - oci_user: &str, -) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), + archive: Vec, +) -> Result<(), Status> { + let stager_name = format!("{}-supervisor-stage", container_name_for_sandbox(sandbox)); + let _ = docker + .remove_container( + &stager_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let created = docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(stager_name.as_str()) + .build(), + ), + ContainerCreateBody { + image: Some(config.supervisor_image_id.clone()), + entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), + labels: Some(docker_auxiliary_container_labels( + sandbox, + config, + LABEL_ISOLATION_ROLE_STAGING, + )), + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + mounts: Some(vec![Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]), + cap_drop: Some(vec!["ALL".to_string()]), + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + ..Default::default() + }), + ..Default::default() + }, + ) + .await + .map_err(|error| { + Status::internal(format!( + "create Docker supervisor staging container: {error}" + )) + })?; + let options = UploadToContainerOptionsBuilder::default() + .path("/") + .copy_uidgid("true") + .build(); + let result = docker + .upload_to_container( + &created.id, + Some(options), + bollard::body_full(Bytes::from(archive)), + ) + .await + .map_err(|error| Status::internal(format!("stage Docker supervisor bundle: {error}"))); + let cleanup = docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + .map_err(|error| { + Status::internal(format!( + "remove Docker supervisor staging container: {error}" + )) + }); + result?; + cleanup +} + +fn docker_auxiliary_container_labels( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + role: &str, +) -> HashMap { + HashMap::from([ ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), ), - ]); + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + (LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + (LABEL_ISOLATION_ROLE.to_string(), role.to_string()), + ]) +} - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, +async fn spawn_docker_control_process( + docker: &Docker, + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, + topology: &BoundaryTopology, + failure_context: DockerRuntimeFailureContext, +) -> Result { + let directory = docker_boundary_state_dir(sandbox, config)?; + let descriptor = topology + .descriptor(DRIVER_ADMITTED_BACKEND) + .map_err(|error| Status::internal(error.to_string()))?; + let main_process_spec = tokio::fs::read_to_string(directory.join(MAIN_PROCESS_SPEC_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker main process spec: {error}")))?; + let workspace_root = tokio::fs::read_to_string(directory.join(WORKSPACE_ROOT_FILE)) + .await + .map_err(|error| Status::internal(format!("read Docker workspace root: {error}")))?; + let supervisor_name = format!("{}-supervisor", container_name_for_sandbox(sandbox)); + let _ = docker + .remove_container( + &supervisor_name, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + let topology_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/topology.payload"); + let token_path = format!("{SUPERVISOR_STATE_MOUNT_PATH}/sandbox.jwt"); + let mut environment = vec![ + format!( + "{}={DRIVER_ADMITTED_BACKEND}", + openshell_core::sandbox_env::ADMITTED_ISOLATION_BACKEND + ), + format!( + "{}={main_process_spec}", + openshell_core::sandbox_env::MAIN_PROCESS_SPEC + ), + format!( + "{}={}", + openshell_core::sandbox_env::ENDPOINT, + config.supervisor_grpc_endpoint + ), + format!("{}={}", openshell_core::sandbox_env::SANDBOX_ID, sandbox.id), + format!("{}={}", openshell_core::sandbox_env::SANDBOX, sandbox.name), + format!( + "{}={token_path}", + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE + ), + format!( + "{}=/run/openshell/ssh.sock", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ), + format!( + "{}=/run/openshell/proxy-tls", + openshell_core::sandbox_env::PROXY_TLS_DIR + ), + format!( + "{}={}", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY + ), + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) + ), + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ]; + if let Some(server_name) = config.gateway_tls_server_name.as_deref() { + environment.push(format!( + "{}={server_name}", + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME + )); + } + if config.guest_tls.is_some() { + environment.extend([ + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/ca.pem", + openshell_core::sandbox_env::TLS_CA + ), + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/cert.pem", + openshell_core::sandbox_env::TLS_CERT + ), + format!( + "{}={SUPERVISOR_STATE_MOUNT_PATH}/tls/key.pem", + openshell_core::sandbox_env::TLS_KEY + ), + ]); + } + let supervisor_archive = docker_supervisor_bundle_archive(sandbox, config).await?; + stage_docker_supervisor_bundle(docker, sandbox, config, supervisor_archive).await?; + let labels = HashMap::from([ + (LABEL_MANAGED_BY.to_string(), "openshell".to_string()), + (LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()), + (LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()), + ( + LABEL_SANDBOX_NAMESPACE.to_string(), + config.sandbox_namespace.clone(), + ), + ( + LABEL_ISOLATION_ROLE.to_string(), + LABEL_ISOLATION_ROLE_SUPERVISOR.to_string(), + ), + ]); + let create = ContainerCreateBody { + image: Some(config.supervisor_image_id.clone()), + user: Some(format!("{SUPERVISOR_UID}:{SUPERVISOR_GID}")), + entrypoint: Some(vec![SUPERVISOR_IMAGE_CONTROL_BINARY_PATH.to_string()]), + cmd: Some(vec![ + format!("--topology-backend-name={}", descriptor.backend_name), + "--topology-payload-file".to_string(), + topology_path.clone(), + "--workdir".to_string(), + workspace_root, + format!("--health-socket-path={SUPERVISOR_HEALTH_SOCKET_PATH}"), + ]), + env: Some(environment), + labels: Some(labels), + host_config: Some(HostConfig { + network_mode: Some(config.network_name.clone()), + mounts: Some(vec![Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(true), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]), + cap_drop: Some(vec!["ALL".to_string()]), + cap_add: None, + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + readonly_rootfs: Some(true), + tmpfs: Some(HashMap::from([ + ( + "/run".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ( + "/tmp".to_string(), + "rw,noexec,nosuid,size=64m,mode=1777".to_string(), + ), + ( + "/var/log".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={SUPERVISOR_UID},gid={SUPERVISOR_GID},mode=0700" + ), + ), + ])), + extra_hosts: Some(vec![ + format!( + "{HOST_OPENSHELL_INTERNAL}:{}", + docker_supervisor_host_alias(&config.gateway_route) + ), + format!( + "{HOST_DOCKER_INTERNAL}:{}", + docker_supervisor_host_alias(&config.gateway_route) + ), + ]), + restart_policy: None, + ..Default::default() + }), + ..Default::default() + }; + let created = docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(supervisor_name.as_str()) + .build(), + ), + create, + ) + .await + .map_err(|error| { + Status::internal(format!("create Docker supervisor container: {error}")) + })?; + if let Err(error) = docker.start_container(&created.id, None).await { + let _ = docker + .remove_container( + &created.id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + return Err(Status::internal(format!( + "start Docker supervisor container: {error}" + ))); + } + let sandbox_id = sandbox.id.clone(); + let (shutdown, mut shutdown_requested) = oneshot::channel(); + let supervisor_id = created.id; + let docker = failure_context.docker.clone(); + let task = tokio::spawn(async move { + let wait = async { + let mut stream = docker.wait_container( + &supervisor_id, + None::, ); + stream.next().await + }; + tokio::select! { + biased; + _ = &mut shutdown_requested => { + let _ = docker.stop_container( + &supervisor_id, + Some(StopContainerOptionsBuilder::default().t(5).build()), + ).await; + let _ = docker.remove_container( + &supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + } + result = wait => { + let mut message = match result { + Some(Ok(status)) => { + warn!(%sandbox_id, status = status.status_code, "Docker supervisor container exited unexpectedly"); + format!("Docker supervisor container exited with status {}", status.status_code) + } + Some(Err(error)) => { + warn!(%sandbox_id, %error, "Failed to wait for Docker supervisor container"); + format!("failed to wait for Docker supervisor container: {error}") + } + None => "Docker supervisor wait stream ended unexpectedly".to_string(), + }; + let log_tail = docker_container_log_tail(&docker, &supervisor_id).await; + if !log_tail.is_empty() { + write!(message, "; log tail: {log_tail}").ok(); + } + let _ = docker.remove_container( + &supervisor_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ).await; + handle_docker_runtime_failure( + failure_context, + "ControlSupervisorExited", + message, + ) + .await; + }, + } + }); + Ok(DockerControlProcess { + shutdown: Some(shutdown), + task, + }) +} + +async fn docker_container_log_tail(docker: &Docker, container_id: &str) -> String { + const MAX_LOG_TAIL_BYTES: usize = 16 * 1024; + let options = LogsOptionsBuilder::default() + .stdout(true) + .stderr(true) + .tail("80") + .build(); + let mut stream = docker.logs(container_id, Some(options)); + let mut output = Vec::new(); + while let Some(result) = stream.next().await { + let Ok(chunk) = result else { + break; + }; + output.extend_from_slice(chunk.as_ref()); + if output.len() > MAX_LOG_TAIL_BYTES { + output.drain(..output.len() - MAX_LOG_TAIL_BYTES); } } + String::from_utf8_lossy(&output).trim().to_string() +} - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox.spec.as_ref()) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), +async fn handle_docker_runtime_failure( + context: DockerRuntimeFailureContext, + reason: &'static str, + message: String, +) { + context.failures.lock().await.insert( + context.sandbox.id.clone(), + DockerRuntimeFailure { + reason, + message: message.clone(), + }, ); - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + + let mut snapshot = pending_sandbox_snapshot( + &context.sandbox, + &context.sandbox_namespace, + error_condition(reason, &message), + false, ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); + if let Some(status) = snapshot.status.as_mut() { + status.instance_id.clone_from(&context.container_id); } + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(snapshot), + }, + )), + }); + let _ = context.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id: context.sandbox.id.clone(), + event: Some(platform_event( + "docker", + "Warning", + reason, + format!("{message}; stopping the isolated workload container"), + )), + }, + )), + }); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - environment.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - oci_user.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - String::new(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - String::new(), - ); + match context + .docker + .stop_container( + &context.container_id, + Some( + StopContainerOptionsBuilder::default() + .t(docker_stop_timeout_secs(context.stop_timeout_secs)) + .build(), + ), + ) + .await + { + Ok(()) => info!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + "Stopped Docker sandbox after control supervisor failure" + ), + Err(error) if is_not_found_error(&error) || is_not_modified_error(&error) => {} + Err(error) => warn!( + sandbox_id = %context.sandbox.id, + container_id = %context.container_id, + %error, + "Failed to stop Docker sandbox after control supervisor failure" + ), + } +} + +async fn stop_docker_control_process(mut process: DockerControlProcess) { + if let Some(shutdown) = process.shutdown.take() { + let _ = shutdown.send(()); + } + let _ = process.task.await; +} + +fn cleanup_docker_boundary_state(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { + cleanup_docker_boundary_state_by_id(&sandbox.id, config); +} - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() +fn cleanup_docker_boundary_state_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { + let Ok(directory) = docker_boundary_state_dir_by_id(sandbox_id, config) else { + return; + }; + if let Err(error) = std::fs::remove_dir_all(&directory) + && error.kind() != std::io::ErrorKind::NotFound { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), + warn!( + %sandbox_id, + path = %directory.display(), + %error, + "Failed to remove Docker boundary state directory" ); } +} - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); + } + token +} + +fn docker_child_environment(sandbox: &DriverSandbox) -> HashMap { + let mut environment = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(HashMap::new, |template| template.environment.clone()); + if let Some(spec) = sandbox.spec.as_ref() { + environment.extend(spec.environment.clone()); + } + for protected in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX, + openshell_core::sandbox_env::SANDBOX_GID, + openshell_core::sandbox_env::SANDBOX_ID, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::SANDBOX_UID, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::USER_ENVIRONMENT, + ] { + environment.remove(protected); + } + environment +} + +fn build_boundary_environment( + sandbox: &DriverSandbox, + config: &DockerDriverRuntimeConfig, +) -> Vec { + vec![ + format!( + "{}={}", + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level) + ), + format!( + "{}={}", + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value() + ), + ] } fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { @@ -2908,6 +4720,14 @@ fn build_container_create_body_with_gpu_devices( .as_ref() .and_then(|spec| spec.template.as_ref()) .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let workload_identity = ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + template.image.clone(), + ) + .map_err(|error| Status::internal(error.to_string()))?; build_container_create_body_for_image( sandbox, config, @@ -2919,6 +4739,7 @@ fn build_container_create_body_with_gpu_devices( working_dir: String::new(), volumes: Vec::new(), }, + &workload_identity, ) } @@ -2928,6 +4749,7 @@ fn build_container_create_body_for_image( driver_config: &DockerSandboxDriverConfig, gpu_device_ids: Option<&[String]>, image: &DockerImageMetadata, + workload_identity: &ResolvedWorkloadIdentity, ) -> Result { let spec = sandbox .spec @@ -2940,7 +4762,7 @@ fn build_container_create_body_for_image( let resource_limits = docker_resource_limits(template)?; let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) .map_err(Status::failed_precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) + driver_mounts::validate_workspace_control_path(&workspace_root, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; for volume in &image.volumes { driver_mounts::validate_container_mount_target(volume).map_err(|error| { @@ -2953,7 +4775,7 @@ fn build_container_create_body_for_image( "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" )) })?; - driver_mounts::validate_mount_control_path(volume, &config.ssh_socket_path) + driver_mounts::validate_mount_control_path(volume, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; } for mount in &driver_config.mounts { @@ -2965,10 +4787,21 @@ fn build_container_create_body_for_image( }; driver_mounts::validate_workspace_mount_target(target, &workspace_root) .map_err(Status::failed_precondition)?; - driver_mounts::validate_mount_control_path(target, &config.ssh_socket_path) + driver_mounts::validate_mount_control_path(target, BOUNDARY_MOUNT_PATH) .map_err(Status::failed_precondition)?; } - let user_mounts = docker_driver_mounts(driver_config)?; + let mut user_mounts = docker_driver_mounts(driver_config)?; + user_mounts.push(Mount { + target: Some(BOUNDARY_MOUNT_PATH.to_string()), + source: Some(docker_channel_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }); let user_bind_strings = docker_driver_bind_strings(driver_config)?; let device_requests = gpu_device_ids.map(|device_ids| { vec![DeviceRequest { @@ -2996,18 +4829,31 @@ fn build_container_create_body_for_image( LABEL_SANDBOX_NAMESPACE.to_string(), config.sandbox_namespace.clone(), ); + labels.insert( + LABEL_ISOLATION_TOPOLOGY.to_string(), + LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE.to_string(), + ); + labels.insert( + LABEL_ISOLATION_ROLE.to_string(), + LABEL_ISOLATION_ROLE_SANDBOX.to_string(), + ); Ok(ContainerCreateBody { image: Some(image.id.clone()), - user: Some("0".to_string()), + user: Some(format!( + "{}:{}", + workload_identity.uid, workload_identity.gid + )), // The image workspace may need to be created or rejected by the // supervisor, so do not let the OCI runtime chdir there first. working_dir: Some("/".to_string()), - env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + env: Some(build_boundary_environment(sandbox, config)), + entrypoint: Some(vec![SANDBOX_BINARY_PATH.to_string()]), + // The image cannot append inherited arguments or select either role. + cmd: Some(vec![ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3015,7 +4861,7 @@ fn build_container_create_body_for_image( pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, device_requests, binds: { - let mut binds = build_binds(sandbox, config)?; + let mut binds = build_binds(sandbox, config); binds.extend(user_bind_strings); Some(binds) }, @@ -3023,33 +4869,33 @@ fn build_container_create_body_for_image( // Canonical main-process exit is terminal. Runtime restart would // silently create a new process generation behind the gateway. restart_policy: None, - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), + group_add: Some( + workload_identity + .supplementary_gids + .iter() + .map(u32::to_string) + .collect(), + ), + cap_drop: Some(vec!["ALL".to_string()]), + cap_add: None, + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + network_mode: Some("none".to_string()), + dns: Some(vec!["127.0.0.53".to_string()]), + tmpfs: Some(HashMap::from([( + "/run".to_string(), + format!( + "rw,noexec,nosuid,size=64m,uid={},gid={},mode=0755", + workload_identity.uid, workload_identity.gid + ), + )])), + sysctls: Some(HashMap::from([( + "net.ipv4.ip_unprivileged_port_start".to_string(), + "0".to_string(), )])), + extra_hosts: None, + ..Default::default() }), + networking_config: None, ..Default::default() }) } @@ -3068,16 +4914,35 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); +fn docker_host_openshell_endpoint( + endpoint: &str, + route: &DockerGatewayRoute, +) -> CoreResult { + let mut url = Url::parse(endpoint) + .map_err(|error| Error::config(format!("invalid docker grpc_endpoint: {error}")))?; + if !matches!( + url.host_str(), + Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) + ) { + return Ok(url.to_string()); + } + let host = match route { + DockerGatewayRoute::Bridge { bind_address, .. } => bind_address.ip(), + DockerGatewayRoute::HostGateway => IpAddr::V4(Ipv4Addr::LOCALHOST), }; + url.set_host(Some(&host.to_string())).map_err(|error| { + Error::config(format!( + "failed to map Docker gateway alias to its host listener: {error}" + )) + })?; + Ok(url.to_string()) +} - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); +fn docker_supervisor_host_alias(route: &DockerGatewayRoute) -> String { + match route { + DockerGatewayRoute::Bridge { bind_address } => bind_address.ip().to_string(), + DockerGatewayRoute::HostGateway => "host-gateway".to_string(), } - - endpoint.to_string() } fn docker_network_name(config: &DockerComputeConfig) -> String { @@ -3125,7 +4990,6 @@ fn docker_gateway_route_for_host( if let Some(host_alias_ip) = host_gateway_ip { return DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, }; } @@ -3134,7 +4998,6 @@ fn docker_gateway_route_for_host( } else { DockerGatewayRoute::Bridge { bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, } } } @@ -3199,19 +5062,6 @@ fn uses_host_gateway_alias(info: &SystemInfo) -> bool { }) } -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { match docker.inspect_network(network_name, None).await { Ok(network) => return validate_bridge_network(network_name, &network), @@ -3599,6 +5449,17 @@ fn label_filters(values: impl IntoIterator) -> HashMap, +) -> HashMap> { + let mut values = vec![format!( + "{LABEL_ISOLATION_ROLE}={LABEL_ISOLATION_ROLE_SANDBOX}" + )]; + values.extend(extra_values); + managed_resource_label_filters(sandbox_namespace, values) +} + +fn managed_resource_label_filters( + sandbox_namespace: &str, + extra_values: impl IntoIterator, ) -> HashMap> { let mut values = vec![ format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), @@ -3677,189 +5538,6 @@ fn sanitize_docker_name(value: &str) -> String { .to_string() } -fn normalize_docker_arch(arch: &str) -> String { - match arch { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - other => other.to_ascii_lowercase(), - } -} - -#[derive(Debug, Eq, PartialEq)] -enum SupervisorBinSource { - Binary(PathBuf), - Image(String), -} - -fn resolve_supervisor_bin_source( - docker_config: &DockerComputeConfig, - current_exe: Option<&Path>, - target_candidates: &[PathBuf], -) -> CoreResult { - // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. - if let Some(path) = docker_config.supervisor_bin.clone() { - let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path).map_err(Error::config)?; - return Ok(SupervisorBinSource::Binary(path)); - } - - // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. - // A configured image should be the source of truth even when a local - // developer build is present under target/. - if let Some(image) = docker_config.supervisor_image.clone() { - return Ok(SupervisorBinSource::Image(image)); - } - - // Tier 3: sibling `openshell-sandbox` next to the running gateway - // (release artifact layout). Linux-only because the sibling must be a - // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") - && let Some(current_exe) = current_exe - && let Some(parent) = current_exe.parent() - { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 4: local cargo target build (developer workflow). Preferred - // over the default registry image when available because it matches - // whatever the developer just built. - for candidate in target_candidates { - if candidate.is_file() { - let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 5: pull the release-matched default supervisor image and extract - // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image( - openshell_core::config::default_supervisor_image(), - )) -} - -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, - docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { - let current_exe = - if cfg!(target_os = "linux") - && docker_config.supervisor_bin.is_none() - && docker_config.supervisor_image.is_none() - { - Some(std::env::current_exe().map_err(|err| { - Error::config(format!("failed to resolve current executable: {err}")) - })?) - } else { - None - }; - let target_candidates = linux_supervisor_candidates(daemon_arch); - - match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? - { - SupervisorBinSource::Binary(path) => Ok(path), - SupervisorBinSource::Image(image) => { - extract_supervisor_bin_from_image(docker, &image).await - } - } -} - -fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { - match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], - _ => Vec::new(), - } -} - -/// Pull the supervisor image (if not already local), extract -/// `/openshell-sandbox` to a host cache keyed by the image's content -/// digest, and return the cache path. -/// -/// The extraction is atomic: the binary is written to a sibling temp file -/// inside the digest-keyed directory and renamed into place, so concurrent -/// gateway starts don't observe a partial file. -async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { - let refresh_attempted = if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - match pull_supervisor_image(docker, image).await { - Ok(()) => true, - Err(err) => { - warn!( - image = image, - error = %err, - "failed to refresh mutable docker supervisor image; falling back to local image if present", - ); - true - } - } - } else { - false - }; - - // Inspect first to see if the image is already present; only pull on miss. - let inspect = match docker.inspect_image(image).await { - Ok(inspect) => inspect, - Err(err) if is_not_found_error(&err) && !refresh_attempted => { - info!(image = image, "Pulling docker supervisor image"); - pull_supervisor_image(docker, image).await?; - docker.inspect_image(image).await.map_err(|err| { - Error::config(format!( - "failed to inspect docker supervisor image '{image}' after pull: {err}", - )) - })? - } - Err(err) if is_not_found_error(&err) => { - return Err(Error::config(format!( - "docker supervisor image '{image}' is not present locally after refresh attempt", - ))); - } - Err(err) => { - return Err(Error::config(format!( - "failed to inspect docker supervisor image '{image}': {err}", - ))); - } - }; - - let digest = inspect.id.clone().ok_or_else(|| { - Error::config(format!( - "docker supervisor image '{image}' inspect response has no Id", - )) - })?; - - let cache_path = - openshell_core::driver_utils::supervisor_cache_path("docker-supervisor", &digest) - .map_err(Error::config)?; - if cache_path.is_file() { - validate_linux_elf_binary(&cache_path).map_err(Error::config)?; - return Ok(cache_path); - } - - info!( - image = image, - digest = digest, - cache_path = %cache_path.display(), - "Extracting supervisor binary from image to host cache", - ); - - let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; - validate_linux_elf_binary(&cache_path).map_err(Error::config)?; - Ok(cache_path) -} - async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -3879,10 +5557,55 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { Ok(()) } +async fn ensure_supervisor_container_image(docker: &Docker, image: &str) -> CoreResult { + let local_image_present = docker.inspect_image(image).await.is_ok(); + if supervisor_image_should_refresh(image) { + info!(image = image, "Refreshing mutable docker supervisor image"); + if let Err(error) = pull_supervisor_image(docker, image).await { + if !local_image_present { + return Err(error); + } + warn!( + image = image, + error = %error, + "failed to refresh mutable Docker supervisor image; using the local image", + ); + } + } else if !local_image_present { + pull_supervisor_image(docker, image).await?; + } + let inspect = docker.inspect_image(image).await.map_err(|error| { + Error::config(format!( + "failed to inspect Docker supervisor image '{image}': {error}" + )) + })?; + inspect.id.filter(|id| !id.is_empty()).ok_or_else(|| { + Error::config(format!( + "Docker supervisor image '{image}' has no immutable image ID" + )) + }) +} + /// Create a short-lived container from `image`, stream out the supervisor /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + let bytes = + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await?; + if !bytes.starts_with(b"\x7fELF") { + return Err(Error::config(format!( + "Docker supervisor image '{image}' contains an invalid sandbox binary" + ))); + } + Ok(bytes) +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -3906,7 +5629,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -3923,12 +5647,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -3942,11 +5668,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index ce6a36c87b..98a2c7eb33 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -5,7 +5,7 @@ use super::*; use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, supervisor_cache_path_with_base, + LABEL_SANDBOX_NAMESPACE, }; use openshell_core::progress::{ PROGRESS_ACTIVE_DETAIL_KEY, PROGRESS_ACTIVE_STEP_KEY, PROGRESS_COMPLETE_LABEL_KEY, @@ -15,16 +15,13 @@ use openshell_core::progress::{ use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, - gateway_listener_requirement::Selector, + WorkloadIdentityRequest, gateway_listener_requirement::Selector, }; use std::fs; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::Arc; use tempfile::TempDir; -const TLS_MOUNT_DIR: &str = "/etc/openshell/tls/client"; -static ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - fn test_sandbox() -> DriverSandbox { // Mirrors the gateway-supplied request: the public `Sandbox` API no // longer carries `namespace`, so the gateway elides the field and the @@ -98,23 +95,23 @@ fn runtime_config() -> DockerDriverRuntimeConfig { default_image: "image:latest".to_string(), image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), - grpc_endpoint: "https://localhost:8443".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), gateway_route: DockerGatewayRoute::Bridge { bind_address: SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, ), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), }, gateway_callback_bind_address: Some(SocketAddr::new( IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), DEFAULT_SERVER_PORT, )), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), - supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + sandbox_binary: Arc::new(b"\x7fELFtest".to_vec()), + supervisor_image_id: "sha256:supervisor-test".to_string(), + network_name: "openshell-test".to_string(), + supervisor_grpc_endpoint: "https://host.openshell.internal:8443".to_string(), + gateway_tls_server_name: None, guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -128,6 +125,17 @@ fn runtime_config() -> DockerDriverRuntimeConfig { } } +fn test_workload_identity() -> ResolvedWorkloadIdentity { + ResolvedWorkloadIdentity::new( + 1234, + 1235, + vec![1236], + "test".to_string(), + "sha256:immutable".to_string(), + ) + .unwrap() +} + fn json_struct(value: serde_json::Value) -> prost_types::Struct { let serde_json::Value::Object(object) = value else { panic!("expected JSON object"); @@ -160,12 +168,14 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr ), config, events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + pending: Arc::new(Mutex::new(HashMap::new())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( CdiGpuInventory::default(), allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + control_processes: Arc::new(Mutex::new(HashMap::new())), + runtime_failures: Arc::new(Mutex::new(HashMap::new())), } } @@ -187,14 +197,14 @@ fn request_with_traceparent(message: T) -> Request { async fn standalone_traced_client() -> ( TestDriverClient, - tokio::sync::oneshot::Sender<()>, + oneshot::Sender<()>, JoinHandle>, ) { use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel(); + let (shutdown, shutdown_rx) = oneshot::channel(); let service = ComputeDriverService::new(test_driver_with_config(runtime_config())); let server = tokio::spawn(async move { tonic::transport::Server::builder() @@ -281,6 +291,78 @@ async fn tracing_standalone_rpc_layer_propagates_context_and_records_errors() { provider.shutdown().unwrap(); } +#[tokio::test] +async fn control_failure_overrides_running_container_readiness() { + let driver = test_driver_with_config(runtime_config()); + driver.runtime_failures.lock().await.insert( + "sbx-123".to_string(), + DockerRuntimeFailure { + reason: "ControlSupervisorExited", + message: "control exited unexpectedly".to_string(), + }, + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "BackendReady".to_string(), + message: "Container is running".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_runtime_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "ControlSupervisorExited"); + assert!(ready.message.contains("control exited unexpectedly")); +} + +#[tokio::test] +async fn control_failure_does_not_hide_a_terminal_container_exit() { + let driver = test_driver_with_config(runtime_config()); + driver.runtime_failures.lock().await.insert( + "sbx-123".to_string(), + DockerRuntimeFailure { + reason: "ControlSupervisorExited", + message: "control exited unexpectedly".to_string(), + }, + ); + let mut sandbox = pending_sandbox_snapshot( + &test_sandbox(), + "default", + DriverCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: CONDITION_EXITED.to_string(), + message: "Container exited".to_string(), + last_transition_time: String::new(), + }, + false, + ); + + driver.apply_runtime_failure(&mut sandbox).await; + + let ready = sandbox + .status + .unwrap() + .conditions + .into_iter() + .find(|condition| condition.r#type == "Ready") + .expect("ready condition"); + assert_eq!(ready.reason, CONDITION_EXITED); +} + #[tokio::test] async fn tracing_in_process_service_preserves_the_driver_rpc_server_boundary() { use opentelemetry_sdk::trace::{InMemorySpanExporterBuilder, SdkTracerProvider}; @@ -499,8 +581,7 @@ async fn tracing_direct_start_exports_a_docker_start_span() { let subscriber = tracing_subscriber::registry().with(otel_tracing::TRACING.layer(&provider)); let driver = test_driver_with_config(runtime_config()); - DockerComputeDriver::start_sandbox(&driver, "", "") - .with_subscriber(subscriber) + Box::pin(DockerComputeDriver::start_sandbox(&driver, "", "").with_subscriber(subscriber)) .await .expect_err("missing identifier should fail"); provider.force_flush().unwrap(); @@ -534,13 +615,15 @@ async fn tracing_image_preparation_failure_exports_nested_failed_spans() { let driver = test_driver_with_config(config); async { - driver - .provision_sandbox_inner(&test_sandbox()) - .instrument(tracing::info_span!( - "docker.provision", - otel.status_code = tracing::field::Empty - )) - .await + Box::pin( + driver + .provision_sandbox_inner(&test_sandbox()) + .instrument(tracing::info_span!( + "docker.provision", + otel.status_code = tracing::field::Empty + )), + ) + .await } .with_subscriber(subscriber) .await @@ -818,34 +901,6 @@ async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { ); } -#[test] -fn container_visible_endpoint_rewrites_loopback_hosts() { - assert_eq!( - docker_container_openshell_endpoint( - "https://localhost:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "http://127.0.0.1:8080", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "http://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "https://gateway.internal:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); -} - #[test] fn docker_bridge_gateway_ip_requires_ipv4_gateway() { let network = bollard::models::NetworkInspect { @@ -910,13 +965,21 @@ fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); +} + +#[test] +fn vm_backed_docker_daemon_uses_daemon_local_companion_transport() { + let desktop = SystemInfo { + operating_system: Some("Docker Desktop".to_string()), + ..Default::default() + }; + let native = SystemInfo { + operating_system: Some("Ubuntu 24.04".to_string()), + ..Default::default() + }; + + assert!(uses_host_gateway_alias(&desktop)); + assert!(!uses_host_gateway_alias(&native)); } #[test] @@ -961,13 +1024,6 @@ fn docker_gateway_route_uses_host_gateway_for_colima() { ), DockerGatewayRoute::HostGateway ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); } #[test] @@ -1052,16 +1108,8 @@ fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { route, DockerGatewayRoute::Bridge { bind_address: "172.18.0.1:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), } ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ] - ); } #[test] @@ -1101,15 +1149,21 @@ fn docker_gateway_route_prefers_configured_host_gateway_ip() { route, DockerGatewayRoute::Bridge { bind_address: "172.20.0.4:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)), } ); +} + +#[test] +fn docker_supervisor_alias_matches_the_trusted_gateway_route() { + assert_eq!( + docker_supervisor_host_alias(&DockerGatewayRoute::Bridge { + bind_address: "172.20.0.4:17670".parse().unwrap(), + }), + "172.20.0.4" + ); assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.20.0.4".to_string(), - "host.openshell.internal:172.20.0.4".to_string() - ] + docker_supervisor_host_alias(&DockerGatewayRoute::HostGateway), + "host-gateway" ); } @@ -1207,91 +1261,44 @@ fn container_create_body_sets_driver_owned_pids_limit() { } #[test] -fn build_environment_sets_docker_tls_paths() { - let env = build_environment(&test_sandbox(), &runtime_config()); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={TLS_CA_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={TLS_CERT_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={TLS_KEY_MOUNT_PATH}"))); - assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); - assert!(env.contains(&"SPEC_ENV=spec".to_string())); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - let encoded = env - .iter() - .find_map(|entry| { - entry - .strip_prefix("OPENSHELL_MAIN_PROCESS_SPEC=") - .map(str::to_string) - }) - .expect("main-process transport"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(&encoded).unwrap(); - // An omitted command is forwarded empty; the supervisor resolves the default - // login shell against the sandbox image at startup. - assert!(main.command.is_empty()); - assert!(main.tty); -} - -#[test] -fn build_environment_keeps_network_capabilities_driver_controlled() { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - "spoofed".to_string(), - ); - let env = build_environment(&sandbox, &runtime_config()); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); -} - -#[test] -fn build_environment_protects_oci_identity_metadata() { +fn docker_child_environment_strips_supervisor_control_keys() { let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - for (key, value) in [ - (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), - (openshell_core::sandbox_env::SANDBOX_UID, "9999"), - (openshell_core::sandbox_env::SANDBOX_GID, "9999"), + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::OCI_IMAGE_USER, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, ] { - spec.environment.insert(key.to_string(), value.to_string()); + spec.environment + .insert(key.to_string(), "spoofed".to_string()); } + spec.environment + .insert("PATH".to_string(), "/agent/bin".to_string()); - let env = build_environment_for_oci_user(&sandbox, &runtime_config(), "app:staff"); + let env = docker_child_environment(&sandbox); - assert!(env.contains(&format!( - "{}=app:staff", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); - assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); + assert_eq!(env.get("PATH").map(String::as_str), Some("/agent/bin")); + assert!(env.contains_key("TEMPLATE_ENV")); + assert!(env.contains_key("SPEC_ENV")); + assert!(!env.values().any(|value| value == "spoofed")); } #[test] -fn build_environment_strips_gateway_tls_server_name() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment.insert( - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), - "evil.attacker.example.com".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); +fn boundary_environment_contains_only_driver_owned_values() { + let env = build_boundary_environment(&test_sandbox(), &runtime_config()); + assert_eq!(env.len(), 2); assert!( - !env.iter().any(|entry| entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - ))), - "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + env.iter() + .any(|entry| entry.starts_with("OPENSHELL_LOG_LEVEL=")) ); + assert!(env.iter().any(|entry| entry.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::TELEMETRY_ENABLED + )))); } #[test] @@ -1309,20 +1316,267 @@ fn container_creation_uses_inspected_immutable_image() { &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap(); assert_eq!(body.image.as_deref(), Some("sha256:immutable")); - assert_eq!(body.user.as_deref(), Some("0")); + assert_eq!(body.user.as_deref(), Some("1234:1235")); assert_eq!(body.working_dir.as_deref(), Some("/")); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_TOPOLOGY)) + .map(String::as_str), + Some(LABEL_ISOLATION_TOPOLOGY_CAPABILITY_FREE) + ); + assert_eq!( + body.labels + .as_ref() + .and_then(|labels| labels.get(LABEL_ISOLATION_ROLE)) + .map(String::as_str), + Some(LABEL_ISOLATION_ROLE_SANDBOX) + ); assert_eq!( body.cmd.as_deref(), - Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) + Some( + &[ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ][..] + ) ); - assert!(body.env.unwrap().contains(&format!( - "{}=1234:1235", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); + assert!(body.env.unwrap().iter().all(|entry| { + !entry.starts_with(&format!("{}=", openshell_core::sandbox_env::OCI_IMAGE_USER)) + })); + let host = body.host_config.unwrap(); + assert_eq!(host.cap_add, None); + assert_eq!(host.cap_drop, Some(vec!["ALL".to_string()])); + assert_eq!(host.group_add, Some(vec!["1236".to_string()])); + assert_eq!( + host.security_opt, + Some(vec!["no-new-privileges:true".to_string()]) + ); + assert_eq!(host.network_mode.as_deref(), Some("none")); + assert_eq!(host.dns, Some(vec!["127.0.0.53".to_string()])); +} + +#[test] +fn docker_outer_fence_accepts_network_none_without_attachments() { + let inspected = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + ..Default::default() + }), + network_settings: Some(bollard::models::NetworkSettings { + networks: Some(HashMap::from([( + "none".to_string(), + bollard::models::EndpointSettings::default(), + )])), + ..Default::default() + }), + ..Default::default() + }; + + assert!(validate_docker_outer_fence(&inspected).is_ok()); +} + +#[test] +fn docker_outer_fence_rejects_network_mode_or_attached_network_drift() { + let bridge_mode = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("bridge".to_string()), + ..Default::default() + }), + ..Default::default() + }; + let attached_network = bollard::models::ContainerInspectResponse { + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + ..Default::default() + }), + network_settings: Some(bollard::models::NetworkSettings { + networks: Some(HashMap::from([( + "unexpected".to_string(), + bollard::models::EndpointSettings::default(), + )])), + ..Default::default() + }), + ..Default::default() + }; + + assert!(validate_docker_outer_fence(&bridge_mode).is_err()); + assert!(validate_docker_outer_fence(&attached_network).is_err()); +} + +#[test] +fn sandbox_bundle_prepares_only_the_driver_managed_workspace() { + let identity = test_workload_identity(); + let default_archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + driver_mounts::DEFAULT_WORKSPACE_ROOT, + ) + .unwrap(); + let mut archive = tar::Archive::new(default_archive.as_slice()); + let sandbox_entry = archive + .entries() + .unwrap() + .map(Result::unwrap) + .find(|entry| entry.path().unwrap().as_ref() == Path::new("sandbox")) + .expect("managed /sandbox entry"); + assert!(sandbox_entry.header().entry_type().is_dir()); + assert_eq!(sandbox_entry.header().mode().unwrap(), 0o700); + assert_eq!( + sandbox_entry.header().uid().unwrap(), + u64::from(identity.uid) + ); + assert_eq!( + sandbox_entry.header().gid().unwrap(), + u64::from(identity.gid) + ); + + let image_archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + "/workspace/project", + ) + .unwrap(); + let mut archive = tar::Archive::new(image_archive.as_slice()); + assert!( + archive + .entries() + .unwrap() + .map(Result::unwrap) + .all(|entry| { entry.path().unwrap().as_ref() != Path::new("workspace/project") }) + ); +} + +#[test] +fn sandbox_bundle_stages_private_mutual_tls_material() { + let identity = test_workload_identity(); + let archive = docker_sandbox_bundle_archive( + b"sandbox-binary", + b"{}", + DockerSandboxTls { + certificate: b"server-cert", + private_key: b"server-key", + client_ca: b"client-ca", + }, + &identity, + driver_mounts::DEFAULT_WORKSPACE_ROOT, + ) + .unwrap(); + let mut archive = tar::Archive::new(archive.as_slice()); + let entries = archive + .entries() + .unwrap() + .map(Result::unwrap) + .filter_map(|entry| { + let path = entry.path().ok()?.into_owned(); + Some(( + path, + ( + entry.header().mode().ok()?, + entry.header().uid().ok()?, + entry.header().gid().ok()?, + ), + )) + }) + .collect::>(); + for path in [ + ".openshell/channel/sandbox/server.crt", + ".openshell/channel/sandbox/server.key", + ".openshell/channel/sandbox/client-ca.crt", + ] { + assert_eq!( + entries.get(Path::new(path)), + Some(&(0o600, u64::from(identity.uid), u64::from(identity.gid))) + ); + } + assert_eq!( + entries.get(Path::new(".openshell/runtime/openshell-sandbox")), + Some(&(0o555, 0, 0)), + "the trusted sandbox executable must not be writable by the workload" + ); + assert_eq!( + entries.get(Path::new(".openshell/channel")), + Some(&(0o755, 0, 0)), + "the workload must not be able to replace the supervisor secret directory" + ); +} + +#[test] +fn docker_identity_resolution_uses_pinned_image_accounts_and_exact_groups() { + let sandbox = test_sandbox(); + let image = DockerImageMetadata { + id: "sha256:image".to_string(), + user: "agent".to_string(), + working_dir: "/sandbox".to_string(), + volumes: Vec::new(), + }; + let resolved = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"root:x:0:0:root:/root:/bin/sh\nagent:x:10001:10002::/sandbox:/bin/sh\n", + b"root:x:0:\nagent:x:10002:\nrender:x:10003:agent\n", + ) + .unwrap(); + + assert_eq!(resolved.uid, 10001); + assert_eq!(resolved.gid, 10002); + assert_eq!(resolved.supplementary_gids, vec![10003]); + assert_eq!(resolved.source, "image"); + assert_eq!(resolved.resource_digest, "sha256:image"); +} + +#[test] +fn docker_identity_resolution_honors_policy_selectors_and_rejects_root() { + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().workload_identity = Some(WorkloadIdentityRequest { + user: "10001".to_string(), + group: "workers".to_string(), + }); + let image = DockerImageMetadata { + id: "sha256:image".to_string(), + user: String::new(), + working_dir: "/sandbox".to_string(), + volumes: Vec::new(), + }; + let resolved = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"agent:x:10001:10002::/sandbox:/bin/sh\n", + b"workers:x:10004:agent\n", + ) + .unwrap(); + assert_eq!((resolved.uid, resolved.gid), (10001, 10004)); + assert_eq!(resolved.source, "policy"); + + sandbox.spec.as_mut().unwrap().workload_identity = Some(WorkloadIdentityRequest { + user: "root".to_string(), + group: "root".to_string(), + }); + let error = resolve_docker_identity_from_accounts( + &sandbox, + &image, + b"root:x:0:0:root:/root:/bin/sh\n", + b"root:x:0:\n", + ) + .unwrap_err(); + assert!(error.message().contains("UID or GID zero")); } #[test] @@ -1339,6 +1593,7 @@ fn container_creation_rejects_invalid_oci_working_dir() { &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap_err(); @@ -1360,6 +1615,7 @@ fn container_creation_rejects_openshell_control_path_working_dir() { &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap_err(); @@ -1383,6 +1639,7 @@ fn container_creation_rejects_image_volume_that_masks_working_dir() { &DockerSandboxDriverConfig::default(), None, &metadata, + &test_workload_identity(), ) .unwrap_err(); @@ -1393,29 +1650,6 @@ fn container_creation_rejects_image_volume_that_masks_working_dir() { ); } -#[test] -fn container_creation_rejects_image_volume_over_configured_ssh_socket() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: vec!["/custom-runtime".to_string()], - }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); - - let error = build_container_create_body_for_image( - &test_sandbox(), - &config, - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - - assert!(error.message().contains("OpenShell control path")); -} - #[test] fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { let metadata = DockerImageMetadata { @@ -1434,6 +1668,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &root_mount, None, &metadata, + &test_workload_identity(), ) .unwrap_err(); assert!( @@ -1456,6 +1691,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &ancestor_mount, None, &nested_metadata, + &test_workload_identity(), ) .unwrap_err(); assert!( @@ -1473,6 +1709,7 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &nested_mount, None, &metadata, + &test_workload_identity(), ) .expect("nested workspace mounts remain supported"); @@ -1487,84 +1724,15 @@ fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts( &compatibility_path_mount, None, &metadata, + &test_workload_identity(), ) .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); } #[test] -fn build_environment_keeps_path_driver_controlled() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment - .insert("PATH".to_string(), "/malicious/spec/bin".to_string()); - spec.template - .as_mut() - .unwrap() - .environment - .insert("PATH".to_string(), "/malicious/template/bin".to_string()); - - let env = build_environment(&sandbox, &runtime_config()); - let path_entries = env - .iter() - .filter(|entry| entry.starts_with("PATH=")) - .collect::>(); - - let expected_path = format!("PATH={SUPERVISOR_PATH}"); - assert_eq!(path_entries.len(), 1); - assert_eq!(path_entries[0], &expected_path); -} - -#[test] -fn build_environment_keeps_telemetry_toggle_driver_controlled() { - let _guard = ENV_LOCK.lock().unwrap(); - temp_env::with_vars( - [( - openshell_core::sandbox_env::TELEMETRY_ENABLED, - Some("false"), - )], - || { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - "true".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - let telemetry_entries = env - .iter() - .filter(|entry| { - entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::TELEMETRY_ENABLED - )) - }) - .collect::>(); - - assert_eq!(telemetry_entries.len(), 1); - assert_eq!( - telemetry_entries[0], - &format!("{}=false", openshell_core::sandbox_env::TELEMETRY_ENABLED) - ); - }, - ); -} - -#[test] -fn build_binds_uses_docker_tls_directory() { - let binds = build_binds(&test_sandbox(), &runtime_config()).unwrap(); - let targets = binds - .iter() - .filter_map(|bind| bind.split(':').nth(1).map(String::from)) - .collect::>(); - assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); - assert!( - targets - .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) - ); +fn build_binds_does_not_expose_host_runtime_material() { + let binds = build_binds(&test_sandbox(), &runtime_config()); + assert!(binds.is_empty()); } #[test] @@ -1597,7 +1765,7 @@ fn build_container_create_body_includes_driver_config_mounts() { .mounts .expect("driver config mounts should be set"); - assert_eq!(mounts.len(), 2); + assert_eq!(mounts.len(), 3); assert_eq!(mounts[0].typ, Some(MountTypeEnum::VOLUME)); assert_eq!(mounts[0].source.as_deref(), Some("work-nfs")); assert_eq!(mounts[0].target.as_deref(), Some("/sandbox/work")); @@ -1611,6 +1779,9 @@ fn build_container_create_body_includes_driver_config_mounts() { ); assert_eq!(mounts[1].typ, Some(MountTypeEnum::TMPFS)); assert_eq!(mounts[1].target.as_deref(), Some("/sandbox/cache")); + assert_eq!(mounts[2].typ, Some(MountTypeEnum::VOLUME)); + assert_eq!(mounts[2].target.as_deref(), Some(BOUNDARY_MOUNT_PATH)); + assert_eq!(mounts[2].read_only, Some(false)); assert_eq!( mounts[1] .tmpfs_options @@ -2036,36 +2207,6 @@ fn driver_config_rejects_reserved_mount_targets() { assert!(err.message().contains("reserved OpenShell path")); } -#[test] -fn driver_config_rejects_mount_over_configured_ssh_socket() { - let mount_config: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{ - "type": "tmpfs", - "target": "/custom-runtime" - }] - })) - .unwrap(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: Vec::new(), - }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); - - let error = build_container_create_body_for_image( - &test_sandbox(), - &config, - &mount_config, - None, - &metadata, - ) - .unwrap_err(); - - assert!(error.message().contains("OpenShell control path")); -} - #[test] fn docker_local_volume_with_bind_option_is_bind_backed() { let volume = inspected_volume( @@ -2118,27 +2259,6 @@ fn docker_nonlocal_volume_with_bind_option_is_not_bind_backed() { assert!(!docker_volume_is_bind_backed(&volume)); } -#[test] -fn build_environment_uses_token_file_without_raw_token_env() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.sandbox_token = "secret.jwt.value".to_string(); - spec.environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "user-provided-token".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - - assert!(!env.iter().any(|entry| { - entry.starts_with(&format!("{}=", openshell_core::sandbox_env::SANDBOX_TOKEN)) - })); - assert!(env.contains(&format!( - "{}={SANDBOX_TOKEN_MOUNT_PATH}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); -} - #[test] fn managed_container_label_filters_include_gateway_namespace() { let filters = @@ -2147,20 +2267,26 @@ fn managed_container_label_filters_include_gateway_namespace() { assert!(labels.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"))); assert!(labels.contains(&format!("{LABEL_SANDBOX_NAMESPACE}=tenant-a"))); + assert!(labels.contains(&format!( + "{LABEL_ISOLATION_ROLE}={LABEL_ISOLATION_ROLE_SANDBOX}" + ))); assert!(labels.contains(&format!("{LABEL_SANDBOX_ID}=sbx-123"))); } #[test] -fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { +fn build_container_create_body_replaces_inherited_cmd_with_sandbox_bootstrap() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); assert_eq!( create_body.entrypoint, - Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]) + Some(vec![SANDBOX_BINARY_PATH.to_string()]) ); assert_eq!( create_body.cmd, - Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) + Some(vec![ + "--bootstrap".to_string(), + BOUNDARY_CONFIG_MOUNT_PATH.to_string(), + ]) ); assert_eq!( create_body @@ -2176,27 +2302,11 @@ fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { ); assert_eq!( host_config.security_opt.as_ref(), - Some(&vec!["apparmor=unconfined".to_string()]) - ); - assert_eq!( - host_config.network_mode.as_deref(), - Some(DEFAULT_DOCKER_NETWORK_NAME) - ); - assert_eq!( - host_config.extra_hosts.as_ref(), - Some(&vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]) - ); - assert_eq!( - create_body - .networking_config - .as_ref() - .and_then(|config| config.endpoints_config.as_ref()) - .and_then(|endpoints| endpoints.get(DEFAULT_DOCKER_NETWORK_NAME)), - Some(&EndpointSettings::default()) + Some(&vec!["no-new-privileges:true".to_string()]) ); + assert_eq!(host_config.network_mode.as_deref(), Some("none")); + assert_eq!(host_config.extra_hosts, None); + assert!(create_body.networking_config.is_none()); } #[test] @@ -2645,23 +2755,17 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() { } #[test] -fn build_container_create_body_uses_bridge_network() { +fn build_container_create_body_disables_docker_networking() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = create_body.host_config.expect("host_config is populated"); assert_eq!( host_config.network_mode, - Some(DEFAULT_DOCKER_NETWORK_NAME.to_string()), - "sandbox should join the driver-managed bridge network" - ); - assert_eq!( - host_config.extra_hosts, - Some(vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]), - "sandbox should expose stable host aliases for gateway callbacks" + Some("none".to_string()), + "the sandbox must not receive direct Docker networking" ); + assert_eq!(host_config.extra_hosts, None); + assert_eq!(host_config.dns, Some(vec!["127.0.0.53".to_string()])); } #[test] @@ -2867,16 +2971,6 @@ fn pending_sandbox_snapshot_uses_docker_namespace_and_starting_condition() { assert_eq!(status.conditions[0].message, "Docker container is starting"); } -#[test] -fn validate_linux_elf_binary_rejects_non_elf_files() { - let tempdir = TempDir::new().unwrap(); - let path = tempdir.path().join("openshell-sandbox"); - fs::write(&path, b"not-elf").unwrap(); - - let err = validate_linux_elf_binary(&path).unwrap_err(); - assert!(err.contains("Linux ELF executable")); -} - #[test] fn docker_guest_tls_paths_require_all_files_for_https() { let tempdir = TempDir::new().unwrap(); @@ -2892,22 +2986,6 @@ fn docker_guest_tls_paths_require_all_files_for_https() { assert!(err.to_string().contains("guest_tls_cert")); } -#[test] -fn linux_supervisor_candidates_follow_daemon_arch() { - assert_eq!( - linux_supervisor_candidates("amd64"), - vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )] - ); - assert_eq!( - linux_supervisor_candidates("arm64"), - vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )] - ); -} - #[test] fn container_name_preserves_id_suffix_for_long_names() { // Names up to 253 chars are permitted by the gRPC layer. The id @@ -3003,36 +3081,6 @@ fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { ); } -#[test] -fn configured_supervisor_image_takes_precedence_over_local_binaries() { - let tempdir = TempDir::new().unwrap(); - let bin_dir = tempdir.path().join("bin"); - fs::create_dir_all(&bin_dir).unwrap(); - let current_exe = bin_dir.join("openshell-gateway"); - let sibling = bin_dir.join("openshell-sandbox"); - fs::write(¤t_exe, b"gateway").unwrap(); - fs::write(&sibling, b"\x7fELFsibling").unwrap(); - - let local_build = tempdir.path().join("target/openshell-sandbox"); - fs::create_dir_all(local_build.parent().unwrap()).unwrap(); - fs::write(&local_build, b"\x7fELFlocal").unwrap(); - - let source = resolve_supervisor_bin_source( - &DockerComputeConfig { - supervisor_image: Some("example.com/openshell/supervisor:test".to_string()), - ..Default::default() - }, - Some(¤t_exe), - &[local_build], - ) - .unwrap(); - - assert_eq!( - source, - SupervisorBinSource::Image("example.com/openshell/supervisor:test".to_string()) - ); -} - #[test] fn docker_supervisor_image_tag_prefers_explicit_build_tags() { use openshell_core::config::resolve_supervisor_image_tag; @@ -3077,63 +3125,6 @@ fn docker_supervisor_image_refreshes_mutable_tags_only() { )); } -#[test] -fn supervisor_cache_path_namespaces_by_digest_under_openshell_data_dir() { - let base = PathBuf::from("/var/cache/share"); - let path = supervisor_cache_path_with_base( - &base, - "docker-supervisor", - "sha256:abc123deadbeef0123456789cafe0123456789fe", - ); - - assert_eq!( - path, - PathBuf::from( - "/var/cache/share/openshell/docker-supervisor/sha256-abc123deadbeef0123456789cafe0123456789fe/openshell-sandbox", - ), - ); -} - -#[test] -fn supervisor_cache_path_isolates_different_digests() { - let base = PathBuf::from("/data"); - let left = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:aaaaaaaa"); - let right = supervisor_cache_path_with_base(&base, "docker-supervisor", "sha256:bbbbbbbb"); - assert_ne!( - left.parent().unwrap(), - right.parent().unwrap(), - "digest-keyed directories must differ so rollouts are isolated", - ); -} - -#[test] -fn write_cache_binary_atomic_materializes_file_with_executable_mode() { - let tempdir = TempDir::new().unwrap(); - let target = tempdir.path().join("nested").join("openshell-sandbox"); - fs::create_dir_all(target.parent().unwrap()).unwrap(); - - write_cache_binary_atomic(&target, b"\x7fELFpayload").unwrap(); - - assert!(target.is_file()); - assert_eq!(fs::read(&target).unwrap(), b"\x7fELFpayload"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mode = fs::metadata(&target).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o755, "expected 0755, got {mode:04o}"); - } -} - -#[test] -fn write_cache_binary_atomic_overwrites_existing_file() { - let tempdir = TempDir::new().unwrap(); - let target = tempdir.path().join("openshell-sandbox"); - fs::write(&target, b"stale").unwrap(); - - write_cache_binary_atomic(&target, b"\x7fELFfresh").unwrap(); - assert_eq!(fs::read(&target).unwrap(), b"\x7fELFfresh"); -} - #[test] fn temp_extract_container_names_are_unique_per_call() { let first = temp_extract_container_name(); diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 65f4871cd9..3cd36a240f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -610,17 +610,14 @@ image_pull_policy = "IfNotPresent" sandbox_namespace = "docker-dev" # Empty auto-detects https://host.openshell.internal: when guest TLS is set. grpc_endpoint = "https://host.openshell.internal:17670" -# Skip the image-pull-and-extract step by pointing at a locally built binary. -supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" -# When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. -# Defaults to the gateway version; override to pin a specific build. +# Contains both /openshell-sandbox and /openshell-supervisor. Defaults to the +# gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" network_name = "openshell-docker" host_gateway_ip = "172.17.0.1" -ssh_socket_path = "/run/openshell/ssh.sock" # Unsafe operator override. Host bind mounts, including Docker local-driver # bind-backed volumes, expose gateway-host paths inside sandboxes and can # negate OpenShell isolation and filesystem controls. diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index e025452e2d..d6ecec92af 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -161,7 +161,7 @@ that already covers loopback. Otherwise, the Docker driver requests a separate For maintainer-level implementation details, refer to the [Docker driver README](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-driver-docker/README.md). -Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_bin`, `supervisor_image`, `image_pull_policy`, `ssh_socket_path`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. +Select Docker with `compute_drivers = ["docker"]` in `[openshell.gateway]`. Configure Docker driver values such as `socket_path`, `grpc_endpoint`, `network_name`, `supervisor_image`, `image_pull_policy`, `sandbox_pids_limit`, and `guest_tls_*` in `[openshell.drivers.docker]`. The supervisor image must contain both `/openshell-sandbox` and `/openshell-supervisor`. When `socket_path` is unset, the driver uses the same responsive local socket selected by auto-detection. An explicitly selected Docker driver falls back to `/var/run/docker.sock` when no candidate responds. When operating `openshell-driver-docker` as an external driver, set `OPENSHELL_OTLP_ENDPOINT` to export its spans. The driver continues W3C trace diff --git a/e2e/rust/tests/credential_gating.rs b/e2e/rust/tests/credential_gating.rs index 9d4cf0325a..2afe9ec231 100644 --- a/e2e/rust/tests/credential_gating.rs +++ b/e2e/rust/tests/credential_gating.rs @@ -221,6 +221,7 @@ enum EndpointMode { TlsSkip, L4OptIn, RestBody { rewrite: bool }, + WebSocket, } #[derive(Clone, Copy)] @@ -244,6 +245,9 @@ fn write_policy( EndpointMode::RestBody { rewrite } => format!( " protocol: rest\n access: full\n request_body_credential_rewrite: {rewrite}\n" ), + EndpointMode::WebSocket => { + " protocol: websocket\n access: read-write\n".to_string() + } }; let credential_binding = match credential_source { CredentialSource::ProviderProfile => String::new(), @@ -268,12 +272,7 @@ network_policies: endpoints: - host: {TEST_HOST} port: {port} -{endpoint_options}{credential_binding} allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - binaries: +{endpoint_options}{credential_binding} binaries: - path: /usr/bin/python* - path: /usr/local/bin/python* - path: /sandbox/.uv/python/*/bin/python* @@ -286,27 +285,6 @@ network_policies: Ok(file) } -fn write_base_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - file.write_all( - br#"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -"#, - ) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - #[derive(Debug, Default, Clone, Copy)] struct BodyObservation { saw_placeholder: bool, @@ -836,7 +814,11 @@ async fn run_body_sandbox( } async fn run_profile_body_sandbox(port: u16) -> Result { - let policy = write_base_policy()?; + let policy = write_policy( + port, + EndpointMode::RestBody { rewrite: false }, + CredentialSource::ProviderProfile, + )?; let policy_path = policy .path() .to_str() @@ -869,7 +851,11 @@ async fn assert_rest_body_backstop(server: &HttpProbeServer) -> Result<(), Strin } async fn assert_websocket_binary_denied(server: &BinaryWebSocketProbeServer) -> Result<(), String> { - let policy = write_base_policy()?; + let policy = write_policy( + server.port, + EndpointMode::WebSocket, + CredentialSource::ProviderProfile, + )?; let policy_path = policy .path() .to_str() diff --git a/e2e/rust/tests/driver_config_volume.rs b/e2e/rust/tests/driver_config_volume.rs index 2d8789edce..8608e1af77 100644 --- a/e2e/rust/tests/driver_config_volume.rs +++ b/e2e/rust/tests/driver_config_volume.rs @@ -296,10 +296,20 @@ fn write_bind_mount_policy() -> Result { let mut file = tempfile::NamedTempFile::new().map_err(|err| format!("create bind policy: {err}"))?; file.write_all( - br"version: 1 + br#"version: 1 filesystem_policy: include_workdir: false + read_only: + - "/bin" + - "/dev" + - "/etc" + - "/lib" + - "/proc" + - "/usr" + read_write: + - "/sandbox/e2e-bind" + - "/tmp" landlock: compatibility: best_effort @@ -307,7 +317,7 @@ landlock: process: run_as_user: sandbox run_as_group: sandbox -", +"#, ) .map_err(|err| format!("write bind policy: {err}"))?; Ok(file) diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs index f5df4f53e3..29261ff635 100644 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ b/e2e/rust/tests/forward_proxy_l7_bypass.rs @@ -10,13 +10,13 @@ use std::io::Write; -use openshell_e2e::harness::container::ContainerHttpServer; +use openshell_e2e::harness::container::HostSupportContainer; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -const TEST_SERVER_ALIAS: &str = "rest-l7.openshell.test"; +const TEST_SERVER_HOST: &str = "host.openshell.internal"; -async fn start_test_server() -> Result { +async fn start_test_server() -> Result { let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer class Handler(BaseHTTPRequestHandler): @@ -34,7 +34,7 @@ class Handler(BaseHTTPRequestHandler): HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() "#; - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await + HostSupportContainer::start_python(script, 8000).await } fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { @@ -100,7 +100,7 @@ network_policies: async fn forward_proxy_allows_l7_permitted_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -129,7 +129,7 @@ for attempt in range(6): break print(json.dumps(last)) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, ); @@ -150,7 +150,7 @@ print(json.dumps(last)) async fn forward_proxy_denies_l7_blocked_request() { let server = start_test_server().await.expect("start test server"); let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); + write_policy_with_l7_rules(TEST_SERVER_HOST, server.port).expect("write custom policy"); let policy_path = policy .path() .to_str() @@ -170,7 +170,7 @@ except urllib.error.HTTPError as e: except Exception as e: print(json.dumps({{"status": -1, "error": str(e)}})) "#, - host = server.host, + host = TEST_SERVER_HOST, port = server.port, ); diff --git a/e2e/rust/tests/gateway_start.rs b/e2e/rust/tests/gateway_start.rs index cca35e3d59..31ffabb003 100644 --- a/e2e/rust/tests/gateway_start.rs +++ b/e2e/rust/tests/gateway_start.rs @@ -26,12 +26,21 @@ const STOPPED_READY_MARKER: &str = "gateway-start-stopped-ready"; const START_FILE: &str = "/sandbox/gateway-start-state"; const SANDBOX_NAMESPACE_LABEL: &str = "openshell.ai/sandbox-namespace"; const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; +const SANDBOX_ROLE_LABEL_FILTER: &str = "label=openshell.ai/isolation-role=sandbox"; fn sandbox_container_id(namespace: &str, sandbox_name: &str) -> Result { let namespace_filter = format!("label={SANDBOX_NAMESPACE_LABEL}={namespace}"); let sandbox_name_filter = format!("label={SANDBOX_NAME_LABEL}={sandbox_name}"); let output = Command::new("docker") - .args(["ps", "-aq", "--filter", MANAGED_BY_LABEL_FILTER, "--filter"]) + .args([ + "ps", + "-aq", + "--filter", + MANAGED_BY_LABEL_FILTER, + "--filter", + SANDBOX_ROLE_LABEL_FILTER, + "--filter", + ]) .arg(namespace_filter) .args(["--filter"]) .arg(sandbox_name_filter) diff --git a/e2e/rust/tests/local_driver_token_restart.rs b/e2e/rust/tests/local_driver_token_restart.rs index 5223e3a704..5c9661d89c 100644 --- a/e2e/rust/tests/local_driver_token_restart.rs +++ b/e2e/rust/tests/local_driver_token_restart.rs @@ -67,6 +67,7 @@ impl LocalDriver { match self { Self::Docker => vec![ "label=openshell.ai/managed-by=openshell".to_string(), + "label=openshell.ai/isolation-role=sandbox".to_string(), format!("label=openshell.ai/sandbox-namespace={namespace}"), format!("label=openshell.ai/sandbox-name={sandbox_name}"), ], @@ -274,8 +275,24 @@ async fn stop_container_sandbox( sandbox_name: &str, ) -> Result<(), String> { let container_id = sandbox_container_id(engine, driver, namespace, sandbox_name)?; - let token = read_bootstrap_token(engine, &container_id)?; - require_non_expiring_token(&token, "local-driver bootstrap JWT")?; + if driver == LocalDriver::Docker { + run_engine( + engine, + &[ + "exec".to_string(), + container_id.clone(), + "sh".to_string(), + "-c".to_string(), + format!("test ! -r {CONTAINER_TOKEN_MOUNT_PATH}"), + ], + ) + .map_err(|error| { + format!("Docker sandbox workload must not be able to read its bootstrap JWT: {error}") + })?; + } else { + let token = read_bootstrap_token(engine, &container_id)?; + require_non_expiring_token(&token, "local-driver bootstrap JWT")?; + } run_engine(engine, &["stop".to_string(), container_id.clone()])?; wait_for_container_running(engine, &container_id, false, Duration::from_secs(60)).await diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index a2b9c49a3f..f841e509c1 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -3,16 +3,16 @@ #![cfg(feature = "e2e")] -//! E2E coverage for the shared explicit-proxy egress pipeline. +//! E2E coverage for the transparent sandbox egress pipeline. //! -//! These tests exercise behavior that must remain identical while CONNECT and -//! forward HTTP converge on shared authorization, destination, and relay -//! primitives: -//! - live policy reloads affect new requests through both adapters and close a -//! pre-existing CONNECT HTTP stream before its next request is forwarded; +//! Workloads connect directly to their requested destinations. Seccomp +//! notification diverts those sockets to the supervisor without proxy +//! environment variables or explicit CONNECT requests. These tests cover: +//! - live policy reloads affect new requests and close a pre-existing HTTP +//! stream before its next request is forwarded; //! - `tls: skip` selects a byte-transparent TCP relay; //! - provider placeholders in HTTP headers and opted-in REST bodies are -//! resolved through both adapters without appearing in test output. +//! resolved without appearing in test output. use std::io::{self, Error, ErrorKind, Write}; use std::process::Stdio; @@ -22,6 +22,7 @@ use std::sync::{ }; use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::container::{SupportContainer, e2e_network_name}; use openshell_e2e::harness::sandbox::SandboxGuard; use serde_json::Value; use tempfile::{Builder as TempFileBuilder, NamedTempFile}; @@ -359,9 +360,9 @@ network_policies: } fn write_ip_literal_success_policy( - ip: &str, - explicit_port: u16, - implicit_port: u16, + explicit_ip: &str, + implicit_ip: &str, + port: u16, ) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = format!( @@ -384,13 +385,14 @@ network_policies: name: destination_successes endpoints: - host: {ip} - port: {explicit_port} - allowed_ips: ["{ip}/32"] - - host: {ip} - port: {implicit_port} + port: {port} + allowed_ips: ["{explicit_ip}/32"] + - host: {implicit_ip} + port: {port} binaries: - path: "/**" -"# +"#, + ip = explicit_ip, ); file.write_all(policy.as_bytes()) .map_err(|error| format!("write policy: {error}"))?; @@ -749,26 +751,15 @@ async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { stream.write_all(response.as_bytes()).await } -fn proxy_status_script(host: &str, port: u16) -> String { +fn transparent_status_script(host: &str, port: u16) -> String { format!( r#" import json -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_headers(sock): data = b"" while b"\r\n\r\n" not in data: @@ -782,44 +773,35 @@ def status(response): parts = response.split(None, 2) return int(parts[1]) if len(parts) > 1 else 0 -def forward_status(): - proxy_host, proxy_port = proxy_parts() - target = f"{{HOST}}:{{PORT}}" - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall( - f"GET http://{{target}}/forward HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - -def connect_status(): - proxy_host, proxy_port = proxy_parts() +def request_status(path): target = f"{{HOST}}:{{PORT}}" - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - code = status(read_headers(sock)) - if code != 200: - return code - sock.sendall( - f"GET /connect HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - -print(json.dumps({{"connect": connect_status(), "forward": forward_status()}}, sort_keys=True)) + try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET {{path}} HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + except OSError as error: + return {{"errno": error.errno, "error": repr(error)}} + +print(json.dumps({{ + "first": request_status("/first"), + "second": request_status("/second"), +}}, sort_keys=True)) "#, host = host, port = port, ) } -fn persistent_connect_script(host: &str, port: u16) -> String { +fn persistent_transparent_script(host: &str, port: u16) -> String { format!( r#" import json import os import socket import time -import urllib.parse HOST = {host:?} PORT = {port} @@ -827,15 +809,6 @@ READY = "/tmp/proxy-reload-ready" GO = "/tmp/proxy-reload-go" RESULT = "/tmp/proxy-reload-result" -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -855,15 +828,11 @@ def read_response(sock): body += chunk return int(headers.split(None, 2)[1]) -proxy_host, proxy_port = proxy_parts() target = f"{{HOST}}:{{PORT}}" failed_closed = False second_status = 0 try: - with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - if read_response(sock) != 200: - raise RuntimeError("initial CONNECT was denied") + with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall( f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() ) @@ -920,7 +889,7 @@ fn parse_json_line(output: &str) -> Value { } #[tokio::test] -async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { +async fn policy_reload_updates_transparent_requests_and_closes_existing_http_stream() { let server = KeepAliveHttpServer::start() .await .expect("start keep-alive HTTP server"); @@ -950,7 +919,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { .await .expect("wait for policy A"); - let persistent_script = persistent_connect_script(TEST_SERVER_HOST, server.port); + let persistent_script = persistent_transparent_script(TEST_SERVER_HOST, server.port); guard .exec(&[ "sh", @@ -960,7 +929,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { &persistent_script, ]) .await - .expect("start persistent CONNECT client"); + .expect("start persistent transparent client"); wait_for_sandbox_file( &guard, "/tmp/proxy-reload-ready", @@ -968,16 +937,19 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { ) .await; - let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); let before = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters before reload"); + .expect("exercise transparent requests before reload"); let before = parse_json_line(&before); - assert_eq!(before["connect"], 200, "CONNECT before reload: {before}"); assert_eq!( - before["forward"], 200, - "forward HTTP before reload: {before}" + before["first"], 200, + "first request before reload: {before}" + ); + assert_eq!( + before["second"], 200, + "second request before reload: {before}" ); run_cli(&[ @@ -996,7 +968,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { guard .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) .await - .expect("release persistent CONNECT client"); + .expect("release persistent transparent client"); let stale_tunnel = wait_for_sandbox_file( &guard, "/tmp/proxy-reload-result", @@ -1006,16 +978,16 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { let stale_tunnel = parse_json_line(&stale_tunnel); assert_eq!( stale_tunnel["failed_closed"], true, - "existing CONNECT HTTP stream forwarded after policy reload: {stale_tunnel}" + "existing transparent HTTP stream forwarded after policy reload: {stale_tunnel}" ); let after = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters after reload"); + .expect("exercise transparent requests after reload"); let after = parse_json_line(&after); - assert_eq!(after["connect"], 403, "CONNECT after reload: {after}"); - assert_eq!(after["forward"], 403, "forward HTTP after reload: {after}"); + assert_ne!(after["first"], 200, "first request after reload: {after}"); + assert_ne!(after["second"], 200, "second request after reload: {after}"); guard.cleanup().await; } @@ -1052,14 +1024,20 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { .await .expect("wait for valid policy"); - let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); let before = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters before invalid update"); + .expect("exercise transparent requests before invalid update"); let before = parse_json_line(&before); - assert_eq!(before["connect"], 200, "CONNECT before update: {before}"); - assert_eq!(before["forward"], 200, "forward before update: {before}"); + assert_eq!( + before["first"], 200, + "first request before update: {before}" + ); + assert_eq!( + before["second"], 200, + "second request before update: {before}" + ); let history_before = run_cli(&["policy", "list", &guard.name]) .await .expect("list policy history before rejected update"); @@ -1093,15 +1071,15 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { let after_rejection = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters after rejected update"); + .expect("exercise transparent requests after rejected update"); let after_rejection = parse_json_line(&after_rejection); assert_eq!( - after_rejection["connect"], 200, - "CONNECT should keep using the active valid policy: {after_rejection}" + after_rejection["first"], 200, + "first request should keep using the active valid policy: {after_rejection}" ); assert_eq!( - after_rejection["forward"], 200, - "forward HTTP should keep using the active valid policy: {after_rejection}" + after_rejection["second"], 200, + "second request should keep using the active valid policy: {after_rejection}" ); assert!( server.connection_count() > connections_before_rejection, @@ -1112,69 +1090,25 @@ async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { } #[tokio::test] -async fn destination_denial_modes_match_across_connect_and_forward_adapters() { +async fn transparent_destination_denials_fail_connect_with_eacces() { let policy = write_destination_denial_policy().expect("write destination denial policy"); let policy_path = policy_path(&policy); let script = r#" import json -import os import socket -import urllib.parse - -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - headers, _, body = data.partition(b"\r\n\r\n") - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - status = int(headers.split(None, 2)[1]) - return {"status": status, "body": json.loads(body.decode())} - -def connect_result(host, port): - with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{host}:{port}" - sock.sendall(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode()) - return read_response(sock) - -def forward_result(host, port): - with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{host}:{port}" - sock.sendall( - f"GET http://{target}/probe HTTP/1.1\r\n" - f"Host: {target}\r\nConnection: close\r\n\r\n".encode() - ) - return read_response(sock) targets = { "metadata": ("169.254.169.254", 80), - "loopback": ("127.0.0.1", 80), "control_plane": ("203.0.113.10", 6443), "outside_allowed_ips": ("203.0.113.10", 8080), } result = {} for name, target in targets.items(): - result[name] = { - "connect": connect_result(*target), - "forward": forward_result(*target), - } + try: + with socket.create_connection(target, timeout=10): + result[name] = 0 + except OSError as error: + result[name] = error.errno print(json.dumps(result, sort_keys=True)) "#; @@ -1182,81 +1116,46 @@ print(json.dumps(result, sort_keys=True)) .await .expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for name in [ - "metadata", - "loopback", - "control_plane", - "outside_allowed_ips", - ] { - for adapter in ["connect", "forward"] { - assert_eq!( - result[name][adapter]["status"], 403, - "{name} {adapter}: {result}" - ); - assert_eq!( - result[name][adapter]["body"]["error"], "ssrf_denied", - "{name} {adapter}: {result}" - ); - } + for name in ["metadata", "control_plane", "outside_allowed_ips"] { + assert_eq!(result[name], 13, "{name} should fail with EACCES: {result}"); } - assert_eq!( - result["metadata"]["connect"]["body"]["detail"], - "CONNECT 169.254.169.254:80 blocked: declared endpoint check failed" - ); - assert_eq!( - result["metadata"]["forward"]["body"]["detail"], - "GET 169.254.169.254:80 blocked: declared endpoint check failed" - ); - assert_eq!( - result["control_plane"]["connect"]["body"]["detail"], - "CONNECT 203.0.113.10:6443 blocked: allowed_ips check failed" - ); - assert_eq!( - result["outside_allowed_ips"]["forward"]["body"]["detail"], - "GET 203.0.113.10:8080 blocked: allowed_ips check failed" - ); } #[tokio::test] -async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adapters() { - let resolver = SandboxGuard::create(&[ - "--", - "python3", - "-c", - "import socket; print('GATEWAY_IP=' + socket.gethostbyname('host.openshell.internal'))", - ]) - .await - .expect("resolve host gateway inside sandbox"); - let gateway_ip = resolver - .create_output - .lines() - .find_map(|line| line.trim().strip_prefix("GATEWAY_IP=")) - .expect("sandbox gateway IPv4 output") - .parse::() - .expect("host gateway must resolve to IPv4 for this e2e"); - - // Rootless Podman with pasta exposes its trusted host-gateway alias as a - // link-local address. The hostname receives a narrow runtime exemption, - // but the equivalent raw IP literal must remain hard-blocked. Other - // drivers still exercise the successful IP-literal path below. - if gateway_ip.is_loopback() || gateway_ip.is_link_local() || gateway_ip.is_unspecified() { - eprintln!( - "skipping IP-literal success assertions: host gateway {gateway_ip} is always blocked" - ); +async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_transparently() { + if e2e_network_name().is_none() { + eprintln!("skipping IP-literal success assertions without a shared container network"); return; } - let gateway_ip = gateway_ip.to_string(); + const HTTP_SERVER: &str = r#" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - let explicit_server = KeepAliveHttpServer::start() - .await - .expect("start explicit allowed_ips server"); - let implicit_server = KeepAliveHttpServer::start() - .await - .expect("start implicit IP-literal server"); - let policy = - write_ip_literal_success_policy(&gateway_ip, explicit_server.port, implicit_server.port) - .expect("write IP literal policy"); +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = b"ok" + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + +ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() +"#; + let explicit_server = + SupportContainer::start_python("explicit-ip.openshell.test", HTTP_SERVER, 8000) + .await + .expect("start explicit allowed_ips support container"); + let implicit_server = + SupportContainer::start_python("implicit-ip.openshell.test", HTTP_SERVER, 8000) + .await + .expect("start implicit IP-literal support container"); + let explicit_ip = explicit_server.ip().expect("explicit support container IP"); + let implicit_ip = implicit_server.ip().expect("implicit support container IP"); + let policy = write_ip_literal_success_policy(&explicit_ip, &implicit_ip, 8000) + .expect("write IP literal policy"); let policy_path = policy_path(&policy); let mut guard = SandboxGuard::create_keep_with_args( &["--policy", &policy_path], @@ -1266,17 +1165,21 @@ async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adap .await .expect("create keep sandbox"); - for (mode, port) in [ - ("explicit_allowed_ips", explicit_server.port), - ("implicit_ip_literal", implicit_server.port), + for (mode, destination) in [ + ("explicit_allowed_ips", explicit_ip.as_str()), + ("implicit_ip_literal", implicit_ip.as_str()), ] { let output = guard - .exec(&["python3", "-c", &proxy_status_script(&gateway_ip, port)]) + .exec(&[ + "python3", + "-c", + &transparent_status_script(destination, 8000), + ]) .await .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); let statuses = parse_json_line(&output); - assert_eq!(statuses["connect"], 200, "{mode} CONNECT: {statuses}"); - assert_eq!(statuses["forward"], 200, "{mode} forward: {statuses}"); + assert_eq!(statuses["first"], 200, "{mode} first request: {statuses}"); + assert_eq!(statuses["second"], 200, "{mode} second request: {statuses}"); } guard.cleanup().await; @@ -1290,28 +1193,13 @@ async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) echoed = b"" while len(echoed) < len(PAYLOAD): @@ -1338,7 +1226,7 @@ print("RAW_RELAY_OK") } #[tokio::test] -async fn middleware_redacts_request_bodies_through_both_adapters() { +async fn middleware_redacts_transparent_request_bodies() { let server = RequestBodyEchoServer::start() .await .expect("start request body echo server"); @@ -1348,21 +1236,12 @@ async fn middleware_redacts_request_bodies_through_both_adapters() { let script = format!( r#" import json -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} SECRET = "sk-1234567890abcdef" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -1395,22 +1274,12 @@ def request_bytes(target): "Connection: close\r\n\r\n" ).encode() + body -target = f"{{HOST}}:{{PORT}}" -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as forward_sock: - forward_sock.sendall(request_bytes(f"http://{{target}}/middleware")) - forward = read_response(forward_sock) - -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as connect_sock: - connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - connect_response = b"" - while b"\r\n\r\n" not in connect_response: - connect_response += connect_sock.recv(4096) - if int(connect_response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") - connect_sock.sendall(request_bytes("/middleware")) - connect = read_response(connect_sock) - -print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +def request_once(): + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request_bytes("/middleware")) + return read_response(sock) + +print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) "#, host = TEST_SERVER_HOST, port = server.port, @@ -1420,44 +1289,29 @@ print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) .await .expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for adapter in ["connect", "forward"] { + for request in ["first", "second"] { assert_eq!( - result[adapter]["api_key"], "[REDACTED]", - "{adapter} did not deliver the middleware-transformed body: {result}" + result[request]["api_key"], "[REDACTED]", + "{request} did not deliver the middleware-transformed body: {result}" ); } } #[tokio::test] -async fn fail_closed_middleware_blocks_uninspectable_connect_payload_before_upstream() { +async fn fail_closed_middleware_blocks_uninspectable_transparent_payload_before_upstream() { let server = EchoServer::start().await.expect("start TCP echo server"); let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") .expect("write fail-closed middleware policy"); let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied before tunnel establishment") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) denial = b"" while True: @@ -1518,7 +1372,7 @@ print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") } #[tokio::test] -async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { +async fn fail_open_middleware_bypasses_uninspectable_transparent_tls_skip() { let server = EchoServer::start().await.expect("start TCP echo server"); let policy = write_middleware_policy( TEST_SERVER_HOST, @@ -1530,28 +1384,13 @@ async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse HOST = {host:?} PORT = {port} PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: - target = f"{{HOST}}:{{PORT}}" - sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - response = b"" - while b"\r\n\r\n" not in response: - response += sock.recv(4096) - if int(response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") +with socket.create_connection((HOST, PORT), timeout=10) as sock: sock.sendall(PAYLOAD) echoed = b"" while len(echoed) < len(PAYLOAD): @@ -1588,7 +1427,7 @@ print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") } #[tokio::test] -async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { +async fn transparent_pipeline_never_reaches_upstream_as_first_request_overflow() { let server = PipelineProbeServer::start() .await .expect("start pipeline probe server"); @@ -1603,26 +1442,18 @@ async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { let policy_path = policy_path(&policy); let script = format!( r#" -import os import socket -import urllib.parse -proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) -) -parsed = urllib.parse.urlparse(proxy_url) target = "{host}:{port}" first = ( - f"GET http://{{target}}/allowed HTTP/1.1\r\n" + f"GET /allowed HTTP/1.1\r\n" f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" ) second = ( - f"POST http://{{target}}/blocked HTTP/1.1\r\n" + f"POST /blocked HTTP/1.1\r\n" f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" ) -with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: +with socket.create_connection(({host:?}, {port}), timeout=10) as sock: sock.sendall((first + second).encode()) response = b"" while True: @@ -1630,9 +1461,11 @@ with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) if not chunk: break response += chunk -if response.count(b"HTTP/1.1 ") != 1 or b" 200 " not in response.split(b"\r\n", 1)[0]: +responses = response.count(b"HTTP/1.1 ") +first_status = response.split(b"\r\n", 1)[0] +if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: raise RuntimeError(f"unexpected pipelined response: {{response!r}}") -print("FORWARD_PIPELINE_CLOSED") +print("TRANSPARENT_PIPELINE_DENIED") "#, host = TEST_SERVER_HOST, port = server.port, @@ -1642,8 +1475,8 @@ print("FORWARD_PIPELINE_CLOSED") .await .expect("sandbox create"); assert!( - guard.create_output.contains("FORWARD_PIPELINE_CLOSED"), - "forward proxy did not close after one response:\n{}", + guard.create_output.contains("TRANSPARENT_PIPELINE_DENIED"), + "transparent HTTP stream did not deny the disallowed pipelined request:\n{}", guard.create_output ); @@ -1657,7 +1490,7 @@ print("FORWARD_PIPELINE_CLOSED") } #[tokio::test] -async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters() { +async fn http_credentials_are_rewritten_in_transparent_headers_and_bodies() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1682,21 +1515,11 @@ async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters( import json import os import socket -import urllib.parse HOST = {host:?} PORT = {port} TOKEN = os.environ[{token_env:?}] -def proxy_parts(): - proxy_url = next( - os.environ[name] - for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") - if os.environ.get(name) - ) - parsed = urllib.parse.urlparse(proxy_url) - return parsed.hostname, parsed.port or 80 - def read_response(sock): data = b"" while b"\r\n\r\n" not in data: @@ -1730,23 +1553,12 @@ def request_bytes(target): "Connection: close\r\n\r\n" ).encode() + body -proxy_host, proxy_port = proxy_parts() -target = f"{{HOST}}:{{PORT}}" -with socket.create_connection((proxy_host, proxy_port), timeout=10) as forward_sock: - forward_sock.sendall(request_bytes(f"http://{{target}}/probe")) - forward = read_response(forward_sock) - -with socket.create_connection((proxy_host, proxy_port), timeout=10) as connect_sock: - connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) - connect_response = b"" - while b"\r\n\r\n" not in connect_response: - connect_response += connect_sock.recv(4096) - if int(connect_response.split(None, 2)[1]) != 200: - raise RuntimeError("CONNECT was denied") - connect_sock.sendall(request_bytes("/probe")) - connect = read_response(connect_sock) - -print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +def request_once(): + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request_bytes("/probe")) + return read_response(sock) + +print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) "#, host = TEST_SERVER_HOST, port = server.port, @@ -1772,18 +1584,18 @@ print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) let guard = result.expect("sandbox create"); let result = parse_json_line(&guard.create_output); - for adapter in ["connect", "forward"] { + for request in ["first", "second"] { assert_eq!( - result[adapter]["header_resolved"], true, - "{adapter} header placeholder was not resolved: {result}" + result[request]["header_resolved"], true, + "{request} header placeholder was not resolved: {result}" ); assert_eq!( - result[adapter]["body_resolved"], true, - "{adapter} body placeholder was not resolved: {result}" + result[request]["body_resolved"], true, + "{request} body placeholder was not resolved: {result}" ); assert_eq!( - result[adapter]["saw_placeholder"], false, - "{adapter} leaked an unresolved placeholder upstream: {result}" + result[request]["saw_placeholder"], false, + "{request} leaked an unresolved placeholder upstream: {result}" ); } assert!( diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 1027654641..6f3f751193 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -356,7 +356,7 @@ print('transparent-tcp-e2e-ok') let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")) - && logs.contains("transparent_tcp_port_mismatch") + && logs.contains("Denied staged transparent connection") }) .await .expect("wait for sandbox logs"); @@ -364,7 +364,10 @@ print('transparent-tcp-e2e-ok') logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")), "{logs}" ); - assert!(logs.contains("transparent_tcp_port_mismatch"), "{logs}"); + assert!( + logs.contains("Denied staged transparent connection"), + "{logs}" + ); sandbox.cleanup().await; } diff --git a/rfc/0003-gateway-configuration/README.md b/rfc/0003-gateway-configuration/README.md index d6d4750f25..981a120b93 100644 --- a/rfc/0003-gateway-configuration/README.md +++ b/rfc/0003-gateway-configuration/README.md @@ -133,8 +133,7 @@ image_pull_policy = "IfNotPresent" sandbox_namespace = "docker-dev" grpc_endpoint = "https://host.openshell.internal:8080" network_name = "openshell" -supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # optional override -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # used to extract bin +supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" # contains sandbox + supervisor guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem"