diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 780cf8b9c6..1b0b6bb957 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -69,28 +69,12 @@ mise run helm:skaffold:dev mise run helm:skaffold:run ``` -**Supervisor sidecar topology** (build once and leave running): -```bash -mise run helm:skaffold:run:sidecar -``` - -**Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running): -```bash -mise run helm:skaffold:run:sidecar-mtls -``` - -Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm -chart. The sidecar profile renders an `openshell-network-init` init container for -nftables setup and an `openshell-supervisor-network` runtime sidecar for proxying. -Binary-aware policy mode runs that sidecar as UID 0 with `SYS_PTRACE` and -`DAC_READ_SEARCH`; relaxed mode can run it as the configured proxy UID, which -must be at least `1000` and distinct from the workload UID. The -sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores -`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install -Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first -install. The default Skaffold values export gateway and Kubernetes-driver traces to -the collector service installed by `helm:k3s:create`. Envoy Gateway opt-in; see the -Optional Add-ons section below. +Both commands build the gateway and trusted runtime image and deploy the +OpenShell Helm chart. Kubernetes sandboxes always use the proxy-pod topology: +a capability-free sandbox Pod behind an empty-egress NetworkPolicy and a +separate capability-free supervisor Deployment. The default Skaffold values +export gateway and Kubernetes-driver traces to the collector service installed +by `helm:k3s:create`. Envoy Gateway is opt-in; see the Optional Add-ons section. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or the unified local forwarding task: @@ -102,8 +86,8 @@ mise run helm:k3s:forward The task forwards OTLP/gRPC to `http://127.0.0.1:4317` and the trace UI to `http://127.0.0.1:18888`. When Skaffold has deployed a Kubernetes gateway, it also forwards the gateway to `http://127.0.0.1:8090`; otherwise it continues -with the collector ports only. A successful plaintext `helm:skaffold:run` or -`helm:skaffold:run:sidecar` registers the gateway under the worktree-specific +with the collector ports only. A successful plaintext `helm:skaffold:run` +registers the gateway under the worktree-specific k3d cluster name and selects it as the active gateway. Keep the forwarding task running while using those endpoints. @@ -134,8 +118,7 @@ create the Secret named `openshell-ha-pg` with a `uri` key, then run ### TLS behaviour `ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run -plaintext by default. To test sidecar topology with TLS enabled, use -`mise run helm:skaffold:run:sidecar-mtls`. +plaintext by default. Remove that override to test gateway TLS/mTLS locally. | Mode | `server.disableTls` | Gateway scheme | |------|---------------------|----------------| @@ -188,12 +171,6 @@ openshell sandbox list --gateway-endpoint https://localhost:8090 mise run helm:skaffold:delete ``` -For a sidecar-profile deployment: - -```bash -mise run helm:skaffold:delete:sidecar -``` - ### Delete the cluster entirely ```bash @@ -259,7 +236,7 @@ SPIFFE JWT-SVIDs for dynamic provider token grants: `openshell.local` and adds a `ClusterSPIFFEID` that maps sandbox pod annotations to `spiffe://openshell.local/openshell/sandbox/`. OpenShell mounts the SPIFFE CSI Workload API socket at -`/spiffe-workload-api/spire-agent.sock` into sandbox pods for provider token +`/spiffe-workload-api/spire-agent.sock` only into supervisor Pods for provider token grants. Supervisor-to-gateway authentication remains on the Kubernetes ServiceAccount bootstrap and gateway-minted sandbox JWT path; the selected Kubernetes compute driver validates the projected token before the gateway @@ -320,7 +297,6 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay | | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | -| `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 4c894e3ea3..e83043ec09 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -370,16 +370,12 @@ jobs: include: - agent_sandbox_api: v1beta1 agent_sandbox_version: v0.5.0 - topology: combined - extra_helm_values: "" + topology: proxy-pod + extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml - agent_sandbox_api: v1alpha1 agent_sandbox_version: v0.4.6 - topology: combined - extra_helm_values: "" - - agent_sandbox_api: v1beta1 - agent_sandbox_version: v0.5.0 - topology: sidecar - extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml + topology: proxy-pod + extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml permissions: actions: read contents: read @@ -404,6 +400,7 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed + extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml conformance-artifact-prefix: openshell-conformance kubernetes-external-driver-e2e: @@ -421,6 +418,7 @@ jobs: gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu external-driver-binary: openshell-driver-kubernetes cluster-images: supervisor + extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml conformance-artifact-prefix: openshell-conformance kubernetes-workspace-operator-e2e: @@ -435,6 +433,7 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator + extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml conformance-artifact-prefix: openshell-conformance kubernetes-ha-e2e: diff --git a/AGENTS.md b/AGENTS.md index d0049c85a3..30e63e59d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,7 +38,8 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner | | `crates/openshell-conformance-cli/` | Conformance CLI | Distributable `list` and `run` entrypoint for gateway conformance | | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | -| `crates/openshell-sandbox/` | Sandbox runtime | Container supervision, policy-enforced egress routing | +| `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O | +| `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking | | `crates/openshell-binary-identity/` | Binary identity | Shared trusted procfs executable identity resolution for isolation backends | | `crates/openshell-isolation-interface/` | Isolation backend interface | RFC 0012 `IsolationBackend` trait + types; the supervisor-facing runtime contract for the boundary | | `crates/openshell-policy/` | Policy engine | Filesystem, network, process, and inference constraints | @@ -59,7 +60,6 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-driver-db-credstore/` | Database credential driver | In-process `CredentialDriver` backend for gateway database credential storage | | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods | | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers | -| `crates/openshell-driver-mxc/` | MXC compute driver | Windows in-process `ComputeDriver` backend for MXC sandbox execution | | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers | | `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | | `crates/openshell-driver-mxc/` | Microsoft MXC compute driver | In-process Windows AppContainer and isolation-session compute backend | diff --git a/Cargo.lock b/Cargo.lock index 6b3102d2db..e3ed1fc8e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3959,6 +3959,7 @@ dependencies = [ "miette", "notify", "openshell-core", + "openshell-isolation-interface", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -3966,6 +3967,8 @@ dependencies = [ "opentelemetry_sdk", "prost", "prost-types", + "rand 0.9.4", + "rcgen", "serde", "serde_json", "temp-env", diff --git a/crates/openshell-core/src/container_paths.rs b/crates/openshell-core/src/container_paths.rs index 39dfe8a871..b2ec3ebf0f 100644 --- a/crates/openshell-core/src/container_paths.rs +++ b/crates/openshell-core/src/container_paths.rs @@ -12,7 +12,6 @@ pub const OPT_ROOT: &str = "/opt/openshell"; pub const ETC_ROOT: &str = "/etc/openshell"; pub const TLS_ROOT: &str = "/etc/openshell-tls"; pub const RUN_ROOT: &str = "/run/openshell"; -pub const SIDECAR_RUN_ROOT: &str = "/run/openshell-sidecar"; pub const NETNS_MOUNT_ROOT: &str = "/run/netns"; pub const NETNS_IPROUTE2_ROOT: &str = "/var/run/netns"; @@ -34,7 +33,6 @@ pub const CONTROL_ROOTS: &[&str] = &[ ETC_ROOT, TLS_ROOT, RUN_ROOT, - SIDECAR_RUN_ROOT, NETNS_MOUNT_ROOT, // The supervisor currently uses the conventional iproute2 spelling. NETNS_IPROUTE2_ROOT, @@ -52,9 +50,6 @@ pub const CONTAINER_POLICY_PATH: &str = "/etc/openshell/policy.yaml"; pub const POLICY_ADVISOR_SKILL_PATH: &str = "/etc/openshell/skills/policy_advisor.md"; pub const SSH_SOCKET_PATH: &str = "/run/openshell/ssh.sock"; -pub const SIDECAR_CONTROL_SOCKET: &str = "/run/openshell-sidecar/control.sock"; -pub const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -pub const SIDECAR_CLIENT_TLS_DIR: &str = "/etc/openshell-tls/proxy/client"; pub const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; pub const SUPERVISOR_CA_CERT_PATH: &str = "/etc/openshell-tls/openshell-ca.pem"; pub const SUPERVISOR_CA_BUNDLE_PATH: &str = "/etc/openshell-tls/ca-bundle.pem"; @@ -92,9 +87,6 @@ mod tests { CONTAINER_POLICY_PATH, POLICY_ADVISOR_SKILL_PATH, SSH_SOCKET_PATH, - SIDECAR_CONTROL_SOCKET, - SIDECAR_TLS_DIR, - SIDECAR_CLIENT_TLS_DIR, CLIENT_TLS_DIR, SUPERVISOR_CA_CERT_PATH, SUPERVISOR_CA_BUNDLE_PATH, diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index b1a3049882..13172e7219 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -296,7 +296,6 @@ mod tests { "/etc/openshell-tls", "/run", "/run/openshell/cache", - "/run/openshell-sidecar/control.sock", "/run/netns/project", "/var/run/netns/project", ] { diff --git a/crates/openshell-core/src/proposals.rs b/crates/openshell-core/src/proposals.rs index 53586629ca..39810f0668 100644 --- a/crates/openshell-core/src/proposals.rs +++ b/crates/openshell-core/src/proposals.rs @@ -4,7 +4,7 @@ //! Shared state controlling agent-driven policy proposals. //! //! Initialised once during sandbox start from the `agent_policy_proposals_enabled` -//! setting and updated by the policy poll loop or authoritative sidecar control +//! setting and updated by the policy poll loop or authoritative supervisor //! when the setting changes. Read by the `policy.local` route handler and by //! the skills installer to gate the agent-controlled mutation surface. @@ -15,7 +15,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; /// /// Clones point at the same atomic value, so the sandbox orchestrator can pass /// this into the process and network supervisors and then update it from the -/// settings poll loop or sidecar control. +/// settings poll loop or supervisor control. #[derive(Clone, Debug)] pub struct AgentProposals { enabled: Arc, diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index d9055fa319..97ff3449e2 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -165,8 +165,8 @@ impl ProviderCredentialState { /// Build a static provider state from an already-prepared child /// environment snapshot. /// - /// Kubernetes sidecar topology uses this in the process-only supervisor: - /// the network sidecar owns provider credential resolvers and sends the + /// The Kubernetes proxy-pod topology uses this in the sandbox process: + /// the supervisor owns provider credential resolvers and sends the /// workload-facing env map over a local control channel. The process leaf /// must inject that map into child processes without re-placeholderizing it /// or holding the gateway-side resolver material. diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c1c91822b6..c6a6824fad 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -131,10 +131,6 @@ impl MainProcessConfig { /// Deployment-controlled telemetry toggle propagated to the sandbox supervisor. pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; -/// Supervisor pod/runtime topology. Kubernetes sidecar mode sets this to -/// `"sidecar"`; the default combined supervisor path omits it. -pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; - /// The isolation backend admitted by the deployment configuration (RFC 0012). /// /// Delivered on a channel separate from the topology descriptor so descriptor @@ -142,9 +138,6 @@ pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; /// whenever a topology descriptor is supplied. pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND"; -/// Network enforcement backend selected by the compute driver. -pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; - /// Comma-separated runtime networking capabilities supplied by the compute /// driver. Capabilities describe substrate the shared supervisor may activate; /// they never move policy evaluation into the driver. @@ -153,20 +146,6 @@ pub const NETWORK_RUNTIME_CAPABILITIES: &str = "OPENSHELL_NETWORK_RUNTIME_CAPABI /// Driver capability for policy-gated DNS and transparent TCP interception. pub const POLICY_DNS_TRANSPARENT_TCP_CAPABILITY: &str = "policy-dns-transparent-tcp"; -/// Whether network policy evaluation must bind requests to the peer binary. -/// -/// The default when unset is `"required"`. Kubernetes sidecar experiments may -/// set this to `"relaxed"` to enforce endpoint and L7 policy without per-binary -/// `/proc` identity binding. -pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; - -/// Unix socket used by Kubernetes sidecar topology for local coordination. -/// -/// The network sidecar owns gateway credentials and serves policy/provider -/// state over this socket instead of exposing gateway credentials to the agent -/// container. -pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; - /// Optional TLS server name override used when connecting to the gateway. pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 3a3d843f1a..b1816de367 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation-interface = { path = "../openshell-isolation-interface" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } @@ -37,6 +38,8 @@ tracing-subscriber = { workspace = true } thiserror = { workspace = true } miette = { workspace = true } notify = "8" +rand = { workspace = true } +rcgen = { workspace = true } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 02dcfe5e87..16d5a2038b 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -45,9 +45,43 @@ not a tenant isolation boundary. ## Runtime Model The gateway stores platform state and delegates sandbox workload creation to -this driver. Kubernetes owns scheduling and pod lifecycle. The -`openshell-sandbox` supervisor inside each workload owns agent isolation, -credential injection, policy polling, logs, and the gateway relay. +this driver. Kubernetes owns scheduling and pod lifecycle. The driver always +uses the proxy-pod topology. The workload Pod runs `openshell-sandbox`, while a +separate Deployment runs `openshell-supervisor`. + +The sandbox owns the agent process, applies Landlock and child seccomp filters, +identifies the binary behind each network syscall, and relays mediated streams +to the supervisor. The supervisor authenticates to the gateway with a JWT, +loads policy and provider state, performs destination and L7 authorization, and +opens upstream connections. The workload receives no gateway credential, +provider identity socket, or corporate-proxy credential. + +Both Pods run as the namespace-resolved non-root UID/GID with +`allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, and the runtime +default seccomp profile. The sandbox installs a nested seccomp user-notification +filter without requesting a capability in the Pod spec. Startup fails closed +when the runtime blocks the required seccomp or Landlock operations. + +The driver creates an empty-egress `NetworkPolicy` before it releases the +workload Pod. A second rule permits only the paired supervisor Pod to reach the +sandbox TLS port. The supervisor has normal egress for gateway, DNS, and +policy-approved upstream connections. Set +`proxy_pod.network_policy_enforced = true` only after verifying that the cluster +CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces. + +Each sandbox generation uses two immutable bootstrap Secrets. A trusted init +container stages the sandbox bootstrap into memory, and the sandbox removes it +before starting untrusted code. The other Secret is mounted only by the +supervisor. The TLS channel binds the namespace, Sandbox CR, workload Pod, +supervisor Deployment, and egress policy identities. Stop deletes the workload +Pod and scales the supervisor to zero. Start rotates both Secrets and the +supervisor Deployment before creating a new workload Pod. Delete retains the +egress fence until workload deletion is confirmed. + +Kubernetes policies are additive, and the API does not attest that the CNI +enforces them. Keep sandbox namespaces administrative: untrusted principals +must not create permissive policies, create Pods, read bootstrap Secrets, or +spoof the pair labels. ## Sandbox Resource @@ -95,71 +129,31 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC. ## Credentials, TLS, and Relay -The driver injects gateway callback configuration, sandbox identity, TLS client -material, and the supervisor SSH socket path into the workload. Driver-owned -values must override image-provided environment variables. +Both Pods set `automountServiceAccountToken: false`. The supervisor receives an +explicit audience-bound projected token for the one-shot `IssueSandboxToken` +exchange. The driver verifies that token and the gateway returns the +sandbox-scoped JWT used by the supervisor session. The sandbox Pod receives +neither token. -Sandbox pods run as `service_account_name` and keep -`automountServiceAccountToken: false`. The only Kubernetes token exposed to the -supervisor is an explicit, audience-bound projected token mounted at -`/var/run/secrets/openshell/token` for the one-shot `IssueSandboxToken` -bootstrap exchange. The Kubernetes driver authenticates that token through the -compute-driver protocol using its own `service_account_name` and workspace-mode -namespace policy; the gateway receives only the verified sandbox ID. +The gateway uses the supervisor relay for connect, exec, logs, and file sync. +Sandbox Pods do not need direct external ingress for SSH. -The gateway uses the supervisor relay for connect, exec, and file sync. Sandbox -pods do not need direct external ingress for SSH. - -The driver forwards the canonical main-process specification to the process -supervisor and sets pod `restartPolicy: Never`. Main-process environment -overrides stay local to that child; the sidecar bootstrap retains the unmodified -provider environment used by later exec, editor, and SFTP sessions. +The driver sends the canonical main-process specification only to the +supervisor. The supervisor passes admitted launch state over the protected +channel. Provider environment updates apply to future exec sessions. ## Container Security Context -The default `combined` supervisor topology grants the sandbox agent container -the Linux capabilities the supervisor needs for namespace setup and process, -filesystem, and network policy enforcement. - -The `sidecar` supervisor topology moves pod-level network setup into a root init -container. In the default process/binary-aware mode, the long-lived network -sidecar runs as UID 0 with `allowPrivilegeEscalation: false`, drops default -Linux capabilities, and adds only `SYS_PTRACE` plus `DAC_READ_SEARCH` for -cross-UID workload `/proc` inspection. The agent container also runs as the -resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and -`capabilities.drop: ["ALL"]`. -Set `sidecar.process_binary_aware_network_policy = false` to run the network -sidecar as the configured non-root `sidecar.proxy_uid`, omit the extra `/proc` -inspection capabilities, and enforce endpoint/L7 network policy without -matching `policy.binaries`. -In this mode OpenShell preserves gateway session and SSH behavior, but the -process supervisor does not perform root-to-sandbox privilege dropping or -supervisor identity mount isolation. It still applies Landlock filesystem policy -and child seccomp filters where the kernel/runtime supports them. Network -endpoint and L7 policy remain enforced by the network sidecar, and -sidecar pods use a shared process namespace so the network sidecar can resolve -process/binary identity through `/proc/`. - -Sidecar mode keeps gateway credentials in the network sidecar. The agent -container does not mount the projected service-account token used for sandbox -token bootstrap, does not mount the sandbox client TLS secret, and does not get -gateway callback environment variables. The process supervisor receives policy -and provider environment state from the sidecar over a local control socket in -the shared sidecar state volume. The sidecar accepts only the pre-workload -process-supervisor connection, authenticates its UID/GID/PID with peer -credentials, and removes the listener afterward. SSH relays use a Linux -abstract socket whose peer PID must match that authenticated supervisor. Both -supervisors exit if the control connection closes, coupling their container -restart lifecycle before a new authoritative client can be established. - -The driver can request a Kubernetes AppArmor profile through -`app_armor_profile`. - -Supported values are `Unconfined`, `RuntimeDefault`, and -`Localhost/`. An empty or unset value omits -`securityContext.appArmorProfile`. Helm deployments default sandbox agent -containers to `Unconfined` because runtime/default AppArmor profiles can block -the supervisor's network namespace mount setup on AppArmor-enabled nodes. +The sandbox, trusted bootstrap init container, and supervisor request no added +Linux capability. They run as the same numeric non-root identity, disable +privilege escalation, drop all capabilities, and inherit `RuntimeDefault` +seccomp. The sandbox and agent must use the same complete UID, GID, and +supplementary-group identity because the capability-free sandbox cannot change +credentials after launch and must inspect its same-identity descendants. + +The workload Pod does not share host network, PID, IPC, or process namespaces. +The driver uses a scheduling gate to inspect the admitted Pod and bind its UID +into the bootstrap claims before kubelet starts it. ## GPU Support diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 805c0314b0..7ebe17d3fa 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -22,76 +22,38 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; -/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. -pub const DEFAULT_PROXY_UID: u32 = 1337; - -/// How the supervisor binary is delivered into sandbox pods. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum SupervisorSideloadMethod { - /// Mount the supervisor OCI image directly as a read-only volume - /// (requires Kubernetes >= v1.33 with the `ImageVolume` feature gate, - /// or >= v1.36 where it is GA). - #[default] - ImageVolume, - /// Copy the binary via an init container and emptyDir volume. - /// Works on all Kubernetes versions. - InitContainer, -} - -impl std::fmt::Display for SupervisorSideloadMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::ImageVolume => f.write_str("image-volume"), - Self::InitContainer => f.write_str("init-container"), - } - } +/// Driver-owned requirements for the cross-pod RFC 0012 topology. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesProxyPodConfig { + /// Explicit operator assertion that the cluster CNI enforces + /// `networking.k8s.io/v1` `NetworkPolicy` for the sandbox namespaces. + pub network_policy_enforced: bool, + /// TCP port exposed by the workload boundary to its paired control pod. + pub boundary_port: u16, } -impl FromStr for SupervisorSideloadMethod { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "image-volume" => Ok(Self::ImageVolume), - "init-container" => Ok(Self::InitContainer), - other => Err(format!( - "unknown supervisor sideload method '{other}'; expected 'image-volume' or 'init-container'" - )), +impl Default for KubernetesProxyPodConfig { + fn default() -> Self { + Self { + network_policy_enforced: false, + boundary_port: 5500, } } } -/// How the supervisor is arranged inside Kubernetes sandbox pods. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum SupervisorTopology { - /// Run networking and process supervision in the agent container. - #[default] - Combined, - /// Run network supervision in a privileged sidecar and process supervision - /// as a low-capability wrapper in the agent container. - Sidecar, -} - -impl std::fmt::Display for SupervisorTopology { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Combined => f.write_str("combined"), - Self::Sidecar => f.write_str("sidecar"), +impl KubernetesProxyPodConfig { + pub fn validate(&self) -> Result<(), String> { + if !self.network_policy_enforced { + return Err( + "proxy-pod topology requires proxy_pod.network_policy_enforced = true after the operator has verified CNI NetworkPolicy enforcement" + .to_string(), + ); } - } -} - -impl FromStr for SupervisorTopology { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "combined" => Ok(Self::Combined), - "sidecar" => Ok(Self::Sidecar), - other => Err(format!("unknown topology '{other}'")), + if self.boundary_port < 1024 { + return Err("proxy_pod.boundary_port must be at least 1024".to_string()); } + Ok(()) } } @@ -137,138 +99,6 @@ impl FromStr for WorkspaceMode { } } -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct KubernetesSidecarConfig { - /// UID used by relaxed long-running network sidecars in `sidecar` - /// topology. The network init container installs nftables rules that - /// exempt this UID, so it must not match the sandbox workload UID. - /// Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants - /// the requested `/proc` inspection capabilities into the effective set. - pub proxy_uid: u32, - /// Require process/binary-aware network policy enforcement in sidecar - /// topology. When disabled, the network sidecar runs as `proxy_uid`, - /// drops the extra `/proc` inspection permissions, and evaluates - /// endpoint/L7 policy without matching `policy.binaries`. - pub process_binary_aware_network_policy: bool, -} - -impl Default for KubernetesSidecarConfig { - fn default() -> Self { - Self { - proxy_uid: DEFAULT_PROXY_UID, - process_binary_aware_network_policy: true, - } - } -} - -impl KubernetesSidecarConfig { - pub fn validate_proxy_uid(&self) -> Result<(), String> { - if !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&self.proxy_uid) - { - return Err(format!( - "sidecar.proxy_uid must be in range [{}, {}]", - openshell_policy::MIN_SANDBOX_PROXY_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - Ok(()) - } -} - -/// Kubernetes `AppArmor` profile requested for the sandbox agent container. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AppArmorProfile { - RuntimeDefault, - Unconfined, - Localhost(String), -} - -impl AppArmorProfile { - #[must_use] - pub fn to_k8s_type(&self) -> &'static str { - match self { - Self::RuntimeDefault => "RuntimeDefault", - Self::Unconfined => "Unconfined", - Self::Localhost(_) => "Localhost", - } - } - - #[must_use] - pub fn localhost_profile(&self) -> Option<&str> { - match self { - Self::Localhost(profile) => Some(profile), - Self::RuntimeDefault | Self::Unconfined => None, - } - } -} - -impl std::fmt::Display for AppArmorProfile { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::RuntimeDefault => f.write_str("RuntimeDefault"), - Self::Unconfined => f.write_str("Unconfined"), - Self::Localhost(profile) => write!(f, "Localhost/{profile}"), - } - } -} - -impl FromStr for AppArmorProfile { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "RuntimeDefault" => Ok(Self::RuntimeDefault), - "Unconfined" => Ok(Self::Unconfined), - other => match other.strip_prefix("Localhost/") { - Some("") => Err( - "invalid AppArmor profile 'Localhost/'; expected non-empty profile name" - .to_string(), - ), - Some(profile) => Ok(Self::Localhost(profile.to_string())), - None => Err(format!( - "unknown AppArmor profile '{other}'; expected 'RuntimeDefault', 'Unconfined', or 'Localhost/'" - )), - }, - } - } -} - -impl Serialize for AppArmorProfile { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -impl<'de> Deserialize<'de> for AppArmorProfile { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let value = String::deserialize(deserializer)?; - Self::from_str(&value).map_err(serde::de::Error::custom) - } -} - -fn deserialize_optional_app_armor_profile<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let value = Option::::deserialize(deserializer)?; - match value.as_deref() { - None | Some("") => Ok(None), - Some(value) => AppArmorProfile::from_str(value) - .map(Some) - .map_err(serde::de::Error::custom), - } -} - fn deserialize_provider_spiffe_workload_api_socket_path<'de, D>( deserializer: D, ) -> Result @@ -302,8 +132,9 @@ pub struct KubernetesComputeConfig { /// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount. #[serde(default, skip_serializing_if = "Option::is_none")] pub operator_namespace_file: Option, - /// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by - /// the driver's `TokenReview` bootstrap authenticator. + /// Kubernetes `ServiceAccount` assigned to both topology Pods. Automatic + /// token mounting is disabled; only the supervisor receives an explicit + /// audience-bound projected token accepted by the bootstrap authenticator. pub service_account_name: String, pub default_image: String, pub image_pull_policy: String, @@ -313,19 +144,14 @@ pub struct KubernetesComputeConfig { /// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222 /// only from gateway pods matching this peer. pub managed_ssh_ingress: ManagedSshIngressConfig, - /// Image that provides the `openshell-sandbox` supervisor binary. - /// Mounted directly as an image volume, or copied via an init container, - /// depending on `supervisor_sideload_method`. + /// Image that provides the trusted `openshell-sandbox` bootstrap binary + /// and the separate `openshell-supervisor` control binary. pub supervisor_image: String, /// Kubernetes `imagePullPolicy` for the supervisor image. /// Empty string delegates to the Kubernetes default. pub supervisor_image_pull_policy: String, - /// How the supervisor binary is delivered into sandbox pods. - pub supervisor_sideload_method: SupervisorSideloadMethod, - /// How the supervisor is arranged for Kubernetes sandbox pods. - pub topology: SupervisorTopology, - /// Sidecar-only settings used when `topology = "sidecar"`. - pub sidecar: KubernetesSidecarConfig, + /// Cross-pod sandbox/supervisor settings. + pub proxy_pod: KubernetesProxyPodConfig, /// Corporate HTTP forward proxy used by the network supervisor for /// policy-approved TLS CONNECT egress. pub https_proxy: Option, @@ -350,14 +176,6 @@ pub struct KubernetesComputeConfig { pub client_tls_secret_name: String, pub host_gateway_ip: String, pub enable_user_namespaces: bool, - /// Kubernetes `AppArmor` profile requested for the sandbox agent container. - /// Empty/None omits the `appArmorProfile` field from sandbox pod specs. - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "deserialize_optional_app_armor_profile" - )] - pub app_armor_profile: Option, pub workspace_default_storage_size: String, /// Kubernetes `StorageClass` name for the default workspace PVC. /// Empty string (default) = omit `storageClassName`, using the cluster's @@ -386,13 +204,10 @@ pub struct KubernetesComputeConfig { deserialize_with = "deserialize_provider_spiffe_workload_api_socket_path" )] pub provider_spiffe_workload_api_socket_path: String, - /// UID used for privilege-drop operations and workspace init container - /// ownership. The supervisor container always runs as UID 0 (root) to - /// create network namespaces and configure Landlock/seccomp; the - /// `sandbox_uid` is injected as the `SANDBOX_UID` environment variable so - /// the supervisor knows which UID to drop to for child processes. + /// Exact UID shared by `openshell-sandbox`, its agent children, the trusted + /// workspace/bootstrap init containers, and `openshell-supervisor`. /// When empty, the driver auto-detects from `OpenShift` SCC annotations on - /// the target namespace; if those are also absent, falls back to `1000`. + /// the target namespace; if those are also absent, falls back to `10001`. #[serde(default, skip_serializing_if = "Option::is_none")] pub sandbox_uid: Option, /// GID used alongside `sandbox_uid` for PVC init container operations. @@ -419,7 +234,7 @@ pub const MAX_SA_TOKEN_TTL_SECS: i64 = 86_400; /// Default sandbox UID used when neither config nor `OpenShift` SCC annotations /// provide a resolved value. -pub(crate) const DEFAULT_SANDBOX_UID: u32 = 1000; +pub(crate) const DEFAULT_SANDBOX_UID: u32 = 10001; /// The annotation key for the `OpenShift` `ServiceAccount` UID range. /// Format: `/` (e.g. `1000000000/10000`). @@ -448,9 +263,7 @@ impl Default for KubernetesComputeConfig { managed_ssh_ingress: ManagedSshIngressConfig::default(), supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), - supervisor_sideload_method: SupervisorSideloadMethod::default(), - topology: SupervisorTopology::default(), - sidecar: KubernetesSidecarConfig::default(), + proxy_pod: KubernetesProxyPodConfig::default(), https_proxy: None, no_proxy: None, proxy_auth_secret_name: None, @@ -462,7 +275,6 @@ impl Default for KubernetesComputeConfig { client_tls_secret_name: String::new(), host_gateway_ip: String::new(), enable_user_namespaces: false, - app_armor_profile: None, workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(), workspace_storage_class: String::new(), default_runtime_class_name: String::new(), @@ -503,7 +315,7 @@ impl KubernetesComputeConfig { } pub fn validate_proxy_uid(&self) -> Result<(), String> { - self.sidecar.validate_proxy_uid() + self.proxy_pod.validate() } /// Validate the operator-owned corporate upstream proxy configuration. @@ -578,12 +390,6 @@ impl KubernetesComputeConfig { if self.proxy_auth_allow_insecure != Some(true) { return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string()); } - if self.topology == SupervisorTopology::Combined { - return Err( - "proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user" - .to_string(), - ); - } } _ => { return Err( @@ -608,7 +414,7 @@ impl KubernetesComputeConfig { /// 2. `OpenShift` SCC namespace annotations (`sa.scc.uid-range`, /// `sa.scc.supplemental-groups`) — passed in as the optional /// `namespace_annotations` map - /// 3. Fallback defaults: UID=`1000`, GID=UID + /// 3. Fallback defaults: UID=`10001`, GID=UID pub fn resolve_sandbox_uid( &self, namespace_annotations: Option<&BTreeMap>, @@ -910,40 +716,16 @@ mod tests { } #[test] - fn default_topology_is_combined() { - let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.topology, SupervisorTopology::Combined); - assert_eq!(cfg.topology.to_string(), "combined"); - } - - #[test] - fn default_proxy_uid_is_dedicated_non_root_uid() { - let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); - } - - #[test] - fn default_sidecar_requires_process_binary_aware_network_policy() { - let cfg = KubernetesComputeConfig::default(); - assert!(cfg.sidecar.process_binary_aware_network_policy); - } - - #[test] - fn serde_override_topology_sidecar() { - let json = serde_json::json!({ - "topology": "sidecar" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.topology, SupervisorTopology::Sidecar); - } + fn proxy_pod_requires_network_policy_enforcement_acknowledgement() { + let mut cfg = KubernetesComputeConfig::default(); + assert!( + cfg.validate_proxy_uid() + .unwrap_err() + .contains("network_policy_enforced") + ); - #[test] - fn serde_override_topology_combined() { - let json = serde_json::json!({ - "topology": "combined" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.topology, SupervisorTopology::Combined); + cfg.proxy_pod.network_policy_enforced = true; + cfg.validate_proxy_uid().unwrap(); } #[test] @@ -957,51 +739,6 @@ mod tests { assert!(err.to_string().contains("unknown field")); } - #[test] - fn serde_override_sidecar_process_binary_aware_network_policy_nested() { - let json = serde_json::json!({ - "sidecar": { - "process_binary_aware_network_policy": false - } - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert!(!cfg.sidecar.process_binary_aware_network_policy); - } - - #[test] - fn serde_override_sidecar_proxy_uid_nested() { - let json = serde_json::json!({ - "sidecar": { - "proxy_uid": 2000 - } - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.sidecar.proxy_uid, 2000); - cfg.validate_proxy_uid().unwrap(); - } - - #[test] - fn validate_proxy_uid_rejects_privileged_uid() { - let cfg = KubernetesComputeConfig { - sidecar: KubernetesSidecarConfig { - proxy_uid: 999, - ..KubernetesSidecarConfig::default() - }, - ..KubernetesComputeConfig::default() - }; - let err = cfg.validate_proxy_uid().unwrap_err(); - assert!(err.contains("proxy_uid")); - } - - #[test] - fn serde_rejects_invalid_topology() { - let json = serde_json::json!({ - "topology": "unsupported" - }); - let err = serde_json::from_value::(json).unwrap_err(); - assert!(err.to_string().contains("unknown variant")); - } - #[test] fn serde_rejects_removed_topology_alias_field() { let mut json = serde_json::Map::new(); @@ -1086,53 +823,6 @@ mod tests { assert!(cfg.default_runtime_class_name.is_empty()); } - #[test] - fn default_app_armor_profile_is_none() { - let cfg = KubernetesComputeConfig::default(); - assert!(cfg.app_armor_profile.is_none()); - } - - #[test] - fn serde_override_app_armor_profile_unconfined() { - let json = serde_json::json!({ - "app_armor_profile": "Unconfined" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::Unconfined)); - } - - #[test] - fn serde_override_app_armor_profile_runtime_default() { - let json = serde_json::json!({ - "app_armor_profile": "RuntimeDefault" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::RuntimeDefault)); - } - - #[test] - fn serde_override_app_armor_profile_localhost() { - let json = serde_json::json!({ - "app_armor_profile": "Localhost/openshell-supervisor" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!( - cfg.app_armor_profile, - Some(AppArmorProfile::Localhost( - "openshell-supervisor".to_string() - )) - ); - } - - #[test] - fn serde_empty_app_armor_profile_disables_field() { - let json = serde_json::json!({ - "app_armor_profile": "" - }); - let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.app_armor_profile, None); - } - #[test] fn serde_accepts_absolute_provider_spiffe_socket_path() { let json = serde_json::json!({ @@ -1162,15 +852,6 @@ mod tests { } } - #[test] - fn serde_rejects_invalid_app_armor_profile() { - let json = serde_json::json!({ - "app_armor_profile": "runtime/default" - }); - let err = serde_json::from_value::(json).unwrap_err(); - assert!(err.to_string().contains("unknown AppArmor profile")); - } - #[test] fn serde_override_image_pull_secrets() { let json = serde_json::json!({ @@ -1377,7 +1058,6 @@ mod tests { #[test] fn upstream_proxy_config_accepts_secret_credentials_with_acknowledgement() { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some("credentials".to_string()), @@ -1388,10 +1068,9 @@ mod tests { } #[test] - fn toml_deserializes_sidecar_upstream_proxy_settings() { + fn toml_deserializes_upstream_proxy_settings() { let cfg: KubernetesComputeConfig = toml::from_str( r#" - topology = "sidecar" https_proxy = "http://proxy.corp.example:8080" no_proxy = ".svc.cluster.local,10.96.0.0/12" proxy_auth_secret_name = "corporate-proxy-auth" @@ -1513,7 +1192,6 @@ mod tests { "bad key".to_string(), // whitespace is outside the allowed charset ] { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some(key.clone()), @@ -1531,7 +1209,6 @@ mod tests { #[test] fn upstream_proxy_config_accepts_max_length_secret_key() { let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Sidecar, https_proxy: Some("http://proxy.corp.example:8080".to_string()), proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), proxy_auth_secret_key: Some("a".repeat(253)), @@ -1541,20 +1218,6 @@ mod tests { assert!(cfg.validate_upstream_proxy_config().is_ok()); } - #[test] - fn upstream_proxy_config_rejects_credentials_in_combined_topology() { - let cfg = KubernetesComputeConfig { - topology: SupervisorTopology::Combined, - https_proxy: Some("http://proxy.corp.example:8080".to_string()), - proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()), - proxy_auth_secret_key: Some("credentials".to_string()), - proxy_auth_allow_insecure: Some(true), - ..KubernetesComputeConfig::default() - }; - let err = cfg.validate_upstream_proxy_config().unwrap_err(); - assert!(err.contains("topology = \"sidecar\""), "{err}"); - } - #[test] fn upstream_proxy_config_allows_explicit_false_acknowledgement_without_credentials() { let cfg = KubernetesComputeConfig { diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 62c102777f..7273ec097a 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3,20 +3,27 @@ //! Kubernetes compute driver. -use super::AppArmorProfile; use crate::config::{ - DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, OperatorNamespaceAllowlist, - SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, is_dns_1123_label, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, + KubernetesComputeConfig, OperatorNamespaceAllowlist, WorkspaceMode, is_dns_1123_label, managed_namespace, managed_namespace_prefix, validate_managed_namespace_name, }; +use crate::isolation::{BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesProxyPodBoundarySpec}; +use crate::proxy_pod::{ + BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CLIENT_CA_PATH, BOUNDARY_CONFIG_PATH, + BOUNDARY_PRIVATE_KEY_PATH, ProxyPodNames, boundary_service, control_deployment, + control_egress_policy, generate_proxy_ca_material, sandbox_bootstrap_secret, + sandbox_owner_reference as proxy_pod_sandbox_owner_reference, supervisor_bootstrap_secret, + workload_fence, +}; use futures::{Stream, StreamExt, TryStreamExt}; +use k8s_openapi::api::apps::v1::{Deployment, ReplicaSet}; use k8s_openapi::api::authentication::v1::{ TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo, }; use k8s_openapi::api::core::v1::{ Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Pod, Secret, - ServiceAccount, Volume, VolumeMount, + Service, ServiceAccount, Volume, VolumeMount, }; use k8s_openapi::api::networking::v1::{ NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, @@ -30,13 +37,13 @@ use kube::api::{ use kube::core::gvk::GroupVersionKind; use kube::core::{DynamicObject, ObjectMeta}; use kube::runtime::WatchStreamExt; +use kube::runtime::wait::await_condition; use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_GATEWAY_ID, LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, - LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - openshell_sandbox_label_selector, + LABEL_SANDBOX_NAME, LABEL_SANDBOX_WORKSPACE, openshell_sandbox_label_selector, }; use openshell_core::gpu::{driver_gpu_requirements, effective_driver_gpu_count}; use openshell_core::progress::{ @@ -52,6 +59,11 @@ use openshell_core::proto::compute::v1::{ watch_sandboxes_event, }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryServerTls, generate_boundary_mutual_tls_material, +}; +use openshell_isolation_interface::contract::ResolvedWorkloadIdentity; +use rand::RngCore as _; use serde::Deserialize; use std::collections::{BTreeMap, HashSet}; use std::path::{Path, PathBuf}; @@ -67,6 +79,13 @@ pub type WatchStream = const MANAGED_SSH_NETWORK_POLICY_NAME: &str = "openshell-sandbox-ssh"; const AGENT_SANDBOX_TRACE_CONTEXT_ANNOTATION: &str = "opentelemetry.io/trace-context"; +const ANNOTATION_PROXY_POD_BOOTSTRAPPING: &str = "openshell.ai/proxy-pod-bootstrapping"; +const ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: &str = + "openshell.ai/proxy-pod-bootstrap-started-at-ms"; +const ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: &str = "openshell.ai/proxy-pod-bootstrap-operation"; +const ANNOTATION_PROXY_POD_GENERATION: &str = "openshell.ai/proxy-pod-generation"; +const ANNOTATION_PROXY_POD_READINESS: &str = "openshell.ai/proxy-pod-readiness"; +const ANNOTATION_PROXY_POD_WORKLOAD_UID: &str = "openshell.ai/proxy-pod-workload-uid"; #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { @@ -85,12 +104,56 @@ pub enum KubernetesDriverError { impl KubernetesDriverError { fn from_kube(err: KubeError) -> Self { match err { - KubeError::Api(api) if api.code == 409 => Self::AlreadyExists, + KubeError::Api(api) if api.code == 409 && api.reason == "AlreadyExists" => { + Self::AlreadyExists + } + KubeError::Api(api) if api.code == 404 => Self::NotFound, other => Self::Message(other.to_string()), } } } +fn is_kube_resource_version_conflict(error: &KubeError) -> bool { + matches!(error, KubeError::Api(api) if api.code == 409 && api.reason == "Conflict") +} + +async fn patch_dynamic_object_with_resource_version_retry( + api: &Api, + name: &str, + mut patch_for_resource_version: impl FnMut(&str) -> serde_json::Value, +) -> Result { + let operation = async { + loop { + let current = api.get(name).await?; + let resource_version = current + .metadata + .resource_version + .as_deref() + .unwrap_or_default(); + let patch = patch_for_resource_version(resource_version); + match api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)) + .await + { + Ok(updated) => return Ok(updated), + Err(error) if is_kube_resource_version_conflict(&error) => { + tokio::task::yield_now().await; + } + Err(error) => return Err(error), + } + } + }; + tokio::time::timeout(KUBE_API_TIMEOUT, operation) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s updating Kubernetes resource {name}", + KUBE_API_TIMEOUT.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube) +} + impl From for openshell_core::ComputeDriverError { fn from(err: KubernetesDriverError) -> Self { match err { @@ -107,6 +170,25 @@ impl From for openshell_core::ComputeDriverError { /// This prevents gRPC handlers from blocking indefinitely when the k8s /// API server is unreachable or slow. const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); +const PROXY_POD_RECONCILE_INTERVAL: Duration = Duration::from_secs(30); +const PROXY_POD_FENCE_QUIESCE_WINDOW: Duration = Duration::from_secs(6); +const PROXY_POD_FENCE_QUIESCE_INTERVAL: Duration = Duration::from_secs(2); +/// A pre-created fence may legitimately have no Sandbox CR while create is in flight. +const PROXY_POD_ORPHAN_FENCE_MIN_AGE: Duration = Duration::from_secs(5 * 60); +/// Bound how long a crash-interrupted, fail-closed bootstrap may remain stranded. +const PROXY_POD_BOOTSTRAP_GRACE: Duration = Duration::from_secs(5 * 60); + +fn random_proxy_pod_token() -> String { + use std::fmt::Write as _; + + let mut bytes = [0_u8; 32]; + rand::rng().fill_bytes(&mut bytes); + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(token, "{byte:02x}").expect("writing to String cannot fail"); + } + token +} /// Kubernetes defaults pod termination to 30 seconds when the pod template /// omits `terminationGracePeriodSeconds`. @@ -288,7 +370,11 @@ const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ UPSTREAM_PROXY_AUTH_VOLUME_NAME, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, SPIFFE_WORKLOAD_API_VOLUME_NAME, - SUPERVISOR_VOLUME_NAME, + SANDBOX_RUNTIME_VOLUME_NAME, + SANDBOX_STATE_VOLUME_NAME, + SANDBOX_BOOTSTRAP_VOLUME_NAME, + SANDBOX_POD_IDENTITY_VOLUME_NAME, + SANDBOX_PROXY_CA_VOLUME_NAME, WORKSPACE_VOLUME_NAME, ]; @@ -446,7 +532,7 @@ const WORKSPACE_MOUNT_PATH: &str = "/sandbox"; /// Mount path for the workspace PVC in the **init** container. A temporary /// path so the init container can see the image's original `/sandbox` and /// copy it into the PVC. -const WORKSPACE_INIT_MOUNT_PATH: &str = "/workspace-pvc"; +const WORKSPACE_INIT_MOUNT_PATH: &str = "/mnt/openshell-workspace"; /// Name of the init container that seeds the workspace PVC. const WORKSPACE_INIT_CONTAINER_NAME: &str = "workspace-init"; @@ -621,7 +707,9 @@ impl KubernetesComputeDriver { })?; validate_pod_uid(&pod, &identity.pod_uid)?; let sandbox_id = pod_sandbox_id(&pod)?; - let owner = sandbox_owner_reference(&pod)?; + let (owner, via_proxy_control) = self + .resolve_sandbox_owner(&pod, &identity.namespace, &sandbox_id) + .await?; let sandboxes = self .supported_agent_sandbox_api(self.client.clone(), &identity.namespace) .await @@ -632,10 +720,81 @@ impl KubernetesComputeDriver { warn!(sandbox = %owner.name, %error, "failed to read authenticated Sandbox resource"); tonic::Status::internal("failed to read authenticated Sandbox resource") })?.ok_or_else(|| tonic::Status::permission_denied("sandbox owner not found"))?; - validate_sandbox_owner_identity(owner, &sandbox_id, &sandbox)?; + validate_sandbox_owner_identity(&owner, &sandbox_id, &sandbox)?; + require_proxy_control_authentication(via_proxy_control)?; Ok(sandbox_id) } + #[allow(clippy::result_large_err)] + async fn resolve_sandbox_owner( + &self, + pod: &Pod, + namespace: &str, + sandbox_id: &str, + ) -> Result<(OwnerReference, bool), tonic::Status> { + if let Ok(owner) = sandbox_owner_reference(pod) { + return Ok((owner.clone(), false)); + } + validate_proxy_control_labels(pod, sandbox_id)?; + let names = ProxyPodNames::new(sandbox_id); + let replica_set_owner = controller_owner_reference( + pod.metadata.owner_references.as_deref().unwrap_or_default(), + ) + .filter(|owner| owner.api_version == "apps/v1" && owner.kind == "ReplicaSet") + .ok_or_else(|| tonic::Status::permission_denied("pod is not controlled by a Sandbox"))?; + let replica_set = Api::::namespaced(self.client.clone(), namespace) + .get(&replica_set_owner.name) + .await + .map_err(|error| { + tonic::Status::internal(format!("ReplicaSet lookup failed: {error}")) + })?; + validate_owner_uid(&replica_set.metadata.uid, replica_set_owner)?; + let deployment_owner = controller_owner_reference( + replica_set + .metadata + .owner_references + .as_deref() + .unwrap_or_default(), + ) + .filter(|owner| owner.api_version == "apps/v1" && owner.kind == "Deployment") + .ok_or_else(|| { + tonic::Status::permission_denied("ReplicaSet is not controlled by a Deployment") + })?; + if deployment_owner.name != names.control_deployment { + return Err(tonic::Status::permission_denied( + "control pod is not owned by the expected proxy-pod Deployment", + )); + } + let deployment = Api::::namespaced(self.client.clone(), namespace) + .get(&deployment_owner.name) + .await + .map_err(|error| { + tonic::Status::internal(format!("Deployment lookup failed: {error}")) + })?; + validate_owner_uid(&deployment.metadata.uid, deployment_owner)?; + validate_proxy_control_labels_from_metadata(&deployment.metadata, sandbox_id)?; + let owner = deployment + .metadata + .owner_references + .as_deref() + .and_then(|owners| { + owners + .iter() + .find(|owner| owner.controller == Some(true) && owner.kind == SANDBOX_KIND) + }) + .filter(|owner| { + matches!( + owner.api_version.as_str(), + "agents.x-k8s.io/v1beta1" | "agents.x-k8s.io/v1alpha1" + ) + }) + .cloned() + .ok_or_else(|| { + tonic::Status::permission_denied("Deployment is not controlled by a Sandbox") + })?; + Ok((owner, true)) + } + fn accepts_auth_namespace(&self, namespace: &str) -> bool { accepts_auth_namespace(&self.config, self.operator_allowlist.as_ref(), namespace) } @@ -1088,17 +1247,9 @@ impl KubernetesComputeDriver { } fn validate_driver_config_for_sandbox( - &self, sandbox: &Sandbox, ) -> Result { - kubernetes_driver_config_for_spec( - sandbox.spec.as_ref(), - self.config.provider_spiffe_enabled().then_some( - self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - ) + kubernetes_driver_config_for_spec(sandbox.spec.as_ref()) } fn agent_sandbox_api( @@ -1283,8 +1434,7 @@ impl KubernetesComputeDriver { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), tonic::Status> { - let _ = self - .validate_driver_config_for_sandbox(sandbox) + let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { WorkspaceMode::Shared => { @@ -1325,20 +1475,23 @@ impl KubernetesComputeDriver { let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => list.items.into_iter().next().map_or_else( - || { + Ok(Ok(list)) => { + let Some(obj) = list.items.into_iter().next() else { debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes"); - Ok(None) - }, - |obj| { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - Ok(sandbox_from_object(&ns, obj).ok().map(|(_, s)| s)) - }, - ), + return Ok(None); + }; + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + Ok( + sandbox_from_object_with_proxy_pod_readiness(&self.client, &ns, obj) + .await + .ok() + .map(|(_, sandbox)| sandbox), + ) + } Ok(Err(err)) => { warn!( sandbox_id = %sandbox_id, @@ -1380,25 +1533,22 @@ impl KubernetesComputeDriver { .await { Ok(Ok(list)) => { - let mut sandboxes: Vec = list - .items - .into_iter() - .filter_map(|obj| { - let name = obj.metadata.name.clone().unwrap_or_default(); - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - match sandbox_from_object(&ns, obj) { - Ok((_, s)) => Some(s), - Err(err) => { - warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); - None - } + let mut sandboxes = Vec::new(); + for obj in list.items { + let name = obj.metadata.name.clone().unwrap_or_default(); + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + match sandbox_from_object_with_proxy_pod_readiness(&self.client, &ns, obj).await + { + Ok((_, sandbox)) => sandboxes.push(sandbox), + Err(err) => { + warn!(object_name = %name, error = %err, "skipping unrecognized Sandbox in list"); } - }) - .collect(); + } + } sandboxes.sort_by(|left, right| { left.name .cmp(&right.name) @@ -1503,53 +1653,57 @@ impl KubernetesComputeDriver { .resolve_sandbox_identity_in_namespace(&target_namespace) .await; + let generation = random_proxy_pod_token(); + let proxy_names = ProxyPodNames::for_generation(&sandbox.id, &generation); let params = SandboxPodParams { default_image: &self.config.default_image, image_pull_policy: &self.config.image_pull_policy, image_pull_secrets: &self.config.image_pull_secrets, supervisor_image: &self.config.supervisor_image, supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, - supervisor_sideload_method: self.config.supervisor_sideload_method, - topology: self.config.topology, - proxy_uid: self.config.sidecar.proxy_uid, - process_binary_aware_network_policy: self - .config - .sidecar - .process_binary_aware_network_policy, - https_proxy: self.config.https_proxy.as_deref(), - no_proxy: self.config.no_proxy.as_deref(), - proxy_auth_secret_name: self.config.proxy_auth_secret_name.as_deref(), - proxy_auth_secret_key: self.config.proxy_auth_secret_key.as_deref(), - proxy_auth_allow_insecure: self.config.proxy_auth_allow_insecure == Some(true), - proxy_connect_by_hostname: self.config.proxy_connect_by_hostname == Some(true), service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, - sandbox_name: &sandbox.name, - grpc_endpoint: &self.config.grpc_endpoint, - ssh_socket_path: self.ssh_socket_path(), - client_tls_secret_name: &self.config.client_tls_secret_name, - host_gateway_ip: &self.config.host_gateway_ip, enable_user_namespaces: self.config.enable_user_namespaces, - app_armor_profile: self.config.app_armor_profile.as_ref(), workspace_default_storage_size: &self.config.workspace_default_storage_size, workspace_storage_class: &self.config.workspace_storage_class, default_runtime_class_name: &self.config.default_runtime_class_name, - sa_token_ttl_secs: self.config.effective_sa_token_ttl_secs(), - provider_spiffe_enabled: self.config.provider_spiffe_enabled(), - provider_spiffe_workload_api_socket_path: &self - .config - .provider_spiffe_workload_api_socket_path, sandbox_uid: resolved_user_id, sandbox_gid: resolved_group_id, + boundary_port: self.config.proxy_pod.boundary_port, + sandbox_secret_name: &proxy_names.sandbox_secret, }; - validate_sidecar_proxy_identity(¶ms)?; - - let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) - .map_err(KubernetesDriverError::InvalidArgument)?; let kube_name = self.config.kube_resource_name(workspace, name); + let mut data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) + .map_err(KubernetesDriverError::InvalidArgument)?; + self.create_proxy_pod_fence(&target_namespace, &proxy_names, sandbox, &kube_name) + .await?; + // A missing bootstrap Secret keeps both pods inert as defense in + // depth, but the CR is also created suspended so the controller + // never races an unfenced workload into execution. + if agent_sandbox_api.resource.version == SANDBOX_VERSION_V1ALPHA1 { + data["spec"]["replicas"] = serde_json::json!(0); + } else { + data["spec"]["operatingMode"] = serde_json::json!("Suspended"); + } let mut obj = DynamicObject::new(&kube_name, &agent_sandbox_api.resource); let mut annotations = sandbox_annotations(sandbox); add_trace_context_annotation(&mut annotations); + annotations.insert( + ANNOTATION_PROXY_POD_BOOTSTRAPPING.to_string(), + "true".to_string(), + ); + annotations.insert( + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT.to_string(), + openshell_core::time::now_ms().to_string(), + ); + annotations.insert( + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION.to_string(), + "create".to_string(), + ); + annotations.insert( + ANNOTATION_PROXY_POD_GENERATION.to_string(), + generation.clone(), + ); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, @@ -1559,27 +1713,27 @@ impl KubernetesComputeDriver { } } obj.metadata = ObjectMeta { - name: Some(kube_name), - namespace: Some(target_namespace), + name: Some(kube_name.clone()), + namespace: Some(target_namespace.clone()), labels: Some(sandbox_labels(sandbox, Some(&self.config.gateway_id))), annotations: Some(annotations), ..Default::default() }; obj.data = data; - match tokio::time::timeout( + let created = match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.create(&PostParams::default(), &obj), ) .await { - Ok(Ok(_result)) => { + Ok(Ok(result)) => { info!( sandbox_id = %sandbox.id, sandbox_name = %name, "Sandbox created in Kubernetes successfully" ); - Ok(()) + result } Ok(Err(err)) => { warn!( @@ -1588,7 +1742,7 @@ impl KubernetesComputeDriver { error = %err, "Failed to create sandbox in Kubernetes" ); - Err(KubernetesDriverError::from_kube(err)) + return Err(KubernetesDriverError::from_kube(err)); } Err(_elapsed) => { warn!( @@ -1597,165 +1751,1157 @@ impl KubernetesComputeDriver { timeout_secs = KUBE_API_TIMEOUT.as_secs(), "Timed out creating sandbox in Kubernetes" ); - Err(KubernetesDriverError::Message(format!( + return Err(KubernetesDriverError::Message(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - ))) + ))); } + }; + if let Err(error) = self + .create_proxy_pod_companions( + sandbox, + &target_namespace, + &kube_name, + &agent_sandbox_api, + &created, + &proxy_names, + &generation, + resolved_user_id, + resolved_group_id, + ) + .await + { + warn!(sandbox_id = %sandbox.id, %error, "proxy-pod provisioning failed; rolling back Sandbox CR"); + let _ = agent_sandbox_api + .api + .delete(&kube_name, &DeleteParams::default()) + .await; + return Err(error); } + Ok(()) } - #[tracing::instrument( - name = "kubernetes.stop_sandbox", - skip(self), - fields( - otel.name = "kubernetes.stop_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, + async fn create_proxy_pod_fence( + &self, + namespace: &str, + names: &ProxyPodNames, + sandbox: &Sandbox, + workload_pod_name: &str, + ) -> Result<(), KubernetesDriverError> { + let mut policy = workload_fence( + namespace, + names, + &sandbox.id, + self.config.proxy_pod.boundary_port, ) - )] - pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self.stop_sandbox_inner(sandbox_id).await; - span_status.finish(result) + .workload_policy; + let labels = policy.metadata.labels.get_or_insert_default(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_GATEWAY_ID.to_string(), self.config.gateway_id.clone()); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + labels.insert( + "openshell.ai/component".to_string(), + "boundary-workload-fence".to_string(), + ); + policy.metadata.annotations.get_or_insert_default().insert( + "openshell.ai/workload-pod-name".to_string(), + workload_pod_name.to_string(), + ); + let policies: Api = Api::namespaced(self.client.clone(), namespace); + create_or_validate_proxy_pod_fence(&policies, &policy).await } - async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self - .patch_sandbox_operating_state(sandbox_id, false) - .await?; - let pod_api = Api::::namespaced(self.client.clone(), &namespace); - - let deadline = tokio::time::Instant::now() + stop_timeout; - let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + async fn wait_for_bootstrap_workload_pod( + &self, + pods: &Api, + pod_name: &str, + sandbox_uid: &str, + ) -> Result { + let deadline = tokio::time::Instant::now() + KUBE_API_TIMEOUT; loop { - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); - } - let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); - let object = tokio::time::timeout( - request_timeout, - agent_sandbox_api.api.get(&kube_name), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API while checking sandbox stop", - request_timeout.as_secs() - )) - })? - .map_err(KubernetesDriverError::from_kube)?; - if let Some(error) = kubernetes_sandbox_stop_failure(&object) { - return Err(KubernetesDriverError::Message(error)); - } - let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) - .await - .map_err(KubernetesDriverError::Message)?; - let stop_is_complete = kubernetes_sandbox_stop_is_complete( - &agent_sandbox_api.resource.version, - &object, - pod_is_gone, - ); - if stop_is_complete { - return Ok(()); - } - let now = tokio::time::Instant::now(); - if now >= deadline { - return Err(KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes sandbox to stop", - stop_timeout.as_secs() - ))); + match pods.get_opt(pod_name).await { + Ok(Some(pod)) => { + let owned = pod + .metadata + .owner_references + .as_deref() + .unwrap_or_default() + .iter() + .any(|owner| { + owner.controller == Some(true) + && owner.kind == SANDBOX_KIND + && owner.uid == sandbox_uid + }); + if !owned { + return Err(KubernetesDriverError::Precondition(format!( + "workload Pod {pod_name} is not controlled by the created Sandbox UID" + ))); + } + return Ok(pod); + } + Ok(None) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(100)).await; + } + Ok(None) => { + return Err(KubernetesDriverError::Message(format!( + "timed out waiting for gated workload Pod {pod_name}" + ))); + } + Err(error) => return Err(KubernetesDriverError::from_kube(error)), } - tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; - poll_interval = next_stop_poll_interval(poll_interval); } } - #[tracing::instrument( - name = "kubernetes.start_sandbox", - skip(self), - fields( - otel.name = "kubernetes.start_sandbox", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox_id, - ) - )] - pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { - let span_status = openshell_otel::ErrorStatusGuard::current(); - let result = self - .patch_sandbox_operating_state(sandbox_id, true) - .await - .map(|_| ()); - span_status.finish(result) - } - - async fn patch_sandbox_operating_state( - &self, + #[allow(clippy::too_many_arguments)] + fn validate_capability_free_workload_pod( + pod: &Pod, + sandbox_resource_uid: &str, sandbox_id: &str, - running: bool, - ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { - let lookup_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await - .map_err(KubernetesDriverError::Message)?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let list = tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api - .api - .list(&ListParams::default().labels(&selector)), - ) - .await - .map_err(|_| { - KubernetesDriverError::Message(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() + uid: u32, + gid: u32, + sandbox_secret_name: &str, + ) -> Result<(), KubernetesDriverError> { + let fail = |message: &str| { + KubernetesDriverError::Precondition(format!( + "admitted workload Pod does not preserve capability-free isolation: {message}" )) - })? - .map_err(KubernetesDriverError::from_kube)?; - let object = list - .items - .into_iter() - .next() - .ok_or(KubernetesDriverError::NotFound)?; - let namespace = object + }; + if pod.metadata.uid.as_deref().is_none() { + return Err(fail("missing Pod UID")); + } + let owner_matches = pod .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let agent_sandbox_api = Self::agent_sandbox_api( - self.client.clone(), - &lookup_api.resource.version, - &namespace, - ); - let stop_timeout = kubernetes_sandbox_stop_timeout(&object); - let kube_name = object.metadata.name.ok_or_else(|| { - KubernetesDriverError::Message("sandbox resource has no name".to_string()) - })?; - let pod_name = object + .owner_references + .as_deref() + .unwrap_or_default() + .iter() + .any(|owner| { + owner.controller == Some(true) + && owner.kind == SANDBOX_KIND + && owner.uid == sandbox_resource_uid + }); + if !owner_matches { + return Err(fail("Sandbox owner UID changed")); + } + let labels = pod .metadata - .annotations + .labels .as_ref() - .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) - .cloned() - .unwrap_or_else(|| kube_name.clone()); - let resource_version = object.metadata.resource_version.unwrap_or_default(); - let desired = sandbox_operating_state_patch( - &agent_sandbox_api.resource.version, - &resource_version, - running, - ); - tokio::time::timeout( - KUBE_API_TIMEOUT, - agent_sandbox_api.api.patch( - &kube_name, - &PatchParams::default(), - &Patch::Merge(&desired), - ), + .ok_or_else(|| fail("missing labels"))?; + let expected_pair = crate::proxy_pod::pair_label_value(sandbox_id); + if labels.get(BOUNDARY_ROLE_LABEL).map(String::as_str) != Some("workload") + || labels.get(BOUNDARY_PAIR_LABEL).map(String::as_str) != Some(expected_pair.as_str()) + { + return Err(fail("pair labels changed")); + } + let spec = pod.spec.as_ref().ok_or_else(|| fail("missing Pod spec"))?; + if spec.host_network == Some(true) + || spec.host_pid == Some(true) + || spec.host_ipc == Some(true) + || spec.share_process_namespace == Some(true) + || spec.automount_service_account_token != Some(false) + { + return Err(fail("host namespaces or ServiceAccount token enabled")); + } + let projected_service_account_token = spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|volume| volume.projected.as_ref()) + .flat_map(|projected| projected.sources.as_deref().unwrap_or_default()) + .any(|source| source.service_account_token.is_some()); + if projected_service_account_token { + return Err(fail( + "workload Pod must not receive a projected ServiceAccount token", + )); + } + if spec.restart_policy.as_deref() != Some("Never") + || spec.dns_policy.as_deref() != Some("None") + || spec + .dns_config + .as_ref() + .is_none_or(|dns| dns.nameservers.as_deref() != Some(&["127.0.0.53".to_string()])) + { + return Err(fail("restart or DNS posture changed")); + } + if !spec + .scheduling_gates + .as_deref() + .unwrap_or_default() + .iter() + .any(|gate| gate.name == SANDBOX_BOOTSTRAP_SCHEDULING_GATE) + { + return Err(fail("bootstrap scheduling gate missing")); + } + let security = spec + .security_context + .as_ref() + .ok_or_else(|| fail("missing Pod security context"))?; + if security.run_as_user != Some(i64::from(uid)) + || security.run_as_group != Some(i64::from(gid)) + || security.run_as_non_root != Some(true) + || security.fs_group != Some(i64::from(gid)) + || security + .supplemental_groups + .as_deref() + .is_some_and(|groups| !groups.is_empty()) + || security + .seccomp_profile + .as_ref() + .is_none_or(|profile| profile.type_ != "RuntimeDefault") + { + return Err(fail("numeric identity, groups, or seccomp profile changed")); + } + let pod_json = serde_json::to_value(pod) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + if let Some(policy) = pod_json.pointer("/spec/securityContext/supplementalGroupsPolicy") + && policy != &serde_json::json!("Strict") + { + return Err(fail("supplementalGroupsPolicy is not Strict")); + } + if pod_json + .pointer("/spec/securityContext/supplementalGroupsPolicy") + .is_none() + { + tracing::warn!( + pod = pod.metadata.name.as_deref().unwrap_or(""), + "Kubernetes omitted supplementalGroupsPolicy; exact runtime groups remain enforced by boundary confirmation" + ); + } + let unprivileged_port_sysctl = pod_json + .pointer("/spec/securityContext/sysctls") + .and_then(serde_json::Value::as_array) + .is_some_and(|sysctls| { + sysctls.iter().any(|sysctl| { + sysctl.get("name").and_then(serde_json::Value::as_str) + == Some("net.ipv4.ip_unprivileged_port_start") + && sysctl.get("value").and_then(serde_json::Value::as_str) == Some("0") + }) + }); + if !unprivileged_port_sysctl { + return Err(fail("safe unprivileged-port sysctl changed")); + } + let check_container = |container: &k8s_openapi::api::core::v1::Container, + name: &str| + -> Result<(), KubernetesDriverError> { + let context = container + .security_context + .as_ref() + .ok_or_else(|| fail(&format!("{name} has no security context")))?; + let drops_all = context + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.drop.as_deref()) + .is_some_and(|drops| drops.iter().any(|capability| capability == "ALL")); + let adds_none = context + .capabilities + .as_ref() + .and_then(|capabilities| capabilities.add.as_deref()) + .is_none_or(<[String]>::is_empty); + if context.run_as_user != Some(i64::from(uid)) + || context.run_as_group != Some(i64::from(gid)) + || context.run_as_non_root != Some(true) + || context.allow_privilege_escalation != Some(false) + || !drops_all + || !adds_none + { + return Err(fail(&format!("{name} security context changed"))); + } + Ok(()) + }; + let agent = spec + .containers + .iter() + .find(|container| container.name == "agent") + .ok_or_else(|| fail("agent container missing"))?; + check_container(agent, "agent")?; + let bootstrap = spec + .init_containers + .as_deref() + .unwrap_or_default() + .iter() + .find(|container| container.name == "openshell-sandbox-bootstrap") + .ok_or_else(|| fail("trusted bootstrap init container missing"))?; + check_container(bootstrap, "bootstrap init container")?; + let mounts_volume = |container: &k8s_openapi::api::core::v1::Container, + volume_name: &str| { + container + .volume_mounts + .as_deref() + .unwrap_or_default() + .iter() + .any(|mount| mount.name == volume_name) + }; + if mounts_volume(agent, SANDBOX_BOOTSTRAP_VOLUME_NAME) + || !mounts_volume(bootstrap, SANDBOX_BOOTSTRAP_VOLUME_NAME) + { + return Err(fail( + "bootstrap Secret must be mounted only by the trusted init container", + )); + } + let secret_matches = spec + .volumes + .as_deref() + .unwrap_or_default() + .iter() + .any(|volume| { + volume.name == SANDBOX_BOOTSTRAP_VOLUME_NAME + && volume + .secret + .as_ref() + .and_then(|secret| secret.secret_name.as_deref()) + == Some(sandbox_secret_name) + }); + if !secret_matches { + return Err(fail("generation-specific sandbox Secret changed")); + } + Ok(()) + } + + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn create_proxy_pod_companions( + &self, + sandbox: &Sandbox, + namespace: &str, + cr_name: &str, + sandbox_api: &AgentSandboxApi, + sandbox_cr: &DynamicObject, + names: &ProxyPodNames, + generation: &str, + agent_uid: u32, + agent_gid: u32, + ) -> Result<(), KubernetesDriverError> { + let cr_uid = sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) + })?; + let namespace_uid = Api::::all(self.client.clone()) + .get(namespace) + .await + .map_err(KubernetesDriverError::from_kube)? + .metadata + .uid + .ok_or_else(|| { + KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) + })?; + let dependent_owner = proxy_pod_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ); + let controller_owner = proxy_pod_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + true, + ); + + let services: Api = Api::namespaced(self.client.clone(), namespace); + let service = services + .create( + &PostParams::default(), + &boundary_service( + namespace, + names, + &sandbox.id, + self.config.proxy_pod.boundary_port, + dependent_owner.clone(), + ), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let service_ip: std::net::IpAddr = service + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + + let main_process_spec = openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec( + sandbox.spec.as_ref(), + ) + .map_err(|error| { + KubernetesDriverError::InvalidArgument(format!("encode main process spec: {error}")) + })?; + let log_level = openshell_core::driver_utils::sandbox_log_level(sandbox, "info"); + let deployments: Api = Api::namespaced(self.client.clone(), namespace); + let deployment = deployments + .create( + &PostParams::default(), + &control_deployment( + namespace, + names, + &sandbox.id, + &sandbox.name, + &self.config.gateway_id, + &self.config.supervisor_image, + &self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.config.image_pull_secrets, + &self.config.grpc_endpoint, + &self.config.client_tls_secret_name, + &main_process_spec, + &log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), + self.config + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + controller_owner, + ), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let deployment_uid = deployment.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("control Deployment has no UID".to_string()) + })?; + + let policies: Api = Api::namespaced(self.client.clone(), namespace); + policies + .create( + &PostParams::default(), + &control_egress_policy(namespace, names, &sandbox.id, dependent_owner.clone()), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let fence = policies + .get(&names.workload_policy) + .await + .map_err(KubernetesDriverError::from_kube)?; + let fence_uid = fence.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("workload NetworkPolicy has no UID".to_string()) + })?; + let fence_resource_version = fence.metadata.resource_version.ok_or_else(|| { + KubernetesDriverError::Message( + "workload NetworkPolicy has no resourceVersion".to_string(), + ) + })?; + + // Release the Sandbox CR only far enough for the controller to create + // the workload Pod. The Pod remains unschedulable because its template + // carries the OpenShell scheduling gate and references a Secret that + // does not exist yet. + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + sandbox_operating_state_patch(&sandbox_api.resource.version, version, true) + }) + .await?; + + let pods: Api = Api::namespaced(self.client.clone(), namespace); + let workload_pod = self + .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) + .await?; + Self::validate_capability_free_workload_pod( + &workload_pod, + cr_uid, + &sandbox.id, + agent_uid, + agent_gid, + &names.sandbox_secret, + )?; + let workload_pod_uid = + workload_pod.metadata.uid.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no UID".to_string()) + })?; + let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no name".to_string()) + })?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + serde_json::json!({ + "metadata": { + "resourceVersion": version, + "annotations": { + ANNOTATION_PROXY_POD_WORKLOAD_UID: workload_pod_uid.clone(), + } + } + }) + }) + .await?; + + let bootstrap_token = random_proxy_pod_token(); + let mut child_env = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(std::collections::HashMap::new, |template| { + template.environment.clone() + }); + if let Some(spec) = sandbox.spec.as_ref() { + child_env.extend(spec.environment.clone()); + } + child_env.retain(|name, _| !name.starts_with("OPENSHELL_")); + let host_gateway_ip = self.config.host_gateway_ip.parse().ok(); + let tls = generate_boundary_mutual_tls_material() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let workload_identity = ResolvedWorkloadIdentity::new( + agent_uid, + agent_gid, + Vec::new(), + "kubernetes-config".to_string(), + format!("sandbox:{cr_uid}"), + ) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let provisioned = KubernetesProxyPodBoundarySpec { + boundary_id: sandbox.id.clone(), + bootstrap_token, + generation: generation.to_string(), + session_epoch: random_proxy_pod_token(), + namespace_uid, + sandbox_resource_uid: cr_uid.to_string(), + workload_pod_uid: workload_pod_uid.clone(), + workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), + control_deployment_uid: deployment_uid.clone(), + egress_policy_uid: fence_uid, + egress_policy_resource_version: fence_resource_version, + boundary_listener: std::net::SocketAddr::new( + if service_ip.is_ipv6() { + std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) + } else { + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) + }, + self.config.proxy_pod.boundary_port, + ), + control_address: std::net::SocketAddr::new( + service_ip, + self.config.proxy_pod.boundary_port, + ), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), + client_ca_certificate_path: PathBuf::from(BOUNDARY_CLIENT_CA_PATH), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name.clone(), + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem.clone(), + private_key_pem: tls.supervisor_private_key_pem.clone(), + }, + host_gateway_ip, + workload_identity, + child_env, + } + .provision(); + let descriptor = provisioned + .topology + .descriptor(crate::isolation::BACKEND_NAME) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let sandbox_secret = sandbox_bootstrap_secret( + namespace, + names, + &sandbox.id, + provisioned + .boundary_config + .encode() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + tls.sandbox_certificate_pem.into_bytes(), + tls.sandbox_private_key_pem.into_bytes(), + tls.ca_certificate_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: workload_pod_name.clone(), + uid: workload_pod_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + &sandbox.id, + descriptor.payload, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + OwnerReference { + api_version: "apps/v1".to_string(), + kind: "Deployment".to_string(), + name: names.control_deployment.clone(), + uid: deployment_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let secrets = Api::::namespaced(self.client.clone(), namespace); + secrets + .create(&PostParams::default(), &sandbox_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + + deployments + .patch( + &names.control_deployment, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"replicas": 1}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + pods.patch( + &workload_pod_name, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + spawn_proxy_pod_bootstrap_completion( + deployments.clone(), + sandbox_api.api.clone(), + names.control_deployment.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ); + // Return while the CR remains explicitly bootstrapping. The gateway + // can now commit the sandbox configuration required by a policy-less + // control process without deadlocking behind this driver call. Only + // boundary PID 1 is running at this point; the agent process cannot + // start until control attaches and confirms enforcement. Reconcile + // removes the marker after the control Deployment becomes Available. + Ok(()) + } + + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn install_proxy_pod_generation( + &self, + namespace: &str, + cr_name: &str, + sandbox_api: &AgentSandboxApi, + sandbox_id: &str, + cr_uid: &str, + names: &ProxyPodNames, + generation: &str, + deployment_uid: &str, + agent_uid: u32, + agent_gid: u32, + child_env: std::collections::HashMap, + ) -> Result<(), KubernetesDriverError> { + let namespace_uid = Api::::all(self.client.clone()) + .get(namespace) + .await + .map_err(KubernetesDriverError::from_kube)? + .metadata + .uid + .ok_or_else(|| { + KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) + })?; + let services = Api::::namespaced(self.client.clone(), namespace); + let service_ip: std::net::IpAddr = services + .get(&names.boundary_service) + .await + .map_err(KubernetesDriverError::from_kube)? + .spec + .and_then(|spec| spec.cluster_ip) + .filter(|ip| ip != "None") + .ok_or_else(|| { + KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) + })? + .parse() + .map_err(|error| { + KubernetesDriverError::Message(format!( + "invalid boundary Service ClusterIP: {error}" + )) + })?; + let policies = Api::::namespaced(self.client.clone(), namespace); + let fence = policies + .get(&names.workload_policy) + .await + .map_err(KubernetesDriverError::from_kube)?; + let fence_uid = fence.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("workload NetworkPolicy has no UID".to_string()) + })?; + let fence_resource_version = fence.metadata.resource_version.ok_or_else(|| { + KubernetesDriverError::Message( + "workload NetworkPolicy has no resourceVersion".to_string(), + ) + })?; + + let pods = Api::::namespaced(self.client.clone(), namespace); + let workload_pod = self + .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) + .await?; + Self::validate_capability_free_workload_pod( + &workload_pod, + cr_uid, + sandbox_id, + agent_uid, + agent_gid, + &names.sandbox_secret, + )?; + let workload_pod_uid = + workload_pod.metadata.uid.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no UID".to_string()) + })?; + let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("workload Pod has no name".to_string()) + })?; + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + serde_json::json!({ + "metadata": { + "resourceVersion": version, + "annotations": { + ANNOTATION_PROXY_POD_WORKLOAD_UID: workload_pod_uid.clone(), + } + } + }) + }) + .await?; + + let tls = generate_boundary_mutual_tls_material() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; + let workload_identity = ResolvedWorkloadIdentity::new( + agent_uid, + agent_gid, + Vec::new(), + "kubernetes-config".to_string(), + format!("sandbox:{cr_uid}"), + ) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let provisioned = KubernetesProxyPodBoundarySpec { + boundary_id: sandbox_id.to_string(), + bootstrap_token: random_proxy_pod_token(), + generation: generation.to_string(), + session_epoch: random_proxy_pod_token(), + namespace_uid, + sandbox_resource_uid: cr_uid.to_string(), + workload_pod_uid: workload_pod_uid.clone(), + workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), + control_deployment_uid: deployment_uid.to_string(), + egress_policy_uid: fence_uid, + egress_policy_resource_version: fence_resource_version, + boundary_listener: std::net::SocketAddr::new( + if service_ip.is_ipv6() { + std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) + } else { + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) + }, + self.config.proxy_pod.boundary_port, + ), + control_address: std::net::SocketAddr::new( + service_ip, + self.config.proxy_pod.boundary_port, + ), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), + private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), + client_ca_certificate_path: PathBuf::from(BOUNDARY_CLIENT_CA_PATH), + }, + supervisor_tls: BoundaryClientTls { + server_name: tls.server_name.clone(), + ca_certificate_pem: tls.ca_certificate_pem.clone(), + certificate_chain_pem: tls.supervisor_certificate_pem.clone(), + private_key_pem: tls.supervisor_private_key_pem.clone(), + }, + host_gateway_ip: self.config.host_gateway_ip.parse().ok(), + workload_identity, + child_env, + } + .provision(); + let descriptor = provisioned + .topology + .descriptor(crate::isolation::BACKEND_NAME) + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; + let sandbox_secret = sandbox_bootstrap_secret( + namespace, + names, + sandbox_id, + provisioned + .boundary_config + .encode() + .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, + tls.sandbox_certificate_pem.into_bytes(), + tls.sandbox_private_key_pem.into_bytes(), + tls.ca_certificate_pem.into_bytes(), + OwnerReference { + api_version: "v1".to_string(), + kind: "Pod".to_string(), + name: workload_pod_name.clone(), + uid: workload_pod_uid, + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let supervisor_secret = supervisor_bootstrap_secret( + namespace, + names, + sandbox_id, + descriptor.payload, + proxy_ca.certificate_pem.into_bytes(), + proxy_ca.private_key_pem.into_bytes(), + OwnerReference { + api_version: "apps/v1".to_string(), + kind: "Deployment".to_string(), + name: names.control_deployment.clone(), + uid: deployment_uid.to_string(), + controller: Some(false), + block_owner_deletion: Some(false), + }, + ); + let secrets = Api::::namespaced(self.client.clone(), namespace); + secrets + .create(&PostParams::default(), &sandbox_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + secrets + .create(&PostParams::default(), &supervisor_secret) + .await + .map_err(KubernetesDriverError::from_kube)?; + + let deployments = Api::::namespaced(self.client.clone(), namespace); + deployments + .patch( + &names.control_deployment, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"replicas": 1}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + pods.patch( + &workload_pod_name, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + spawn_proxy_pod_bootstrap_completion( + deployments, + sandbox_api.api.clone(), + names.control_deployment.clone(), + cr_name.to_string(), + Some(cr_uid.to_string()), + ); + Ok(()) + } + + #[tracing::instrument( + name = "kubernetes.stop_sandbox", + skip(self), + fields( + otel.name = "kubernetes.stop_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn stop_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = self.stop_sandbox_inner(sandbox_id).await; + span_status.finish(result) + } + + async fn stop_sandbox_inner(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let (agent_sandbox_api, kube_name, pod_name, namespace, stop_timeout) = self + .patch_sandbox_operating_state(sandbox_id, false) + .await?; + let pod_api = Api::::namespaced(self.client.clone(), &namespace); + + let deadline = tokio::time::Instant::now() + stop_timeout; + let mut poll_interval = STOP_INITIAL_POLL_INTERVAL; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + let request_timeout = KUBE_API_TIMEOUT.min(deadline.saturating_duration_since(now)); + let object = tokio::time::timeout( + request_timeout, + agent_sandbox_api.api.get(&kube_name), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes API while checking sandbox stop", + request_timeout.as_secs() + )) + })? + .map_err(KubernetesDriverError::from_kube)?; + if let Some(error) = kubernetes_sandbox_stop_failure(&object) { + return Err(KubernetesDriverError::Message(error)); + } + let pod_is_gone = kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline) + .await + .map_err(KubernetesDriverError::Message)?; + let stop_is_complete = kubernetes_sandbox_stop_is_complete( + &agent_sandbox_api.resource.version, + &object, + pod_is_gone, + ); + if stop_is_complete { + if let Err(error) = self + .scale_proxy_pod_control(sandbox_id, &namespace, 0) + .await + { + // The workload is already stopped. Treat a stranded + // control replica as resource drift and let periodic + // reconciliation retry rather than failing the stop. + warn!(sandbox_id, %error, "failed to scale stopped proxy-pod control Deployment to zero"); + } + return Ok(()); + } + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(KubernetesDriverError::Message(format!( + "timed out after {}s waiting for Kubernetes sandbox to stop", + stop_timeout.as_secs() + ))); + } + tokio::time::sleep(poll_interval.min(deadline.saturating_duration_since(now))).await; + poll_interval = next_stop_poll_interval(poll_interval); + } + } + + #[tracing::instrument( + name = "kubernetes.start_sandbox", + skip(self), + fields( + otel.name = "kubernetes.start_sandbox", + otel.status_code = tracing::field::Empty, + sandbox.id = %sandbox_id, + ) + )] + pub async fn start_sandbox(&self, sandbox_id: &str) -> Result<(), KubernetesDriverError> { + let span_status = openshell_otel::ErrorStatusGuard::current(); + let result = Box::pin(self.start_proxy_pod_generation(sandbox_id)).await; + span_status.finish(result) + } + + #[allow(clippy::similar_names)] + async fn start_proxy_pod_generation( + &self, + sandbox_id: &str, + ) -> Result<(), KubernetesDriverError> { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let mut objects = lookup_api + .api + .list(&ListParams::default().labels(&selector)) + .await + .map_err(KubernetesDriverError::from_kube)? + .items; + let object = objects.pop().ok_or(KubernetesDriverError::NotFound)?; + if proxy_pod_bootstrap_in_progress(&object) { + return Err(KubernetesDriverError::Precondition( + "sandbox bootstrap has not completed; wait for reconciliation or recreate the sandbox" + .to_string(), + )); + } + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let cr_name = object.metadata.name.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; + let cr_uid = object.metadata.uid.as_deref().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no UID".to_string()) + })?; + let sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let pods = Api::::namespaced(self.client.clone(), &namespace); + if pods + .get_opt(cr_name) + .await + .map_err(KubernetesDriverError::from_kube)? + .is_some() + { + return Err(KubernetesDriverError::Precondition( + "cannot rotate a running workload Pod; stop the sandbox first".to_string(), + )); + } + + let generation = random_proxy_pod_token(); + let names = ProxyPodNames::for_generation(sandbox_id, &generation); + self.create_proxy_pod_fence( + &namespace, + &names, + &Sandbox { + id: sandbox_id.to_string(), + ..Default::default() + }, + cr_name, + ) + .await?; + let deployments = Api::::namespaced(self.client.clone(), &namespace); + let old_deployment = deployments + .get(&names.control_deployment) + .await + .map_err(KubernetesDriverError::from_kube)?; + let old_uid = old_deployment.metadata.uid.clone().ok_or_else(|| { + KubernetesDriverError::Message("control Deployment has no UID".to_string()) + })?; + let owner_references = old_deployment.metadata.owner_references.clone(); + let labels = old_deployment.metadata.labels.clone(); + let annotations = old_deployment.metadata.annotations.clone(); + let mut deployment_spec = old_deployment.spec.ok_or_else(|| { + KubernetesDriverError::Message("control Deployment has no spec".to_string()) + })?; + deployment_spec.replicas = Some(0); + let bootstrap_volume = deployment_spec + .template + .spec + .as_mut() + .and_then(|spec| spec.volumes.as_mut()) + .and_then(|volumes| volumes.iter_mut().find(|volume| volume.name == "bootstrap")) + .and_then(|volume| volume.secret.as_mut()) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "control Deployment is missing its immutable bootstrap Secret volume" + .to_string(), + ) + })?; + bootstrap_volume.secret_name = Some(names.supervisor_secret.clone()); + deployments + .delete( + &names.control_deployment, + &DeleteParams::default().preconditions(Preconditions { + uid: Some(old_uid), + resource_version: None, + }), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + let deadline = tokio::time::Instant::now() + KUBE_API_TIMEOUT; + loop { + if deployments + .get_opt(&names.control_deployment) + .await + .map_err(KubernetesDriverError::from_kube)? + .is_none() + { + break; + } + if tokio::time::Instant::now() >= deadline { + return Err(KubernetesDriverError::Message( + "timed out waiting for old control Deployment deletion".to_string(), + )); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + let replacement = Deployment { + metadata: ObjectMeta { + name: Some(names.control_deployment.clone()), + namespace: Some(namespace.clone()), + owner_references, + labels, + annotations, + ..Default::default() + }, + spec: Some(deployment_spec), + status: None, + }; + let replacement = deployments + .create(&PostParams::default(), &replacement) + .await + .map_err(KubernetesDriverError::from_kube)?; + let deployment_uid = replacement.metadata.uid.ok_or_else(|| { + KubernetesDriverError::Message("replacement control Deployment has no UID".to_string()) + })?; + + let mut volumes = object + .data + .pointer("/spec/podTemplate/spec/volumes") + .and_then(serde_json::Value::as_array) + .cloned() + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox pod template has no volume list".to_string(), + ) + })?; + let sandbox_secret = volumes + .iter_mut() + .find(|volume| { + volume.get("name").and_then(serde_json::Value::as_str) + == Some(SANDBOX_BOOTSTRAP_VOLUME_NAME) + }) + .and_then(|volume| volume.get_mut("secret")) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Sandbox pod template is missing its bootstrap Secret volume".to_string(), + ) + })?; + sandbox_secret["secretName"] = serde_json::json!(names.sandbox_secret); + patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { + let mut running_patch = + sandbox_operating_state_patch(&sandbox_api.resource.version, version, true); + running_patch["metadata"]["annotations"] = serde_json::json!({ + ANNOTATION_PROXY_POD_BOOTSTRAPPING: "true", + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: openshell_core::time::now_ms().to_string(), + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: "restart", + ANNOTATION_PROXY_POD_GENERATION: generation, + }); + running_patch["spec"]["podTemplate"]["spec"]["volumes"] = + serde_json::Value::Array(volumes.clone()); + running_patch + }) + .await?; + + let (agent_uid, agent_gid, _) = + self.resolve_sandbox_identity_in_namespace(&namespace).await; + let child_env = child_environment_from_sandbox_object(&object); + self.install_proxy_pod_generation( + &namespace, + cr_name, + &sandbox_api, + sandbox_id, + cr_uid, + &names, + &generation, + &deployment_uid, + agent_uid, + agent_gid, + child_env, + ) + .await + } + + async fn patch_sandbox_operating_state( + &self, + sandbox_id: &str, + running: bool, + ) -> Result<(AgentSandboxApi, String, String, String, Duration), KubernetesDriverError> { + let lookup_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let selector = self.sandbox_lookup_selector(sandbox_id); + let list = tokio::time::timeout( + KUBE_API_TIMEOUT, + lookup_api + .api + .list(&ListParams::default().labels(&selector)), ) .await .map_err(|_| { @@ -1765,6 +2911,50 @@ impl KubernetesComputeDriver { )) })? .map_err(KubernetesDriverError::from_kube)?; + let object = list + .items + .into_iter() + .next() + .ok_or(KubernetesDriverError::NotFound)?; + let namespace = object + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let agent_sandbox_api = Self::agent_sandbox_api( + self.client.clone(), + &lookup_api.resource.version, + &namespace, + ); + let stop_timeout = kubernetes_sandbox_stop_timeout(&object); + let kube_name = object.metadata.name.clone().ok_or_else(|| { + KubernetesDriverError::Message("sandbox resource has no name".to_string()) + })?; + let pod_name = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(SANDBOX_POD_NAME_ANNOTATION)) + .cloned() + .unwrap_or_else(|| kube_name.clone()); + if running { + if proxy_pod_bootstrap_in_progress(&object) { + return Err(KubernetesDriverError::Precondition( + "proxy-pod bootstrap has not completed; wait for reconciliation or recreate the sandbox" + .to_string(), + )); + } + self.scale_proxy_pod_control(sandbox_id, &namespace, 1) + .await?; + } + patch_dynamic_object_with_resource_version_retry( + &agent_sandbox_api.api, + &kube_name, + |version| { + sandbox_operating_state_patch(&agent_sandbox_api.resource.version, version, running) + }, + ) + .await?; info!( sandbox_id, @@ -1781,6 +2971,24 @@ impl KubernetesComputeDriver { )) } + async fn scale_proxy_pod_control( + &self, + sandbox_id: &str, + namespace: &str, + replicas: i32, + ) -> Result<(), KubernetesDriverError> { + let names = ProxyPodNames::new(sandbox_id); + Api::::namespaced(self.client.clone(), namespace) + .patch( + &names.control_deployment, + &PatchParams::default(), + &Patch::Merge(&serde_json::json!({"spec": {"replicas": replicas}})), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + Ok(()) + } + #[tracing::instrument( name = "kubernetes.delete_sandbox", skip(self), @@ -1808,115 +3016,736 @@ impl KubernetesComputeDriver { .await?; let selector = self.sandbox_lookup_selector(sandbox_id); let lp = ListParams::default().labels(&selector); - let (kube_name, obj_namespace, _workspace, preconditions) = match tokio::time::timeout( - KUBE_API_TIMEOUT, - lookup_api.api.list(&lp), - ) - .await - { - Ok(Ok(list)) => { - if let Some(obj) = list.items.into_iter().next() { - match obj.metadata.name { - Some(name) => { - let ns = obj - .metadata - .namespace - .clone() - .unwrap_or_else(|| self.config.namespace.clone()); - let ws = obj - .metadata - .labels - .as_ref() - .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) - .unwrap_or_default(); - let pc = Preconditions { - uid: obj.metadata.uid, - resource_version: obj.metadata.resource_version, - }; - (name, ns, ws, pc) + let (kube_name, obj_namespace, _workspace, preconditions, pod_name) = + match tokio::time::timeout(KUBE_API_TIMEOUT, lookup_api.api.list(&lp)).await { + Ok(Ok(list)) => { + if let Some(obj) = list.items.into_iter().next() { + match obj.metadata.name.clone() { + Some(name) => { + let ns = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| self.config.namespace.clone()); + let ws = obj + .metadata + .labels + .as_ref() + .and_then(|l| l.get(LABEL_SANDBOX_WORKSPACE).cloned()) + .unwrap_or_default(); + let pc = Preconditions { + uid: obj.metadata.uid, + resource_version: obj.metadata.resource_version, + }; + let pod_name = obj + .metadata + .annotations + .as_ref() + .and_then(|annotations| { + annotations.get(SANDBOX_POD_NAME_ANNOTATION) + }) + .cloned() + .unwrap_or_else(|| name.clone()); + (name, ns, ws, pc, pod_name) + } + None => return Ok(false), + } + } else { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); + return Ok(false); + } + } + Ok(Err(err)) => { + warn!( + sandbox_id = %sandbox_id, + error = %err, + "Failed to list sandbox for deletion from Kubernetes" + ); + return Err(err.to_string()); + } + Err(_elapsed) => { + warn!( + sandbox_id = %sandbox_id, + timeout_secs = KUBE_API_TIMEOUT.as_secs(), + "Timed out listing sandbox for deletion from Kubernetes" + ); + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + }; + + let delete_api = self + .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + .await?; + let dp = DeleteParams::default().preconditions(preconditions); + match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { + Ok(Ok(_response)) => { + info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); + { + let pod_api = Api::::namespaced(self.client.clone(), &obj_namespace); + let deadline = tokio::time::Instant::now() + + DEFAULT_POD_TERMINATION_GRACE_PERIOD + + KUBE_API_TIMEOUT; + loop { + match kubernetes_sandbox_pod_is_gone(&pod_api, &pod_name, deadline).await { + Ok(true) => break, + Ok(false) if tokio::time::Instant::now() < deadline => { + tokio::time::sleep(STOP_INITIAL_POLL_INTERVAL).await; + } + Ok(false) | Err(_) => { + warn!( + sandbox_id, + "retaining proxy-pod workload fence because workload Pod deletion was not confirmed" + ); + return Ok(true); + } + } + } + // API acceptance of DELETE does not mean the CR is gone. + // Keep the unowned fence while finalizers can still leave + // the controller able to reconcile a workload Pod. + if !matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.get(&kube_name)) + .await, + Ok(Err(KubeError::Api(kube::core::ErrorResponse { + code: 404, + .. + }))) + ) { + warn!( + sandbox_id, + "retaining proxy-pod workload fence because Sandbox CR deletion was not confirmed" + ); + return Ok(true); + } + let quiesce_deadline = + tokio::time::Instant::now() + PROXY_POD_FENCE_QUIESCE_WINDOW; + while tokio::time::Instant::now() < quiesce_deadline { + tokio::time::sleep(PROXY_POD_FENCE_QUIESCE_INTERVAL).await; + let cr_is_gone = matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.get(&kube_name)) + .await, + Ok(Err(KubeError::Api(kube::core::ErrorResponse { + code: 404, + .. + }))) + ); + let pod_is_gone = kubernetes_sandbox_pod_is_gone( + &pod_api, + &pod_name, + tokio::time::Instant::now() + KUBE_API_TIMEOUT, + ) + .await + .unwrap_or(false); + if !cr_is_gone || !pod_is_gone { + warn!( + sandbox_id, + "retaining proxy-pod workload fence because deletion did not remain quiescent" + ); + return Ok(true); + } + } + let names = ProxyPodNames::new(sandbox_id); + let policies: Api = + Api::namespaced(self.client.clone(), &obj_namespace); + match policies + .delete(&names.workload_policy, &DeleteParams::default()) + .await + { + Ok(_) + | Err(KubeError::Api(kube::core::ErrorResponse { code: 404, .. })) => {} + Err(error) => { + warn!(sandbox_id, %error, "failed to delete proxy-pod workload fence; reconciliation will retry"); } - None => return Ok(false), } - } else { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted)"); - return Ok(false); } + Ok(true) + } + Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { + debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); + Ok(false) } Ok(Err(err)) => { warn!( sandbox_id = %sandbox_id, error = %err, - "Failed to list sandbox for deletion from Kubernetes" + "Failed to delete sandbox from Kubernetes" ); - return Err(err.to_string()); + Err(err.to_string()) } Err(_elapsed) => { warn!( sandbox_id = %sandbox_id, timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out listing sandbox for deletion from Kubernetes" + "Timed out deleting sandbox from Kubernetes" ); - return Err(format!( + Err(format!( "timed out after {}s waiting for Kubernetes API", KUBE_API_TIMEOUT.as_secs() - )); + )) } - }; + } + } - let delete_api = self - .supported_agent_sandbox_api(self.client.clone(), &obj_namespace) + pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { + let agent_sandbox_api = self + .supported_sandbox_api_for_lookup(self.client.clone()) .await?; - let dp = DeleteParams::default().preconditions(preconditions); - match tokio::time::timeout(KUBE_API_TIMEOUT, delete_api.api.delete(&kube_name, &dp)).await { - Ok(Ok(_response)) => { - info!(sandbox_id = %sandbox_id, namespace = %obj_namespace, "Sandbox deleted from Kubernetes"); - Ok(true) + let selector = self.sandbox_lookup_selector(sandbox_id); + let lp = ListParams::default().labels(&selector); + match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { + Ok(Ok(list)) => Ok(!list.items.is_empty()), + Ok(Err(err)) => Err(err.to_string()), + Err(_elapsed) => Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )), + } + } + + /// Repair lifecycle drift and reap unowned workload fences while the + /// gateway's sandbox watch is alive. Bootstrap material is immutable and + /// intentionally not read by the gateway, so this pass only repairs state + /// that can be proven from the Sandbox CR and named companion objects. + async fn reconcile_proxy_pod_resources(&self) { + let lookup_api = match self + .supported_sandbox_api_for_lookup(self.client.clone()) + .await + { + Ok(api) => api, + Err(error) => { + warn!(%error, "skipping proxy-pod reconciliation: Sandbox API unavailable"); + return; + } + }; + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + lookup_api + .api + .list(&ListParams::default().labels(&self.openshell_sandbox_selector())), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(error)) => { + warn!(%error, "skipping proxy-pod reconciliation: Sandbox list failed"); + return; + } + Err(_) => { + warn!("skipping proxy-pod reconciliation: Sandbox list timed out"); + return; + } + }; + + let mut live_ids = HashSet::new(); + for object in list.items { + let Ok(sandbox_id) = sandbox_id_from_object(&object) else { + continue; + }; + live_ids.insert(sandbox_id.clone()); + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let cr_name = object.metadata.name.as_deref().unwrap_or_default(); + let names = ProxyPodNames::new(&sandbox_id); + let policies = Api::::namespaced(self.client.clone(), namespace); + let fence_was_missing = matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(&names.workload_policy)) + .await, + Ok(Ok(None)) + ); + let sandbox = Sandbox { + id: sandbox_id.clone(), + ..Default::default() + }; + match self + .create_proxy_pod_fence(namespace, &names, &sandbox, cr_name) + .await + { + Ok(()) => {} + Err(KubernetesDriverError::Precondition(error)) => { + warn!(sandbox_id, %error, "proxy-pod workload fence is altered; suspending workload"); + self.suspend_proxy_pod_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + Err(error) => { + warn!(sandbox_id, %error, "could not verify proxy-pod workload fence; reconciliation will retry"); + continue; + } + } + if fence_was_missing { + // Recreating the deny-all fence closes direct egress first; + // suspend next because the immutable boundary descriptor names + // the prior policy UID/resourceVersion and can no longer be + // re-confirmed after a control restart. + warn!( + sandbox_id, + "recreated missing proxy-pod workload fence; suspending stale boundary" + ); + self.suspend_proxy_pod_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + if proxy_pod_bootstrap_in_progress(&object) { + if proxy_pod_control_availability(&self.client, namespace, &sandbox_id).await + == ProxyPodControlAvailability::Available + { + self.complete_proxy_pod_bootstrap(&lookup_api, &object) + .await; + } else { + self.reap_stale_proxy_pod_bootstrap(&lookup_api, &object) + .await; + } + continue; + } + let desired = desired_proxy_pod_control_replicas(&object); + if desired > 0 { + match proxy_pod_workload_generation_matches( + &self.client, + namespace, + cr_name, + &object, + ) + .await + { + ProxyPodControlAvailability::Available => {} + ProxyPodControlAvailability::Unavailable => { + warn!( + sandbox_id, + "proxy-pod workload generation changed; suspending stale boundary" + ); + self.suspend_proxy_pod_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + ProxyPodControlAvailability::Unknown => { + warn!( + sandbox_id, + "could not verify proxy-pod workload generation; reconciliation will retry" + ); + continue; + } + } + } + match self + .reconcile_proxy_pod_control_replicas(&sandbox_id, namespace, desired) + .await + { + Ok(()) => {} + Err(KubernetesDriverError::Precondition(error)) => { + warn!(sandbox_id, %error, "proxy-pod companion is missing; suspending workload"); + self.suspend_proxy_pod_after_dependency_failure(&lookup_api, &object) + .await; + } + Err(error) => { + warn!(sandbox_id, %error, "failed to reconcile proxy-pod control Deployment"); + } + } + let availability = + proxy_pod_control_availability(&self.client, namespace, &sandbox_id).await; + self.publish_proxy_pod_readiness_transition(&lookup_api, &object, availability) + .await; + } + + self.reap_orphaned_proxy_pod_fences(&live_ids).await; + } + + async fn publish_proxy_pod_readiness_transition( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + availability: ProxyPodControlAvailability, + ) { + let state = match availability { + ProxyPodControlAvailability::Available => "ready", + ProxyPodControlAvailability::Unavailable => "unavailable", + ProxyPodControlAvailability::Unknown => return, + }; + if object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_PROXY_POD_READINESS)) + .is_some_and(|current| current == state) + { + return; + } + let (Some(name), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.resource_version.as_deref(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = proxy_pod_readiness_transition_patch(resource_version, state); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + debug!(sandbox = name, %error, "proxy-pod readiness transition publication raced; reconciliation will retry"); + } + Err(_) => { + warn!( + sandbox = name, + "timed out publishing proxy-pod readiness transition" + ); + } + } + } + + async fn suspend_proxy_pod_after_dependency_failure( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + let Some(name) = object.metadata.name.as_deref() else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = sandbox_operating_state_patch( + &lookup_api.resource.version, + object + .metadata + .resource_version + .as_deref() + .unwrap_or_default(), + false, + ); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + warn!(sandbox = name, %error, "failed to suspend proxy-pod after fence failure"); + } + Err(error) => { + warn!(sandbox = name, %error, "timed out suspending proxy-pod after fence failure"); + } + } + } + + async fn reap_stale_proxy_pod_bootstrap( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + if !proxy_pod_bootstrap_is_stale(object, SystemTime::now(), PROXY_POD_BOOTSTRAP_GRACE) { + return; + } + if proxy_pod_bootstrap_operation(object) != Some("create") { + warn!( + sandbox = object.metadata.name.as_deref().unwrap_or(""), + "suspending stale proxy-pod restart bootstrap" + ); + self.suspend_proxy_pod_after_dependency_failure(lookup_api, object) + .await; + return; + } + let (Some(name), Some(uid), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.uid.clone(), + object.metadata.resource_version.clone(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let params = DeleteParams::default().preconditions(Preconditions { + uid: Some(uid), + resource_version: Some(resource_version), + }); + match tokio::time::timeout(KUBE_API_TIMEOUT, api.api.delete(name, ¶ms)).await { + Ok(Ok(_)) => warn!( + sandbox = name, + "rolled back stale fail-closed proxy-pod bootstrap" + ), + Ok(Err(KubeError::Api(error))) if error.code == 404 || error.code == 409 => {} + Ok(Err(error)) => { + warn!(sandbox = name, %error, "failed to roll back stale proxy-pod bootstrap"); + } + Err(_) => { + warn!( + sandbox = name, + "timed out rolling back stale proxy-pod bootstrap" + ); + } + } + } + + async fn complete_proxy_pod_bootstrap( + &self, + lookup_api: &AgentSandboxApi, + object: &DynamicObject, + ) { + let (Some(name), Some(resource_version)) = ( + object.metadata.name.as_deref(), + object.metadata.resource_version.as_deref(), + ) else { + return; + }; + let namespace = object + .metadata + .namespace + .as_deref() + .unwrap_or(&self.config.namespace); + let api = + Self::agent_sandbox_api(self.client.clone(), &lookup_api.resource.version, namespace); + let patch = proxy_pod_bootstrap_completion_patch(resource_version); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + api.api + .patch(name, &PatchParams::default(), &Patch::Merge(&patch)), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + debug!(sandbox = name, %error, "proxy-pod bootstrap completion raced; reconciliation will retry"); + } + Err(_) => warn!(sandbox = name, "timed out completing proxy-pod bootstrap"), + } + } + + async fn reconcile_proxy_pod_control_replicas( + &self, + sandbox_id: &str, + namespace: &str, + desired: i32, + ) -> Result<(), KubernetesDriverError> { + let names = ProxyPodNames::new(sandbox_id); + let services = Api::::namespaced(self.client.clone(), namespace); + let service_exists = + tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(&names.boundary_service)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading proxy-pod boundary Service".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + .is_some(); + if !service_exists { + return Err(KubernetesDriverError::Precondition(format!( + "proxy-pod boundary Service {} is missing and its allocated address cannot be safely reconstructed", + names.boundary_service + ))); + } + let deployments = Api::::namespaced(self.client.clone(), namespace); + let deployment = tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.get_opt(&names.control_deployment), + ) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading proxy-pod control Deployment".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + .ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "proxy-pod control Deployment {} is missing and cannot be safely reconstructed from the Sandbox CR", + names.control_deployment + )) + })?; + if deployment.spec.as_ref().and_then(|spec| spec.replicas) == Some(desired) { + return Ok(()); + } + self.scale_proxy_pod_control(sandbox_id, namespace, desired) + .await + } + + async fn reap_orphaned_proxy_pod_fences(&self, live_ids: &HashSet) { + let policies: Api = if self.config.is_multi_namespace() { + Api::all(self.client.clone()) + } else { + Api::namespaced(self.client.clone(), &self.config.namespace) + }; + let selector = format!( + "{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE},{LABEL_GATEWAY_ID}={},openshell.ai/component=boundary-workload-fence", + self.config.gateway_id + ); + let list = match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.list(&ListParams::default().labels(&selector)), + ) + .await + { + Ok(Ok(list)) => list, + Ok(Err(error)) => { + warn!(%error, "failed to list proxy-pod workload fences for orphan cleanup"); + return; } - Ok(Err(KubeError::Api(err))) if err.code == 404 || err.code == 409 => { - debug!(sandbox_id = %sandbox_id, "Sandbox not found in Kubernetes (already deleted or replaced)"); - Ok(false) + Err(_) => { + warn!("timed out listing proxy-pod workload fences for orphan cleanup"); + return; } - Ok(Err(err)) => { - warn!( - sandbox_id = %sandbox_id, - error = %err, - "Failed to delete sandbox from Kubernetes" + }; + for policy in list.items { + let sandbox_id = policy + .metadata + .labels + .as_ref() + .and_then(|labels| labels.get(LABEL_SANDBOX_ID)) + .cloned() + .unwrap_or_default(); + if sandbox_id.is_empty() || live_ids.contains(&sandbox_id) { + continue; + } + if !proxy_pod_fence_is_old_enough(&policy, SystemTime::now()) { + debug!( + sandbox_id, + "retaining young proxy-pod fence while Sandbox creation may be in flight" ); - Err(err.to_string()) + continue; } - Err(_elapsed) => { + // Reconfirm against the API after the two list snapshots. A sandbox + // created between them must never lose the fence that precedes its CR. + if self.sandbox_exists(&sandbox_id).await != Ok(false) { + continue; + } + let Some(namespace) = policy.metadata.namespace.as_deref() else { + continue; + }; + let Some(pod_name) = policy + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get("openshell.ai/workload-pod-name")) + else { warn!( - sandbox_id = %sandbox_id, - timeout_secs = KUBE_API_TIMEOUT.as_secs(), - "Timed out deleting sandbox from Kubernetes" + sandbox_id, + "retaining orphaned proxy-pod fence without workload Pod annotation" ); - Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )) + continue; + }; + let pods = Api::::namespaced(self.client.clone(), namespace); + if !matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await, + Ok(Ok(None)) + ) { + debug!( + sandbox_id, + pod_name, "retaining proxy-pod fence until workload Pod absence is confirmed" + ); + continue; + } + let quiesce_deadline = tokio::time::Instant::now() + PROXY_POD_FENCE_QUIESCE_WINDOW; + let mut quiescent = true; + while tokio::time::Instant::now() < quiesce_deadline { + tokio::time::sleep(PROXY_POD_FENCE_QUIESCE_INTERVAL).await; + if self.sandbox_exists(&sandbox_id).await != Ok(false) + || !matches!( + tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await, + Ok(Ok(None)) + ) + { + quiescent = false; + break; + } + } + if !quiescent { + debug!( + sandbox_id, + pod_name, "retaining orphaned proxy-pod fence after quiescence recheck" + ); + continue; + } + let Some(name) = policy.metadata.name.as_deref() else { + continue; + }; + let scoped = Api::::namespaced(self.client.clone(), namespace); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + scoped.delete(name, &DeleteParams::default()), + ) + .await + { + Ok(Ok(_) | Err(KubeError::Api(kube::core::ErrorResponse { code: 404, .. }))) => { + info!( + sandbox_id, + policy = name, + "reaped orphaned proxy-pod workload fence" + ); + } + Ok(Err(error)) => { + warn!(sandbox_id, policy = name, %error, "failed to reap orphaned proxy-pod workload fence"); + } + Err(_) => warn!( + sandbox_id, + policy = name, + "timed out reaping orphaned proxy-pod workload fence" + ), } } } - pub async fn sandbox_exists(&self, sandbox_id: &str) -> Result { - let agent_sandbox_api = self - .supported_sandbox_api_for_lookup(self.client.clone()) - .await?; - let selector = self.sandbox_lookup_selector(sandbox_id); - let lp = ListParams::default().labels(&selector); - match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.list(&lp)).await { - Ok(Ok(list)) => Ok(!list.items.is_empty()), - Ok(Err(err)) => Err(err.to_string()), - Err(_elapsed) => Err(format!( - "timed out after {}s waiting for Kubernetes API", - KUBE_API_TIMEOUT.as_secs() - )), - } + fn spawn_proxy_pod_periodic_reconcile( + &self, + tx: mpsc::Sender>, + ) { + let driver = self.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(PROXY_POD_RECONCILE_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = interval.tick() => { + driver.reconcile_proxy_pod_resources().await; + if let Ok(sandboxes) = driver.list_sandboxes().await { + for sandbox in sandboxes { + if tx.send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { sandbox: Some(sandbox) }, + )), + })).await.is_err() { + return; + } + } + } + } + () = tx.closed() => return, + } + } + }); } // Kept `async` to match the gRPC handler signature in `grpc.rs`, which awaits this method. #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { + self.reconcile_proxy_pod_resources().await; if self.config.is_multi_namespace() { self.watch_sandboxes_cluster_wide().await } else { @@ -1942,6 +3771,8 @@ impl KubernetesComputeDriver { ) .boxed(); let (tx, rx) = mpsc::channel(256); + self.spawn_proxy_pod_periodic_reconcile(tx.clone()); + let readiness_client = self.watch_client.clone(); tokio::spawn(async move { let mut sandbox_name_to_id = std::collections::HashMap::::new(); @@ -1951,7 +3782,7 @@ impl KubernetesComputeDriver { tokio::select! { event = sandbox_stream.next() => match event { Some(Event::Applied(obj)) => { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_proxy_pod_readiness(&readiness_client, &namespace, obj).await { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -1980,7 +3811,7 @@ impl KubernetesComputeDriver { } Some(Event::Restarted(objs)) => { for obj in objs { - if let Ok((kube_name, sandbox)) = sandbox_from_object(&namespace, obj) { + if let Ok((kube_name, sandbox)) = sandbox_from_object_with_proxy_pod_readiness(&readiness_client, &namespace, obj).await { update_indexes(&mut sandbox_name_to_id, &mut agent_pod_to_id, &kube_name, &sandbox); let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( @@ -2053,15 +3884,149 @@ impl KubernetesComputeDriver { Ok(cluster_wide_watch_stream( sandbox_stream, self.config.namespace.clone(), + self.watch_client.clone(), + self.clone(), )) } } -fn cluster_wide_watch_stream(mut sandbox_stream: S, default_namespace: String) -> WatchStream +fn proxy_pod_bootstrap_completion_patch(resource_version: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_PROXY_POD_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_PROXY_POD_READINESS: "ready", + } + } + }) +} + +fn spawn_proxy_pod_bootstrap_completion( + deployments: Api, + sandboxes: Api, + deployment_name: String, + sandbox_name: String, + expected_sandbox_uid: Option, +) { + tokio::spawn(async move { + let available = |deployment: Option<&Deployment>| { + deployment.is_some_and(|deployment| { + proxy_pod_control_availability_from_deployment(deployment) + == ProxyPodControlAvailability::Available + }) + }; + match tokio::time::timeout( + PROXY_POD_BOOTSTRAP_GRACE, + await_condition(deployments, &deployment_name, available), + ) + .await + { + Ok(Ok(Some(_))) => {} + Ok(Ok(None)) | Err(_) => return, + Ok(Err(error)) => { + debug!(%error, sandbox = sandbox_name, "proxy-pod bootstrap availability watch failed; reconciliation will retry"); + return; + } + } + + let Ok(object) = sandboxes.get(&sandbox_name).await else { + return; + }; + if object.metadata.uid != expected_sandbox_uid || !proxy_pod_bootstrap_in_progress(&object) + { + return; + } + let Some(resource_version) = object.metadata.resource_version.as_deref() else { + return; + }; + let patch = proxy_pod_bootstrap_completion_patch(resource_version); + if let Err(error) = sandboxes + .patch( + &sandbox_name, + &PatchParams::default(), + &Patch::Merge(&patch), + ) + .await + { + debug!(%error, sandbox = sandbox_name, "proxy-pod bootstrap completion raced; reconciliation will retry"); + } + }); +} + +fn proxy_pod_fence_is_old_enough(policy: &NetworkPolicy, now: SystemTime) -> bool { + metadata_is_older_than(&policy.metadata, now, PROXY_POD_ORPHAN_FENCE_MIN_AGE) +} + +fn proxy_pod_bootstrap_in_progress(object: &DynamicObject) -> bool { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_PROXY_POD_BOOTSTRAPPING)) + .is_some_and(|value| value == "true") +} + +fn proxy_pod_bootstrap_operation(object: &DynamicObject) -> Option<&str> { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION)) + .map(String::as_str) +} + +fn proxy_pod_bootstrap_is_stale( + object: &DynamicObject, + now: SystemTime, + minimum_age: Duration, +) -> bool { + let Some(started_at_ms) = object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT)) + .and_then(|value| value.parse::().ok()) + else { + return true; + }; + now.duration_since(SystemTime::UNIX_EPOCH + Duration::from_millis(started_at_ms)) + .is_ok_and(|age| age >= minimum_age) +} + +fn proxy_pod_readiness_transition_patch(resource_version: &str, state: &str) -> serde_json::Value { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { ANNOTATION_PROXY_POD_READINESS: state }, + } + }) +} + +fn metadata_is_older_than(metadata: &ObjectMeta, now: SystemTime, minimum_age: Duration) -> bool { + let Some(created) = metadata.creation_timestamp.as_ref() else { + return false; + }; + let Ok(created_seconds) = u64::try_from(created.0.timestamp()) else { + return false; + }; + now.duration_since(SystemTime::UNIX_EPOCH + Duration::from_secs(created_seconds)) + .is_ok_and(|age| age >= minimum_age) +} + +fn cluster_wide_watch_stream( + mut sandbox_stream: S, + default_namespace: String, + readiness_client: Client, + driver: KubernetesComputeDriver, +) -> WatchStream where S: Stream> + Send + Unpin + 'static, { let (tx, rx) = mpsc::channel(256); + driver.spawn_proxy_pod_periodic_reconcile(tx.clone()); tokio::spawn(async move { loop { @@ -2070,7 +4035,7 @@ where Some(Event::Applied(obj)) => { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_proxy_pod_readiness(&readiness_client, &ns, obj).await { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -2099,7 +4064,7 @@ where for obj in objs { let ns = obj.metadata.namespace.clone() .unwrap_or_else(|| default_namespace.clone()); - if let Ok((_kube_name, sandbox)) = sandbox_from_object(&ns, obj) { + if let Ok((_kube_name, sandbox)) = sandbox_from_object_with_proxy_pod_readiness(&readiness_client, &ns, obj).await { let event = WatchSandboxesEvent { payload: Some(watch_sandboxes_event::Payload::Sandbox( WatchSandboxesSandboxEvent { sandbox: Some(sandbox) } @@ -2437,6 +4402,48 @@ fn validate_pod_uid(pod: &Pod, expected_uid: &str) -> Result<(), tonic::Status> )) } +fn require_proxy_control_authentication(via_proxy_control: bool) -> Result<(), tonic::Status> { + if via_proxy_control { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "sandbox JWT authentication must originate from the paired supervisor Pod", + )) + } +} + +#[allow(clippy::result_large_err)] +fn validate_proxy_control_labels(pod: &Pod, sandbox_id: &str) -> Result<(), tonic::Status> { + validate_proxy_control_labels_from_metadata(&pod.metadata, sandbox_id) +} + +#[allow(clippy::result_large_err)] +fn validate_proxy_control_labels_from_metadata( + metadata: &ObjectMeta, + sandbox_id: &str, +) -> Result<(), tonic::Status> { + let labels = metadata.labels.as_ref().ok_or_else(|| { + tonic::Status::permission_denied("control workload has no proxy-pod labels") + })?; + let expected_pair = crate::proxy_pod::pair_label_value(sandbox_id); + let matches = labels + .get(BOUNDARY_ROLE_LABEL) + .is_some_and(|role| role == "supervisor") + && labels + .get(BOUNDARY_PAIR_LABEL) + .is_some_and(|pair| pair == &expected_pair) + && labels + .get(LABEL_SANDBOX_ID) + .is_some_and(|actual| actual == sandbox_id); + if matches { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "control workload proxy-pod labels do not match the sandbox identity", + )) + } +} + #[allow(clippy::result_large_err)] fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> { let mut owners = pod @@ -2467,6 +4474,24 @@ fn sandbox_owner_reference(pod: &Pod) -> Result<&OwnerReference, tonic::Status> Ok(owner) } +fn controller_owner_reference(owners: &[OwnerReference]) -> Option<&OwnerReference> { + owners.iter().find(|owner| owner.controller == Some(true)) +} + +#[allow(clippy::result_large_err)] +fn validate_owner_uid( + actual_uid: &Option, + expected_owner: &OwnerReference, +) -> Result<(), tonic::Status> { + if actual_uid.as_deref() == Some(expected_owner.uid.as_str()) { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "Kubernetes controller owner UID mismatch", + )) + } +} + #[allow(clippy::result_large_err)] fn validate_sandbox_owner_identity( owner: &OwnerReference, @@ -2550,17 +4575,242 @@ fn sandbox_from_object(namespace: &str, obj: DynamicObject) -> Result<(String, S .unwrap_or_else(|| namespace.to_string()); let status = status_from_object(&obj); - Ok(( - kube_name, - Sandbox { - id, - name, - namespace, - spec: None, - status, - workspace, - }, - )) + Ok(( + kube_name, + Sandbox { + id, + name, + namespace, + spec: None, + status, + workspace, + }, + )) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProxyPodControlAvailability { + Available, + Unavailable, + Unknown, +} + +fn proxy_pod_control_availability_from_deployment( + deployment: &Deployment, +) -> ProxyPodControlAvailability { + if deployment + .status + .as_ref() + .and_then(|status| status.available_replicas) + .unwrap_or_default() + > 0 + { + ProxyPodControlAvailability::Available + } else { + ProxyPodControlAvailability::Unavailable + } +} + +async fn proxy_pod_control_availability( + client: &Client, + namespace: &str, + sandbox_id: &str, +) -> ProxyPodControlAvailability { + let names = ProxyPodNames::new(sandbox_id); + let deployments = Api::::namespaced(client.clone(), namespace); + let services = Api::::namespaced(client.clone(), namespace); + let policies = Api::::namespaced(client.clone(), namespace); + let deployment = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + deployments.get_opt(&names.control_deployment), + )); + let service = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + services.get_opt(&names.boundary_service), + )); + let fence = Box::pin(tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.get_opt(&names.workload_policy), + )); + let (deployment, service, fence) = tokio::join!(deployment, service, fence); + let control = match deployment { + Ok(Ok(Some(deployment))) => proxy_pod_control_availability_from_deployment(&deployment), + Ok(Ok(None)) => ProxyPodControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine proxy-pod control availability"); + ProxyPodControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking proxy-pod control availability" + ); + ProxyPodControlAvailability::Unknown + } + }; + let service = match service { + Ok(Ok(Some(_))) => ProxyPodControlAvailability::Available, + Ok(Ok(None)) => ProxyPodControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine proxy-pod boundary Service availability"); + ProxyPodControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking proxy-pod boundary Service availability" + ); + ProxyPodControlAvailability::Unknown + } + }; + + // Deployment availability alone is not sufficient: deletion of the + // unowned fence would otherwise leave a live boundary with direct pod + // egress while the driver continued to publish Ready. + let fence = match fence { + Ok(Ok(Some(_))) => ProxyPodControlAvailability::Available, + Ok(Ok(None)) => ProxyPodControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(sandbox_id, %error, "could not determine proxy-pod workload fence availability"); + ProxyPodControlAvailability::Unknown + } + Err(_) => { + warn!( + sandbox_id, + "timed out checking proxy-pod workload fence availability" + ); + ProxyPodControlAvailability::Unknown + } + }; + if [control, service, fence].contains(&ProxyPodControlAvailability::Unavailable) { + ProxyPodControlAvailability::Unavailable + } else if [control, service, fence].contains(&ProxyPodControlAvailability::Unknown) { + ProxyPodControlAvailability::Unknown + } else { + ProxyPodControlAvailability::Available + } +} + +async fn proxy_pod_workload_generation_matches( + client: &Client, + namespace: &str, + pod_name: &str, + sandbox: &DynamicObject, +) -> ProxyPodControlAvailability { + let Some(expected_uid) = sandbox + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_PROXY_POD_WORKLOAD_UID)) + else { + return ProxyPodControlAvailability::Unavailable; + }; + let pods = Api::::namespaced(client.clone(), namespace); + match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await { + Ok(Ok(Some(pod))) if pod.metadata.uid.as_deref() == Some(expected_uid.as_str()) => { + ProxyPodControlAvailability::Available + } + Ok(Ok(_)) => ProxyPodControlAvailability::Unavailable, + Ok(Err(error)) => { + warn!(pod = pod_name, %error, "could not verify proxy-pod workload generation"); + ProxyPodControlAvailability::Unknown + } + Err(_) => { + warn!( + pod = pod_name, + "timed out checking proxy-pod workload generation" + ); + ProxyPodControlAvailability::Unknown + } + } +} + +async fn sandbox_from_object_with_proxy_pod_readiness( + client: &Client, + namespace: &str, + obj: DynamicObject, +) -> Result<(String, Sandbox), String> { + let bootstrapping = proxy_pod_bootstrap_in_progress(&obj); + let sandbox_id = sandbox_id_from_object(&obj).unwrap_or_default(); + let object_namespace = obj + .metadata + .namespace + .clone() + .unwrap_or_else(|| namespace.to_string()); + let (name, mut sandbox) = sandbox_from_object(namespace, obj.clone())?; + if bootstrapping { + mark_proxy_pod_bootstrapping(&mut sandbox); + } + if !sandbox_id.is_empty() { + let dependencies = Box::pin(proxy_pod_control_availability( + client, + &object_namespace, + &sandbox_id, + )); + let workload_generation = Box::pin(proxy_pod_workload_generation_matches( + client, + &object_namespace, + &name, + &obj, + )); + let (dependencies, workload_generation) = tokio::join!(dependencies, workload_generation); + if dependencies != ProxyPodControlAvailability::Available + || workload_generation != ProxyPodControlAvailability::Available + { + mark_proxy_pod_control_unavailable(&mut sandbox); + } + } + Ok((name, sandbox)) +} + +fn mark_proxy_pod_bootstrapping(sandbox: &mut Sandbox) { + if let Some(status) = sandbox.status.as_mut() { + status + .conditions + .retain(|condition| condition.r#type != SANDBOX_SUSPENDED_CONDITION); + } + mark_proxy_pod_control_unavailable(sandbox); +} + +fn mark_proxy_pod_control_unavailable(sandbox: &mut Sandbox) { + const REASON: &str = "DependenciesNotReady"; + const MESSAGE: &str = "proxy-pod enforcement dependencies are not ready"; + let Some(status) = sandbox.status.as_mut() else { + return; + }; + if let Some(ready) = status + .conditions + .iter_mut() + .find(|condition| condition.r#type == "Ready") + { + ready.status = "False".to_string(); + ready.reason = REASON.to_string(); + ready.message = MESSAGE.to_string(); + } else { + status.conditions.push(SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: REASON.to_string(), + message: MESSAGE.to_string(), + last_transition_time: String::new(), + }); + } +} + +fn desired_proxy_pod_control_replicas(obj: &DynamicObject) -> i32 { + if let Some(mode) = obj + .data + .get("spec") + .and_then(|spec| spec.get("operatingMode")) + .and_then(serde_json::Value::as_str) + { + return i32::from(!mode.eq_ignore_ascii_case("Suspended")); + } + obj.data + .get("spec") + .and_then(|spec| spec.get("replicas")) + .and_then(serde_json::Value::as_i64) + .map_or(1, |replicas| i32::from(replicas > 0)) } fn update_indexes( @@ -2692,705 +4942,240 @@ fn extract_image_size(message: &str) -> Option { rest[..end].parse().ok() } -/// Path where the supervisor binary is mounted inside the agent container. -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; - -/// Name of the volume used to side-load the supervisor binary. -const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; - -/// Name of the init container that installs the supervisor binary. -const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; - -/// Name of the init container that prepares pod-level sidecar networking. -const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; - -/// Container name for the network-only supervisor sidecar. -const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; - -/// UID used by strict process/binary-aware sidecars so Kubernetes grants the -/// requested capability set into the effective set without privilege escalation. -const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; - -/// Shared volume used by the network sidecar and process-only supervisor for -/// local coordination in sidecar topology. -const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; -const SIDECAR_STATE_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_CONTROL_SOCKET: &str = openshell_core::container_paths::SIDECAR_CONTROL_SOCKET; -// Linux abstract socket names are scoped to the pod's shared network namespace. -// Unlike a filesystem socket in the shared state volume, the workload cannot -// unlink and replace this relay endpoint after the trusted supervisor binds it. -const SIDECAR_SSH_SOCKET_FILE: &str = "@openshell-sidecar-ssh"; - -/// Shared TLS work directory. The network sidecar writes the proxy CA bundle -/// here, while the agent container consumes it after sidecar bootstrap. -const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; -const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; - -/// Build the emptyDir volume that holds the supervisor binary. -/// -/// The init container writes the binary here; the agent container reads it. -fn supervisor_volume() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "emptyDir": {} - }) -} - -/// Build the read-only volume mount for the supervisor binary in the agent container. -fn supervisor_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": true - }) -} - -/// Build an image volume that mounts the supervisor OCI image directly. -/// -/// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). -/// The entire image filesystem is mounted read-only, making the binary -/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. -fn supervisor_image_volume( - supervisor_image: &str, - supervisor_image_pull_policy: &str, -) -> serde_json::Value { - let mut image_spec = serde_json::json!({ - "reference": supervisor_image, - }); - if !supervisor_image_pull_policy.is_empty() { - image_spec["pullPolicy"] = serde_json::json!(supervisor_image_pull_policy); - } - serde_json::json!({ - "name": SUPERVISOR_VOLUME_NAME, - "image": image_spec - }) -} - -/// Build the init container that copies the supervisor binary into the emptyDir. -/// -/// The supervisor image contains the supervisor binary at `/openshell-sandbox`. -/// We invoke that binary with the `copy-self` subcommand so it copies itself -/// into the shared emptyDir volume, where the agent container then executes it -/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring -/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's -/// emissary executor. -fn supervisor_init_container( - supervisor_image: &str, - supervisor_image_pull_policy: &str, -) -> serde_json::Value { - let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); - let mut spec = serde_json::json!({ - "name": SUPERVISOR_INIT_CONTAINER_NAME, - "image": supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "copy-self", - installed_path, - ], - "securityContext": {"runAsUser": 0}, - "volumeMounts": [{ - "name": SUPERVISOR_VOLUME_NAME, - "mountPath": SUPERVISOR_MOUNT_PATH, - "readOnly": false - }] - }); - if !supervisor_image_pull_policy.is_empty() { - spec["imagePullPolicy"] = serde_json::json!(supervisor_image_pull_policy); - } - spec -} - -fn apply_supervisor_binary_source( - spec: &mut serde_json::Map, - supervisor_image: &str, - supervisor_image_pull_policy: &str, - method: SupervisorSideloadMethod, -) { - let volumes = spec - .entry("volumes") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - match method { - SupervisorSideloadMethod::ImageVolume => { - volumes.push(supervisor_image_volume( - supervisor_image, - supervisor_image_pull_policy, - )); - } - SupervisorSideloadMethod::InitContainer => { - volumes.push(supervisor_volume()); - } - } - } - - if method == SupervisorSideloadMethod::InitContainer { - let init_containers = spec - .entry("initContainers") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_init_container( - supervisor_image, - supervisor_image_pull_policy, - )); - } - } -} +const SANDBOX_RUNTIME_VOLUME_NAME: &str = "openshell-runtime"; +const SANDBOX_STATE_VOLUME_NAME: &str = "openshell-runtime-state"; +const SANDBOX_BOOTSTRAP_VOLUME_NAME: &str = "openshell-sandbox-bootstrap"; +const SANDBOX_POD_IDENTITY_VOLUME_NAME: &str = "openshell-pod-identity"; +const SANDBOX_RUNTIME_MOUNT_PATH: &str = "/.openshell/runtime"; +const SANDBOX_STATE_MOUNT_PATH: &str = "/.openshell/state"; +const SANDBOX_POD_IDENTITY_MOUNT_PATH: &str = "/.openshell/pod-identity"; +const SANDBOX_POD_UID_PATH: &str = "/.openshell/pod-identity/uid"; +const SANDBOX_PROXY_CA_VOLUME_NAME: &str = "openshell-run"; +const SANDBOX_PROXY_CA_MOUNT_PATH: &str = "/run"; +const SANDBOX_BOOTSTRAP_SCHEDULING_GATE: &str = "openshell.ai/bootstrap"; -/// Apply supervisor side-load transforms to an already-built pod template JSON. +/// Render the workload pod half of the RFC 0012 proxy-pod topology. /// -/// Depending on the sideload method: -/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only -/// volume (no init container needed, requires K8s >= v1.33). -/// - **`InitContainer`**: injects an emptyDir volume and an init container that -/// copies the supervisor binary from the supervisor image into that volume. -/// -/// In both cases, the agent container gets a command override to run the -/// side-loaded binary as root so it can create network namespaces, set up the -/// proxy, and configure Landlock/seccomp. -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload_with_params( +/// The pod receives no gateway credential or endpoint. Its non-root sandbox +/// owns the workload process and seccomp listener; only the paired supervisor +/// Deployment can reach its TLS listener through `NetworkPolicy`. +fn apply_supervisor_proxy_pod_boundary( pod_template: &mut serde_json::Value, params: &SandboxPodParams<'_>, ) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, + let metadata = pod_template + .as_object_mut() + .expect("pod template must be an object") + .entry("metadata") + .or_insert_with(|| serde_json::json!({})); + let labels = metadata + .as_object_mut() + .expect("pod metadata must be an object") + .entry("labels") + .or_insert_with(|| serde_json::json!({})); + let labels = labels + .as_object_mut() + .expect("pod labels must be an object"); + labels.insert( + BOUNDARY_PAIR_LABEL.to_string(), + serde_json::json!(crate::proxy_pod::pair_label_value(params.sandbox_id)), + ); + labels.insert( + BOUNDARY_ROLE_LABEL.to_string(), + serde_json::json!("workload"), ); - // Find the agent container and add volume mount + command override - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + let Some(spec) = pod_template + .get_mut("spec") + .and_then(serde_json::Value::as_object_mut) + else { return; }; - - let mut target_index = None; - for (i, c) in containers.iter().enumerate() { - if c.get("name").and_then(|v| v.as_str()) == Some("agent") { - target_index = Some(i); - break; - } - } - let index = target_index.unwrap_or(0); - - if let Some(container) = containers.get_mut(index).and_then(|v| v.as_object_mut()) { - // Override command to use the side-loaded supervisor binary - let mut command = vec![ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--workdir".to_string(), - driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), - ]; - command.extend(upstream_proxy_cli_args(params)); - container.insert("command".to_string(), serde_json::json!(command)); - - // Force the supervisor to run as root (UID 0). Sandbox images may set - // a non-root USER directive (e.g. `USER sandbox`), but the supervisor - // needs root to create network namespaces, set up the proxy, and - // configure Landlock/seccomp. The supervisor itself drops privileges - // for child processes via the policy's `run_as_user`/`run_as_group`. - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert("runAsUser".to_string(), serde_json::json!(0)); - } - - // Add volume mount - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(supervisor_volume_mount()); - } - - // Inject the protected resolved identity contract. Clearing the OCI - // input prevents image or user environment from selecting a - // conflicting identity path. - let env = container - .entry("env") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(env) = env { - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); - } - if has_upstream_proxy_credentials(params) { - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - volume_mounts.push(upstream_proxy_auth_volume_mount()); - } - } - } -} - -#[cfg(test)] -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload( - pod_template: &mut serde_json::Value, - supervisor_image: &str, - supervisor_image_pull_policy: &str, - method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, -) { - let params = SandboxPodParams { - supervisor_image, - supervisor_image_pull_policy, - supervisor_sideload_method: method, - sandbox_uid, - sandbox_gid, - ..SandboxPodParams::default() - }; - apply_supervisor_sideload_with_params(pod_template, ¶ms); -} - -fn upstream_proxy_cli_args(params: &SandboxPodParams<'_>) -> Vec { - let mut args = Vec::new(); - if let Some(url) = params.https_proxy { - args.extend(["--upstream-proxy".to_string(), url.to_string()]); - } - if let Some(list) = params.no_proxy { - args.extend(["--upstream-no-proxy".to_string(), list.to_string()]); - } - if has_upstream_proxy_credentials(params) { - args.extend([ - "--upstream-proxy-auth-file".to_string(), - openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), - ]); - } - if params.proxy_auth_allow_insecure { - args.push("--upstream-proxy-auth-allow-insecure".to_string()); - } - if params.proxy_connect_by_hostname { - args.push("--upstream-proxy-connect-by-hostname".to_string()); - } - args -} - -fn upstream_proxy_auth_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "mountPath": upstream_proxy_auth_volume_mount_path(), - "readOnly": true, - }) -} - -fn upstream_proxy_auth_volume_mount_path() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .parent() - .and_then(Path::to_str) - .expect("upstream proxy auth path has a parent directory") -} - -fn upstream_proxy_auth_file_name() -> &'static str { - Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH) - .file_name() - .and_then(|name| name.to_str()) - .expect("upstream proxy auth path has a UTF-8 file name") -} - -fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { - params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() -} - -fn sidecar_state_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "mountPath": SIDECAR_STATE_MOUNT_PATH, - }) -} - -fn sidecar_tls_volume_mount() -> serde_json::Value { - serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "mountPath": SIDECAR_TLS_MOUNT_PATH, - }) -} - -fn copy_log_level_env( - env: &mut Vec, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, -) { - if let Some(value) = spec_environment - .get(openshell_core::sandbox_env::LOG_LEVEL) - .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) - { - upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); - } -} - -fn supervisor_sidecar_env( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) -> Vec { - let mut env = Vec::new(); - apply_required_env( - &mut env, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - "", - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), - ); - if !params.client_tls_secret_name.is_empty() { - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CA, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_CERT, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::TLS_KEY, - &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), - ); - } - copy_log_level_env(&mut env, template_environment, spec_environment); - upsert_env( - &mut env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, - ); - upsert_env( - &mut env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, + spec.insert("hostNetwork".to_string(), serde_json::json!(false)); + spec.insert("hostPID".to_string(), serde_json::json!(false)); + spec.insert("hostIPC".to_string(), serde_json::json!(false)); + spec.insert( + "shareProcessNamespace".to_string(), + serde_json::json!(false), ); - upsert_env( - &mut env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, + spec.insert("dnsPolicy".to_string(), serde_json::json!("None")); + spec.insert( + "schedulingGates".to_string(), + serde_json::json!([{"name": SANDBOX_BOOTSTRAP_SCHEDULING_GATE}]), ); - apply_resolved_identity_env(&mut env, params.sandbox_uid, params.sandbox_gid); - if !params.process_binary_aware_network_policy { - upsert_env( - &mut env, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, - "relaxed", - ); - } - env -} - -fn supervisor_sidecar_container( - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let capabilities = if params.process_binary_aware_network_policy { + spec.insert( + "dnsConfig".to_string(), serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - } else { + "nameservers": ["127.0.0.53"], + "options": [ + {"name": "ndots", "value": "5"}, + {"name": "timeout", "value": "2"}, + {"name": "attempts", "value": "2"} + ] + }), + ); + spec.insert( + "securityContext".to_string(), serde_json::json!({ - "drop": ["ALL"] - }) - }; - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_SIDECAR_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network", - ], - "env": supervisor_sidecar_env(template_environment, spec_environment, params), - "securityContext": { - "runAsUser": proxy_uid, + "runAsUser": params.sandbox_uid, "runAsGroup": params.sandbox_gid, - "runAsNonRoot": proxy_uid != 0, - "allowPrivilegeEscalation": false, - "capabilities": capabilities - }, - "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), - { - "name": "openshell-sa-token", - "mountPath": "/var/run/secrets/openshell", - "readOnly": true - } - ] - }); - container["command"] - .as_array_mut() - .expect("network supervisor command is an array") - .extend( - upstream_proxy_cli_args(params) - .into_iter() - .map(serde_json::Value::String), - ); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); - } - if params.provider_spiffe_enabled { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - if has_upstream_proxy_credentials(params) { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(upstream_proxy_auth_volume_mount()); - } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container -} - -fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { - if params.process_binary_aware_network_policy { - BINARY_AWARE_SIDECAR_PROXY_UID - } else { - params.proxy_uid - } -} - -fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { - let proxy_uid = effective_sidecar_proxy_uid(params); - let mut container = serde_json::json!({ - "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, - "image": params.supervisor_image, - "command": [ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - proxy_uid.to_string(), - "--proxy-gid", - params.sandbox_gid.to_string(), - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH, - ], - "securityContext": { - "runAsUser": 0, - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - } - }, - "volumeMounts": [ - sidecar_state_volume_mount(), - sidecar_tls_volume_mount(), - ] - }); - if !params.supervisor_image_pull_policy.is_empty() { - container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); - } - if !params.client_tls_secret_name.is_empty() { - container["volumeMounts"] - .as_array_mut() - .expect("volumeMounts is an array") - .push(serde_json::json!({ - "name": "openshell-client-tls", - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); - } - if let Some(profile) = params.app_armor_profile { - container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container -} - -fn apply_supervisor_sidecar_topology( - pod_template: &mut serde_json::Value, - template_environment: &std::collections::HashMap, - spec_environment: &std::collections::HashMap, - params: &SandboxPodParams<'_>, -) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - let pod_security_context = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = pod_security_context.as_object_mut() { - sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); - } - - spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); - - apply_supervisor_binary_source( - spec, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, + "runAsNonRoot": true, + "fsGroup": params.sandbox_gid, + "fsGroupChangePolicy": "OnRootMismatch", + "supplementalGroups": [], + "supplementalGroupsPolicy": "Strict", + "seccompProfile": {"type": "RuntimeDefault"}, + "sysctls": [{"name": "net.ipv4.ip_unprivileged_port_start", "value": "0"}] + }), ); - let volumes = spec .entry("volumes") .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volumes) = volumes { - volumes.push(serde_json::json!({ - "name": SIDECAR_STATE_VOLUME_NAME, - "emptyDir": {} - })); - volumes.push(serde_json::json!({ - "name": SIDECAR_TLS_VOLUME_NAME, - "emptyDir": {} - })); - } + .as_array_mut() + .expect("pod volumes must be an array"); + volumes.retain(|volume| { + !matches!( + volume.get("name").and_then(serde_json::Value::as_str), + Some( + "openshell-sa-token" + | "openshell-client-tls" + | "spiffe-workload-api" + | SANDBOX_RUNTIME_VOLUME_NAME + | SANDBOX_STATE_VOLUME_NAME + | SANDBOX_BOOTSTRAP_VOLUME_NAME + | SANDBOX_POD_IDENTITY_VOLUME_NAME + | SANDBOX_PROXY_CA_VOLUME_NAME + ) + ) + }); + volumes.extend([ + serde_json::json!({"name": SANDBOX_RUNTIME_VOLUME_NAME, "emptyDir": {"medium": "Memory"}}), + serde_json::json!({"name": SANDBOX_STATE_VOLUME_NAME, "emptyDir": {"medium": "Memory"}}), + serde_json::json!({ + "name": SANDBOX_BOOTSTRAP_VOLUME_NAME, + "secret": {"secretName": params.sandbox_secret_name, "defaultMode": 0o440} + }), + serde_json::json!({ + "name": SANDBOX_POD_IDENTITY_VOLUME_NAME, + "downwardAPI": { + "items": [{"path": "uid", "fieldRef": {"fieldPath": "metadata.uid"}}] + } + }), + serde_json::json!({ + "name": SANDBOX_PROXY_CA_VOLUME_NAME, + "emptyDir": {"medium": "Memory"} + }), + ]); let init_containers = spec .entry("initContainers") .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(init_containers) = init_containers { - init_containers.push(supervisor_network_init_container(params)); + .as_array_mut() + .expect("pod init containers must be an array"); + let mut bootstrap = serde_json::json!({ + "name": "openshell-sandbox-bootstrap", + "image": params.supervisor_image, + "command": ["/openshell-sandbox", "bootstrap"], + "securityContext": { + "runAsUser": params.sandbox_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": [ + {"name": SANDBOX_BOOTSTRAP_VOLUME_NAME, "mountPath": crate::proxy_pod::SANDBOX_BOOTSTRAP_INPUT_PATH, "readOnly": true}, + {"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH}, + {"name": SANDBOX_STATE_VOLUME_NAME, "mountPath": SANDBOX_STATE_MOUNT_PATH} + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + bootstrap["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); } + init_containers.push(bootstrap); - let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { - return; - }; - - let target_index = containers + let containers = spec + .get_mut("containers") + .and_then(serde_json::Value::as_array_mut) + .expect("pod containers must be an array"); + let index = containers .iter() - .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .position(|container| { + container.get("name").and_then(serde_json::Value::as_str) == Some("agent") + }) .unwrap_or(0); - - if let Some(container) = containers - .get_mut(target_index) - .and_then(|v| v.as_object_mut()) - { - container.insert( - "command".to_string(), - serde_json::json!([ - format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]), - ); - - let security_context = container - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(sc) = security_context.as_object_mut() { - sc.insert( - "runAsUser".to_string(), - serde_json::json!(params.sandbox_uid), - ); - sc.insert( - "runAsGroup".to_string(), - serde_json::json!(params.sandbox_gid), - ); - sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); - sc.insert( - "allowPrivilegeEscalation".to_string(), - serde_json::json!(false), - ); - sc.insert( - "capabilities".to_string(), - serde_json::json!({ - "drop": ["ALL"] - }), - ); - } - - let volume_mounts = container - .entry("volumeMounts") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(volume_mounts) = volume_mounts { - remove_volume_mount(volume_mounts, "openshell-sa-token"); - remove_volume_mount(volume_mounts, "openshell-client-tls"); - remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); - volume_mounts.push(supervisor_volume_mount()); - volume_mounts.push(sidecar_state_volume_mount()); - volume_mounts.push(sidecar_tls_volume_mount()); - } - - let env = container - .entry("env") - .or_insert_with(|| serde_json::json!([])) - .as_array_mut(); - if let Some(env) = env { - remove_env(env, openshell_core::sandbox_env::ENDPOINT); - remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - remove_env(env, openshell_core::sandbox_env::TLS_CA); - remove_env(env, openshell_core::sandbox_env::TLS_CERT); - remove_env(env, openshell_core::sandbox_env::TLS_KEY); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); - remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); - remove_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, - "sidecar", - ); - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, - "sidecar-nftables", - ); - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - SIDECAR_SSH_SOCKET_FILE, - ); - upsert_env( - env, - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, - SIDECAR_CONTROL_SOCKET, - ); - upsert_env( - env, - openshell_core::sandbox_env::PROXY_TLS_DIR, - SIDECAR_TLS_MOUNT_PATH, - ); - apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); - } + let container = containers[index] + .as_object_mut() + .expect("agent container must be an object"); + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "--bootstrap", + BOUNDARY_CONFIG_PATH, + ]), + ); + container.insert( + "securityContext".to_string(), + serde_json::json!({ + "runAsUser": params.sandbox_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }), + ); + container.insert( + "ports".to_string(), + serde_json::json!([{ + "name": "sandbox-control", + "containerPort": params.boundary_port, + "protocol": "TCP" + }]), + ); + let mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut() + .expect("agent volume mounts must be an array"); + mounts.retain(|mount| { + !matches!( + mount.get("name").and_then(serde_json::Value::as_str), + Some( + "openshell-sa-token" + | "openshell-client-tls" + | "spiffe-workload-api" + | SANDBOX_BOOTSTRAP_VOLUME_NAME + ) + ) + }); + mounts.extend([ + serde_json::json!({"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH, "readOnly": true}), + serde_json::json!({"name": SANDBOX_STATE_VOLUME_NAME, "mountPath": SANDBOX_STATE_MOUNT_PATH}), + serde_json::json!({"name": SANDBOX_POD_IDENTITY_VOLUME_NAME, "mountPath": SANDBOX_POD_IDENTITY_MOUNT_PATH, "readOnly": true}), + serde_json::json!({"name": SANDBOX_PROXY_CA_VOLUME_NAME, "mountPath": SANDBOX_PROXY_CA_MOUNT_PATH}), + ]); + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut() + .expect("agent environment must be an array"); + for key in [ + openshell_core::sandbox_env::ENDPOINT, + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, + openshell_core::sandbox_env::SANDBOX_TOKEN, + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + openshell_core::sandbox_env::MAIN_PROCESS_SPEC, + ] { + remove_env(env, key); } - - containers.push(supervisor_sidecar_container( - template_environment, - spec_environment, - params, - )); + apply_resolved_identity_env(env, params.sandbox_uid, params.sandbox_gid); } /// Apply workspace persistence transforms to an already-built pod template. @@ -3413,7 +5198,8 @@ fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, image_pull_policy: &str, - sandbox_gid: u32, + sandbox_gid: Option, + workspace_owner: Option<(u32, u32)>, ) { let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { return; @@ -3421,11 +5207,13 @@ fn apply_workspace_persistence( // fsGroup is a pod-level field — it instructs kubelet to chown mounted // volumes to this GID. It is invalid at the container securityContext level. - let pod_sc = spec - .entry("securityContext") - .or_insert_with(|| serde_json::json!({})); - if let Some(pod_sc_obj) = pod_sc.as_object_mut() { - pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + if let Some(sandbox_gid) = sandbox_gid { + let pod_sc = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(pod_sc_obj) = pod_sc.as_object_mut() { + pod_sc_obj.insert("fsGroup".to_string(), serde_json::json!(sandbox_gid)); + } } // 1. Add workspace volume mount to the agent container @@ -3490,18 +5278,36 @@ fn apply_workspace_persistence( fi" ); - let mut init_spec = serde_json::json!({ - "name": WORKSPACE_INIT_CONTAINER_NAME, - "image": image, - "command": ["sh", "-c", copy_cmd], - "securityContext": { - "runAsUser": 0, - }, - "volumeMounts": [{ - "name": WORKSPACE_VOLUME_NAME, - "mountPath": WORKSPACE_INIT_MOUNT_PATH - }] - }); + let mut init_spec = if let Some((uid, gid)) = workspace_owner { + serde_json::json!({ + "name": WORKSPACE_INIT_CONTAINER_NAME, + "image": image, + "command": [format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), "seed-workspace"], + "securityContext": { + "runAsUser": uid, + "runAsGroup": gid, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": [ + {"name": WORKSPACE_VOLUME_NAME, "mountPath": WORKSPACE_INIT_MOUNT_PATH}, + {"name": SANDBOX_RUNTIME_VOLUME_NAME, "mountPath": SANDBOX_RUNTIME_MOUNT_PATH, "readOnly": true} + ] + }) + } else { + serde_json::json!({ + "name": WORKSPACE_INIT_CONTAINER_NAME, + "image": image, + "command": ["sh", "-c", copy_cmd], + "securityContext": {"runAsUser": 0}, + "volumeMounts": [{ + "name": WORKSPACE_VOLUME_NAME, + "mountPath": WORKSPACE_INIT_MOUNT_PATH + }] + }) + }; if !image_pull_policy.is_empty() { init_spec["imagePullPolicy"] = serde_json::json!(image_pull_policy); } @@ -3554,37 +5360,20 @@ struct SandboxPodParams<'a> { image_pull_secrets: &'a [String], supervisor_image: &'a str, supervisor_image_pull_policy: &'a str, - supervisor_sideload_method: SupervisorSideloadMethod, - topology: SupervisorTopology, - proxy_uid: u32, - process_binary_aware_network_policy: bool, - https_proxy: Option<&'a str>, - no_proxy: Option<&'a str>, - proxy_auth_secret_name: Option<&'a str>, - proxy_auth_secret_key: Option<&'a str>, - proxy_auth_allow_insecure: bool, - proxy_connect_by_hostname: bool, service_account_name: &'a str, sandbox_id: &'a str, - sandbox_name: &'a str, - grpc_endpoint: &'a str, - ssh_socket_path: &'a str, - client_tls_secret_name: &'a str, - host_gateway_ip: &'a str, enable_user_namespaces: bool, - app_armor_profile: Option<&'a AppArmorProfile>, workspace_default_storage_size: &'a str, workspace_storage_class: &'a str, default_runtime_class_name: &'a str, - /// Lifetime (seconds) of the projected `ServiceAccount` token used - /// for the bootstrap `IssueSandboxToken` exchange. - sa_token_ttl_secs: i64, - provider_spiffe_enabled: bool, - provider_spiffe_workload_api_socket_path: &'a str, /// Resolved sandbox UID for supervisor `runAsUser` and env var. sandbox_uid: u32, /// Resolved sandbox GID for PVC init container operations. sandbox_gid: u32, + /// TLS listener port exposed only to the paired supervisor Pod. + boundary_port: u16, + /// Immutable Secret name for this workload Pod generation. + sandbox_secret_name: &'a str, } impl Default for SandboxPodParams<'_> { @@ -3595,49 +5384,20 @@ impl Default for SandboxPodParams<'_> { image_pull_secrets: &[], supervisor_image: "", supervisor_image_pull_policy: "", - supervisor_sideload_method: SupervisorSideloadMethod::default(), - topology: SupervisorTopology::default(), - proxy_uid: DEFAULT_PROXY_UID, - process_binary_aware_network_policy: true, - https_proxy: None, - no_proxy: None, - proxy_auth_secret_name: None, - proxy_auth_secret_key: None, - proxy_auth_allow_insecure: false, - proxy_connect_by_hostname: false, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", - sandbox_name: "", - grpc_endpoint: "", - ssh_socket_path: "", - client_tls_secret_name: "", - host_gateway_ip: "", enable_user_namespaces: false, - app_armor_profile: None, workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE, workspace_storage_class: "", default_runtime_class_name: "", - sa_token_ttl_secs: 3600, - provider_spiffe_enabled: false, - provider_spiffe_workload_api_socket_path: "", sandbox_uid: DEFAULT_SANDBOX_UID, sandbox_gid: DEFAULT_SANDBOX_UID, + boundary_port: 5500, + sandbox_secret_name: "os-sandbox-test-generation", } } } -fn validate_sidecar_proxy_identity( - params: &SandboxPodParams<'_>, -) -> Result<(), KubernetesDriverError> { - if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { - return Err(KubernetesDriverError::Precondition(format!( - "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", - params.proxy_uid, params.sandbox_uid - ))); - } - Ok(()) -} - fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { @@ -3651,22 +5411,15 @@ fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap, - provider_spiffe_workload_api_socket_path: Option<&str>, ) -> Result { let config = spec .and_then(|spec| spec.template.as_ref()) .map(KubernetesSandboxDriverConfig::from_template) .transpose()? .unwrap_or_default(); - let mut protected_paths = KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS.to_vec(); - let provider_spiffe_mount_path; - if let Some(socket_path) = provider_spiffe_workload_api_socket_path { - provider_spiffe_mount_path = spiffe_socket_mount_path(socket_path); - protected_paths.push(&provider_spiffe_mount_path); - } validate_kubernetes_protected_path_conflicts( &config.containers.agent.volume_mounts, - &protected_paths, + KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS, )?; Ok(config) } @@ -3675,8 +5428,7 @@ fn sandbox_to_k8s_spec( spec: Option<&SandboxSpec>, params: &SandboxPodParams<'_>, ) -> Result { - let driver_config = - kubernetes_driver_config_for_spec(spec, provider_spiffe_socket_path(params))?; + let driver_config = kubernetes_driver_config_for_spec(spec)?; let mut root = serde_json::Map::new(); // Determine early whether OpenShell should inject its default workspace @@ -3696,7 +5448,6 @@ fn sandbox_to_k8s_spec( template, driver_gpu_requirements(spec.resource_requirements.as_ref()), &pod_env, - Some(spec), &driver_config, inject_workspace, params, @@ -3730,7 +5481,6 @@ fn sandbox_to_k8s_spec( &SandboxTemplate::default(), driver_gpu_requirements(spec.and_then(|s| s.resource_requirements.as_ref())), &pod_env, - spec, &driver_config, inject_workspace, params, @@ -3758,7 +5508,6 @@ fn sandbox_template_to_k8s( template, gpu_requirements.as_ref(), spec_environment, - None, &driver_config, inject_workspace, params, @@ -3779,7 +5528,6 @@ fn sandbox_template_to_k8s_with_gpu_requirements( template, gpu_requirements, spec_environment, - None, &driver_config, inject_workspace, params, @@ -3790,29 +5538,16 @@ fn sandbox_template_to_k8s_with_validated_config( template: &SandboxTemplate, gpu_requirements: Option<&GpuResourceRequirements>, spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, driver_config: &KubernetesSandboxDriverConfig, inject_workspace: bool, params: &SandboxPodParams<'_>, ) -> serde_json::Value { let mut metadata = serde_json::Map::new(); - let mut pod_labels = template + let pod_labels = template .labels .iter() .map(|(key, value)| (key.clone(), serde_json::Value::String(value.clone()))) .collect::>(); - if params.provider_spiffe_enabled { - pod_labels.insert( - LABEL_MANAGED_BY.to_string(), - serde_json::Value::String(LABEL_MANAGED_BY_VALUE.to_string()), - ); - if !params.sandbox_id.is_empty() { - pod_labels.insert( - LABEL_SANDBOX_ID.to_string(), - serde_json::Value::String(params.sandbox_id.to_string()), - ); - } - } if !pod_labels.is_empty() { metadata.insert("labels".to_string(), serde_json::Value::Object(pod_labels)); } @@ -3927,71 +5662,17 @@ fn sandbox_template_to_k8s_with_validated_config( } } - // Build environment variables - start with OpenShell-required vars - let env = build_env_list( - None, - &template.environment, - spec_environment, - sandbox_spec, - params.sandbox_id, - params.sandbox_name, - params.grpc_endpoint, - params.ssh_socket_path, - !params.client_tls_secret_name.is_empty(), - provider_spiffe_socket_path(params), - ); + let env = build_sandbox_env(&template.environment, spec_environment); container.insert("env".to_string(), serde_json::Value::Array(env)); - let mut capabilities: Vec<&str> = vec!["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"]; - if use_user_namespaces { - // In a user namespace the bounding set is reset. SETUID/SETGID are - // needed for the supervisor to drop privileges to the sandbox user. - // DAC_READ_SEARCH is needed for cross-UID /proc//fd/ access - // for process identity resolution in network policy enforcement. - capabilities.extend(["SETUID", "SETGID", "DAC_READ_SEARCH"]); - } - let mut security_context = serde_json::json!({ - "capabilities": { - "add": capabilities - } - }); - if let Some(profile) = params.app_armor_profile { - security_context["appArmorProfile"] = app_armor_profile_to_k8s(profile); - } - container.insert("securityContext".to_string(), security_context); - - // Mount client TLS secret for mTLS to the server. Gateway identity uses - // the projected ServiceAccount bootstrap token. Provider token grants may - // additionally mount the SPIFFE Workload API socket. - let mut volume_mounts: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - volume_mounts.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "mountPath": openshell_core::container_paths::CLIENT_TLS_DIR, - "readOnly": true - })); - } - if params.provider_spiffe_enabled { - volume_mounts.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), - "readOnly": true, - })); - } - volume_mounts.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "mountPath": SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, - "readOnly": true, - })); - volume_mounts.extend( - driver_config - .containers - .agent - .volume_mounts - .iter() - .map(kubernetes_driver_volume_mount_to_k8s), - ); + let volume_mounts = driver_config + .containers + .agent + .volume_mounts + .iter() + .map(kubernetes_driver_volume_mount_to_k8s) + .collect::>(); container.insert( "volumeMounts".to_string(), serde_json::Value::Array(volume_mounts), @@ -4006,100 +5687,13 @@ fn sandbox_template_to_k8s_with_validated_config( serde_json::Value::Array(vec![serde_json::Value::Object(container)]), ); - // Add TLS secret volume. Combined mode uses mode 0400 because the - // supervisor starts as root and drops privileges before running workload - // children. Sidecar mode keeps the process supervisor non-root, so it uses - // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. - let mut volumes: Vec = Vec::new(); - if !params.client_tls_secret_name.is_empty() { - let client_tls_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": CLIENT_TLS_VOLUME_NAME, - "secret": { - "secretName": params.client_tls_secret_name, - "defaultMode": client_tls_default_mode - } - })); - } - if has_upstream_proxy_credentials(params) { - let secret_name = params - .proxy_auth_secret_name - .expect("complete proxy credential reference has a Secret name"); - let secret_key = params - .proxy_auth_secret_key - .expect("complete proxy credential reference has a Secret key"); - // The credential volume is mounted only into the container that runs - // network supervision. Sidecar mode uses the pod fsGroup already - // required for its non-root network supervisor. - let default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": UPSTREAM_PROXY_AUTH_VOLUME_NAME, - "secret": { - "secretName": secret_name, - "defaultMode": default_mode, - "items": [{ - "key": secret_key, - "path": upstream_proxy_auth_file_name(), - }] - } - })); - } - if params.provider_spiffe_enabled { - volumes.push(serde_json::json!({ - "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, - "csi": { - "driver": "csi.spiffe.io", - "readOnly": true - } - })); - } - // Projected ServiceAccountToken volume — kubelet writes a short-lived - // audience-bound JWT into /var/run/secrets/openshell/token and rotates - // it automatically. The supervisor exchanges this for a gateway-minted - // JWT via `IssueSandboxToken` once at startup. In sidecar topology both - // supervisor containers run with the sandbox GID and need group-read access. - let sa_token_default_mode = match params.topology { - SupervisorTopology::Combined => 0o400, - SupervisorTopology::Sidecar => 0o440, - }; - volumes.push(serde_json::json!({ - "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, - "projected": { - "sources": [{ - "serviceAccountToken": { - "audience": "openshell-gateway", - "expirationSeconds": params.sa_token_ttl_secs, - "path": "token" - } - }], - "defaultMode": sa_token_default_mode - } - })); - volumes.extend( - driver_config - .volumes - .iter() - .map(kubernetes_driver_volume_to_k8s), - ); + let volumes = driver_config + .volumes + .iter() + .map(kubernetes_driver_volume_to_k8s) + .collect::>(); spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); - // Add hostAliases so sandbox pods can reach the Docker host. - if !params.host_gateway_ip.is_empty() { - spec.insert( - "hostAliases".to_string(), - serde_json::json!([{ - "ip": params.host_gateway_ip, - "hostnames": ["host.docker.internal", "host.openshell.internal"] - }]), - ); - } - let mut template_value = serde_json::Map::new(); if !metadata.is_empty() { template_value.insert("metadata".to_string(), serde_json::Value::Object(metadata)); @@ -4108,19 +5702,7 @@ fn sandbox_template_to_k8s_with_validated_config( let mut result = serde_json::Value::Object(template_value); - match params.topology { - SupervisorTopology::Combined => { - apply_supervisor_sideload_with_params(&mut result, params); - } - SupervisorTopology::Sidecar => { - apply_supervisor_sidecar_topology( - &mut result, - &template.environment, - spec_environment, - params, - ); - } - } + apply_supervisor_proxy_pod_boundary(&mut result, params); // Inject workspace persistence (init container + PVC volume mount) so // that /sandbox data survives pod rescheduling. Skipped when the user @@ -4130,7 +5712,8 @@ fn sandbox_template_to_k8s_with_validated_config( &mut result, image, params.image_pull_policy, - params.sandbox_gid, + None, + Some((params.sandbox_uid, params.sandbox_gid)), ); } @@ -4223,16 +5806,6 @@ fn image_pull_secret_refs(secrets: &[String]) -> Vec { .collect() } -fn app_armor_profile_to_k8s(profile: &AppArmorProfile) -> serde_json::Value { - let mut value = serde_json::json!({ - "type": profile.to_k8s_type() - }); - if let Some(localhost_profile) = profile.localhost_profile() { - value["localhostProfile"] = serde_json::json!(localhost_profile); - } - value -} - fn container_resources( template: &SandboxTemplate, gpu_requirements: Option<&GpuResourceRequirements>, @@ -4297,144 +5870,22 @@ fn apply_gpu_limit(resources: &mut serde_json::Value, quantity: &str) { limits_obj.insert(GPU_RESOURCE_NAME.to_string(), serde_json::json!(quantity)); } -#[allow(clippy::too_many_arguments)] -fn build_env_list( - existing_env: Option<&Vec>, +fn build_sandbox_env( template_environment: &std::collections::HashMap, spec_environment: &std::collections::HashMap, - sandbox_spec: Option<&openshell_core::proto::compute::v1::DriverSandboxSpec>, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, ) -> Vec { - let mut env = existing_env.cloned().unwrap_or_default(); - apply_env_map(&mut env, template_environment); - apply_env_map(&mut env, spec_environment); - let mut user_env = template_environment.clone(); - user_env.extend(spec_environment.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - upsert_env( - &mut env, - openshell_core::sandbox_env::USER_ENVIRONMENT, - &json, - ); + let mut env = Vec::new(); + for (name, value) in template_environment.iter().chain(spec_environment) { + if !name.starts_with("OPENSHELL_") || name == openshell_core::sandbox_env::LOG_LEVEL { + upsert_env(&mut env, name, value); + } } - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox_spec) - .expect("main process config serialization cannot fail"); upsert_env( &mut env, - openshell_core::sandbox_env::MAIN_PROCESS_SPEC, - &main_process, - ); - apply_required_env( - &mut env, - sandbox_id, - sandbox_name, - grpc_endpoint, - ssh_socket_path, - tls_enabled, - provider_spiffe_socket_path, - ); - env -} - -fn apply_env_map( - env: &mut Vec, - values: &std::collections::HashMap, -) { - for (key, value) in values { - upsert_env(env, key, value); - } -} - -// Required env vars are passed individually for clarity at call sites; grouping into a struct -// would not improve readability for this internal helper. -fn apply_required_env( - env: &mut Vec, - sandbox_id: &str, - sandbox_name: &str, - grpc_endpoint: &str, - ssh_socket_path: &str, - tls_enabled: bool, - provider_spiffe_socket_path: Option<&str>, -) { - upsert_env(env, openshell_core::sandbox_env::SANDBOX_ID, sandbox_id); - upsert_env(env, openshell_core::sandbox_env::SANDBOX, sandbox_name); - upsert_env(env, openshell_core::sandbox_env::ENDPOINT, grpc_endpoint); - upsert_env( - env, openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), ); - // Runtime capabilities are driver-owned. Kubernetes topologies do not yet - // provide the complete policy DNS and transparent TCP substrate. - upsert_env( - env, - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - "", - ); - if !ssh_socket_path.is_empty() { - upsert_env( - env, - openshell_core::sandbox_env::SSH_SOCKET_PATH, - ssh_socket_path, - ); - } - // TLS cert paths for sandbox-to-server mTLS. Only set when TLS is enabled - // and the client TLS secret is mounted into the sandbox pod. - if tls_enabled { - upsert_env( - env, - openshell_core::sandbox_env::TLS_CA, - "/etc/openshell-tls/client/ca.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_CERT, - "/etc/openshell-tls/client/tls.crt", - ); - upsert_env( - env, - openshell_core::sandbox_env::TLS_KEY, - "/etc/openshell-tls/client/tls.key", - ); - } - // Projected ServiceAccount token written by kubelet (see the volume - // definition in `sandbox_template_to_k8s`). The supervisor reads this - // and exchanges it for a gateway-minted JWT via `IssueSandboxToken`. - upsert_env( - env, - openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, - "/var/run/secrets/openshell/token", - ); - if let Some(socket_path) = provider_spiffe_socket_path { - upsert_env( - env, - openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - socket_path, - ); - } -} - -fn provider_spiffe_socket_path<'a>(params: &'a SandboxPodParams<'a>) -> Option<&'a str> { - params - .provider_spiffe_enabled - .then_some(params.provider_spiffe_workload_api_socket_path) -} - -fn spiffe_socket_mount_path(socket_path: &str) -> String { - Path::new(socket_path) - .parent() - .and_then(Path::to_str) - .filter(|path| !path.is_empty() && *path != "/") - .expect("provider SPIFFE socket path should be validated before pod rendering") - .to_string() + env } fn upsert_env(env: &mut Vec, name: &str, value: &str) { @@ -4470,10 +5921,6 @@ fn remove_env(env: &mut Vec, name: &str) { env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); } -fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { - volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); -} - /// Extract a string value from the template's `platform_config` Struct. fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { let config = template.platform_config.as_ref()?; @@ -4508,6 +5955,23 @@ fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option std::collections::HashMap { + object + .data + .pointer("/spec/podTemplate/spec/containers/0/env") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(|entry| { + let name = entry.get("name")?.as_str()?; + let value = entry.get("value")?.as_str()?; + (!name.starts_with("OPENSHELL_")).then(|| (name.to_string(), value.to_string())) + }) + .collect() +} + fn status_from_object(obj: &DynamicObject) -> Option { let status = obj.data.get("status")?; let status_obj = status.as_object()?; @@ -4550,6 +6014,100 @@ fn status_from_object(obj: &DynamicObject) -> Option { }) } +async fn create_or_validate_proxy_pod_fence( + policies: &Api, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + match tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(name)).await { + Ok(Ok(Some(existing))) => { + return validate_proxy_pod_fence(&existing, expected); + } + Ok(Ok(None)) => {} + Ok(Err(error)) => return Err(KubernetesDriverError::from_kube(error)), + Err(_) => { + return Err(KubernetesDriverError::Message( + "timed out reading proxy-pod workload fence".to_string(), + )); + } + } + + match tokio::time::timeout( + KUBE_API_TIMEOUT, + policies.create(&PostParams::default(), expected), + ) + .await + { + Ok(Ok(_)) => Ok(()), + Ok(Err(KubeError::Api(error))) if error.code == 409 => { + let existing = tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(name)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out validating existing proxy-pod workload fence".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_proxy_pod_fence(&existing, expected) + } + Ok(Err(error)) => Err(KubernetesDriverError::from_kube(error)), + Err(_) => Err(KubernetesDriverError::Message( + "timed out creating proxy-pod workload fence".to_string(), + )), + } +} + +fn validate_proxy_pod_fence( + existing: &NetworkPolicy, + expected: &NetworkPolicy, +) -> Result<(), KubernetesDriverError> { + if proxy_pod_fence_matches(existing, expected) { + Ok(()) + } else { + let name = expected.metadata.name.as_deref().unwrap_or_default(); + Err(KubernetesDriverError::Precondition(format!( + "proxy-pod workload fence {name} exists but does not match the intended enforcement" + ))) + } +} + +fn proxy_pod_fence_matches(existing: &NetworkPolicy, expected: &NetworkPolicy) -> bool { + fn normalized_spec(mut spec: Option) -> Option { + if let Some(spec) = spec.as_mut() { + // The Kubernetes API server omits explicitly empty rule arrays when it + // persists a NetworkPolicy. For a policy type named in `policyTypes`, + // an omitted rule array and an empty rule array both deny all traffic. + if spec.egress.as_ref().is_some_and(Vec::is_empty) { + spec.egress = None; + } + if spec.ingress.as_ref().is_some_and(Vec::is_empty) { + spec.ingress = None; + } + } + spec + } + + fn contains_required_metadata( + actual: &Option>, + required: &Option>, + ) -> bool { + required.as_ref().is_none_or(|required| { + actual.as_ref().is_some_and(|actual| { + required + .iter() + .all(|(key, value)| actual.get(key) == Some(value)) + }) + }) + } + + normalized_spec(existing.spec.clone()) == normalized_spec(expected.spec.clone()) + && contains_required_metadata(&existing.metadata.labels, &expected.metadata.labels) + && contains_required_metadata( + &existing.metadata.annotations, + &expected.metadata.annotations, + ) +} + fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { obj.data .get("status") @@ -4656,15 +6214,45 @@ fn sandbox_operating_state_patch( running: bool, ) -> serde_json::Value { if api_version == SANDBOX_VERSION_V1BETA1 { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"operatingMode": if running { "Running" } else { "Suspended" }} - }) + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"operatingMode": "Running"} + }) + } else { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_PROXY_POD_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_PROXY_POD_WORKLOAD_UID: serde_json::Value::Null, + }, + }, + "spec": {"operatingMode": "Suspended"} + }) + } } else { - serde_json::json!({ - "metadata": {"resourceVersion": resource_version}, - "spec": {"replicas": i32::from(running)} - }) + if running { + serde_json::json!({ + "metadata": {"resourceVersion": resource_version}, + "spec": {"replicas": 1} + }) + } else { + serde_json::json!({ + "metadata": { + "resourceVersion": resource_version, + "annotations": { + ANNOTATION_PROXY_POD_BOOTSTRAPPING: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: serde_json::Value::Null, + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: serde_json::Value::Null, + ANNOTATION_PROXY_POD_WORKLOAD_UID: serde_json::Value::Null, + }, + }, + "spec": {"replicas": 0} + }) + } } } @@ -5038,6 +6626,32 @@ mod tests { }) } + #[test] + fn resource_version_conflicts_are_not_reported_as_duplicate_sandboxes() { + let conflict = KubeError::Api(kube::core::ErrorResponse { + status: "Failure".to_string(), + message: "the object has been modified".to_string(), + reason: "Conflict".to_string(), + code: 409, + }); + assert!(is_kube_resource_version_conflict(&conflict)); + assert!(matches!( + KubernetesDriverError::from_kube(conflict), + KubernetesDriverError::Message(_) + )); + + let duplicate = KubeError::Api(kube::core::ErrorResponse { + status: "Failure".to_string(), + message: "already exists".to_string(), + reason: "AlreadyExists".to_string(), + code: 409, + }); + assert!(matches!( + KubernetesDriverError::from_kube(duplicate), + KubernetesDriverError::AlreadyExists + )); + } + fn expired_watch_error() -> watcher::Error { watcher::Error::WatchError(kube::core::ErrorResponse { status: "Failure".to_string(), @@ -5107,7 +6721,13 @@ mod tests { ]) .chain(futures::stream::pending()); let sandbox_stream = recovering_watcher_stream(source, "sandbox-resource").boxed(); - let mut outward = cluster_wide_watch_stream(sandbox_stream, "default".to_string()); + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + let mut outward = cluster_wide_watch_stream( + sandbox_stream, + "default".to_string(), + driver.watch_client.clone(), + driver, + ); let event = outward .next() @@ -5307,6 +6927,156 @@ mod tests { assert_eq!(error.code(), tonic::Code::PermissionDenied); } + #[test] + fn proxy_control_identity_requires_exact_pair_and_role_labels() { + let sandbox_id = "sandbox-id-a"; + let mut pod = Pod { + metadata: ObjectMeta { + labels: Some(BTreeMap::from([ + (LABEL_SANDBOX_ID.to_string(), sandbox_id.to_string()), + ( + BOUNDARY_PAIR_LABEL.to_string(), + crate::proxy_pod::pair_label_value(sandbox_id), + ), + (BOUNDARY_ROLE_LABEL.to_string(), "supervisor".to_string()), + ])), + ..Default::default() + }, + ..Default::default() + }; + validate_proxy_control_labels(&pod, sandbox_id).unwrap(); + + pod.metadata + .labels + .as_mut() + .unwrap() + .insert(BOUNDARY_ROLE_LABEL.to_string(), "workload".to_string()); + assert_eq!( + validate_proxy_control_labels(&pod, sandbox_id) + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + } + + #[test] + fn sandbox_authentication_requires_the_paired_supervisor() { + require_proxy_control_authentication(true).expect("paired supervisor is trusted"); + assert_eq!( + require_proxy_control_authentication(false) + .expect_err("workload JWT must not authenticate directly") + .code(), + tonic::Code::PermissionDenied + ); + } + + #[test] + fn orphan_fence_reaping_requires_a_creation_age_floor() { + let policy: NetworkPolicy = serde_json::from_value(serde_json::json!({ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": {"creationTimestamp": "2026-01-01T00:00:00Z"}, + "spec": {"podSelector": {}} + })) + .unwrap(); + let created = SystemTime::UNIX_EPOCH + Duration::from_secs(1_767_225_600); + assert!(!proxy_pod_fence_is_old_enough( + &policy, + created + PROXY_POD_ORPHAN_FENCE_MIN_AGE - Duration::from_secs(1) + )); + assert!(proxy_pod_fence_is_old_enough( + &policy, + created + PROXY_POD_ORPHAN_FENCE_MIN_AGE + )); + assert!(!proxy_pod_fence_is_old_enough( + &NetworkPolicy::default(), + created + PROXY_POD_ORPHAN_FENCE_MIN_AGE + )); + } + + #[test] + fn proxy_pod_fence_validation_accepts_api_normalization_and_injected_metadata() { + let names = ProxyPodNames::new("sandbox-id-a"); + let mut expected = + workload_fence("namespace-a", &names, "sandbox-id-a", 5000).workload_policy; + expected.metadata.labels = Some(BTreeMap::from([( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + )])); + expected.metadata.annotations = Some(BTreeMap::from([( + "openshell.ai/workload-pod-name".to_string(), + "sandbox-a".to_string(), + )])); + + let mut persisted = expected.clone(); + persisted.spec.as_mut().unwrap().egress = None; + persisted + .metadata + .labels + .as_mut() + .unwrap() + .insert("admission.example/injected".to_string(), "true".to_string()); + assert!(proxy_pod_fence_matches(&persisted, &expected)); + assert!(validate_proxy_pod_fence(&persisted, &expected).is_ok()); + + persisted.spec.as_mut().unwrap().policy_types = Some(vec!["Ingress".to_string()]); + assert!(!proxy_pod_fence_matches(&persisted, &expected)); + assert!(matches!( + validate_proxy_pod_fence(&persisted, &expected), + Err(KubernetesDriverError::Precondition(_)) + )); + } + + #[test] + fn proxy_pod_bootstrap_marker_and_age_gate_rollback() { + let started = Duration::from_secs(1_767_225_600); + let mut object: DynamicObject = serde_json::from_value(serde_json::json!({ + "apiVersion": "agents.x-k8s.io/v1beta1", + "kind": "Sandbox", + "metadata": { + "name": "sandbox-a", + "creationTimestamp": "2020-01-01T00:00:00Z", + "annotations": { + ANNOTATION_PROXY_POD_BOOTSTRAPPING: "true", + ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT: started.as_millis().to_string(), + ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION: "restart", + } + } + })) + .unwrap(); + assert!(proxy_pod_bootstrap_in_progress(&object)); + assert_eq!(proxy_pod_bootstrap_operation(&object), Some("restart")); + let started_at = SystemTime::UNIX_EPOCH + started; + assert!(!proxy_pod_bootstrap_is_stale( + &object, + started_at + PROXY_POD_BOOTSTRAP_GRACE - Duration::from_secs(1), + PROXY_POD_BOOTSTRAP_GRACE + )); + assert!(proxy_pod_bootstrap_is_stale( + &object, + started_at + PROXY_POD_BOOTSTRAP_GRACE, + PROXY_POD_BOOTSTRAP_GRACE + )); + object + .metadata + .annotations + .as_mut() + .unwrap() + .remove(ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT); + assert!(proxy_pod_bootstrap_is_stale( + &object, + started_at, + PROXY_POD_BOOTSTRAP_GRACE + )); + object + .metadata + .annotations + .as_mut() + .unwrap() + .remove(ANNOTATION_PROXY_POD_BOOTSTRAPPING); + assert!(!proxy_pod_bootstrap_in_progress(&object)); + } + #[test] fn sandbox_owner_identity_requires_matching_uid_and_sandbox_id() { let owner = sandbox_owner_for_test("sandbox-a", "sandbox-uid-a"); @@ -5338,11 +7108,16 @@ mod tests { fn lifecycle_patch_uses_version_specific_operating_state() { let beta_stop = sandbox_operating_state_patch(SANDBOX_VERSION_V1BETA1, "42", false); assert_eq!(beta_stop["metadata"]["resourceVersion"], "42"); + assert_eq!( + beta_stop["metadata"]["annotations"][ANNOTATION_PROXY_POD_BOOTSTRAPPING], + serde_json::Value::Null + ); assert_eq!(beta_stop["spec"]["operatingMode"], "Suspended"); assert!(beta_stop["spec"].get("replicas").is_none()); let alpha_start = sandbox_operating_state_patch(SANDBOX_VERSION_V1ALPHA1, "43", true); assert_eq!(alpha_start["metadata"]["resourceVersion"], "43"); + assert!(alpha_start["metadata"].get("annotations").is_none()); assert_eq!(alpha_start["spec"]["replicas"], 1); assert!(alpha_start["spec"].get("operatingMode").is_none()); } @@ -5544,7 +7319,7 @@ mod tests { ..Default::default() }; - let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref(), None).unwrap_err(); + let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref()).unwrap_err(); assert!(err.contains("unknown field")); assert!(err.contains("gpu_device_ids")); } @@ -5893,7 +7668,7 @@ mod tests { ..SandboxSpec::default() }; - let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + let err = kubernetes_driver_config_for_spec(Some(&spec)).unwrap_err(); assert!(err.contains("/var/run/secrets/openshell")); } @@ -5921,7 +7696,7 @@ mod tests { ..SandboxSpec::default() }; - kubernetes_driver_config_for_spec(Some(&spec), None) + kubernetes_driver_config_for_spec(Some(&spec)) .expect("SPIFFE workload path should only be protected when SPIFFE is enabled"); } @@ -6036,7 +7811,11 @@ mod tests { CLIENT_TLS_VOLUME_NAME, SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, SPIFFE_WORKLOAD_API_VOLUME_NAME, - SUPERVISOR_VOLUME_NAME, + SANDBOX_RUNTIME_VOLUME_NAME, + SANDBOX_STATE_VOLUME_NAME, + SANDBOX_BOOTSTRAP_VOLUME_NAME, + SANDBOX_POD_IDENTITY_VOLUME_NAME, + SANDBOX_PROXY_CA_VOLUME_NAME, WORKSPACE_VOLUME_NAME, ] { let template = SandboxTemplate { @@ -6059,12 +7838,7 @@ mod tests { #[test] fn reserved_kubernetes_volume_names_cover_managed_pod_volumes() { - let params = SandboxPodParams { - client_tls_secret_name: "openshell-client-tls-secret", - provider_spiffe_enabled: true, - provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", - ..SandboxPodParams::default() - }; + let params = SandboxPodParams::default(); let pod_template = sandbox_template_to_k8s( &SandboxTemplate::default(), false, @@ -6087,36 +7861,6 @@ mod tests { } } - #[test] - fn driver_config_rejects_runtime_provider_spiffe_mount_path() { - let spec = SandboxSpec { - template: Some(SandboxTemplate { - driver_config: Some(json_struct(serde_json::json!({ - "volumes": [{ - "name": "user-data", - "persistent_volume_claim": {"claim_name": "pvc-user-data"} - }], - "containers": { - "agent": { - "volume_mounts": [{ - "name": "user-data", - "mount_path": "/custom-spiffe" - }] - } - } - }))), - ..SandboxTemplate::default() - }), - ..SandboxSpec::default() - }; - - let err = - kubernetes_driver_config_for_spec(Some(&spec), Some("/custom-spiffe/spire-agent.sock")) - .unwrap_err(); - - assert!(err.contains("/custom-spiffe")); - } - #[test] fn validate_rejects_zero_gpu_count() { let sandbox = Sandbox { @@ -6169,757 +7913,148 @@ mod tests { ); assert_eq!( - metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_PULLING_IMAGE) - ); - assert_eq!( - metadata - .get(PROGRESS_COMPLETE_LABEL_KEY) - .map(String::as_str), - Some("Image pulled (42 MB)") - ); - assert_eq!( - metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), - Some(PROGRESS_STEP_STARTING_SANDBOX) - ); - } - - #[test] - fn supervisor_sideload_injects_run_as_user_zero() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "securityContext": { - "capabilities": { - "add": ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] - } - } - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "custom-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1500, // sandbox_uid - 1500, // sandbox_gid - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0, "runAsUser must be 0 for supervisor"); - // Capabilities should be preserved - assert!( - sc["capabilities"]["add"] - .as_array() - .unwrap() - .contains(&serde_json::json!("SYS_ADMIN")) - ); - } - - #[test] - fn supervisor_sideload_replaces_spoofed_identity_environment() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest", - "env": [ - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "spoofed"}, - {"name": openshell_core::sandbox_env::SANDBOX_UID, "value": "9999"}, - {"name": openshell_core::sandbox_env::SANDBOX_GID, "value": "9999"}, - {"name": openshell_core::sandbox_env::OCI_IMAGE_USER, "value": "duplicate"} - ] - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1500, - 1600, - ); - - let agent = &pod_template["spec"]["containers"][0]; - let env = agent["env"].as_array().unwrap(); - for name in [ - openshell_core::sandbox_env::OCI_IMAGE_USER, - openshell_core::sandbox_env::SANDBOX_UID, - openshell_core::sandbox_env::SANDBOX_GID, - ] { - assert_eq!( - env.iter().filter(|item| item["name"] == name).count(), - 1, - "{name} must have one driver-owned value" - ); - } - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_GID), - Some("1600") - ); - } - - #[test] - fn supervisor_sideload_adds_security_context_when_missing() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!( - sc["runAsUser"], 0, - "runAsUser must be 0 even when no prior securityContext" - ); - } - - #[test] - fn supervisor_sideload_injects_emptydir_volume_init_container_and_mount() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::InitContainer, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - // Volume should be an emptyDir - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert!( - volumes[0]["emptyDir"].is_object(), - "volume should be emptyDir, not hostPath" - ); - - // Init container should use the supervisor image, not the sandbox image - let init_containers = pod_template["spec"]["initContainers"] - .as_array() - .expect("initContainers should exist"); - assert_eq!(init_containers.len(), 1); - assert_eq!(init_containers[0]["name"], SUPERVISOR_INIT_CONTAINER_NAME); - assert_eq!(init_containers[0]["image"], "supervisor-image:latest"); - assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); - - // The init container must invoke the binary directly with - // `copy-self ` rather than depending on shell utilities. - let init_command = init_containers[0]["command"] - .as_array() - .expect("init container command should be set"); - assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); - assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); - assert_eq!(init_command[1], "copy-self"); - assert_eq!( - init_command[2].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - assert!( - !init_command.iter().any(|v| v == "sh"), - "init container must not depend on a shell" - ); - - // `--workdir` is optional for standalone supervisor invocations and - // has no implicit default, so Kubernetes must pass its fixed workspace. - let command = pod_template["spec"]["containers"][0]["command"] - .as_array() - .expect("command should be set"); - assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - assert_eq!( - command, - serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT - ]) - .as_array() - .unwrap() - ); - - // Agent volume mount should be read-only - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts.len(), 1); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); - } - - #[test] - fn supervisor_sideload_image_volume_injects_image_source_without_init_container() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "IfNotPresent", - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid - ); - - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - assert_eq!(volumes.len(), 1); - assert_eq!(volumes[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(volumes[0]["image"]["reference"], "supervisor-image:latest"); - assert_eq!(volumes[0]["image"]["pullPolicy"], "IfNotPresent"); - assert!( - volumes[0]["emptyDir"].is_null(), - "image volume method must not use emptyDir" - ); - - assert!( - pod_template["spec"]["initContainers"].is_null(), - "image volume method must not inject init containers" - ); - - let command = pod_template["spec"]["containers"][0]["command"] - .as_array() - .expect("command should be set"); - assert_eq!( - command[0].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") - ); - - let sc = &pod_template["spec"]["containers"][0]["securityContext"]; - assert_eq!(sc["runAsUser"], 0); - - let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] - .as_array() - .expect("volumeMounts should exist"); - assert_eq!(mounts[0]["name"], SUPERVISOR_VOLUME_NAME); - assert_eq!(mounts[0]["mountPath"], SUPERVISOR_MOUNT_PATH); - assert_eq!(mounts[0]["readOnly"], true); - } - - #[test] - fn supervisor_image_volume_omits_pull_policy_when_empty() { - let mut pod_template = serde_json::json!({ - "spec": { - "containers": [{ - "name": "agent", - "image": "custom-image:latest" - }] - } - }); - - apply_supervisor_sideload( - &mut pod_template, - "supervisor-image:latest", - "", - SupervisorSideloadMethod::ImageVolume, - 1000, // sandbox_uid - 1000, // sandbox_gid + metadata.get(PROGRESS_COMPLETE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_PULLING_IMAGE) ); - - let volume = &pod_template["spec"]["volumes"][0]; - assert_eq!(volume["image"]["reference"], "supervisor-image:latest"); - assert!( - volume["image"].get("pullPolicy").is_none(), - "pullPolicy should be omitted when empty" + assert_eq!( + metadata + .get(PROGRESS_COMPLETE_LABEL_KEY) + .map(String::as_str), + Some("Image pulled (42 MB)") + ); + assert_eq!( + metadata.get(PROGRESS_ACTIVE_STEP_KEY).map(String::as_str), + Some(PROGRESS_STEP_STARTING_SANDBOX) ); } #[test] - fn sidecar_topology_renders_process_agent_and_network_sidecar() { + fn proxy_pod_topology_renders_credential_free_boundary_workload() { let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, supervisor_image: "supervisor-image:latest", - supervisor_image_pull_policy: "IfNotPresent", - grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", - client_tls_secret_name: "openshell-client-tls", - proxy_uid: 2200, + sandbox_id: "sandbox-123", sandbox_uid: 1500, sandbox_gid: 1500, ..SandboxPodParams::default() }; let pod_template = sandbox_template_to_k8s( - &SandboxTemplate { - image: "agent-image:latest".to_string(), - environment: std::collections::HashMap::from([ - ( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - "spoofed".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - "9999".to_string(), - ), - ( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - "9999".to_string(), - ), - ]), - ..SandboxTemplate::default() - }, + &SandboxTemplate::default(), false, &std::collections::HashMap::new(), - false, + true, ¶ms, ); + let agent = &pod_template["spec"]["containers"][0]; - assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); - assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - assert_eq!(containers.len(), 2); - - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); assert_eq!( agent["command"], serde_json::json!([ - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), - "--mode=process", - "--workdir", - driver_mounts::DEFAULT_WORKSPACE_ROOT + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "--bootstrap", + BOUNDARY_CONFIG_PATH ]) ); assert_eq!(agent["securityContext"]["runAsUser"], 1500); assert_eq!(agent["securityContext"]["runAsGroup"], 1500); assert_eq!(agent["securityContext"]["runAsNonRoot"], true); - assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); assert_eq!( agent["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::TLS_CA), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), - None - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) - ); - assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); - assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); - assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None - ); - assert_eq!( - rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), - None + serde_json::json!({"drop": ["ALL"]}) ); + assert!(agent["securityContext"]["capabilities"]["add"].is_null()); + assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) + pod_template["spec"]["securityContext"]["seccompProfile"]["type"], + "RuntimeDefault" ); + assert_eq!(pod_template["spec"]["dnsPolicy"], "None"); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") + pod_template["spec"]["dnsConfig"]["nameservers"], + serde_json::json!(["127.0.0.53"]) ); assert_eq!( - rendered_env(agent, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") + pod_template["spec"]["schedulingGates"], + serde_json::json!([{"name": SANDBOX_BOOTSTRAP_SCHEDULING_GATE}]) ); - - let sidecar = containers + let workspace_init = pod_template["spec"]["initContainers"] + .as_array() + .unwrap() .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .find(|container| container["name"] == WORKSPACE_INIT_CONTAINER_NAME) .unwrap(); - assert_eq!(sidecar["image"], "supervisor-image:latest"); - assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); - assert_eq!( - sidecar["command"], - serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) - ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), - Some("https://openshell-gateway.openshell.svc:8080") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), - Some(SIDECAR_SSH_SOCKET_FILE) - ); - assert!( - SIDECAR_SSH_SOCKET_FILE.starts_with('@'), - "sidecar SSH relay must use a Linux abstract socket" - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_UID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_GID), - Some("1500") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::OCI_IMAGE_USER), - Some("") - ); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), - Some(SIDECAR_CONTROL_SOCKET) + workspace_init["command"], + serde_json::json!([ + format!("{SANDBOX_RUNTIME_MOUNT_PATH}/openshell-sandbox"), + "seed-workspace" + ]) ); + assert_eq!(workspace_init["securityContext"]["runAsUser"], 1500); assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), - None + workspace_init["securityContext"]["capabilities"], + serde_json::json!({"drop": ["ALL"]}) ); assert_eq!( - rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), None ); assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), None ); - assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); - assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), - Some(SIDECAR_TLS_MOUNT_PATH) - ); assert_eq!( - rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), - Some("/etc/openshell-tls/proxy/client/ca.crt") + pod_template["metadata"]["labels"][BOUNDARY_ROLE_LABEL], + "workload" ); - let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); + + let mounts = agent["volumeMounts"].as_array().unwrap(); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_RUNTIME_VOLUME_NAME && mount["readOnly"] == true + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_STATE_VOLUME_NAME && mount["readOnly"].is_null() + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_POD_IDENTITY_VOLUME_NAME + && mount["mountPath"] == SANDBOX_POD_IDENTITY_MOUNT_PATH + && mount["readOnly"] == true + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SANDBOX_PROXY_CA_VOLUME_NAME + && mount["mountPath"] == SANDBOX_PROXY_CA_MOUNT_PATH + && mount["readOnly"].is_null() + })); assert!( - !sidecar_mounts + !mounts .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" + .any(|mount| mount["name"] == SANDBOX_BOOTSTRAP_VOLUME_NAME) ); - let agent_mounts = agent["volumeMounts"].as_array().unwrap(); assert!( - !agent_mounts + !mounts .iter() - .any(|mount| mount["name"] == "openshell-sa-token"), - "agent container must not mount gateway bootstrap token in sidecar topology" + .any(|mount| mount["name"] == CLIENT_TLS_VOLUME_NAME) ); assert!( - !agent_mounts + !mounts .iter() - .any(|mount| mount["name"] == "openshell-client-tls"), - "agent container must not mount gateway client TLS secret in sidecar topology" - ); - let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); - let sa_token = volumes - .iter() - .find(|volume| volume["name"] == "openshell-sa-token") - .unwrap(); - assert_eq!(sa_token["projected"]["defaultMode"], 0o440); - let client_tls = volumes - .iter() - .find(|volume| volume["name"] == "openshell-client-tls") - .unwrap(); - assert_eq!(client_tls["secret"]["defaultMode"], 0o440); - - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["image"], "supervisor-image:latest"); - assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); - assert_eq!( - network_init["command"], - serde_json::json!([ - SUPERVISOR_IMAGE_BINARY_PATH, - "--mode=network-init", - "--proxy-uid", - "0", - "--proxy-gid", - "1500", - "--sidecar-state-dir", - SIDECAR_STATE_MOUNT_PATH, - "--sidecar-tls-dir", - SIDECAR_TLS_MOUNT_PATH - ]) - ); - assert_eq!( - network_init["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] - }) - ); - let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); - assert!(network_init_mounts.iter().any(|mount| { - mount["name"] == "openshell-client-tls" - && mount["mountPath"] == "/etc/openshell-tls/client" - })); - } - - #[test] - fn sidecar_topology_can_relax_process_binary_aware_network_policy() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - proxy_uid: 2200, - sandbox_uid: 1500, - sandbox_gid: 1500, - process_binary_aware_network_policy: false, - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate { - image: "agent-image:latest".to_string(), - ..SandboxTemplate::default() - }, - false, - &std::collections::HashMap::new(), - false, - ¶ms, - ); - - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false - ); - assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"] - }) - ); - assert_eq!( - rendered_env( - sidecar, - openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY - ), - Some("relaxed") - ); - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "2200"); - } - - #[test] - fn sidecar_topology_adds_shared_state_and_tls_volumes() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, - supervisor_image: "supervisor-image:latest", - grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - false, - ¶ms, + .any(|mount| mount["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME) ); let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); - assert!( - volumes - .iter() - .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) - ); - assert!( - volumes - .iter() - .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) - ); - assert!(volumes.iter().any(|volume| { - volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() - })); - - let containers = pod_template["spec"]["containers"].as_array().unwrap(); - let sidecar = containers + let pod_identity = volumes .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .find(|volume| volume["name"] == SANDBOX_POD_IDENTITY_VOLUME_NAME) .unwrap(); assert_eq!( - sidecar["securityContext"]["capabilities"], - serde_json::json!({ - "drop": ["ALL"], - "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] - }) - ); - assert_eq!(sidecar["securityContext"]["runAsUser"], 0); - assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); - assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); - assert_eq!( - sidecar["securityContext"]["allowPrivilegeEscalation"], - false + pod_identity["downwardAPI"]["items"], + serde_json::json!([{"path": "uid", "fieldRef": {"fieldPath": "metadata.uid"}}]) ); - - for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { - let container = containers - .iter() - .find(|container| container["name"] == container_name) - .unwrap(); - let mounts = container["volumeMounts"].as_array().unwrap(); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_STATE_VOLUME_NAME - && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH - })); - assert!(mounts.iter().any(|mount| { - mount["name"] == SIDECAR_TLS_VOLUME_NAME - && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH - })); - } - let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); - let network_init = init_containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert_eq!(network_init["command"][3], "0"); - } - - #[test] - fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - proxy_uid: 1500, - sandbox_uid: 1500, - ..SandboxPodParams::default() - }; - - let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); - assert!(matches!(err, KubernetesDriverError::Precondition(_))); - assert!(err.to_string().contains("proxy_uid")); } /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start /// with "No such file or directory" errors. - #[test] - fn tls_env_vars_match_volume_mount_path() { - // The mount path used in pod template construction - const TLS_MOUNT_PATH: &str = "/etc/openshell-tls/client"; - - // Build env with TLS enabled - let mut env = Vec::new(); - apply_required_env( - &mut env, - "sandbox-1", - "my-sandbox", - "https://endpoint:8080", - "0.0.0.0:2222", - true, // tls_enabled - None, - ); - - // Extract the TLS-related env vars - let get_env = |name: &str| -> Option { - env.iter() - .find(|e| e.get("name").and_then(|v| v.as_str()) == Some(name)) - .and_then(|e| e.get("value").and_then(|v| v.as_str()).map(String::from)) - }; - - let tls_ca = get_env("OPENSHELL_TLS_CA").expect("OPENSHELL_TLS_CA must be set"); - let tls_cert = get_env("OPENSHELL_TLS_CERT").expect("OPENSHELL_TLS_CERT must be set"); - let tls_key = get_env("OPENSHELL_TLS_KEY").expect("OPENSHELL_TLS_KEY must be set"); - - // All TLS paths must be within the mount path - assert!( - tls_ca.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_CA path '{tls_ca}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - assert!( - tls_cert.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_CERT path '{tls_cert}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - assert!( - tls_key.starts_with(TLS_MOUNT_PATH), - "OPENSHELL_TLS_KEY path '{tls_key}' must start with mount path '{TLS_MOUNT_PATH}'" - ); - } #[test] fn gpu_sandbox_adds_runtime_class_and_gpu_limit() { @@ -7209,111 +8344,40 @@ mod tests { resources: Some(DriverResourceRequirements { cpu_limit: "2".to_string(), ..Default::default() - }), - ..SandboxTemplate::default() - }; - - let pod_template = { - let params = SandboxPodParams::default(); - sandbox_template_to_k8s( - &template, - true, - &std::collections::HashMap::new(), - true, - ¶ms, - ) - }; - - let limits = &pod_template["spec"]["containers"][0]["resources"]["limits"]; - assert_eq!(limits["cpu"], serde_json::json!("2")); - assert_eq!(limits[GPU_RESOURCE_NAME], serde_json::json!("1")); - } - - #[test] - fn cpu_and_memory_limits_are_mirrored_to_requests() { - use openshell_core::proto::compute::v1::DriverResourceRequirements; - let template = SandboxTemplate { - resources: Some(DriverResourceRequirements { - cpu_limit: "500m".to_string(), - memory_limit: "2Gi".to_string(), - ..Default::default() - }), - ..SandboxTemplate::default() - }; - - let pod_template = { - let params = SandboxPodParams::default(); - sandbox_template_to_k8s( - &template, - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ) - }; - - let resources = &pod_template["spec"]["containers"][0]["resources"]; - assert_eq!(resources["limits"]["cpu"], serde_json::json!("500m")); - assert_eq!(resources["limits"]["memory"], serde_json::json!("2Gi")); - assert_eq!(resources["requests"]["cpu"], serde_json::json!("500m")); - assert_eq!(resources["requests"]["memory"], serde_json::json!("2Gi")); - } - - #[test] - fn host_aliases_injected_when_gateway_ip_set() { - let pod_template = { - let params = SandboxPodParams { - host_gateway_ip: "172.17.0.1", - ..Default::default() - }; - sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ) + }), + ..SandboxTemplate::default() }; - let host_aliases = pod_template["spec"]["hostAliases"] - .as_array() - .expect("hostAliases should exist"); - assert_eq!(host_aliases.len(), 1); - assert_eq!(host_aliases[0]["ip"], "172.17.0.1"); - let hostnames = host_aliases[0]["hostnames"] - .as_array() - .expect("hostnames should exist"); - assert!(hostnames.contains(&serde_json::json!("host.docker.internal"))); - assert!(hostnames.contains(&serde_json::json!("host.openshell.internal"))); - } - - #[test] - fn host_aliases_not_injected_when_gateway_ip_empty() { let pod_template = { let params = SandboxPodParams::default(); sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, + &template, + true, &std::collections::HashMap::new(), true, ¶ms, ) }; - assert!( - pod_template["spec"]["hostAliases"].is_null(), - "hostAliases should not be present when host_gateway_ip is empty" - ); + let limits = &pod_template["spec"]["containers"][0]["resources"]["limits"]; + assert_eq!(limits["cpu"], serde_json::json!("2")); + assert_eq!(limits[GPU_RESOURCE_NAME], serde_json::json!("1")); } #[test] - fn tls_secret_volume_uses_restrictive_default_mode() { - let template = SandboxTemplate::default(); - let pod_template = { - let params = SandboxPodParams { - client_tls_secret_name: "my-tls-secret", + fn cpu_and_memory_limits_are_mirrored_to_requests() { + use openshell_core::proto::compute::v1::DriverResourceRequirements; + let template = SandboxTemplate { + resources: Some(DriverResourceRequirements { + cpu_limit: "500m".to_string(), + memory_limit: "2Gi".to_string(), ..Default::default() - }; + }), + ..SandboxTemplate::default() + }; + + let pod_template = { + let params = SandboxPodParams::default(); sandbox_template_to_k8s( &template, false, @@ -7323,18 +8387,11 @@ mod tests { ) }; - let volumes = pod_template["spec"]["volumes"] - .as_array() - .expect("volumes should exist"); - let tls_vol = volumes - .iter() - .find(|v| v["name"] == CLIENT_TLS_VOLUME_NAME) - .expect("TLS volume should exist"); - assert_eq!( - tls_vol["secret"]["defaultMode"], - 256, // 0o400 - "TLS secret volume must use mode 0400 to prevent sandbox user from reading the private key" - ); + let resources = &pod_template["spec"]["containers"][0]["resources"]; + assert_eq!(resources["limits"]["cpu"], serde_json::json!("500m")); + assert_eq!(resources["limits"]["memory"], serde_json::json!("2Gi")); + assert_eq!(resources["requests"]["cpu"], serde_json::json!("500m")); + assert_eq!(resources["requests"]["memory"], serde_json::json!("2Gi")); } // ----------------------------------------------------------------------- @@ -7356,7 +8413,8 @@ mod tests { &mut pod_template, "openshell/sandbox:latest", "IfNotPresent", - 1000, // sandbox_gid + Some(1000), // sandbox_gid + None, ); // Init container @@ -7415,7 +8473,8 @@ mod tests { &mut pod_template, "my-custom-image:v2", "IfNotPresent", - 1000, + Some(1000), + None, ); let init_image = pod_template["spec"]["initContainers"][0]["image"] @@ -7438,7 +8497,7 @@ mod tests { } }); - apply_workspace_persistence(&mut pod_template, "img:latest", "Always", 1000); + apply_workspace_persistence(&mut pod_template, "img:latest", "Always", Some(1000), None); let cmd = pod_template["spec"]["initContainers"][0]["command"] .as_array() @@ -7466,10 +8525,7 @@ mod tests { #[test] fn workspace_persistence_skipped_when_inject_workspace_false() { - let params = SandboxPodParams { - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - ..SandboxPodParams::default() - }; + let params = SandboxPodParams::default(); let pod_template = sandbox_template_to_k8s( &SandboxTemplate::default(), false, @@ -7517,65 +8573,6 @@ mod tests { ) } - #[test] - fn app_armor_profile_omitted_by_default() { - let pod_template = default_template_to_k8s(false); - assert!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"].is_null(), - "appArmorProfile must be omitted when no profile is configured" - ); - } - - #[test] - fn app_armor_profile_renders_unconfined() { - let profile = AppArmorProfile::Unconfined; - let params = SandboxPodParams { - app_armor_profile: Some(&profile), - ..Default::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"], - serde_json::json!({ "type": "Unconfined" }) - ); - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"][0], - serde_json::json!("SYS_ADMIN"), - "AppArmor rendering must preserve required capabilities" - ); - } - - #[test] - fn app_armor_profile_renders_localhost_profile() { - let profile = AppArmorProfile::Localhost("openshell-supervisor".to_string()); - let params = SandboxPodParams { - app_armor_profile: Some(&profile), - ..Default::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - assert_eq!( - pod_template["spec"]["containers"][0]["securityContext"]["appArmorProfile"], - serde_json::json!({ - "type": "Localhost", - "localhostProfile": "openshell-supervisor" - }) - ); - } - #[test] fn user_namespaces_disabled_by_default() { let pod_template = default_template_to_k8s(false); @@ -7583,11 +8580,10 @@ mod tests { pod_template["spec"]["hostUsers"].is_null(), "hostUsers must not be set when user namespaces are disabled" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert_eq!(caps.len(), 4); - assert!(!caps.contains(&serde_json::json!("SETUID"))); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7601,19 +8597,12 @@ mod tests { } #[test] - fn user_namespaces_adds_extra_capabilities() { + fn user_namespaces_preserve_capability_free_posture() { let pod_template = default_template_to_k8s(true); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert!(caps.contains(&serde_json::json!("SYS_ADMIN"))); - assert!(caps.contains(&serde_json::json!("NET_ADMIN"))); - assert!(caps.contains(&serde_json::json!("SYS_PTRACE"))); - assert!(caps.contains(&serde_json::json!("SYSLOG"))); - assert!(caps.contains(&serde_json::json!("SETUID"))); - assert!(caps.contains(&serde_json::json!("SETGID"))); - assert!(caps.contains(&serde_json::json!("DAC_READ_SEARCH"))); - assert_eq!(caps.len(), 7); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7637,10 +8626,10 @@ mod tests { serde_json::json!(false), "per-sandbox user namespace intent must set hostUsers: false" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert!(caps.contains(&serde_json::json!("SETUID"))); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7666,14 +8655,10 @@ mod tests { pod_template["spec"]["hostUsers"].is_null(), "per-sandbox user namespace intent must override the cluster default" ); - let caps = pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]["add"] - .as_array() - .unwrap(); - assert_eq!( - caps.len(), - 4, - "extra capabilities must not be added when user namespaces are disabled" - ); + let capabilities = + &pod_template["spec"]["containers"][0]["securityContext"]["capabilities"]; + assert!(capabilities["add"].is_null()); + assert_eq!(capabilities["drop"], serde_json::json!(["ALL"])); } #[test] @@ -7845,51 +8830,6 @@ mod tests { ); } - #[test] - fn provider_spiffe_mounts_csi_socket_and_keeps_sa_token_bootstrap() { - let params = SandboxPodParams { - sandbox_id: "sandbox-123", - sandbox_name: "sandbox", - provider_spiffe_enabled: true, - provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", - ..SandboxPodParams::default() - }; - let pod_template = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - true, - ¶ms, - ); - - let env = pod_template["spec"]["containers"][0]["env"] - .as_array() - .expect("env"); - assert!(env.iter().any(|e| { - e["name"] == openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - && e["value"] == "/spiffe-workload-api/spire-agent.sock" - })); - assert!(env.iter().any(|e| { - e["name"] == openshell_core::sandbox_env::K8S_SA_TOKEN_FILE - && e["value"] == "/var/run/secrets/openshell/token" - })); - - let volumes = pod_template["spec"]["volumes"].as_array().expect("volumes"); - assert!(volumes.iter().any(|volume| { - volume["name"] == SPIFFE_WORKLOAD_API_VOLUME_NAME - && volume["csi"]["driver"] == "csi.spiffe.io" - })); - assert!(volumes.iter().any(|volume| { - volume["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME - && volume["projected"]["sources"][0]["serviceAccountToken"]["path"] == "token" - })); - - assert_eq!( - pod_template["metadata"]["labels"][LABEL_MANAGED_BY], - serde_json::json!(LABEL_MANAGED_BY_VALUE) - ); - } - #[test] fn log_level_propagates_as_env_var_to_sandbox_pod() { let spec = SandboxSpec { @@ -7939,7 +8879,7 @@ mod tests { } #[test] - fn sandbox_pod_clears_unsupported_network_capabilities() { + fn sandbox_pod_drops_legacy_network_capability_environment() { let spec = SandboxSpec { environment: std::collections::HashMap::from([( openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), @@ -7951,14 +8891,9 @@ mod tests { let env = cr["spec"]["podTemplate"]["spec"]["containers"][0]["env"] .as_array() .unwrap(); - let entries = env - .iter() - .filter(|entry| { - entry["name"] == openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES - }) - .collect::>(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0]["value"], serde_json::json!("")); + assert!(!env.iter().any(|entry| { + entry["name"] == openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES + })); } #[test] @@ -8407,103 +9342,6 @@ mod tests { ); } - #[test] - fn upstream_proxy_is_injected_only_into_network_supervisors() { - let params = SandboxPodParams { - topology: SupervisorTopology::Sidecar, - supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, - supervisor_image: "supervisor-image:latest", - https_proxy: Some("http://proxy.corp.example:8080"), - no_proxy: Some(".svc.cluster.local,10.96.0.0/12"), - proxy_auth_secret_name: Some("corporate-proxy-auth"), - proxy_auth_secret_key: Some("credentials"), - proxy_auth_allow_insecure: true, - proxy_connect_by_hostname: true, - sandbox_uid: 1500, - sandbox_gid: 1500, - ..SandboxPodParams::default() - }; - let pod = sandbox_template_to_k8s( - &SandboxTemplate::default(), - false, - &std::collections::HashMap::new(), - false, - ¶ms, - ); - let containers = pod["spec"]["containers"].as_array().unwrap(); - let network = containers - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) - .unwrap(); - let command = network["command"].as_array().unwrap(); - assert!(command.iter().any(|arg| arg == "--upstream-proxy")); - assert!(command.iter().any(|arg| arg == "--upstream-no-proxy")); - let auth_file_index = command - .iter() - .position(|arg| arg == "--upstream-proxy-auth-file") - .unwrap(); - assert_eq!( - command[auth_file_index + 1], - openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH - ); - assert!( - command - .iter() - .any(|arg| arg == "--upstream-proxy-auth-allow-insecure") - ); - assert!( - command - .iter() - .any(|arg| arg == "--upstream-proxy-connect-by-hostname") - ); - assert!( - network["volumeMounts"] - .as_array() - .unwrap() - .iter() - .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - ); - - let init = pod["spec"]["initContainers"] - .as_array() - .unwrap() - .iter() - .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) - .unwrap(); - assert!(!init["command"].as_array().unwrap().iter().any(|arg| { - arg.as_str() - .is_some_and(|arg| arg.starts_with("--upstream-")) - })); - let agent = containers - .iter() - .find(|container| container["name"] == "agent") - .unwrap(); - assert!( - !agent["volumeMounts"] - .as_array() - .unwrap() - .iter() - .any(|mount| mount["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - ); - assert!(!agent["env"].as_array().unwrap().iter().any(|entry| { - entry["value"] == "corporate-proxy-auth" || entry["value"] == "credentials" - })); - - let volume = pod["spec"]["volumes"] - .as_array() - .unwrap() - .iter() - .find(|volume| volume["name"] == UPSTREAM_PROXY_AUTH_VOLUME_NAME) - .unwrap(); - assert_eq!(volume["secret"]["secretName"], "corporate-proxy-auth"); - assert_eq!(volume["secret"]["items"][0]["key"], "credentials"); - assert_eq!( - volume["secret"]["items"][0]["path"], - upstream_proxy_auth_file_name() - ); - assert_eq!(volume["secret"]["defaultMode"], 0o440); - } - #[test] fn sandbox_lookup_selector_always_includes_gateway_id() { let sel = sandbox_lookup_selector_for("sb-123", "gw-42"); @@ -8719,4 +9557,151 @@ mod tests { namespace_watcher_retry_delay(3, 2) ); } + + #[test] + fn proxy_pod_control_availability_requires_an_available_replica() { + let mut deployment = Deployment::default(); + assert_eq!( + proxy_pod_control_availability_from_deployment(&deployment), + ProxyPodControlAvailability::Unavailable + ); + deployment.status = Some(k8s_openapi::api::apps::v1::DeploymentStatus { + available_replicas: Some(1), + ..Default::default() + }); + assert_eq!( + proxy_pod_control_availability_from_deployment(&deployment), + ProxyPodControlAvailability::Available + ); + } + + #[test] + fn proxy_pod_readiness_transitions_bump_the_watched_cr() { + let unavailable = proxy_pod_readiness_transition_patch("42", "unavailable"); + let ready = proxy_pod_readiness_transition_patch("42", "ready"); + + assert_eq!(unavailable["metadata"]["resourceVersion"], "42"); + assert_eq!(ready["metadata"]["resourceVersion"], "42"); + assert_eq!( + unavailable["metadata"]["annotations"][ANNOTATION_PROXY_POD_READINESS], + "unavailable" + ); + assert_eq!( + ready["metadata"]["annotations"][ANNOTATION_PROXY_POD_READINESS], + "ready" + ); + assert_ne!(unavailable, ready); + } + + #[test] + fn proxy_pod_bootstrap_completion_is_resource_version_guarded_and_publishes_ready() { + let patch = proxy_pod_bootstrap_completion_patch("42"); + + assert_eq!(patch["metadata"]["resourceVersion"], "42"); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_PROXY_POD_BOOTSTRAPPING], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_PROXY_POD_BOOTSTRAP_STARTED_AT], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_PROXY_POD_BOOTSTRAP_OPERATION], + serde_json::Value::Null + ); + assert_eq!( + patch["metadata"]["annotations"][ANNOTATION_PROXY_POD_READINESS], + "ready" + ); + } + + #[test] + fn proxy_pod_readiness_is_downgraded_with_a_transient_reason() { + let mut sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + mark_proxy_pod_control_unavailable(&mut sandbox); + let ready = &sandbox.status.unwrap().conditions[0]; + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "DependenciesNotReady"); + } + + #[test] + fn proxy_pod_bootstrap_does_not_publish_a_terminal_suspension() { + let mut sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SANDBOX_SUSPENDED_CONDITION.to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + mark_proxy_pod_bootstrapping(&mut sandbox); + + let conditions = &sandbox.status.unwrap().conditions; + assert!( + !conditions + .iter() + .any(|condition| condition.r#type == SANDBOX_SUSPENDED_CONDITION) + ); + assert!(conditions.iter().any(|condition| { + condition.r#type == "Ready" + && condition.status == "False" + && condition.reason == "DependenciesNotReady" + })); + } + + #[test] + fn completed_proxy_pod_bootstrap_preserves_real_suspension() { + let sandbox = Sandbox { + status: Some(SandboxStatus { + conditions: vec![SandboxCondition { + r#type: SANDBOX_SUSPENDED_CONDITION.to_string(), + status: "True".to_string(), + reason: "PodTerminated".to_string(), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + }; + + assert!(sandbox.status.unwrap().conditions.iter().any(|condition| { + condition.r#type == SANDBOX_SUSPENDED_CONDITION && condition.status == "True" + })); + } + + #[test] + fn desired_proxy_pod_control_replicas_tracks_both_sandbox_apis() { + let resource = ApiResource::from_gvk(&GroupVersionKind::gvk( + SANDBOX_GROUP, + SANDBOX_VERSION_V1BETA1, + SANDBOX_KIND, + )); + let mut beta = DynamicObject::new("beta", &resource); + beta.data = serde_json::json!({"spec": {"operatingMode": "Suspended"}}); + assert_eq!(desired_proxy_pod_control_replicas(&beta), 0); + beta.data = serde_json::json!({"spec": {"operatingMode": "Running"}}); + assert_eq!(desired_proxy_pod_control_replicas(&beta), 1); + + let mut alpha = DynamicObject::new("alpha", &resource); + alpha.data = serde_json::json!({"spec": {"replicas": 0}}); + assert_eq!(desired_proxy_pod_control_replicas(&alpha), 0); + alpha.data = serde_json::json!({"spec": {"replicas": 1}}); + assert_eq!(desired_proxy_pod_control_replicas(&alpha), 1); + } } diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 095752d842..aadbf1e481 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -158,19 +158,21 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { - self.rpc_tracer - .trace(openshell_otel::rpc::CREATE_SANDBOX, async { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .create_sandbox(&sandbox) - .await - .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; - Ok(Response::new(CreateSandboxResponse {})) - }) - .await + Box::pin( + self.rpc_tracer + .trace(openshell_otel::rpc::CREATE_SANDBOX, async { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.driver + .create_sandbox(&sandbox) + .await + .map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?; + Ok(Response::new(CreateSandboxResponse {})) + }), + ) + .await } async fn stop_sandbox( @@ -198,21 +200,22 @@ impl ComputeDriver for ComputeDriverService { &self, request: Request, ) -> Result, Status> { - self.rpc_tracer - .trace(openshell_otel::rpc::START_SANDBOX, async { - let request = request.into_inner(); - if request.sandbox_id.is_empty() { - return Err(Status::invalid_argument("sandbox_id is required")); - } - self.driver - .start_sandbox(&request.sandbox_id) - .await - .map_err(|error| { - Status::from(openshell_core::ComputeDriverError::from(error)) - })?; - Ok(Response::new(StartSandboxResponse {})) - }) - .await + Box::pin( + self.rpc_tracer + .trace(openshell_otel::rpc::START_SANDBOX, async { + let request = request.into_inner(); + if request.sandbox_id.is_empty() { + return Err(Status::invalid_argument("sandbox_id is required")); + } + Box::pin(self.driver.start_sandbox(&request.sandbox_id)) + .await + .map_err(|error| { + Status::from(openshell_core::ComputeDriverError::from(error)) + })?; + Ok(Response::new(StartSandboxResponse {})) + }), + ) + .await } async fn delete_sandbox( diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs new file mode 100644 index 0000000000..da59384751 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -0,0 +1,392 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Kubernetes provisioning for the shared authenticated boundary protocol. +//! +//! This module deliberately contains no lifecycle, process, network, identity, +//! or wire implementation. The driver chooses the proxy-pod placement, binds +//! immutable Kubernetes resource identities, and provisions TCP coordinates; +//! `openshell-isolation-interface` and `openshell-sandbox` provide the common +//! control and boundary behavior. + +use std::collections::{BTreeMap, HashMap}; +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; + +use k8s_openapi::api::networking::v1::{ + NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort, + NetworkPolicySpec, +}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use kube::core::ObjectMeta; +use openshell_isolation_interface::boundary_protocol::{ + BoundaryClientTls, BoundaryConfig, BoundaryListener, BoundaryServerTls, BoundaryTopology, + BoundaryTransport, +}; +use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity}; + +/// Registered RFC 0012 backend name for the proxy-pod topology. +pub const BACKEND_NAME: &str = "kubernetes-proxy-pod"; + +/// Label that binds the workload and control pods in one unique pair. +pub const BOUNDARY_PAIR_LABEL: &str = "openshell.ai/boundary-pair"; + +/// Label distinguishing the two pods in a boundary pair. +pub const BOUNDARY_ROLE_LABEL: &str = "openshell.ai/boundary-role"; + +const WORKLOAD_ROLE: &str = "workload"; +const SUPERVISOR_ROLE: &str = "supervisor"; + +/// Driver-owned inputs for the workload pod's Kubernetes network fence. +/// +/// This is the first phase of proxy-pod provisioning. The driver applies the +/// returned labels to the respective pods and creates the returned policy. It +/// then observes the policy UID and resourceVersion and supplies both to +/// `KubernetesProxyPodBoundarySpec`. +pub struct KubernetesProxyPodNetworkFenceSpec { + pub namespace: String, + pub policy_name: String, + /// A unique, Kubernetes-label-safe value generated for this pod pair. + pub pair_label_value: String, + pub boundary_port: u16, +} + +/// Labels and policy needed to remove direct workload-pod egress. +pub struct KubernetesProxyPodNetworkFence { + pub workload_labels: BTreeMap, + pub control_labels: BTreeMap, + pub workload_policy: NetworkPolicy, +} + +impl KubernetesProxyPodNetworkFenceSpec { + /// Render a default-deny workload fence with one control-to-boundary path. + /// + /// Kubernetes `NetworkPolicy` is connection-aware: traffic returning over + /// the control-initiated boundary connection is allowed even though the + /// workload pod has no egress rules. The control pod remains responsible + /// for opening policy-approved upstream connections. + #[must_use] + pub fn provision(self) -> KubernetesProxyPodNetworkFence { + let workload_labels = boundary_pair_labels(&self.pair_label_value, WORKLOAD_ROLE); + let control_labels = boundary_pair_labels(&self.pair_label_value, SUPERVISOR_ROLE); + + let workload_policy = NetworkPolicy { + metadata: ObjectMeta { + name: Some(self.policy_name), + namespace: Some(self.namespace), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(workload_labels.clone()), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string(), "Egress".to_string()]), + // Only the exactly paired control pod in this namespace may + // establish the authenticated boundary connection. + ingress: Some(vec![NetworkPolicyIngressRule { + from: Some(vec![NetworkPolicyPeer { + pod_selector: Some(LabelSelector { + match_labels: Some(control_labels.clone()), + ..Default::default() + }), + ..Default::default() + }]), + ports: Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(i32::from(self.boundary_port))), + protocol: Some("TCP".to_string()), + ..Default::default() + }]), + }]), + // An explicit empty list selects the pod for egress and allows + // no new workload-initiated connections, including DNS and the + // Kubernetes API. Reply traffic for allowed ingress remains + // permitted by conforming NetworkPolicy implementations. + egress: Some(Vec::new()), + }), + status: None, + }; + + KubernetesProxyPodNetworkFence { + workload_labels, + control_labels, + workload_policy, + } + } +} + +fn boundary_pair_labels(pair: &str, role: &str) -> BTreeMap { + BTreeMap::from([ + (BOUNDARY_PAIR_LABEL.to_string(), pair.to_string()), + (BOUNDARY_ROLE_LABEL.to_string(), role.to_string()), + ]) +} + +/// Driver-owned inputs that bind one workload/proxy pair to one boundary. +/// +/// The driver constructs this only after Kubernetes has assigned every UID and +/// after it has observed the exact egress policy resource version. The workload +/// stays held until the matching boundary config and control topology have been +/// installed. +pub struct KubernetesProxyPodBoundarySpec { + pub boundary_id: String, + pub bootstrap_token: String, + pub generation: String, + pub session_epoch: String, + pub namespace_uid: String, + pub sandbox_resource_uid: String, + pub workload_pod_uid: String, + pub workload_pod_uid_path: PathBuf, + pub control_deployment_uid: String, + pub egress_policy_uid: String, + pub egress_policy_resource_version: String, + pub boundary_listener: SocketAddr, + pub control_address: SocketAddr, + pub sandbox_tls: BoundaryServerTls, + pub supervisor_tls: BoundaryClientTls, + pub host_gateway_ip: Option, + pub workload_identity: ResolvedWorkloadIdentity, + pub child_env: HashMap, +} + +/// Protected workload-pod config and matching proxy-pod descriptor. +pub struct KubernetesProxyPodBoundaryProvisioning { + pub boundary_config: BoundaryConfig, + pub topology: BoundaryTopology, +} + +impl KubernetesProxyPodBoundarySpec { + /// Produce both sides of the common protocol from one observed Kubernetes + /// resource set so a stale or recreated object cannot be attached. + #[must_use] + pub fn provision(self) -> KubernetesProxyPodBoundaryProvisioning { + let resource_claims = BTreeMap::from([ + ("kubernetes.namespace_uid".to_string(), self.namespace_uid), + ( + "kubernetes.sandbox_resource_uid".to_string(), + self.sandbox_resource_uid, + ), + ( + "kubernetes.workload_pod_uid".to_string(), + self.workload_pod_uid, + ), + ( + "kubernetes.control_deployment_uid".to_string(), + self.control_deployment_uid, + ), + ( + "kubernetes.egress_policy_uid".to_string(), + self.egress_policy_uid, + ), + ( + "kubernetes.egress_policy_resource_version".to_string(), + self.egress_policy_resource_version, + ), + ]); + let driver_fence = DriverFenceEvidence::Kubernetes { + network_policy_uid: resource_claims["kubernetes.egress_policy_uid"].clone(), + network_policy_resource_version: + resource_claims["kubernetes.egress_policy_resource_version"].clone(), + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + }; + KubernetesProxyPodBoundaryProvisioning { + boundary_config: BoundaryConfig { + boundary_id: self.boundary_id.clone(), + generation: self.generation.clone(), + session_epoch: self.session_epoch.clone(), + bootstrap_token: self.bootstrap_token.clone(), + listener: BoundaryListener::TlsTcp { + address: self.boundary_listener, + tls: self.sandbox_tls, + }, + resource_claims: resource_claims.clone(), + resource_claim_files: BTreeMap::from([( + "kubernetes.workload_pod_uid".to_string(), + self.workload_pod_uid_path, + )]), + workload_identity: self.workload_identity.clone(), + driver_fence: driver_fence.clone(), + child_env: self.child_env, + }, + topology: BoundaryTopology { + boundary_id: self.boundary_id, + generation: self.generation, + session_epoch: self.session_epoch, + workload_identity: self.workload_identity, + transport: BoundaryTransport::TlsTcp { + address: self.control_address, + tls: self.supervisor_tls, + }, + host_gateway_ip: self.host_gateway_ip, + resource_claims, + driver_fence, + bootstrap_token: self.bootstrap_token, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn spec() -> KubernetesProxyPodBoundarySpec { + KubernetesProxyPodBoundarySpec { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "a".repeat(64), + generation: "generation-1".to_string(), + session_epoch: "epoch-1".to_string(), + namespace_uid: "namespace-uid".to_string(), + sandbox_resource_uid: "sandbox-resource-uid".to_string(), + workload_pod_uid: "pod-uid".to_string(), + workload_pod_uid_path: PathBuf::from("/.openshell/pod-identity/uid"), + control_deployment_uid: "control-deployment-uid".to_string(), + egress_policy_uid: "network-policy-uid".to_string(), + egress_policy_resource_version: "1945".to_string(), + boundary_listener: "0.0.0.0:5500".parse().expect("valid listener"), + control_address: "10.42.0.7:5500".parse().expect("valid target"), + sandbox_tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), + private_key_path: PathBuf::from("/run/boundary/tls.key"), + client_ca_certificate_path: PathBuf::from("/run/boundary/client-ca.crt"), + }, + supervisor_tls: BoundaryClientTls { + server_name: "boundary.sandbox.openshell".to_string(), + ca_certificate_pem: "test-ca".to_string(), + certificate_chain_pem: "test-client-cert".to_string(), + private_key_pem: "test-client-key".to_string(), + }, + host_gateway_ip: Some("10.42.0.1".parse().expect("valid gateway IP")), + workload_identity: ResolvedWorkloadIdentity::new( + 1000, + 1000, + vec![1000], + "kubernetes-config".to_string(), + "sandbox:sandbox-resource-uid".to_string(), + ) + .unwrap(), + child_env: HashMap::new(), + } + } + + #[test] + fn provisioning_binds_identical_kubernetes_resource_claims() { + let provisioned = spec().provision(); + + assert_eq!( + provisioned.boundary_config.resource_claims, + provisioned.topology.resource_claims + ); + assert_eq!( + provisioned.topology.resource_claims["kubernetes.sandbox_resource_uid"], + "sandbox-resource-uid" + ); + assert_eq!( + provisioned.topology.resource_claims["kubernetes.egress_policy_resource_version"], + "1945" + ); + assert_eq!( + provisioned.boundary_config.driver_fence, + provisioned.topology.driver_fence + ); + assert!( + provisioned + .topology + .driver_fence + .validate_for_backend(BACKEND_NAME) + .is_ok() + ); + } + + #[test] + fn provisioning_uses_one_shared_tcp_protocol_across_pods() { + let provisioned = spec().provision(); + + assert_eq!( + provisioned.boundary_config.listener, + BoundaryListener::TlsTcp { + address: "0.0.0.0:5500".parse().expect("valid listener"), + tls: BoundaryServerTls { + certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), + private_key_path: PathBuf::from("/run/boundary/tls.key"), + client_ca_certificate_path: PathBuf::from("/run/boundary/client-ca.crt",), + }, + } + ); + assert_eq!( + provisioned.topology.transport, + BoundaryTransport::TlsTcp { + address: "10.42.0.7:5500".parse().expect("valid target"), + tls: BoundaryClientTls { + server_name: "boundary.sandbox.openshell".to_string(), + ca_certificate_pem: "test-ca".to_string(), + certificate_chain_pem: "test-client-cert".to_string(), + private_key_pem: "test-client-key".to_string(), + }, + } + ); + } + + #[test] + fn network_fence_denies_all_workload_initiated_egress() { + let fence = KubernetesProxyPodNetworkFenceSpec { + namespace: "sandbox-ns".to_string(), + policy_name: "openshell-boundary-sandbox-1".to_string(), + pair_label_value: "pair-1".to_string(), + boundary_port: 5500, + } + .provision(); + + let policy_spec = fence.workload_policy.spec.expect("policy has a spec"); + assert_eq!( + policy_spec.policy_types, + Some(vec!["Ingress".to_string(), "Egress".to_string()]) + ); + assert_eq!(policy_spec.egress, Some(Vec::new())); + assert_eq!( + policy_spec.pod_selector.match_labels, + Some(fence.workload_labels) + ); + } + + #[test] + fn network_fence_allows_only_paired_control_to_boundary_port() { + let fence = KubernetesProxyPodNetworkFenceSpec { + namespace: "sandbox-ns".to_string(), + policy_name: "openshell-boundary-sandbox-1".to_string(), + pair_label_value: "pair-1".to_string(), + boundary_port: 5500, + } + .provision(); + + let policy_spec = fence.workload_policy.spec.expect("policy has a spec"); + let ingress = policy_spec + .ingress + .expect("policy has ingress rules") + .pop() + .expect("policy has one ingress rule"); + let peer = ingress + .from + .expect("rule has peers") + .pop() + .expect("rule has one peer"); + assert_eq!( + peer.pod_selector + .expect("peer has a pod selector") + .match_labels, + Some(fence.control_labels) + ); + assert!(peer.namespace_selector.is_none()); + + let port = ingress + .ports + .expect("rule has ports") + .pop() + .expect("rule has one port"); + assert_eq!(port.protocol.as_deref(), Some("TCP")); + assert_eq!(port.port, Some(IntOrString::Int(5500))); + } +} diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 28d3c77a7d..8257b8372c 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -4,12 +4,13 @@ pub mod config; pub mod driver; pub mod grpc; +pub mod isolation; pub mod otel_tracing; +mod proxy_pod; pub use config::{ - AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, + KubernetesComputeConfig, KubernetesProxyPodConfig, ManagedSshIngressConfig, WorkspaceMode, managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 9b11f5da2a..76127c2fd9 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -11,10 +11,9 @@ use tracing::info; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver, - KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, - WorkspaceMode, + ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesProxyPodConfig, + ManagedSshIngressConfig, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -118,29 +117,17 @@ struct Args { #[arg( long, - env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", - default_value = "image-volume" + env = "OPENSHELL_K8S_PROXY_POD_NETWORK_POLICY_ENFORCED", + default_value_t = false )] - supervisor_sideload_method: SupervisorSideloadMethod, - - #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] - topology: SupervisorTopology, - - #[arg( - long = "sidecar-proxy-uid", - alias = "proxy-uid", - env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", - default_value_t = DEFAULT_PROXY_UID - )] - sidecar_proxy_uid: u32, + proxy_pod_network_policy_enforced: bool, #[arg( - long = "sidecar-process-binary-aware-network-policy", - env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", - default_value_t = true, - action = ArgAction::Set + long, + env = "OPENSHELL_K8S_PROXY_POD_BOUNDARY_PORT", + default_value_t = 5500 )] - sidecar_process_binary_aware_network_policy: bool, + proxy_pod_boundary_port: u16, /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] @@ -169,9 +156,6 @@ struct Args { #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, - #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] - app_armor_profile: Option, - /// Lifetime (seconds) of the projected `ServiceAccount` token /// kubelet writes into each sandbox pod for the `IssueSandboxToken` /// bootstrap exchange. Kubelet enforces a minimum of 600s; the @@ -259,12 +243,9 @@ async fn main() -> Result<()> { .supervisor_image .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args - .sidecar_process_binary_aware_network_policy, + proxy_pod: KubernetesProxyPodConfig { + network_policy_enforced: args.proxy_pod_network_policy_enforced, + boundary_port: args.proxy_pod_boundary_port, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, @@ -277,7 +258,6 @@ async fn main() -> Result<()> { client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, workspace_default_storage_size: std::env::var( "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", ) diff --git a/crates/openshell-driver-kubernetes/src/proxy_pod.rs b/crates/openshell-driver-kubernetes/src/proxy_pod.rs new file mode 100644 index 0000000000..749a709aae --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/proxy_pod.rs @@ -0,0 +1,736 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Kubernetes resources owned by the RFC 0012 proxy-pod topology. + +use std::collections::BTreeMap; +use std::path::Path; + +use k8s_openapi::ByteString; +use k8s_openapi::api::apps::v1::Deployment; +use k8s_openapi::api::core::v1::{Secret, Service}; +use k8s_openapi::api::networking::v1::{NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicySpec}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, OwnerReference}; +use kube::core::ObjectMeta; +use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; + +use crate::isolation::{ + BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesProxyPodNetworkFence, + KubernetesProxyPodNetworkFenceSpec, +}; + +pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap"; +pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap"; +pub const BOUNDARY_CONFIG_KEY: &str = "boundary.json"; +pub const TOPOLOGY_PAYLOAD_KEY: &str = "topology.json"; +pub const BOUNDARY_CERTIFICATE_KEY: &str = "tls.crt"; +pub const BOUNDARY_PRIVATE_KEY: &str = "tls.key"; +pub const BOUNDARY_CLIENT_CA_KEY: &str = "client-ca.crt"; +pub const PROXY_CA_CERTIFICATE_KEY: &str = "proxy-ca.crt"; +pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key"; +pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input"; +pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json"; +pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt"; +pub const BOUNDARY_PRIVATE_KEY_PATH: &str = "/.openshell/state/bootstrap/tls.key"; +pub const BOUNDARY_CLIENT_CA_PATH: &str = "/.openshell/state/bootstrap/client-ca.crt"; +pub const TOPOLOGY_PAYLOAD_PATH: &str = "/.openshell/supervisor/topology.json"; +pub const PROXY_CA_CERTIFICATE_PATH: &str = "/.openshell/supervisor/proxy-ca.crt"; +pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key"; +pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock"; + +pub struct ProxyCaMaterial { + pub certificate_pem: String, + pub private_key_pem: String, +} + +pub fn generate_proxy_ca_material() -> Result { + let key = KeyPair::generate().map_err(|error| format!("generate proxy CA key: {error}"))?; + let mut params = CertificateParams::default(); + params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + params + .distinguished_name + .push(DnType::CommonName, "OpenShell Sandbox CA"); + params + .distinguished_name + .push(DnType::OrganizationName, "OpenShell"); + params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign]; + let certificate = params + .self_signed(&key) + .map_err(|error| format!("generate proxy CA certificate: {error}"))?; + Ok(ProxyCaMaterial { + certificate_pem: certificate.pem(), + private_key_pem: key.serialize_pem(), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProxyPodNames { + pub sandbox_secret: String, + pub supervisor_secret: String, + pub boundary_service: String, + pub control_deployment: String, + pub workload_policy: String, + pub control_policy: String, +} + +impl ProxyPodNames { + #[must_use] + pub fn new(sandbox_id: &str) -> Self { + let suffix = sandbox_id.to_ascii_lowercase(); + Self { + sandbox_secret: format!("os-sandbox-{suffix}"), + supervisor_secret: format!("os-supervisor-{suffix}"), + boundary_service: format!("os-boundary-{suffix}"), + control_deployment: format!("os-supervisor-{suffix}"), + workload_policy: format!("os-boundary-{suffix}"), + control_policy: format!("os-supervisor-{suffix}"), + } + } + + /// Return stable companion names plus generation-specific immutable + /// bootstrap Secret names. + #[must_use] + pub fn for_generation(sandbox_id: &str, generation: &str) -> Self { + let mut names = Self::new(sandbox_id); + let generation = generation + .chars() + .filter(char::is_ascii_alphanumeric) + .take(12) + .collect::() + .to_ascii_lowercase(); + names.sandbox_secret = format!("{}-{generation}", names.sandbox_secret); + names.supervisor_secret = format!("{}-{generation}", names.supervisor_secret); + names + } +} + +#[must_use] +pub fn pair_label_value(sandbox_id: &str) -> String { + sandbox_id.to_ascii_lowercase() +} + +#[must_use] +pub fn workload_fence( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + boundary_port: u16, +) -> KubernetesProxyPodNetworkFence { + KubernetesProxyPodNetworkFenceSpec { + namespace: namespace.to_string(), + policy_name: names.workload_policy.clone(), + pair_label_value: pair_label_value(sandbox_id), + boundary_port, + } + .provision() +} + +#[must_use] +pub fn boundary_service( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + boundary_port: u16, + owner: OwnerReference, +) -> Service { + serde_json::from_value(serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": names.boundary_service, + "namespace": namespace, + "ownerReferences": [owner], + "labels": common_labels(sandbox_id, "boundary-service"), + }, + "spec": { + "selector": pair_labels(sandbox_id, "workload"), + "ports": [{"name": "boundary", "protocol": "TCP", "port": boundary_port, "targetPort": boundary_port}], + } + })) + .expect("boundary Service renderer must produce a valid object") +} + +#[allow(clippy::too_many_arguments, clippy::similar_names)] +#[must_use] +pub fn control_deployment( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + sandbox_name: &str, + gateway_id: &str, + supervisor_image: &str, + supervisor_pull_policy: &str, + service_account_name: &str, + control_uid: u32, + control_gid: u32, + image_pull_secrets: &[String], + grpc_endpoint: &str, + client_tls_secret_name: &str, + main_process_spec: &str, + log_level: &str, + sa_token_ttl_secs: i64, + https_proxy: Option<&str>, + no_proxy: Option<&str>, + proxy_auth_secret: Option<(&str, &str)>, + proxy_auth_allow_insecure: bool, + proxy_connect_by_hostname: bool, + provider_spiffe_socket_path: Option<&str>, + owner: OwnerReference, +) -> Deployment { + let labels = control_labels(sandbox_id, gateway_id); + let mut environment = vec![ + env_var( + "OPENSHELL_ADMITTED_ISOLATION_BACKEND", + crate::isolation::BACKEND_NAME, + ), + env_var("OPENSHELL_ENDPOINT", grpc_endpoint), + env_var("OPENSHELL_SANDBOX_ID", sandbox_id), + env_var("OPENSHELL_SANDBOX", sandbox_name), + env_var("OPENSHELL_MAIN_PROCESS_SPEC", main_process_spec), + env_var( + "OPENSHELL_K8S_SA_TOKEN_FILE", + "/var/run/secrets/openshell/token", + ), + env_var("OPENSHELL_SSH_SOCKET_PATH", "/run/openshell/ssh.sock"), + env_var(openshell_core::sandbox_env::SSH_SOCKET_SHARED, "true"), + env_var("OPENSHELL_PROXY_TLS_DIR", "/run/openshell/proxy-tls"), + env_var( + openshell_core::sandbox_env::PROXY_CA_CERT, + PROXY_CA_CERTIFICATE_PATH, + ), + env_var( + openshell_core::sandbox_env::PROXY_CA_KEY, + PROXY_CA_PRIVATE_KEY_PATH, + ), + env_var("OPENSHELL_LOG_LEVEL", log_level), + env_var( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ), + env_var( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + "", + ), + ]; + let mut volume_mounts = vec![ + serde_json::json!({"name": "bootstrap", "mountPath": "/.openshell/supervisor", "readOnly": true}), + serde_json::json!({"name": "sa-token", "mountPath": "/var/run/secrets/openshell", "readOnly": true}), + serde_json::json!({"name": "run", "mountPath": "/run/openshell"}), + serde_json::json!({"name": "logs", "mountPath": "/var/log"}), + ]; + let mut volumes = vec![ + serde_json::json!({"name": "bootstrap", "secret": {"secretName": names.supervisor_secret, "defaultMode": 0o440}}), + serde_json::json!({"name": "sa-token", "projected": {"sources": [{"serviceAccountToken": {"audience": "openshell-gateway", "expirationSeconds": sa_token_ttl_secs, "path": "token"}}], "defaultMode": 0o440}}), + serde_json::json!({"name": "run", "emptyDir": {}}), + serde_json::json!({"name": "logs", "emptyDir": {}}), + ]; + if !client_tls_secret_name.is_empty() { + environment.extend([ + env_var("OPENSHELL_TLS_CA", "/var/run/secrets/openshell-tls/ca.crt"), + env_var( + "OPENSHELL_TLS_CERT", + "/var/run/secrets/openshell-tls/tls.crt", + ), + env_var( + "OPENSHELL_TLS_KEY", + "/var/run/secrets/openshell-tls/tls.key", + ), + ]); + volume_mounts.push(serde_json::json!({"name": "client-tls", "mountPath": "/var/run/secrets/openshell-tls", "readOnly": true})); + volumes.push(serde_json::json!({"name": "client-tls", "secret": {"secretName": client_tls_secret_name, "defaultMode": 0o440}})); + } + let mut command = vec![ + "/openshell-supervisor".to_string(), + "--topology-backend-name".to_string(), + crate::isolation::BACKEND_NAME.to_string(), + "--topology-payload-file".to_string(), + TOPOLOGY_PAYLOAD_PATH.to_string(), + "--workdir".to_string(), + "/sandbox".to_string(), + "--health-socket-path".to_string(), + CONTROL_HEALTH_SOCKET_PATH.to_string(), + ]; + if let Some(url) = https_proxy { + command.extend(["--upstream-proxy".to_string(), url.to_string()]); + } + if let Some(hosts) = no_proxy { + command.extend(["--upstream-no-proxy".to_string(), hosts.to_string()]); + } + if proxy_auth_secret.is_some() { + command.extend([ + "--upstream-proxy-auth-file".to_string(), + openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(), + ]); + } + if proxy_auth_allow_insecure { + command.push("--upstream-proxy-auth-allow-insecure".to_string()); + } + if proxy_connect_by_hostname { + command.push("--upstream-proxy-connect-by-hostname".to_string()); + } + if let Some((secret_name, secret_key)) = proxy_auth_secret { + let auth_path = Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH); + volume_mounts.push(serde_json::json!({ + "name": "upstream-proxy-auth", + "mountPath": auth_path.parent().and_then(Path::to_str).expect("auth path has parent"), + "readOnly": true + })); + volumes.push(serde_json::json!({ + "name": "upstream-proxy-auth", + "secret": {"secretName": secret_name, "defaultMode": 0o440, "items": [{ + "key": secret_key, + "path": auth_path.file_name().and_then(|name| name.to_str()).expect("auth path has file name") + }]} + })); + } + if let Some(socket_path) = provider_spiffe_socket_path { + environment.push(env_var( + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + socket_path, + )); + volume_mounts.push(serde_json::json!({ + "name": "spiffe-workload-api", + "mountPath": Path::new(socket_path).parent().and_then(Path::to_str).expect("SPIFFE socket has parent"), + "readOnly": true + })); + volumes.push(serde_json::json!({ + "name": "spiffe-workload-api", + "csi": {"driver": "csi.spiffe.io", "readOnly": true} + })); + } + let mut container = serde_json::json!({ + "name": "supervisor", + "image": supervisor_image, + "command": command, + "terminationMessagePolicy": "FallbackToLogsOnError", + "env": environment, + "readinessProbe": { + "exec": {"command": [ + "/openshell-supervisor", + "health", + "--socket", + CONTROL_HEALTH_SOCKET_PATH + ]}, + "periodSeconds": 1, + "failureThreshold": 3 + }, + "securityContext": { + "runAsUser": control_uid, + "runAsGroup": control_gid, + "runAsNonRoot": true, + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": {"drop": ["ALL"]} + }, + "volumeMounts": volume_mounts, + }); + if !supervisor_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(supervisor_pull_policy); + } + serde_json::from_value(serde_json::json!({ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": {"name": names.control_deployment, "namespace": namespace, "ownerReferences": [owner], "labels": labels}, + "spec": { + "replicas": 0, + "strategy": {"type": "Recreate"}, + "selector": {"matchLabels": pair_labels(sandbox_id, "supervisor")}, + "template": { + "metadata": {"labels": control_labels(sandbox_id, gateway_id), "annotations": {"openshell.ai/sandbox-id": sandbox_id}}, + "spec": { + "serviceAccountName": service_account_name, + "imagePullSecrets": image_pull_secrets.iter().map(|name| serde_json::json!({"name": name})).collect::>(), + "automountServiceAccountToken": false, + "securityContext": { + "fsGroup": control_gid, + "fsGroupChangePolicy": "OnRootMismatch", + "seccompProfile": {"type": "RuntimeDefault"} + }, + "restartPolicy": "Always", + "containers": [container], + "volumes": volumes + } + } + } + })) + .expect("control Deployment renderer must produce a valid object") +} + +#[must_use] +pub fn control_egress_policy( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + owner: OwnerReference, +) -> NetworkPolicy { + NetworkPolicy { + metadata: ObjectMeta { + name: Some(names.control_policy.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, "control-egress")), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(pair_labels(sandbox_id, "supervisor")), + ..Default::default() + }, + policy_types: Some(vec!["Egress".to_string()]), + // Control is the policy-enforcing egress principal. Namespace-wide + // default-deny policies must not prevent its gateway/DNS/upstream dials. + egress: Some(vec![NetworkPolicyEgressRule::default()]), + ..Default::default() + }), + status: None, + } +} + +#[allow(clippy::too_many_arguments)] +#[must_use] +pub fn sandbox_bootstrap_secret( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + boundary_config: Vec, + boundary_certificate: Vec, + boundary_private_key: Vec, + boundary_client_ca: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.sandbox_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SANDBOX_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([ + (BOUNDARY_CONFIG_KEY.to_string(), ByteString(boundary_config)), + ( + BOUNDARY_CERTIFICATE_KEY.to_string(), + ByteString(boundary_certificate), + ), + ( + BOUNDARY_PRIVATE_KEY.to_string(), + ByteString(boundary_private_key), + ), + ( + BOUNDARY_CLIENT_CA_KEY.to_string(), + ByteString(boundary_client_ca), + ), + ])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +#[must_use] +pub fn supervisor_bootstrap_secret( + namespace: &str, + names: &ProxyPodNames, + sandbox_id: &str, + topology_payload: Vec, + proxy_ca_certificate: Vec, + proxy_ca_private_key: Vec, + owner: OwnerReference, +) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(names.supervisor_secret.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner]), + labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)), + ..Default::default() + }, + data: Some(BTreeMap::from([ + ( + TOPOLOGY_PAYLOAD_KEY.to_string(), + ByteString(topology_payload), + ), + ( + PROXY_CA_CERTIFICATE_KEY.to_string(), + ByteString(proxy_ca_certificate), + ), + ( + PROXY_CA_PRIVATE_KEY.to_string(), + ByteString(proxy_ca_private_key), + ), + ])), + immutable: Some(true), + type_: Some("Opaque".to_string()), + ..Default::default() + } +} + +#[must_use] +pub fn sandbox_owner_reference( + name: &str, + uid: &str, + api_version: &str, + controller: bool, +) -> OwnerReference { + OwnerReference { + api_version: api_version.to_string(), + kind: "Sandbox".to_string(), + name: name.to_string(), + uid: uid.to_string(), + controller: controller.then_some(true), + // The driver's RBAC intentionally does not permit mutating Sandbox + // finalizers. Kubernetes garbage collection does not require this bit, + // and setting it would make admission fail under + // OwnerReferencesPermissionEnforcement. + block_owner_deletion: Some(false), + } +} + +fn pair_labels(sandbox_id: &str, role: &str) -> BTreeMap { + BTreeMap::from([ + ( + BOUNDARY_PAIR_LABEL.to_string(), + pair_label_value(sandbox_id), + ), + (BOUNDARY_ROLE_LABEL.to_string(), role.to_string()), + ]) +} + +fn common_labels(sandbox_id: &str, component: &str) -> BTreeMap { + BTreeMap::from([ + ( + "openshell.ai/managed-by".to_string(), + "openshell".to_string(), + ), + ( + "openshell.ai/sandbox-id".to_string(), + sandbox_id.to_string(), + ), + ("openshell.ai/component".to_string(), component.to_string()), + ]) +} + +fn control_labels(sandbox_id: &str, gateway_id: &str) -> BTreeMap { + let mut labels = common_labels(sandbox_id, "supervisor"); + labels.extend(pair_labels(sandbox_id, "supervisor")); + labels.insert( + "openshell.ai/gateway-id".to_string(), + gateway_id.to_string(), + ); + labels +} + +fn env_var(name: &str, value: &str) -> serde_json::Value { + serde_json::json!({"name": name, "value": value}) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn owner() -> OwnerReference { + sandbox_owner_reference("demo", "uid-1", "agents.x-k8s.io/v1beta1", true) + } + + #[test] + fn service_selects_only_the_workload_boundary() { + let names = ProxyPodNames::new("4b67c0d0-1111-2222-3333-444444444444"); + let service = boundary_service("sandbox", &names, "pair", 5500, owner()); + assert_eq!( + service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL], + "workload" + ); + } + + #[test] + fn owner_reference_does_not_require_finalizer_mutation_permission() { + assert_eq!(owner().block_owner_deletion, Some(false)); + } + + #[test] + fn control_policy_explicitly_allows_control_egress() { + let names = ProxyPodNames::new("pair"); + let policy = control_egress_policy("sandbox", &names, "pair", owner()); + let spec = policy.spec.unwrap(); + assert_eq!(spec.policy_types.unwrap(), ["Egress"]); + assert_eq!(spec.egress.unwrap().len(), 1); + } + + #[test] + fn control_deployment_is_singleton_ready_and_unprivileged() { + let names = ProxyPodNames::new("pair"); + let deployment = control_deployment( + "sandbox", + &names, + "pair", + "demo", + "gateway", + "supervisor:latest", + "IfNotPresent", + "sandbox-sa", + 1000, + 1000, + &["registry-credentials".to_string()], + "https://gateway:8080", + "client-tls", + "{}", + "info", + 600, + None, + None, + None, + false, + false, + None, + owner(), + ); + let spec = deployment.spec.as_ref().unwrap(); + assert_eq!( + spec.strategy + .as_ref() + .and_then(|strategy| strategy.type_.as_deref()), + Some("Recreate") + ); + let container = &spec.template.spec.as_ref().unwrap().containers[0]; + let pod_spec = spec.template.spec.as_ref().unwrap(); + assert_eq!(pod_spec.automount_service_account_token, Some(false)); + assert_eq!( + pod_spec.image_pull_secrets.as_ref().unwrap()[0] + .name + .as_deref(), + Some("registry-credentials") + ); + let pod_security = + serde_json::to_value(pod_spec.security_context.as_ref().unwrap()).unwrap(); + assert_eq!(pod_security["fsGroup"], 1000); + assert_eq!(pod_security["seccompProfile"]["type"], "RuntimeDefault"); + let container_security = + serde_json::to_value(container.security_context.as_ref().unwrap()).unwrap(); + assert_eq!(container_security["runAsUser"], 1000); + assert_eq!(container_security["runAsNonRoot"], true); + assert_eq!(container_security["readOnlyRootFilesystem"], true); + assert_eq!( + container_security["capabilities"]["drop"], + serde_json::json!(["ALL"]) + ); + assert_eq!( + container + .readiness_probe + .as_ref() + .and_then(|probe| probe.exec.as_ref()) + .and_then(|exec| exec.command.as_ref()), + Some(&vec![ + "/openshell-supervisor".to_string(), + "health".to_string(), + "--socket".to_string(), + CONTROL_HEALTH_SOCKET_PATH.to_string(), + ]) + ); + let command = container.command.as_ref().unwrap(); + assert!( + command + .windows(2) + .any(|args| args == ["--health-socket-path", CONTROL_HEALTH_SOCKET_PATH]) + ); + let env = container.env.as_ref().unwrap(); + let env_value = |name: &str| { + env.iter() + .find(|variable| variable.name == name) + .and_then(|variable| variable.value.as_deref()) + }; + assert_eq!( + env_value(openshell_core::sandbox_env::SSH_SOCKET_SHARED), + Some("true") + ); + assert_eq!( + env_value(openshell_core::sandbox_env::PROXY_CA_CERT), + Some(PROXY_CA_CERTIFICATE_PATH) + ); + assert_eq!( + env_value(openshell_core::sandbox_env::PROXY_CA_KEY), + Some(PROXY_CA_PRIVATE_KEY_PATH) + ); + let mount = container + .volume_mounts + .as_ref() + .unwrap() + .iter() + .find(|mount| mount.name == "bootstrap") + .expect("durable supervisor material is mounted into supervisor"); + assert_eq!(mount.mount_path, "/.openshell/supervisor"); + assert_eq!(mount.read_only, Some(true)); + } + + #[test] + fn bootstrap_secrets_are_immutable_and_split_by_trust_domain() { + let names = ProxyPodNames::new("pair"); + let sandbox = sandbox_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + owner(), + ); + assert_eq!( + sandbox.metadata.labels.as_ref().unwrap()["openshell.ai/component"], + SANDBOX_SECRET_COMPONENT + ); + assert_eq!(sandbox.immutable, Some(true)); + let sandbox_keys = sandbox + .data + .unwrap() + .into_keys() + .collect::>(); + assert_eq!( + sandbox_keys, + std::collections::BTreeSet::from([ + BOUNDARY_CERTIFICATE_KEY.to_string(), + BOUNDARY_CLIENT_CA_KEY.to_string(), + BOUNDARY_CONFIG_KEY.to_string(), + BOUNDARY_PRIVATE_KEY.to_string(), + ]) + ); + + let supervisor = supervisor_bootstrap_secret( + "sandbox", + &names, + "pair", + Vec::new(), + Vec::new(), + Vec::new(), + owner(), + ); + assert_eq!( + supervisor.metadata.labels.as_ref().unwrap()["openshell.ai/component"], + SUPERVISOR_SECRET_COMPONENT + ); + assert_eq!(supervisor.immutable, Some(true)); + let supervisor_keys = supervisor + .data + .unwrap() + .into_keys() + .collect::>(); + assert_eq!( + supervisor_keys, + std::collections::BTreeSet::from([ + PROXY_CA_CERTIFICATE_KEY.to_string(), + PROXY_CA_PRIVATE_KEY.to_string(), + TOPOLOGY_PAYLOAD_KEY.to_string(), + ]) + ); + } + + #[test] + fn generated_proxy_ca_material_is_pem_encoded() { + let material = generate_proxy_ca_material().unwrap(); + assert!( + material + .certificate_pem + .starts_with("-----BEGIN CERTIFICATE-----") + ); + assert!(material.private_key_pem.contains("PRIVATE KEY")); + } +} diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index fb3161a604..ec66fb6a46 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -43,12 +43,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ --set server.disableTls=true \ @@ -230,7 +227,6 @@ discovery endpoint or its TLS CA. | securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. | | securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. | | securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. | -| server.appArmorProfile | string | `"Unconfined"` | Kubernetes AppArmor profile requested for sandbox agent containers. Default Unconfined avoids runtime/default AppArmor blocking the supervisor's network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit the field, "RuntimeDefault" to force the runtime default profile, or "Localhost/profile-name" for an operator-managed localhost profile. | | server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. | | server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace | bool | `false` | Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. | | server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. | @@ -300,10 +296,8 @@ discovery endpoint or its TLS CA. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | -| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | -| supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | -| supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | +| supervisor.proxyPod.boundaryPort | int | `5500` | Workload boundary TLS listener port. | +| supervisor.proxyPod.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. | | upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. | diff --git a/deploy/helm/openshell/README.md.gotmpl b/deploy/helm/openshell/README.md.gotmpl index cf8677741e..ad4782716c 100644 --- a/deploy/helm/openshell/README.md.gotmpl +++ b/deploy/helm/openshell/README.md.gotmpl @@ -43,12 +43,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -n openshell \ --set server.disableTls=true \ diff --git a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml index 35532440b2..70fdec8d7b 100644 --- a/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml +++ b/deploy/helm/openshell/ci/values-corporate-proxy-e2e.yaml @@ -3,9 +3,6 @@ # The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL # and creates `openshell-e2e-proxy-auth` before Helm installs the gateway. -supervisor: - topology: sidecar - upstreamProxy: authSecret: name: openshell-e2e-proxy-auth diff --git a/deploy/helm/openshell/ci/values-proxy-pod.yaml b/deploy/helm/openshell/ci/values-proxy-pod.yaml new file mode 100644 index 0000000000..85535aef1b --- /dev/null +++ b/deploy/helm/openshell/ci/values-proxy-pod.yaml @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +supervisor: + proxyPod: + networkPolicyEnforced: true diff --git a/deploy/helm/openshell/ci/values-sidecar-kata.yaml b/deploy/helm/openshell/ci/values-sidecar-kata.yaml deleted file mode 100644 index 2e23a1009e..0000000000 --- a/deploy/helm/openshell/ci/values-sidecar-kata.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology under -# a Kata RuntimeClass. -# -# Use with e2e/with-kube-gateway.sh by setting: -# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar-kata.yaml -# The e2e wrapper supplies the image repository and tag through OPENSHELL_REGISTRY -# and IMAGE_TAG for existing-cluster runs. - -supervisor: - # Use the sidecar topology under Kata so network enforcement runs in the - # sidecar and the sandbox agent container stays low-privilege. - topology: sidecar - sidecar: - # Keep strict process/binary-aware network policy enabled for the Kata - # validation path. Set this false only when intentionally validating the - # documented endpoint/L7-only downgrade mode. - processBinaryAwareNetworkPolicy: true - -# Kata validation clusters normally install this RuntimeClass. -server: - defaultRuntimeClassName: kata-qemu diff --git a/deploy/helm/openshell/ci/values-sidecar.yaml b/deploy/helm/openshell/ci/values-sidecar.yaml deleted file mode 100644 index ba8dc50ef1..0000000000 --- a/deploy/helm/openshell/ci/values-sidecar.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology. -# -# Merge after values.yaml and ci/values-skaffold.yaml: -# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar.yaml -# -# Or set: -# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar.yaml -# before running `mise run e2e:kubernetes`. -supervisor: - topology: sidecar - sidecar: - # The strict sidecar default requires cross-container /proc identity access. - # CI/dev e2e uses the explicit downgraded mode so endpoint and L7 policy - # coverage remains runnable on local k3d while that path is hardened. - processBinaryAwareNetworkPolicy: false diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index ce32c72132..fa3762c4f4 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -119,8 +119,6 @@ deploy: # To enable SPIFFE/SPIRE provider token grants (requires the # spire-crds and spire releases above): #- ci/values-spire.yaml - # To exercise the Kubernetes supervisor sidecar topology: - #- ci/values-sidecar.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -129,20 +127,6 @@ deploy: supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' profiles: - - name: sidecar - patches: - - op: add - path: /deploy/helm/releases/0/valuesFiles/- - value: ci/values-sidecar.yaml - - name: sidecar-mtls - patches: - - op: add - path: /deploy/helm/releases/0/valuesFiles/- - value: ci/values-sidecar.yaml - - op: add - path: /deploy/helm/releases/0/setValues - value: - server.disableTls: "false" - name: credential-driver-kubernetes-secrets patches: - op: add diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 3d9f2f3e0b..1488f940f3 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -194,25 +194,6 @@ the in-cluster Service DNS, release namespace, service port, and disableTls flag — so the default value works for any release name or namespace without override. */}} -{{/* -Supervisor sideload method. When supervisor.sideloadMethod is set, use it -verbatim. Otherwise auto-detect from the cluster version: the ImageVolume -feature gate is enabled by default starting in K8s v1.35 (GA in v1.36). -Clusters on v1.33-v1.34 can opt in by setting sideloadMethod explicitly -after enabling the feature gate. -*/}} -{{- define "openshell.supervisorSideloadMethod" -}} -{{- if .Values.supervisor.sideloadMethod -}} -{{- .Values.supervisor.sideloadMethod -}} -{{- else -}} -{{- if semverCompare ">=1.35-0" .Capabilities.KubeVersion.Version -}} -image-volume -{{- else -}} -init-container -{{- end -}} -{{- end -}} -{{- end }} - {{- define "openshell.grpcEndpoint" -}} {{- if .Values.server.grpcEndpoint -}} {{- .Values.server.grpcEndpoint -}} diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index eb1ed8e1d0..36281406df 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -72,6 +72,7 @@ rules: - pods verbs: - get + - patch {{- end }} {{- $copiedSecretNames := list }} {{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }} @@ -84,6 +85,23 @@ rules: {{- end }} {{- end }} {{- end }} + {{- if ne $workspaceMode "shared" }} + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "delete", "get", "list"] + {{- end }} {{- $copiedSecretNames = uniq $copiedSecretNames }} {{- if $copiedSecretNames }} # Copy only explicitly configured TLS and image-pull Secrets into workspace diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 083748aee3..67e882b0a9 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -158,8 +158,6 @@ data: {{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }} operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }} {{- end }} - supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} - topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.upstreamProxy.url }} https_proxy = {{ .Values.upstreamProxy.url | quote }} @@ -203,9 +201,6 @@ data: {{- if .Values.server.defaultRuntimeClassName }} default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }} {{- end }} - {{- if .Values.server.appArmorProfile }} - app_armor_profile = {{ .Values.server.appArmorProfile | quote }} - {{- end }} {{- if .Values.supervisor.image.pullPolicy }} supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} @@ -215,9 +210,9 @@ data: gateway_namespace = {{ .Release.Namespace | quote }} gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} } - [openshell.drivers.kubernetes.sidecar] - proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} - process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} + [openshell.drivers.kubernetes.proxy_pod] + network_policy_enforced = {{ .Values.supervisor.proxyPod.networkPolicyEnforced }} + boundary_port = {{ .Values.supervisor.proxyPod.boundaryPort | default 5500 }} {{- if not $credentialDrivers }} diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index dfd6423615..24978060b6 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -33,14 +33,29 @@ rules: - watch # Per-sandbox identity: TokenReview authenticates the projected token from # the configured sandbox service account, then the gateway resolves the - # returned pod name and UID to the pod's `openshell.ai/sandbox-id` - # annotation. patch is intentionally NOT granted — the annotation is set - # once at pod create and must remain immutable for the lifetime of the - # sandbox. + # returned pod name and UID to its immutable sandbox labels. PATCH is used + # only to remove the driver-owned bootstrap scheduling gate after the + # immutable bootstrap material has been created. - apiGroups: - "" resources: - pods verbs: - get + - patch + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["services"] + verbs: ["create", "delete", "get"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + - apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "delete", "get", "list"] {{- end }} diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index afecada9b6..87fb25a173 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -20,17 +20,23 @@ tests: resources: ["networkpolicies"] verbs: ["get", "create", "patch", "update"] - - it: omits managed NetworkPolicy permissions when isolation is disabled + - it: preserves proxy-pod fence permissions when gateway isolation is disabled set: server.drivers.kubernetes.workspaceMode: managed networkPolicy.enabled: false asserts: + - contains: + path: rules + content: + apiGroups: ["networking.k8s.io"] + resources: ["networkpolicies"] + verbs: ["create", "delete", "get", "list"] - notContains: path: rules content: apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] - any: true + verbs: ["get", "create", "patch", "update"] - it: grants broad secret access when credential driver is enabled (operator) set: @@ -57,17 +63,23 @@ tests: resourceNames: ["custom-client-tls"] verbs: ["get", "patch"] - - it: omits operator secret access when TLS and credential storage are disabled + - it: limits operator bootstrap secret access to create when TLS and credential storage are disabled set: server.drivers.kubernetes.workspaceMode: operator server.disableTls: true asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] - notContains: path: rules content: apiGroups: [""] resources: ["secrets"] - any: true + verbs: ["get", "create", "patch", "delete"] - it: restricts managed copies to TLS and configured image-pull secrets set: @@ -122,3 +134,37 @@ tests: apiGroups: [""] resources: ["secrets"] any: true + + - it: grants managed proxy-pod companion permissions + set: + server.drivers.kubernetes.workspaceMode: managed + asserts: + - contains: + path: rules + content: + apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + + - it: grants operator proxy-pod companion permissions + set: + server.drivers.kubernetes.workspaceMode: operator + asserts: + - contains: + path: rules + content: + apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create", "delete", "get", "list", "patch", "watch"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 1380cb18c5..81bc4bb1ff 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -145,16 +145,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?service_account_name\s*=\s*"openshell-sandbox"' - - it: renders combined supervisor topology by default under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"combined"' - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'supervisor[_]topology\s*=' - - it: renders operator-owned upstream proxy settings under the Kubernetes driver template: templates/gateway-config.yaml set: @@ -220,36 +210,6 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' - - it: renders sidecar supervisor topology under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - set: - supervisor.topology: sidecar - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"' - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'supervisor[_]topology\s*=' - - - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] - template: templates/gateway-config.yaml - set: - supervisor.sidecar.proxyUid: 2200 - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' - - - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] - template: templates/gateway-config.yaml - set: - supervisor.sidecar.processBinaryAwareNetworkPolicy: false - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' - - it: configures managed SSH isolation with the gateway peer template: templates/gateway-config.yaml asserts: @@ -268,22 +228,6 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_secrets\s*=\s*\["regcred", "backup-regcred"\]' - - it: renders the default sandbox AppArmor profile under [openshell.drivers.kubernetes] - template: templates/gateway-config.yaml - asserts: - - matchRegex: - path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?app_armor_profile\s*=\s*"Unconfined"' - - - it: omits sandbox AppArmor profile when disabled - template: templates/gateway-config.yaml - set: - server.appArmorProfile: "" - asserts: - - notMatchRegex: - path: data["gateway.toml"] - pattern: 'app_armor_profile\s*=' - - it: does not reuse gateway image pull secrets for sandbox pods template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 453654f790..9cc728eff0 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -26,9 +26,8 @@ image: # -- Gateway image tag. Defaults to the chart appVersion when empty. tag: "" -# Supervisor image for the openshell-sandbox binary injected into sandbox pods. -# The default repository and empty tag use the version-pinned image built into -# the gateway. Changing the repository or setting a tag enables a Helm override. +# Trusted runtime image. Its `openshell-sandbox` binary stages into the workload +# runtime volume, while its `openshell-supervisor` binary runs in a separate Pod. supervisor: image: # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. @@ -37,30 +36,11 @@ supervisor: pullPolicy: "" # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" - # -- How the supervisor binary is delivered into sandbox pods. - # Empty (default) = auto-detect from cluster version: - # K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) - # K8s < v1.35 -> "init-container" (copies via init container + emptyDir) - # On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, - # set this to "image-volume" explicitly. - sideloadMethod: "" - # -- Supervisor pod topology for Kubernetes sandboxes. - # "combined" runs the current single supervisor container in the agent pod. - # "sidecar" runs network enforcement in a dedicated sidecar and the process - # supervisor as a low-capability wrapper in the agent container. - topology: "combined" - sidecar: - # -- UID for relaxed long-running network sidecars in sidecar topology. - # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants - # the required /proc inspection capabilities into the effective set. The - # network init container installs nftables rules that exempt the effective - # sidecar UID. - proxyUid: 1337 - # -- Keep process/binary-aware network policy enabled in sidecar topology. - # When false, the network sidecar runs as proxyUid, drops the extra /proc - # inspection capabilities, and enforces endpoint/L7 policy without matching - # policy.binaries. - processBinaryAwareNetworkPolicy: true + proxyPod: + # -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. + networkPolicyEnforced: false + # -- Workload boundary TLS listener port. + boundaryPort: 5500 # -- Operator-owned corporate forward proxy for policy-approved TLS egress # from Kubernetes sandboxes. The workload cannot select or override it. @@ -265,12 +245,6 @@ server: # Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host # UID and capabilities become namespaced. enableUserNamespaces: false - # -- Kubernetes AppArmor profile requested for sandbox agent containers. - # Default Unconfined avoids runtime/default AppArmor blocking the supervisor's - # network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit - # the field, "RuntimeDefault" to force the runtime default profile, or - # "Localhost/profile-name" for an operator-managed localhost profile. - appArmorProfile: "Unconfined" # Kubernetes compute driver settings. drivers: kubernetes: diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 43e7d0338b..a59367dc72 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -3,140 +3,66 @@ # SPDX-License-Identifier: Apache-2.0 title: "OpenShift" sidebar-title: "OpenShift" -description: "Install the OpenShell Helm chart on OpenShift, including the SCC binding and chart overrides required by OpenShift's Security Context Constraints." +description: "Install the OpenShell Helm chart on OpenShift with capability-free sandbox workloads." keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" position: 6 --- +The Kubernetes driver resolves the UID range assigned to each OpenShift +namespace and renders the sandbox and supervisor with a numeric non-root +identity from that range. OpenShell does not require the `privileged` SCC or any +added Linux capability. + -The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS disabled. Use only for evaluation on a private network. +Verify that the selected OpenShift runtime profile permits an unprivileged +process to install a nested seccomp user-notification filter and use Landlock. +OpenShell fails sandbox startup when either capability-free runtime probe fails. -OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. - -OpenShell installs sandbox nftables rules as individual commands. On OpenShift -nodes where optional conntrack or packet log expressions are unavailable, those -optional rules can fail without rolling back the required proxy bypass reject -rules. - ## Prerequisites -- OpenShift 4.x cluster with `oc` configured -- Helm 3.x -- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs installed - -## Install - - +- OpenShift 4.x cluster with `oc` configured. +- Helm 3.x. +- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs. +- A CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces. -## Create the namespace +## Install OpenShell -Pre-create the namespace so the SCC binding can be applied before the chart installs: +Pre-create the namespace, then install the chart. Keep the default restricted +security posture and acknowledge NetworkPolicy only after validating the CNI. ```shell oc create ns openshell -``` - -## Grant the privileged SCC to sandbox pods - -Sandbox pods run under the `openshell-sandbox` service account in the `openshell` namespace and require the `privileged` SCC: - -```shell -oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell -``` - -## Install the chart with OpenShift overrides - -```shell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set server.disableTls=true \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null + --set supervisor.proxyPod.networkPolicyEnforced=true ``` -| Override | Reason | -|---|---| -| `server.disableTls=true` | Runs the gateway over plaintext HTTP for simpler evaluation. | -| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Clear the chart's hardcoded UID and fsGroup so OpenShift's SCC admission can assign them. | +The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and +uses the resulting UID/GID for the sandbox, agent, trusted init containers, and +supervisor. Each container sets `allowPrivilegeEscalation: false`, drops all +Linux capabilities, and uses `RuntimeDefault` seccomp. -## Wait for the gateway to be ready +Wait for the gateway: ```shell oc -n openshell rollout status statefulset/openshell ``` -If you set `workload.kind=deployment`, use -`oc -n openshell rollout status deployment/openshell` instead. - - +If you set `workload.kind=deployment`, wait for `deployment/openshell` instead. -## Connect to the gateway +## Connect to the Gateway -The gateway is now running over plaintext HTTP. Connect with `oc port-forward`: +Forward the gateway port for local evaluation: ```shell oc -n openshell port-forward svc/openshell 8080:8080 -``` - -Register the gateway with the CLI: - -```shell -openshell gateway add http://127.0.0.1:8080 --local --name openshift +openshell gateway add https://127.0.0.1:8080 --local --name openshift openshell status ``` -## Production: expose externally with a real certificate - -The steps above run the gateway over plaintext HTTP for quick evaluation. For -a real deployment, cert-manager can issue the gateway's server certificate -from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an -OpenShift Route with TLS passthrough exposes it externally while the gateway -keeps terminating its own TLS and mTLS. - -Install cert-manager and configure a working `ClusterIssuer` first — see -[Managing Certificates](/kubernetes/managing-certificates) for the -`certManager.serverIssuerRef` details. Configure an OIDC provider as described -in [Access Control](/kubernetes/access-control) — remote gateways authenticate -CLI users via OIDC, not mTLS, so the gateway must know the OIDC issuer URL. -Install the chart with: - -```shell -helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ - --version \ - --namespace openshell \ - --set podSecurityContext.fsGroup=null \ - --set securityContext.runAsUser=null \ - --set server.disableTls=false \ - --set certManager.enabled=true \ - --set certManager.serverIssuerRef.name= \ - --set certManager.serverIssuerRef.kind=ClusterIssuer \ - --set certManager.serverDnsNames[0]= \ - --set openshiftRoute.enabled=true \ - --set openshiftRoute.host= \ - --set server.oidc.issuer= \ - --set server.oidc.audience= -``` - -| Override | Reason | -|---|---| -| `certManager.serverIssuerRef` | Creates a second server certificate from your Issuer or ClusterIssuer for external clients. The gateway uses SNI to present this cert for the external hostname while continuing to present the internal (chart CA) cert to supervisors. The internal certificate's `ca.crt` is the chart CA that also signed the client cert, so the default `clientCaFromServerTlsSecret=true` is correct. | -| `openshiftRoute.enabled` / `openshiftRoute.host` | Creates an OpenShift Route with TLS passthrough — the router forwards the encrypted connection by SNI without decrypting, so the gateway uses the SNI hostname to select the external certificate. | -| `server.oidc.issuer` / `server.oidc.audience` | Configures server-side OIDC validation. Without these, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). | - -Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI -users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control): - -```shell -openshell gateway add https:// \ - --name openshift \ - --oidc-issuer -openshell gateway login openshift -``` - -## Next Steps - -- For more on certificate provisioning modes, refer to [Managing Certificates](/kubernetes/managing-certificates). -- To expose the gateway externally through the Kubernetes Gateway API instead of a Route, refer to [Ingress](/kubernetes/ingress). -- To configure OIDC authentication, refer to [Access Control](/kubernetes/access-control). +For a remote deployment, configure an OpenShift Route with TLS passthrough and +an OIDC provider. Refer to [Ingress](/kubernetes/ingress), [Managing +Certificates](/kubernetes/managing-certificates), and [Access +Control](/kubernetes/access-control). diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index fb7881af2d..4203a2bd20 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -199,14 +199,12 @@ The most commonly changed values are: | `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. | | `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. | -| `server.appArmorProfile` | AppArmor profile requested for sandbox agent containers. Defaults to `Unconfined`. | | `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. | | `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. | | `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. | | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | -| `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | -| `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | -| `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | +| `supervisor.proxyPod.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. | +| `supervisor.proxyPod.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. | | `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). | Use a values file for repeatable deployments: @@ -219,13 +217,6 @@ helm upgrade --install openshell \ --values my-values.yaml ``` -The chart defaults `server.appArmorProfile` to `Unconfined` because -runtime/default AppArmor profiles can block the supervisor's network namespace -mount setup on AppArmor-enabled nodes. Set `server.appArmorProfile` to an empty -string to omit the field, `RuntimeDefault` to force the runtime default, or -`Localhost/` when you load and manage a localhost profile on each -node. - To use private sandbox images, create a `kubernetes.io/dockerconfigjson` Secret in the sandbox namespace and reference its name: @@ -265,13 +256,12 @@ upstreamProxy: key: credentials authAllowInsecure: true -supervisor: - topology: sidecar ``` Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy. -Proxy credentials require `sidecar` topology. It mounts the credential only into the dedicated network supervisor container. OpenShell rejects credential Secrets with `combined` topology because Kubernetes `fsGroup` volume permission handling can make a shared credential mount readable by the sandbox group. +The credential mounts only in the separately scheduled supervisor Pod. The +sandbox workload cannot read it through its environment or volumes. ## RBAC @@ -321,7 +311,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). +- Kubernetes sandboxes use a separate sandbox Pod and supervisor Deployment; refer to [Topology](/kubernetes/topology). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index 869fc07f1b..406e2606fe 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -3,247 +3,119 @@ # SPDX-License-Identifier: Apache-2.0 title: "Kubernetes Sandbox Topology" sidebar-title: "Topology" -description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." -keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" +description: "Understand the capability-free sandbox and supervisor topology used for Kubernetes workloads." +keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Network Policy, Seccomp, Landlock" position: 2 --- -Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or -`sidecar` topology. Choose the topology based on which controls you need inside -the pod and how much privilege your cluster allows on the agent container. +OpenShell runs each Kubernetes sandbox as two separately scheduled workloads. +The sandbox owns the agent process. The supervisor owns gateway credentials, +policy decisions, and upstream connections. -## Choose a Topology +## Understand the Components -The default `combined` topology preserves the full OpenShell enforcement model. -Use `sidecar` only when you accept network-focused enforcement in exchange for a -lower-privilege agent container. - -| Topology | Use when | Main tradeoff | -|---|---|---| -| `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | -| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | - -## Privilege Model - -The long-running container permissions differ by topology: - -| Topology | Pod or container | UID/GID | Privilege escalation | Capabilities | Result | -|---|---|---|---|---|---| -| `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | -| `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | -| `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | -| `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | - -Short-lived setup containers still have the permissions needed to prepare the -pod: - -| Topology | Setup container | UID/GID | Privilege escalation | Capabilities | Purpose | -|---|---|---|---|---|---| -| `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | -| `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | - -## Combined Topology - -Combined topology is the original Kubernetes mode and remains the default. The -agent container starts the OpenShell supervisor, and the supervisor launches the -workload after applying sandbox setup. - -```mermaid -flowchart TB - Sandbox["agents.x-k8s.io Sandbox"] - - subgraph Pod["Sandbox pod"] - subgraph Agent["agent container"] - Supervisor["OpenShell supervisor
network + process + filesystem"] - Workload["Agent workload"] - end - end - - Gateway["OpenShell Gateway"] - External["External services"] - - Sandbox --> Pod - Supervisor --> Workload - Supervisor -->|"gateway callback / SSH relay"| Gateway - Supervisor -->|"policy-enforced egress"| External -``` - -Combined topology keeps these controls in one supervisor path: - -- Network endpoint and L7 policy enforcement. -- Filesystem policy enforcement. -- Process and binary identity checks. -- Privilege drop into the sandbox user. -- Gateway relay, SSH sessions, exec, and file sync. - -Because the supervisor performs network namespace setup and process/filesystem -controls from the agent container, Kubernetes grants that container elevated -Linux capabilities. Use this mode when you need the complete OpenShell sandbox -contract and your cluster policy permits those capabilities. - -## Sidecar Topology - -Sidecar topology splits the supervisor into a network sidecar and a -low-privilege process supervisor in the agent container. +The Kubernetes driver always uses this topology: ```mermaid -flowchart TB - Sandbox["agents.x-k8s.io Sandbox"] - - subgraph Pod["Sandbox pod"] - Init["network init container
root setup capabilities"] - State["shared state + TLS volumes"] - NetNS["pod network namespace"] - - subgraph Agent["agent container"] - ProcessSupervisor["process supervisor
network-only"] - Workload["Agent workload"] - end - - NetworkSidecar["network supervisor sidecar
UID 0 by default"] - SshEndpoint["abstract SSH relay socket
peer-PID authenticated"] - end - - Gateway["OpenShell Gateway"] - External["External services"] - - Sandbox --> Pod - Init -->|"installs nftables rules"| NetNS - ProcessSupervisor --> Workload - Workload -->|"egress redirected on loopback"| NetworkSidecar - NetworkSidecar -->|"gateway session + relays"| Gateway - NetworkSidecar -->|"policy-enforced egress"| External - NetworkSidecar -->|"control socket + proxy TLS"| State - ProcessSupervisor -->|"bootstrap + updates"| State - ProcessSupervisor --> SshEndpoint - NetworkSidecar -->|"SSH relay"| SshEndpoint - NetworkSidecar --- State -``` - -The pod contains these OpenShell-managed pieces: - -| Component | Runs as | Purpose | -|---|---|---| -| Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | -| Network sidecar | UID 0 by default; `supervisor.sidecar.proxyUid` when binary-aware policy is disabled | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and serves local policy/provider state over the sidecar control socket. | -| Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | - -In this topology, the agent container defaults to `runAsNonRoot: true`, -`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The -default binary-aware network sidecar runs as UID 0, drops default Linux -capabilities, and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for cross-UID workload -process identity resolution. Setting -`supervisor.sidecar.processBinaryAwareNetworkPolicy=false` runs the sidecar as -the configured non-root `proxyUid`, omits both capabilities, and downgrades -network policy to endpoint/L7 enforcement without binary matching. The root -init container keeps the setup capabilities needed to configure pod networking. - -Sidecar mode preserves gateway session behavior, including SSH connectivity, -because the network sidecar owns the gateway session and bridges relay requests -to a Linux abstract SSH socket owned by the process supervisor. The relay -verifies the socket peer PID against the authenticated control connection, so -the workload cannot replace the relay endpoint. The agent container does not get a -gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the -default sidecar path. - - -Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still -enforces network endpoint and L7 policy through the sidecar, and the process -supervisor applies Landlock filesystem policy and child seccomp filters where -the kernel/runtime supports them. The process supervisor does not perform -root-to-sandbox privilege dropping because Kubernetes starts the container as -the sandbox UID/GID, and it does not perform supervisor identity mount -isolation because gateway credentials are not mounted into the agent container. -Sidecar pods use `shareProcessNamespace: true` so the network sidecar can -resolve workload process and binary identity through `/proc/`. - - -## Credential Exposure - -Sidecar topology keeps gateway credentials in the network sidecar. The agent -container does not mount the projected ServiceAccount token used for sandbox -token bootstrap, does not mount the sandbox client TLS secret, and does not get -gateway callback environment variables. - -The network sidecar serves the policy and workload-facing provider environment -over a Unix control socket in the shared sidecar state volume. Before launching -the workload, the process supervisor establishes the only accepted connection. -The sidecar validates its UID, GID, and PID with peer credentials, unlinks the -listener, derives the SSH target from trusted configuration, and rejects later -clients. The connection receives bootstrap state and provider-environment -updates after settings polls. If it closes, the network sidecar exits so -Kubernetes recreates the one-client bootstrap listener, and the process -supervisor exits so Kubernetes terminates the workload and restarts the agent -container. This symmetric failure behavior prevents a surviving workload from -claiming the new control listener after an isolated sidecar restart. Future -child processes can see refreshed provider env without giving the agent -container gateway authentication material. This does not mutate the environment -of the already-running workload entrypoint. Use `combined` topology when you -need the full single-supervisor enforcement path; use additional runtime -isolation when you need a stronger container boundary around sidecar workloads. - -## RuntimeClass Isolation - -Sidecar topology has been validated with Kata Containers. It does not currently -support gVisor because sidecar mode requires pod-local nftables setup, which -gVisor does not provide to the init container. A supported sandboxed runtime -strengthens the container boundary while OpenShell focuses on network policy -enforcement from the sidecar. - -Runtime classes do not re-enable the OpenShell privilege-drop or supervisor -mount-isolation controls that sidecar mode relaxes. Use them as an additional -workload boundary, not as a replacement for the combined topology's full -supervisor controls. - -You can set a default runtime class in the Kubernetes driver configuration or -override it per sandbox with driver config: - -```shell -openshell sandbox create \ - --driver-config-json '{"kubernetes":{"pod":{"runtime_class_name":"kata-containers"}}}' \ - -- claude +flowchart LR + Gateway[OpenShell gateway] + Supervisor[Supervisor Deployment] + Sandbox[Sandbox workload Pod] + Agent[Agent processes] + External[External services] + + Gateway <-->|JWT-authenticated session| Supervisor + Supervisor <-->|TLS and bootstrap token| Sandbox + Sandbox --> Agent + Supervisor --> External ``` -## Enable Sidecar Mode +`openshell-sandbox` runs as PID 1 in the workload container. It launches the +agent, applies Landlock and child seccomp filters, identifies the process behind +each network operation, and relays approved streams. `openshell-supervisor` runs +in a separate Deployment. It authenticates to the gateway, evaluates policy, +handles L7 and provider transformations, and opens upstream connections. -For direct gateway TOML configuration, set the Kubernetes driver fields: +Both containers run as the same namespace-resolved non-root UID and GID. Their +Pod specs set `allowPrivilegeEscalation: false`, drop every Linux capability, +and use `RuntimeDefault` seccomp. The sandbox installs an additional nested +seccomp user-notification filter without requesting a capability. Startup fails +closed if the runtime blocks the required seccomp or Landlock operations. -```toml -[openshell.drivers.kubernetes] -topology = "sidecar" +## Enforce Network Isolation -[openshell.drivers.kubernetes.sidecar] -proxy_uid = 1337 -``` - -`proxy_uid` configures only the relaxed endpoint/L7-only sidecar. It must be at -least `1000` and must not match the sandbox UID. The default binary-aware mode -runs the sidecar as UID 0 instead. The network init container exempts the -effective sidecar UID from proxy redirection so the sidecar can reach the -gateway. - -When the Helm chart renders `gateway.toml`, set the equivalent chart values: +The driver creates the workload fence before it releases the workload Pod: ```yaml -supervisor: - topology: sidecar - sidecar: - proxyUid: 1337 - processBinaryAwareNetworkPolicy: true +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +spec: + podSelector: + matchLabels: + openshell.ai/boundary-role: workload + openshell.ai/boundary-pair: SANDBOX_ID + policyTypes: [Ingress, Egress] + ingress: + - from: + - podSelector: + matchLabels: + openshell.ai/boundary-role: supervisor + openshell.ai/boundary-pair: SANDBOX_ID + ports: + - protocol: TCP + port: 5500 + egress: [] ``` -Leave `topology` unset, or set it to `combined`, to keep the original -single-container supervisor path. For Helm installs, leave -`supervisor.topology` unset or set it to `combined`. - -Set `supervisor.sidecar.processBinaryAwareNetworkPolicy=false` only when you -accept downgrading sidecar network policy to endpoint/L7 enforcement without -matching `policy.binaries`. This changes the sidecar from UID 0 to `proxyUid` -and removes its `SYS_PTRACE` and `DAC_READ_SEARCH` capabilities, which are used -for cross-UID `/proc` inspection. - -## Next Steps - -- To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup). -- To configure gateway authentication, refer to [Access Control](/kubernetes/access-control). -- To review the driver fields, refer to [Gateway Configuration File](/reference/gateway-config). +The empty egress list blocks direct DNS, gateway, node, metadata, and Internet +connections from the workload. The ingress rule permits only the paired +supervisor to reach the sandbox TLS listener. A separate policy selects the +supervisor and permits egress so it can reach cluster DNS, the gateway, and +policy-approved upstream destinations. + +Kubernetes policies are additive. Keep sandbox namespaces under administrative +control so another principal cannot add permissive policies, create Pods with +OpenShell labels, or read bootstrap Secrets. Set +`supervisor.proxyPod.networkPolicyEnforced: true` only after you verify that the +cluster CNI enforces both ingress and egress policies for these namespaces. + +## Bootstrap a Sandbox + +The driver creates each sandbox generation in a fail-closed order: + +1. Create and validate the workload egress fence. +2. Create the Sandbox resource with a scheduling gate. +3. Inspect the admitted Pod identity, security context, DNS settings, and + generation-specific Secret reference. +4. Create separate immutable Secrets for sandbox and supervisor trust material. +5. Start the supervisor and remove the workload scheduling gate. +6. Publish readiness only after the supervisor attaches, confirms enforcement, + and registers the gateway relay. + +The trusted sandbox init container copies its Secret into a memory-backed +volume. The main container never mounts the projected Secret and removes the +staged bootstrap before it launches untrusted code. The supervisor receives an +audience-bound Kubernetes token, exchanges it for a sandbox-scoped JWT, and +keeps gateway and provider credentials outside the workload Pod. + +Stopping a sandbox removes the workload Pod and scales the supervisor to zero. +Starting it creates a new generation with new Secrets and a replacement +supervisor Deployment while preserving the workspace PVC. Deletion retains the +egress fence until Kubernetes confirms that the workload Pod is gone. + +## Check Cluster Requirements + +The topology requires the following cluster behavior: + +- Linux nodes and a container runtime that permits an unprivileged process to + install a nested seccomp user-notification filter under `RuntimeDefault`. +- Landlock enabled and usable by the non-root sandbox process. +- A CNI that enforces `networking.k8s.io/v1` ingress and egress policies, + including node-local and metadata destinations. +- Support for Pod scheduling gates and the safe + `net.ipv4.ip_unprivileged_port_start=0` sysctl. +- Administrative control of sandbox namespaces and OpenShell pair labels. + +OpenShell actively probes the Linux primitives and validates the admitted Pod +before starting the agent. Treat a failed probe or changed security posture as +an unsupported runtime, not a degraded mode. diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 3cd36a240f..1efe953ada 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -488,13 +488,13 @@ image_pull_secrets = ["regcred"] # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" -# Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" -# on older clusters or where the ImageVolume feature gate is off. -supervisor_sideload_method = "image-volume" -# "combined" runs the existing single supervisor container with full process, -# filesystem, and network enforcement in the agent container. "sidecar" moves -# pod-level network enforcement and gateway session handling into a network sidecar. -topology = "combined" +[openshell.drivers.kubernetes.proxy_pod] +# Set this only after verifying that the cluster CNI enforces both ingress and +# egress NetworkPolicy in every sandbox namespace. +network_policy_enforced = true +boundary_port = 5500 + +[openshell.drivers.kubernetes] # Optional corporate HTTP forward proxy for policy-approved TLS egress. The # sandbox workload cannot select or override these settings. Only http:// proxy # endpoints and TLS CONNECT traffic are supported; plain HTTP egress remains @@ -512,9 +512,7 @@ topology = "combined" # Secret prevents that Pod from starting; unreadable or malformed `user:pass` # content is validated fail-closed by the supervisor at startup and never # falls back to direct egress. -# Proxy credential Secrets require `topology = "sidecar"`. Combined topology -# shares its credential mount with the workload and can make it readable by the -# sandbox group through Kubernetes `fsGroup` volume permission handling. +# Proxy credentials mount only in the separately scheduled supervisor Pod. # Required with a credential Secret: Basic authentication to an http:// proxy # is cleartext on the connection to that proxy. # proxy_auth_allow_insecure = true @@ -526,7 +524,6 @@ ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" host_gateway_ip = "10.0.0.1" enable_user_namespaces = false -app_armor_profile = "Unconfined" workspace_default_storage_size = "10Gi" # Kubernetes StorageClass for the workspace PVC. Empty (default) omits the # field, using the cluster's default StorageClass. Set this on clusters with no @@ -561,19 +558,12 @@ enabled = true gateway_namespace = "openshell" gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } -[openshell.drivers.kubernetes.sidecar] -# UID used by relaxed long-running network sidecars. Strict process/binary-aware -# sidecars run as UID 0 so Kubernetes grants the required /proc inspection -# capabilities into the effective set. In sidecar topology the network init -# container installs nftables rules that exempt the effective sidecar UID, so -# this dedicated infrastructure UID must remain at least 1000 and must not -# match the sandbox workload UID. -proxy_uid = 1337 -# Keep process/binary-aware network policy enabled in sidecar topology. Set -# false to run the sidecar as proxy_uid, drop the sidecar's extra /proc -# inspection capabilities, and enforce endpoint/L7 policy without matching -# policy.binaries. -process_binary_aware_network_policy = true +[openshell.drivers.kubernetes.proxy_pod] +# Required acknowledgement that the cluster CNI enforces NetworkPolicy and the +# sandbox namespaces prevent untrusted policy, pod, label, and Secret changes. +network_policy_enforced = true +# TLS-protected boundary listener reached only by the paired control pod. +boundary_port = 5500 ``` In managed workspace mode, the Kubernetes driver copies each explicitly named diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index d6ecec92af..48ce29e749 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -228,7 +228,7 @@ OpenShell rejects mount `source`, `target`, and Docker volume `subpath` values with surrounding whitespace. OpenShell also rejects mount targets that replace the workspace root or container root, or contain or are contained by the configured SSH socket or reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network +`/etc/openshell-tls`, `/run/openshell`, and network namespace roots. These checks do not make host bind mounts safe. ## Podman Driver @@ -407,19 +407,16 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | -| `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | -| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `proxy_pod.network_policy_enforced` | `supervisor.proxyPod.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. | +| `proxy_pod.boundary_port` | `supervisor.proxyPod.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. | | `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. | | `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. | -| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. | -| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. Requires `sidecar` topology. | +| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. | +| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. | | `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. | | `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. | -| `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Dedicated UID of at least `1000` used by the relaxed sidecar when process/binary-aware network policy is disabled. It must not match the workload UID. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | -| `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | -| `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | @@ -431,30 +428,15 @@ remain limited to the explicitly configured TLS and image-pull Secret names. The driver creates copies only in gateway-owned managed namespaces. Do not reuse the gateway ServiceAccount for unrelated workloads. -In `combined` topology, the agent container carries the Linux capabilities -needed by the supervisor for network namespace setup, Landlock filesystem -policy, process privilege changes, and network policy enforcement. In `sidecar` -topology, the agent container runs as the resolved sandbox UID/GID with no added -Linux capabilities. A root init container performs the nftables setup, and the -long-running binary-aware sidecar runs as UID 0, drops default capabilities, -and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for workload process identity -resolution through shared `/proc`. The -`sidecar.process_binary_aware_network_policy = false` setting runs it as the -configured non-root `proxy_uid`, removes both capabilities, and relaxes network -policy to endpoint/L7 matching only. The -network sidecar owns gateway authentication and writes local policy/provider -state to the process supervisor over a local control socket, so the agent -container does not mount the sandbox bootstrap token or client TLS secret in -the default sidecar path. The provider environment is refreshed by the network -sidecar after settings polls and streamed to the process supervisor so future -child processes can see updated provider env without gateway access in the -agent container. -Sidecar mode keeps gateway session and SSH behavior. The process supervisor -applies Landlock filesystem policy and child seccomp filters where supported, -but it does not perform root-to-sandbox privilege dropping or supervisor -identity mount isolation. Network policy still runs in the sidecar, and sidecar -pods set `shareProcessNamespace: true` so the network sidecar can resolve -process/binary identity through `/proc/`. +The Kubernetes driver always uses a capability-free proxy-pod topology. The +workload Pod runs `openshell-sandbox` as the same non-root UID/GID as the agent +and requests no added Linux capabilities. A separate Deployment runs +`openshell-supervisor`, authenticates to the gateway with a sandbox JWT, and +owns upstream connections. Both containers disable privilege escalation, drop +all capabilities, and use `RuntimeDefault` seccomp. The sandbox adds a nested +seccomp user-notification filter and Landlock restrictions before it launches +the agent. An empty-egress `NetworkPolicy` is the mandatory outer fence for the +workload Pod. The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh index 4d9d2eeff2..43ee52140a 100755 --- a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # Helm post-renderer for the external Kubernetes compute-driver smoke test. -# It keeps the test-only sidecar and Unix socket plumbing out of the chart. +# It keeps the test-only external-driver container and Unix socket plumbing out +# of the chart. set -euo pipefail diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index e6948acbb6..e0a1f35898 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -7,6 +7,7 @@ //! is dropped, replacing the `trap cleanup EXIT` pattern from the bash tests. use std::process::Stdio; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -31,6 +32,23 @@ fn extract_sandbox_name(output: &str) -> Option { /// startup. const SANDBOX_READY_TIMEOUT: Duration = Duration::from_secs(600); +static NEXT_SANDBOX_NAME: AtomicU64 = AtomicU64::new(1); + +fn has_explicit_sandbox_name(args: &[&str]) -> bool { + args.iter() + .any(|arg| *arg == "--name" || arg.starts_with("--name=")) +} + +fn add_unique_name_if_missing(command: &mut tokio::process::Command, args: &[&str]) { + if !has_explicit_sandbox_name(args) { + command.arg("--name").arg(format!( + "e2e-{}-{}", + std::process::id(), + NEXT_SANDBOX_NAME.fetch_add(1, Ordering::Relaxed) + )); + } +} + /// RAII guard that deletes a sandbox on drop. /// /// For sandboxes created with `--keep` (long-running background command), the @@ -92,6 +110,7 @@ impl SandboxGuard { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, create_args); for arg in create_args { cmd.arg(arg); } @@ -166,10 +185,9 @@ impl SandboxGuard { /// which lets tests control competing and reconnecting clients directly. pub async fn create_detached_main(command: &[&str]) -> Result { let mut cmd = openshell_cmd(); - cmd.arg("sandbox") - .arg("create") - .arg("--detach") - .arg("--") + cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, &[]); + cmd.arg("--") .args(command) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -216,6 +234,7 @@ impl SandboxGuard { ) -> Result { let mut create_cmd = openshell_cmd(); create_cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut create_cmd, create_args); for arg in create_args { create_cmd.arg(arg); } @@ -354,6 +373,7 @@ impl SandboxGuard { ) -> Result { let mut cmd = openshell_cmd(); cmd.arg("sandbox").arg("create").arg("--detach"); + add_unique_name_if_missing(&mut cmd, &[]); for (local, dest) in uploads { cmd.arg("--upload").arg(format!("{local}:{dest}")); } @@ -680,3 +700,15 @@ impl Drop for SandboxGuard { }); } } + +#[cfg(test)] +mod tests { + use super::has_explicit_sandbox_name; + + #[test] + fn detects_explicit_sandbox_names() { + assert!(has_explicit_sandbox_name(&["--name", "example"])); + assert!(has_explicit_sandbox_name(&["--name=example"])); + assert!(!has_explicit_sandbox_name(&["--policy", "policy.yaml"])); + } +} diff --git a/e2e/rust/tests/kubernetes_corporate_proxy.rs b/e2e/rust/tests/kubernetes_corporate_proxy.rs index 0e83a1a799..e1c01dd56d 100644 --- a/e2e/rust/tests/kubernetes_corporate_proxy.rs +++ b/e2e/rust/tests/kubernetes_corporate_proxy.rs @@ -6,7 +6,7 @@ //! Kubernetes wiring coverage for authenticated corporate forward proxies. //! //! The shell wrapper configures the gateway before it starts, creates the -//! credential Secret, and forces sidecar topology. This test starts a proxy +//! credential Secret, and enables proxy-pod topology. This test starts a proxy //! and an HTTPS upstream on the host visible to sandbox pods, then proves that //! a permitted request uses authenticated CONNECT while a policy-denied port //! never reaches the proxy. diff --git a/e2e/rust/tests/sandbox_labels.rs b/e2e/rust/tests/sandbox_labels.rs index 4de545d7bb..90351266c5 100644 --- a/e2e/rust/tests/sandbox_labels.rs +++ b/e2e/rust/tests/sandbox_labels.rs @@ -115,23 +115,26 @@ async fn delete_sandbox(name: &str) { #[tokio::test] #[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle async fn sandbox_labels_are_stored_and_filterable() { + // Keep the unique names below the public 19-character routable-name + // limit. Hex keeps the per-process suffix compact on busy CI hosts. + let suffix = format!("{:x}", std::process::id()); + let dev_backend = format!("lbl-db-{suffix}"); + let staging_backend = format!("lbl-sb-{suffix}"); + let prod_frontend = format!("lbl-pf-{suffix}"); + let dev_data = format!("lbl-dd-{suffix}"); + // Create sandboxes with different labels let name1 = - create_sandbox_with_labels("e2e-lbl-dev-back", &[("env", "dev"), ("team", "backend")]) - .await; + create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await; - let name2 = create_sandbox_with_labels( - "e2e-lbl-stg-back", - &[("env", "staging"), ("team", "backend")], - ) - .await; + let name2 = + create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")]) + .await; let name3 = - create_sandbox_with_labels("e2e-lbl-prd-frnt", &[("env", "prod"), ("team", "frontend")]) - .await; + create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await; - let name4 = - create_sandbox_with_labels("e2e-lbl-dev-data", &[("env", "dev"), ("team", "data")]).await; + let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await; // Test 1: Verify labels are stored in sandbox metadata let details = get_sandbox_details(&name1).await; diff --git a/e2e/rust/tests/user_namespaces.rs b/e2e/rust/tests/user_namespaces.rs index 53f2c6ec1f..abe29fbc9f 100644 --- a/e2e/rust/tests/user_namespaces.rs +++ b/e2e/rust/tests/user_namespaces.rs @@ -8,8 +8,8 @@ //! Enables `OPENSHELL_ENABLE_USER_NAMESPACES` on the gateway, triggers sandbox //! creation, and inspects the resulting pod spec to confirm: //! 1. `spec.hostUsers` is `false` -//! 2. The container security context includes the extra capabilities -//! (SETUID, SETGID, DAC_READ_SEARCH) required for user namespace operation +//! 2. The container security context requests no added capabilities and +//! drops every capability //! //! The sandbox pod may fail to start in Docker-in-Docker dev clusters where the //! filesystem does not support ID-mapped mounts. The test inspects the pod spec @@ -211,7 +211,7 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { .await; // Inspect capabilities on the agent container. - let caps = kubectl(&[ + let cap_add = kubectl(&[ "get", "pod", &sandbox_name, @@ -221,6 +221,16 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.add}", ]) .await; + let cap_drop = kubectl(&[ + "get", + "pod", + &sandbox_name, + "-n", + "openshell", + "-o", + "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.drop}", + ]) + .await; // Clean up. stop_child(&mut child).await; @@ -235,18 +245,14 @@ async fn sandbox_pod_spec_has_user_namespace_fields() { "sandbox pod must have spec.hostUsers=false when user namespaces are enabled" ); - // Assert extra capabilities are present. - let caps_val = caps.expect("failed to get capabilities from pod spec"); - for cap in ["SETUID", "SETGID", "DAC_READ_SEARCH"] { - assert!( - caps_val.contains(cap), - "sandbox pod must include {cap} in capabilities when user namespaces are enabled, got: {caps_val}" - ); - } - for cap in ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] { - assert!( - caps_val.contains(cap), - "sandbox pod must include {cap} in capabilities, got: {caps_val}" - ); - } + let cap_add = cap_add.expect("failed to get added capabilities from pod spec"); + assert!( + cap_add.trim().is_empty(), + "sandbox pod must not request added capabilities, got: {cap_add}" + ); + let cap_drop = cap_drop.expect("failed to get dropped capabilities from pod spec"); + assert!( + cap_drop.contains("ALL"), + "sandbox pod must drop every capability, got: {cap_drop}" + ); } diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 533a000aeb..711e675408 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -186,7 +186,7 @@ Common findings: - Sandbox fails before readiness with an OCI workspace validation error: inspect the image's `WorkingDir` using the immutable image ID reported by the gateway. Empty, `/`, and explicit `/sandbox` use the managed `/sandbox` compatibility workspace. Any other workdir must be an absolute normalized directory with no symlink components; the final policy UID, primary GID, and supplementary groups must pass the kernel's effective traverse/write checks, including POSIX ACL and LSM decisions. OpenShell does not create, chown, or chmod a non-default image workdir. - Docker also rejects an image `VOLUME` that covers the workdir or one of its parents because the runtime would mask the immutable path before validation. Move the `VOLUME` below the workspace or remove the declaration. - A workdir rejected as a special filesystem or OpenShell control-path collision cannot be made valid with permissions. Move the image workdir away from kernel-backed mounts and the concrete supervisor, TLS, token, runtime, and socket paths named in the error. -- Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify `OPENSHELL_DOCKER_SUPERVISOR_BIN`, the sibling binary next to `openshell-gateway`, or the configured supervisor image contains `/openshell-sandbox`. +- Docker driver cannot initialize because its trusted runtime image is incomplete: verify `supervisor_image` contains both `/openshell-sandbox` and `/openshell-supervisor` with the expected release digest. - Sandbox never registers: check gateway logs and supervisor callback endpoint. - On macOS, repeated `Policy fetch failed after 5 attempts` messages with a Homebrew gateway bound to `[::1]:17670` indicate that the Docker @@ -195,8 +195,8 @@ Common findings: `127.0.0.1:17670` primary listener, and reuse it for authenticated sandbox callbacks. On an older release, set `bind_address = "127.0.0.1:17670"` or upgrade. -- Supervisor image exits before printing `openshell-sandbox --version`: verify the configured supervisor image contains a static executable at `/openshell-sandbox`. -- A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the Docker or Podman driver supplied the `policy-dns-transparent-tcp` runtime capability and inspect supervisor logs for missing `nft`, synthetic-route overlap, or namespace-local DNS/TCP listener bind failures. Kubernetes, VM, sidecar, and out-of-tree drivers must reject this policy until they provide the complete substrate; use omitted protocol with an explicit proxy on those runtimes. +- Trusted runtime validation fails: verify the configured image contains static `/openshell-sandbox` and `/openshell-supervisor` executables from the same release. +- The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. During a graceful gateway restart, Docker, Podman, and VM sandboxes with @@ -385,7 +385,7 @@ If `server.providerTokenGrants.spiffe.enabled=true`, the gateway should still render `[openshell.gateway.gateway_jwt]` and mount the `sandbox-jwt` Secret. SPIRE is used by both the gateway and sandbox supervisors for dynamic provider token grants. The gateway pod must mount the `spiffe-workload-api` CSI volume -and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; sandbox pods must +and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; supervisor Pods must receive the matching Workload API socket from the Kubernetes driver config. The gateway verifies supervisor JWT-SVIDs from JWT bundles fetched through this Workload API socket, not from the SPIRE OIDC discovery endpoint. @@ -501,58 +501,46 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\ kubectl -n get sandbox -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}' ``` -If `topology = "sidecar"` is rendered under `[openshell.drivers.kubernetes]`, -sandbox pods should have an `openshell-network-init` init container running -`--mode=network-init`, an `agent` container running -`openshell-sandbox --mode=process`, and an `openshell-supervisor-network` -container running `--mode=network`. The init container owns nftables setup and -should be the only sidecar topology container with `NET_ADMIN`. It also needs -`CHOWN`/`FOWNER` to hand shared emptyDir state to the effective sidecar UID. The -default binary-aware network sidecar runs as UID 0 with primary GID -`sandbox_gid` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH`. When -`process_binary_aware_network_policy = false`, it runs as the configured -non-root `proxy_uid` without those inspection capabilities. That dedicated -proxy UID must remain at least `1000` and must not match the workload UID -because the pod egress fence exempts its traffic. The pod `fsGroup` is set to -`sandbox_gid` in both modes. - -In sidecar topology only the network sidecar should mount the gateway bootstrap -credentials (`openshell-sa-token` and `openshell-client-tls`). The process -container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the -sandbox token file, or those credential mounts. Instead, the network sidecar -serves policy and provider environment state over the Unix control socket from -`OPENSHELL_SIDECAR_CONTROL_SOCKET` (`/run/openshell-sidecar/control.sock` by -default). The process supervisor must be the first and only client. After -validating its peer UID, GID, and PID, the sidecar unlinks the listener. If the -connection later closes, the network sidecar exits non-zero so Kubernetes can -restart it with a fresh listener. If the process supervisor fails before -launching the workload, -inspect both containers for control-socket bind, connect, bootstrap, or update -errors. If new SSH/exec sessions do not pick up refreshed provider environment, -inspect the network sidecar settings-poll logs and the process container logs -for provider environment update handling; the process container should consume -newer provider-env revisions without receiving gateway credentials. - -The process container reports the workload entrypoint PID over the same control -socket, and the network sidecar uses that PID for binary-scoped policy -decisions through `/proc`. If rules with `policy.binaries` are unexpectedly -denied, inspect the sidecar control logs and confirm the pod has -`shareProcessNamespace: true`. -The shared state directory should preserve `sandbox_gid` inheritance -(`02775`). Sidecar SSH uses the Linux abstract socket -`@openshell-sidecar-ssh`; the network sidecar verifies its peer PID before -bridging gateway relay requests. No `ssh.sock` file should appear in the shared -state directory. -Inspect all three when sandbox registration or egress enforcement fails: +Kubernetes always uses the proxy-pod topology. Helm must render +`network_policy_enforced = true`. This is an explicit operator assertion that +the cluster CNI enforces Kubernetes NetworkPolicy; the Kubernetes API cannot +attest enforcement. Run sandboxes only in a trusted namespace +where tenants cannot create pods, copy the OpenShell pair labels, or read the +bootstrap Secret. + +The workload Pod runs `/openshell-sandbox`. It has no +gateway credentials and no direct egress. A deny-all workload NetworkPolicy is +created before the suspended Sandbox resource, then a per-sandbox Service, +split immutable bootstrap Secrets, supervisor Deployment, supervisor-egress +policy, and sandbox-ingress policy are provisioned before the workload is +released. The supervisor Pod runs `/openshell-supervisor`. Both Pods use the +same resolved non-root identity, request no capabilities, drop `ALL`, disable +privilege escalation, and use `RuntimeDefault` seccomp. The supervisor reaches +the sandbox over per-sandbox TLS with server-certificate verification plus +bootstrap-token client authentication, and owns gateway policy, provider +credentials, DNS, and mediated upstream connections. + +Inspect the complete companion set when a proxy-pod sandbox remains Starting +or loses readiness: ```bash -kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*=' -kubectl -n get pod -o jsonpath='{range .spec.initContainers[*]}{.name}{" "}{.command}{"\n"}{end}' -kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' -kubectl -n logs -c openshell-network-init --tail=200 -kubectl -n logs -c openshell-supervisor-network --tail=200 -kubectl -n logs -c agent --tail=200 -``` +kubectl -n get sandbox,pod,deployment,replicaset,service,secret,networkpolicy -l openshell.ai/sandbox-id= +kubectl -n describe deployment -l openshell.ai/sandbox-id= +kubectl -n logs deployment/ --tail=200 +kubectl -n get pod -l openshell.ai/sandbox-id=,openshell.ai/boundary-role=workload -o yaml +kubectl -n get networkpolicy -l openshell.ai/sandbox-id= -o yaml +``` + +Creation and recovery fail closed. A missing Secret leaves both pods inert; a +missing or unobserved workload fence must prevent the driver from releasing the +Sandbox; and readiness requires both Agent Sandbox readiness and an Available +supervisor Deployment. Its exec readiness check succeeds only after the +supervisor has attached, confirmed enforcement, started or resumed the +workload, and registered the gateway access plane. Use both Pod logs for +bootstrap errors. An `EPERM` during enforcement setup means the runtime blocked +a required unprivileged seccomp, task-memory, or Landlock operation. Do not add +capabilities, gateway egress, or credentials to the workload Pod as a +workaround. #### Corporate upstream proxy @@ -569,26 +557,21 @@ helm -n openshell get values openshell | grep -A8 upstreamProxy ``` Only `http://host:port` forward proxies are supported; `https://` proxy URLs and -plain-HTTP egress are out of scope and rejected. Proxy credentials require -`topology = "sidecar"` — combined topology shares the credential mount with the -workload, so the gateway rejects credentials there. The credential Secret named -by `proxy_auth_secret_name` must exist in the sandbox namespace with the key -named by `proxy_auth_secret_key`, and Kubernetes will not create keys longer -than 253 bytes or named `.`/`..`. - -The proxy arguments and credential mount are injected only into the container -that runs network supervision (the `agent` container in combined topology, the -`openshell-supervisor-network` sidecar in sidecar topology). The one-shot -`openshell-network-init` container and the process `agent` container in sidecar -topology must never receive them. The credential is projected read-only as the +plain-HTTP egress are out of scope and rejected. The credential Secret named by +`proxy_auth_secret_name` must exist in the sandbox namespace with the key named +by `proxy_auth_secret_key`, and Kubernetes will not create keys longer than 253 +bytes or named `.`/`..`. + +The proxy arguments and credential mount are injected only into the supervisor +Pod. The sandbox Pod must never receive them. The credential is projected read-only as the `openshell-upstream-proxy-auth` volume at `/run/openshell/upstream-proxy-auth` and passed as `--upstream-proxy-auth-file`; it must never appear in env, annotations, or command arguments. ```bash kubectl -n get secret -o jsonpath='{.data}' >/dev/null && echo "secret present" -kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' | grep -- '--upstream-' -kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{": "}{range .volumeMounts[*]}{.name}{" "}{end}{"\n"}{end}' | grep upstream-proxy-auth +kubectl -n get deployment -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' | grep -- '--upstream-' +kubectl -n get deployment -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{": "}{range .volumeMounts[*]}{.name}{" "}{end}{"\n"}{end}' | grep upstream-proxy-auth kubectl -n get events --sort-by=.lastTimestamp | grep -Ei 'secret|MountVolume' | tail -n 20 ``` @@ -601,7 +584,7 @@ destination that should be direct is missing from `no_proxy`. Inspect the network supervisor logs for CONNECT and upstream-proxy decisions: ```bash -kubectl -n logs -c openshell-supervisor-network --tail=200 | grep -Ei 'upstream|connect|proxy' +kubectl -n logs deployment/ --tail=200 | grep -Ei 'upstream|connect|proxy' ``` ### Step 7: Check VM-Backed Gateways diff --git a/tasks/helm.toml b/tasks/helm.toml index 525e9c2e51..43f01bf3f1 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -71,46 +71,16 @@ description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)" dir = "deploy/helm/openshell" run = "skaffold dev" -["helm:skaffold:dev:sidecar"] -description = "Run skaffold dev with the Kubernetes supervisor sidecar topology" -dir = "deploy/helm/openshell" -run = "skaffold dev -p sidecar" - -["helm:skaffold:dev:sidecar-mtls"] -description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold dev -p sidecar-mtls" - ["helm:skaffold:run"] description = "Run a one-shot Skaffold deploy and register its local plaintext gateway" dir = "deploy/helm/openshell" run = "skaffold run && ../../../tasks/scripts/helm-k3s-local.sh register" -["helm:skaffold:run:sidecar"] -description = "Run a sidecar Skaffold deploy and register its local plaintext gateway" -dir = "deploy/helm/openshell" -run = "skaffold run -p sidecar && ../../../tasks/scripts/helm-k3s-local.sh register" - -["helm:skaffold:run:sidecar-mtls"] -description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold run -p sidecar-mtls" - ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" run = "skaffold delete" -["helm:skaffold:delete:sidecar"] -description = "Run skaffold delete for the Kubernetes supervisor sidecar topology" -dir = "deploy/helm/openshell" -run = "skaffold delete -p sidecar" - -["helm:skaffold:delete:sidecar-mtls"] -description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled" -dir = "deploy/helm/openshell" -run = "skaffold delete -p sidecar-mtls" - ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/test.toml b/tasks/test.toml index 4a5cda0890..0b5eee8069 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -180,9 +180,9 @@ run = [ "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] -["e2e:kubernetes:sidecar"] -description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" -env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } +["e2e:kubernetes:proxy-pod"] +description = "Run Kubernetes e2e with the sandbox/supervisor proxy-pod topology" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" @@ -238,7 +238,7 @@ depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" ["e2e:kubernetes:external-driver"] -description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver sidecar" +description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" } depends = ["e2e:conformance:build"] run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"