diff --git a/Cargo.lock b/Cargo.lock index 00efaf157a..2117029c12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3401,6 +3401,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.3" @@ -3573,6 +3582,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -3942,12 +3952,17 @@ dependencies = [ name = "openshell-driver-docker" version = "0.0.0" dependencies = [ + "async-trait", + "base64 0.22.1", "bollard", "bytes", "clap", "futures", + "libc", "miette", + "nix 0.29.0", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -3955,6 +3970,7 @@ dependencies = [ "prost-types", "serde", "serde_json", + "sha2 0.10.9", "tar", "temp-env", "tempfile", @@ -3966,6 +3982,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "uuid", ] [[package]] @@ -4285,6 +4302,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-driver-docker", "openshell-isolation", "openshell-isolation-vm", "openshell-ocsf", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index eadb06ad43..69ff12da5b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -211,7 +211,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | +| Docker | Local Linux development with Docker available. | Network-disabled workload container plus native host supervisor. | Uses RFC 0012 supervisor-owned creation and OCI seccomp notification to carry policy DNS and transparent TCP without a proxy port or an OpenShell binary in the workload. | | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 8f40555dfb..bd4f888328 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -139,6 +139,14 @@ generation-pinned authorization form the transparent TCP security boundary. Docker and Podman do not currently advertise usable IPv6 egress for this substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. +Isolation backends may provide the same substrate without namespace listeners. +Their DNS source carries bounded wire exchanges directly to policy DNS, and +their connection source supplies the captured synthetic destination and calling +binary identity with each stream. The Docker host-supervisor backend uses OCI +seccomp user notification to inject those streams while Docker networking stays +disabled. The supervisor therefore binds no workload-visible proxy or DNS port; +the synthetic mapping and pinned-destination checks remain unchanged. + Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static credential resolves only when the request host, port, and path match an endpoint diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 1c9e675f77..2a1473c2e2 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -29,12 +30,20 @@ tracing-subscriber = { workspace = true } bytes = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } url = { workspace = true } clap = { workspace = true } miette = { workspace = true } toml = { workspace = true } +async-trait = "0.1" +base64 = { workspace = true } +uuid = { workspace = true } + +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" +nix = { workspace = true, features = ["socket", "uio"] } [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 17ae1cfa03..de75130c87 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -1,193 +1,78 @@ # openshell-driver-docker -Docker-backed compute driver for local OpenShell gateways. - -When the gateway configures `[openshell.gateway.otlp]`, Docker compute-driver -spans export to the same OTLP/gRPC collector with the service name -`openshell-driver-docker`. The in-process driver preserves the gateway trace -context and emits the compute-driver RPC boundary that a standalone driver -would expose. - -The driver manages sandbox containers through the local Docker daemon with the -`bollard` client. It is intended for developer environments where Docker is -already available and running Kubernetes would be unnecessary. +Docker-backed compute and isolation driver for local Linux OpenShell gateways. The driver connects to `[openshell.drivers.docker].socket_path` when configured. -Otherwise, it uses the first standard local Docker socket that responds to an -API ping, which is the same selection mechanism used by gateway auto-detection. -An explicitly selected Docker driver falls back to `/var/run/docker.sock` when -no candidate responds. - -## Runtime Model - -The gateway runs as a host process. The Docker driver creates one container per -sandbox and starts the `openshell-sandbox` supervisor inside that container. The -supervisor then creates the nested sandbox namespace for the agent process. - -## Stop and Start - -Stop stops the managed container without removing it. Docker retains the -container writable layer, attached volumes, labels, token material, and restart -policy. Start starts that same container, so files in the resolved OCI -workspace remain available. A durably stopped sandbox is excluded from -gateway startup recovery and stays stopped across gateway restarts. Delete -continues to force-remove the container and clean up driver-owned material. -Graceful gateway shutdown sends `StopSandbox` for each sandbox whose persisted -phase requires running compute without changing that persisted intent. On -startup, the gateway sends an idempotent `StartSandbox` request for the same -sandboxes, restarting their retained containers. Explicitly stopped sandboxes -remain excluded. - -Before creating the container, the driver inspects the final sandbox image and -captures its immutable image ID, raw OCI `Config.User`, and OCI -`Config.WorkingDir`. Container creation uses that image ID, preventing a -mutable tag from changing between inspection and launch. The supervisor runs as -root, resolves omitted policy identity fields from the image declaration, and -drops only agent children to the resulting identity. Named OCI components -remain names after validation; a missing group is filled with the user's -numeric primary GID. Explicit `process.run_as_user` and -`process.run_as_group` values take precedence independently. - -An absolute OCI working directory becomes the agent workspace. An empty, -root (`/`), or explicit `/sandbox` declaration uses `/sandbox`, which OpenShell -creates when necessary and owns as a compatibility workspace. Any other image -workdir must already exist without symlink components. The completed identity, -including supplementary groups, must already be able to traverse every parent -and write and enter the workdir. OpenShell does not change its ownership or -mode. - -OpenShell deliberately asks the Linux kernel to make this access decision -under the completed sandbox identity instead of reproducing permission rules -from ownership and mode bits. Mode-bit inspection alone can reject authority -granted by a POSIX ACL or overlook a denial imposed by a Linux Security Module -such as SELinux or AppArmor. OpenShell does not configure or otherwise manage -ACLs or LSM policy here; the one-shot validator only observes the kernel's -effective decision. This keeps the no-authority-expansion invariant aligned -with the access the eventual workload will receive without adding a separate, -incomplete permission model to OpenShell. - -Image `VOLUME` declarations must not cover the workdir or one of its parents -because Docker would mount the volume before the supervisor could validate the -immutable image path. -Workdirs under the standard OCI runtime namespaces `/proc`, `/sys`, and `/dev` -are rejected, as are paths that overlap concrete OpenShell control resources. -The workspace is the child cwd and `HOME`. The supervisor starts from `/`, then -reports an invalid workdir as a readiness failure. - -Docker containers join an OpenShell-managed bridge network. The driver injects -`host.openshell.internal` and `host.docker.internal` so supervisors have stable -names for reaching the gateway host. On Docker Desktop, Colima, Rancher -Desktop, OrbStack, and macOS-hosted gateways, those names use Docker's -`host-gateway` alias. The driver requests a separate IPv4 loopback callback -listener when the primary listener does not already cover it. On native Linux -Docker, the gateway also binds the bridge gateway IP so containers can call -back to the host process. - -## Container Contract - -The driver-controlled container settings are part of the sandbox security -contract: - -| Setting | Purpose | -|---|---| -| `user = "0"` | The supervisor needs root inside the container to prepare namespaces, mounts, Landlock, and seccomp. | -| `network_mode = openshell` | Places the supervisor on the managed Docker bridge network. | -| `cap_add` | Grants supervisor-only capabilities required for namespace setup and process inspection. | -| `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. | -| `restart_policy = no` | A canonical main-process exit remains terminal and is not silently restarted by Docker. | -| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | -| CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | -| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | - -The agent child process does not retain these supervisor privileges. - -## Driver Config Mounts - -The gateway forwards the `docker` block from `--driver-config-json` to this -driver. The driver accepts user-supplied `mounts` entries with these Docker -mount types: - -- `bind`: mounts an absolute host path when `[openshell.drivers.docker]` - has `enable_bind_mounts = true`. -- `volume`: mounts an existing Docker named volume. The driver validates that - the volume exists before provisioning and never creates or removes it. - Docker local-driver volumes created with bind options are treated as host - bind mounts and require `enable_bind_mounts = true`. -- `tmpfs`: mounts an in-memory filesystem with optional `options`, - `size_bytes`, and `mode`. - -Host bind mounts are disabled by default because they expose gateway host -paths to sandbox requests. Image mounts are not part of the Docker -driver-config schema. The driver still uses internal bind mounts for -OpenShell-owned supervisor, token, and TLS material. - -Docker `bind` mounts accept `source`, `target`, optional `read_only`, and an -optional `selinux_label` of `shared` (applies `:z`) or `private` (applies -`:Z`) for SELinux-enforcing hosts. Docker `volume` mounts may include -`subpath`. User-supplied bind and volume mounts are read-only by default; set -`read_only: false` to make them writable. Mount `source`, `target`, and -`subpath` values must not contain surrounding whitespace. Mount targets must be -absolute container paths and must not replace or contain the resolved workspace -root. Nested workspace mounts remain valid. Mounts also must not overlap the -configured SSH socket or the reserved `/opt/openshell`, `/etc/openshell`, -`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network -namespace roots. - -Example named-volume usage: - -```shell -docker volume create openshell-work - -openshell sandbox create \ - --driver-config-json '{"docker":{"mounts":[{"type":"volume","source":"openshell-work","target":"/sandbox/work"}]}}' \ - -- claude -``` - -## Supervisor Binary Resolution - -The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into -each sandbox container. Resolution order is: +Otherwise, it selects a standard local Docker socket and falls back to +`/var/run/docker.sock` when Docker is explicitly enabled. + +## Runtime model + +Docker uses RFC 0012 supervisor-owned boundary creation. The compute driver +resolves the immutable workload image and launches a native +`openshell-sandbox` process on the gateway host. That supervisor registers the +Docker `IsolationBackend`, creates the container, confirms enforcement, starts +the workload, and exposes exec through the normal supervisor relay. + +The workload container contains no OpenShell binary, gateway credential, TLS +private key, or supervisor capability set. The dedicated mode runs the +workload as the gateway user's numeric UID and GID so the unprivileged host +supervisor can capture syscall arguments and hash the calling executable. It +uses `/` as the initial working directory and does not provide driver mounts. + +The backend binds a private host Unix socket before container creation and +passes OCI seccomp `listenerPath`, `listenerMetadata`, and `SCMP_ACT_NOTIFY` +through the Docker seccomp profile. runc sends the listener FD directly to the +host supervisor with `SCM_RIGHTS` before the workload starts. The container +drops every Linux capability and has direct networking disabled. + +The listener injects connected sockets for workload TCP and DNS operations. +DNS queries travel directly to the supervisor policy-DNS service and return +short-lived synthetic addresses. A later connection to a synthetic address is +bound to the queried hostname, calling-binary identity, allowed port, policy +generation, and pinned real addresses. The supervisor consumes these streams +without binding a container-visible proxy port or setting proxy environment +variables. Direct real-IP and non-mediated UDP traffic fail closed. + +For TLS termination, the backend read-only mounts the generated public CA and +combined public trust bundle. CA private key material remains in host +supervisor memory. + +The current implementation supports create, confirm, start, wait, +signal-main-process, delete, Docker exec, policy DNS, transparent TCP, and TLS +termination. Port forwarding, exec signaling, GPU devices, resource limits, +driver mounts, images that require root, and durable running-boundary recovery +remain unsupported and fail closed. + +## Supervisor binary resolution + +The native host supervisor is resolved in this order: 1. `supervisor_bin` in `[openshell.drivers.docker]`. 2. `supervisor_image` in `[openshell.drivers.docker]`, extracting - `/openshell-sandbox` from that image. -3. A sibling `openshell-sandbox` next to the running `openshell-gateway` binary. + `/openshell-sandbox` to a host cache. +3. A sibling `openshell-sandbox` next to `openshell-gateway`. 4. A local Linux cargo target build for the Docker daemon architecture. -5. The release-matched default supervisor image, extracting `/openshell-sandbox`. - -Release and Docker-image gateway builds bake the matching supervisor image tag -into the binary at compile time. The default Docker supervisor image is not -`:latest` unless a custom build explicitly sets that tag. - -## Callback and TLS +5. The release-matched default supervisor image. -`OPENSHELL_ENDPOINT` is injected from the gateway's configured gRPC endpoint. -When no endpoint is configured, the driver uses -`host.openshell.internal:` with the appropriate HTTP or HTTPS -scheme. Set `host_gateway_ip` only when the host has an explicit, locally -assigned address that containers should use for callbacks; package-managed -macOS gateways should leave it unset. +The resolved binary executes on the host; it is never mounted into the +workload container. -For HTTPS endpoints, the server certificate must include the endpoint host as a -subject alternative name. Docker sandboxes also need the client TLS bundle -mounted into the container and exposed with: +## Gateway authentication -- `OPENSHELL_TLS_CA` -- `OPENSHELL_TLS_CERT` -- `OPENSHELL_TLS_KEY` +The compute driver writes the sandbox JWT to a host-only state directory and +passes that path only to the native supervisor. HTTPS CA, certificate, and key +paths likewise remain on the host. The supervisor connects to the gateway over +the configured `grpc_endpoint`; host aliases such as +`host.openshell.internal` are normalized to loopback for the native process. -HTTP endpoints reject TLS material because the supervisor would not use it. +## Testing -## Environment Ownership +The standard Docker runner exercises this implementation: -The driver merges template environment and sandbox spec environment first, then -overwrites security-critical keys: - -- `OPENSHELL_ENDPOINT` -- `OPENSHELL_SANDBOX_ID` -- `OPENSHELL_SANDBOX` -- `OPENSHELL_SSH_SOCKET_PATH` -- `OPENSHELL_MAIN_PROCESS_SPEC` -- TLS path variables when HTTPS is enabled +```shell +mise run e2e:docker +``` -Do not allow sandbox images or templates to override these values. +Set `OPENSHELL_E2E_SANDBOX_IMAGE` to test another workload image supported by +the smoke scenario. diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs new file mode 100644 index 0000000000..8de9aad80d --- /dev/null +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -0,0 +1,1971 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor-created Docker isolation boundary. +//! +//! The workload has no Docker network. OCI seccomp user notification injects +//! supervisor-owned sockets and transports DNS and transparent TCP through the +//! RFC 0012 mediation sources without an in-container `OpenShell` process. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +use std::collections::hash_map::DefaultHasher; +use std::fmt; +use std::fs::{File, OpenOptions}; +use std::hash::{Hash, Hasher}; +use std::io::{IoSliceMut, Read as _, Write as _}; +use std::mem::size_of; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::os::fd::{AsRawFd as _, FromRawFd as _, OwnedFd, RawFd}; +use std::os::unix::fs::PermissionsExt as _; +use std::os::unix::net::{UnixDatagram, UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use bollard::Docker; +use bollard::container::LogOutput; +use bollard::errors::Error as BollardError; +use bollard::exec::{CreateExecOptions, ResizeExecOptions, StartExecOptions, StartExecResults}; +use bollard::models::{ + ContainerCreateBody, ContainerStateStatusEnum, ContainerWaitResponse, HostConfig, +}; +use bollard::query_parameters::{ + CreateContainerOptionsBuilder, KillContainerOptionsBuilder, RemoveContainerOptionsBuilder, +}; +use futures::StreamExt as _; +use nix::cmsg_space; +use nix::sys::socket::{ControlMessageOwned, MsgFlags, recvmsg}; +use openshell_core::driver_utils::{LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID}; +use openshell_isolation::contract::{ + BackendError, BinaryIdentity, BoundBoundary, BoundaryDuplexStream, BoundaryExec, + BoundaryExitStatus, BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, + BoundarySignal, BoundaryTerminal, CreatedBoundary, DnsMediationSource, DnsTransport, + ExecSession, ExecSpec, INTERFACE_VERSION, IsolationBackend, LoopbackTarget, MediatedConnection, + MediatedDnsQuery, NetworkMediationSource, ReadyBoundary, ResolveError, RunningBoundary, + SandboxContext, TopologyDescriptor, VerifiedBoundaryCreatePlan, VerifiedTopologyDescriptor, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use tokio::io::AsyncWriteExt as _; +use tokio::sync::{Mutex, mpsc, oneshot}; + +const BACKEND_NAME: &str = "docker"; +pub const DOCKER_SOCKET_ENV: &str = "OPENSHELL_DOCKER_SOCKET_PATH"; +pub const DOCKER_LISTENER_DIR_ENV: &str = "OPENSHELL_DOCKER_LISTENER_DIR"; +const CONTAINER_TLS_DIR: &str = "/etc/openshell-tls"; +const LISTENER_ACCEPT_TIMEOUT: Duration = Duration::from_secs(15); +const MAX_OCI_STATE_BYTES: usize = 128 * 1024; +const LABEL_LAUNCH_GENERATION: &str = "ai.openshell.prototype.launch-generation"; +const LABEL_EXPERIMENTAL: &str = "ai.openshell.prototype.isolation"; +const LABEL_PLAN_FINGERPRINT: &str = "ai.openshell.prototype.create-plan"; +const MIN_LISTENER_TOKEN_BYTES: usize = 32; + +/// Prepared inputs for the Docker creation path. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DockerBoundaryCreatePlan { + /// Resolved image ID or immutable image reference. + pub image: String, + /// Stable generation used with sandbox ID as the create idempotency key. + pub launch_generation: String, + /// Driver-generated secret stable for idempotent retries of this launch. + pub listener_token: String, + /// Stable compute-driver-selected container name. + pub container_name: String, + /// Trusted labels used by compute-driver discovery and ownership checks. + #[serde(default)] + pub labels: HashMap, + /// Environment passed directly to the admitted agent. + #[serde(default)] + pub env: Vec, + /// Optional container user. The image default is used when absent. + pub user: Option, +} + +impl fmt::Debug for DockerBoundaryCreatePlan { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DockerBoundaryCreatePlan") + .field("image", &self.image) + .field("launch_generation", &self.launch_generation) + .field("listener_token", &"") + .field("env", &self.env) + .field("user", &self.user) + .finish() + } +} + +impl DockerBoundaryCreatePlan { + /// Encode this backend-private plan into the common RFC envelope. + pub fn into_boundary_plan( + self, + ) -> Result { + let payload = serde_json::to_vec(&self).map_err(|error| { + BackendError::Descriptor(format!("encode Docker create plan: {error}")) + })?; + Ok(openshell_isolation::contract::BoundaryCreatePlan { + version: INTERFACE_VERSION, + backend_name: BACKEND_NAME.to_string(), + payload, + }) + } +} + +#[derive(Clone, Serialize, Deserialize)] +struct DockerTopology { + sandbox_id: String, + launch_generation: String, + container_id: String, + container_name: String, + listener_path: PathBuf, + listener_metadata: String, + plan_fingerprint: String, +} + +/// Linux Docker backend driven by a native host supervisor. +#[derive(Clone)] +pub struct DockerIsolationBackend { + docker: Arc, + listener_dir: PathBuf, + provider_env: HashMap, + proxy_tls_dir: Option, +} + +impl DockerIsolationBackend { + /// Construct a backend for a Docker daemon on this host. + /// + /// `listener_dir` must be visible in the Docker daemon/runc host mount + /// namespace. A remote Docker daemon therefore requires a colocated + /// backend rather than a client-local path. + #[must_use] + pub fn new(docker: Arc, listener_dir: PathBuf) -> Self { + Self { + docker, + listener_dir, + provider_env: HashMap::new(), + proxy_tls_dir: None, + } + } + + /// Connect to the Docker daemon selected by the compute driver and use a + /// per-sandbox listener directory owned by this host supervisor process. + pub fn from_host_environment( + provider_env: HashMap, + ) -> Result { + let socket_path = std::env::var_os(DOCKER_SOCKET_ENV) + .map(PathBuf::from) + .or_else(openshell_core::config::detect_docker_socket) + .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); + let socket = socket_path.to_str().ok_or_else(|| { + BackendError::Descriptor(format!( + "Docker socket path is not valid UTF-8: {}", + socket_path.display() + )) + })?; + let listener_dir = std::env::var_os(DOCKER_LISTENER_DIR_ENV) + .map(PathBuf::from) + .ok_or_else(|| { + BackendError::Descriptor(format!("{DOCKER_LISTENER_DIR_ENV} is required")) + })?; + let proxy_tls_dir = + std::env::var_os(openshell_core::sandbox_env::PROXY_TLS_DIR).map(PathBuf::from); + let docker = Docker::connect_with_socket(socket, 120, bollard::API_DEFAULT_VERSION) + .map_err(|error| docker_error("connect to Docker daemon", error))?; + Ok(Self { + docker: Arc::new(docker), + listener_dir, + provider_env, + proxy_tls_dir, + }) + } +} + +#[async_trait] +impl IsolationBackend for DockerIsolationBackend { + fn backend_name(&self) -> &str { + BACKEND_NAME + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn create( + &self, + plan: VerifiedBoundaryCreatePlan, + sandbox: SandboxContext, + ) -> Result { + let mut plan: DockerBoundaryCreatePlan = + serde_json::from_slice(plan.payload()).map_err(|error| { + BackendError::Descriptor(format!("decode Docker create plan: {error}")) + })?; + validate_create_plan(&plan)?; + for (name, value) in &self.provider_env { + plan.env.push(format!("{name}={value}")); + } + if self.proxy_tls_dir.is_some() { + plan.env.extend([ + format!("SSL_CERT_FILE={CONTAINER_TLS_DIR}/ca-bundle.pem"), + format!("REQUESTS_CA_BUNDLE={CONTAINER_TLS_DIR}/ca-bundle.pem"), + format!("CURL_CA_BUNDLE={CONTAINER_TLS_DIR}/ca-bundle.pem"), + format!("NODE_EXTRA_CA_CERTS={CONTAINER_TLS_DIR}/openshell-ca.pem"), + ]); + } + + let resource_key = resource_key(&sandbox.sandbox_id, &plan.launch_generation); + let container_name = plan.container_name.clone(); + let listener_path = self.listener_dir.join(format!("{resource_key}.sock")); + let listener_metadata = format!( + "openshell:{}:{}:{}", + sandbox.sandbox_id, plan.launch_generation, plan.listener_token + ); + let plan_fingerprint = plan_fingerprint(&plan)?; + let (listener, listener_guard) = bind_private_listener(&listener_path)?; + + let mut labels = plan.labels.clone(); + labels.extend([ + ( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ), + (LABEL_SANDBOX_ID.to_string(), sandbox.sandbox_id.clone()), + ( + LABEL_LAUNCH_GENERATION.to_string(), + plan.launch_generation.clone(), + ), + (LABEL_EXPERIMENTAL.to_string(), "docker-create".to_string()), + (LABEL_PLAN_FINGERPRINT.to_string(), plan_fingerprint.clone()), + ]); + let create_body = build_create_body( + &plan, + &sandbox, + &listener_path, + &listener_metadata, + labels, + self.proxy_tls_dir.as_deref(), + )?; + + let container_id = match self + .docker + .create_container( + Some( + CreateContainerOptionsBuilder::default() + .name(&container_name) + .build(), + ), + create_body, + ) + .await + { + Ok(response) => response.id, + Err(BollardError::DockerResponseServerError { + status_code: 409, .. + }) => { + let existing = self + .docker + .inspect_container(&container_name, None) + .await + .map_err(|error| docker_error("inspect idempotent Docker boundary", error))?; + validate_existing_container( + &existing, + &sandbox.sandbox_id, + &plan.launch_generation, + &plan_fingerprint, + )?; + existing.id.ok_or_else(|| { + BackendError::Attach("existing Docker boundary has no container ID".to_string()) + })? + } + Err(error) => return Err(docker_error("create Docker boundary", error)), + }; + + let topology = DockerTopology { + sandbox_id: sandbox.sandbox_id.clone(), + launch_generation: plan.launch_generation, + container_id, + container_name, + listener_path, + listener_metadata, + plan_fingerprint, + }; + let descriptor = topology_descriptor(&topology)?; + let mediation = Arc::new(DockerMediation::new()); + let bound = DockerBound { + docker: self.docker.clone(), + topology, + listener, + listener_guard, + mediation, + }; + Ok(CreatedBoundary::new(descriptor, Box::new(bound))) + } + + async fn destroy( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox_id: &str, + ) -> Result<(), BackendError> { + let topology = decode_topology(&descriptor)?; + validate_topology_identity(&topology, sandbox_id, &self.listener_dir)?; + + let inspected = match self + .docker + .inspect_container(&topology.container_id, None) + .await + { + Ok(inspected) => Some(inspected), + Err(BollardError::DockerResponseServerError { + status_code: 404, .. + }) => None, + Err(error) => return Err(docker_error("inspect Docker boundary for destroy", error)), + }; + if let Some(inspected) = inspected { + validate_container_labels( + &inspected, + &topology.sandbox_id, + &topology.launch_generation, + &topology.plan_fingerprint, + )?; + remove_container(&self.docker, &topology.container_id).await?; + } + remove_listener_artifacts(&topology.listener_path)?; + Ok(()) + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology = decode_topology(&descriptor)?; + validate_topology_identity(&topology, &sandbox.sandbox_id, &self.listener_dir)?; + let inspected = self + .docker + .inspect_container(&topology.container_id, None) + .await + .map_err(|error| docker_error("inspect Docker boundary for attach", error))?; + validate_existing_container( + &inspected, + &topology.sandbox_id, + &topology.launch_generation, + &topology.plan_fingerprint, + )?; + let (listener, listener_guard) = bind_private_listener(&topology.listener_path)?; + Ok(Box::new(DockerBound { + docker: self.docker.clone(), + topology, + listener, + listener_guard, + mediation: Arc::new(DockerMediation::new()), + })) + } +} + +fn validate_create_plan(plan: &DockerBoundaryCreatePlan) -> Result<(), BackendError> { + if plan.image.trim().is_empty() { + return Err(BackendError::Descriptor( + "Docker create plan image must not be empty".to_string(), + )); + } + if plan.launch_generation.trim().is_empty() { + return Err(BackendError::Descriptor( + "Docker create plan launch generation must not be empty".to_string(), + )); + } + if plan.container_name.trim().is_empty() { + return Err(BackendError::Descriptor( + "Docker create plan container name must not be empty".to_string(), + )); + } + if plan.listener_token.len() < MIN_LISTENER_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "Docker create plan listener token must be at least {MIN_LISTENER_TOKEN_BYTES} bytes" + ))); + } + Ok(()) +} + +fn plan_fingerprint(plan: &DockerBoundaryCreatePlan) -> Result { + let bytes = serde_json::to_vec(plan).map_err(|error| { + BackendError::Descriptor(format!("encode Docker plan fingerprint: {error}")) + })?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} + +fn resource_key(sandbox_id: &str, launch_generation: &str) -> String { + let mut hasher = DefaultHasher::new(); + sandbox_id.hash(&mut hasher); + launch_generation.hash(&mut hasher); + format!("{:016x}", hasher.finish()) +} + +fn build_create_body( + plan: &DockerBoundaryCreatePlan, + sandbox: &SandboxContext, + listener_path: &Path, + listener_metadata: &str, + labels: HashMap, + proxy_tls_dir: Option<&Path>, +) -> Result { + let seccomp = serde_json::json!({ + "defaultAction": "SCMP_ACT_ALLOW", + "listenerPath": listener_path, + "listenerMetadata": listener_metadata, + "syscalls": [{ + "names": ["socket", "connect", "sendto", "bpf"], + "action": "SCMP_ACT_NOTIFY" + }] + }); + let seccomp = serde_json::to_string(&seccomp).map_err(|error| { + BackendError::Descriptor(format!("encode Docker seccomp profile: {error}")) + })?; + + Ok(ContainerCreateBody { + image: Some(plan.image.clone()), + user: plan.user.clone(), + working_dir: sandbox.agent.workdir.clone(), + env: Some(plan.env.clone()), + entrypoint: Some(vec![sandbox.agent.program.clone()]), + cmd: Some(sandbox.agent.args.clone()), + tty: Some(sandbox.agent.interactive), + open_stdin: Some(sandbox.agent.interactive), + network_disabled: Some(true), + labels: Some(labels), + host_config: Some(HostConfig { + network_mode: Some("none".to_string()), + cap_drop: Some(vec!["ALL".to_string()]), + security_opt: Some(vec![ + "no-new-privileges=true".to_string(), + format!("seccomp={seccomp}"), + ]), + restart_policy: None, + binds: proxy_tls_dir + .map(|path| vec![format!("{}:{CONTAINER_TLS_DIR}:ro", path.display())]), + ..Default::default() + }), + ..Default::default() + }) +} + +fn topology_descriptor(topology: &DockerTopology) -> Result { + Ok(TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: BACKEND_NAME.to_string(), + payload: serde_json::to_vec(topology).map_err(|error| { + BackendError::Descriptor(format!("encode Docker topology: {error}")) + })?, + }) +} + +fn decode_topology( + descriptor: &VerifiedTopologyDescriptor, +) -> Result { + serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode Docker topology: {error}"))) +} + +fn validate_topology_identity( + topology: &DockerTopology, + sandbox_id: &str, + listener_dir: &Path, +) -> Result<(), BackendError> { + if topology.sandbox_id != sandbox_id { + return Err(BackendError::Denied(format!( + "Docker boundary sandbox {:?} does not match admitted sandbox {sandbox_id:?}", + topology.sandbox_id + ))); + } + let key = resource_key(&topology.sandbox_id, &topology.launch_generation); + let expected_listener = listener_dir.join(format!("{key}.sock")); + let expected_metadata_prefix = format!( + "openshell:{}:{}:", + topology.sandbox_id, topology.launch_generation + ); + if topology.container_id.is_empty() + || topology.container_name.is_empty() + || topology.listener_path != expected_listener + || !topology + .listener_metadata + .starts_with(&expected_metadata_prefix) + || topology.listener_metadata.len() + < expected_metadata_prefix.len() + MIN_LISTENER_TOKEN_BYTES + { + return Err(BackendError::Denied( + "Docker topology identity does not match its trusted resource key".to_string(), + )); + } + Ok(()) +} + +fn validate_existing_container( + inspected: &bollard::models::ContainerInspectResponse, + sandbox_id: &str, + launch_generation: &str, + plan_fingerprint: &str, +) -> Result<(), BackendError> { + validate_container_labels(inspected, sandbox_id, launch_generation, plan_fingerprint)?; + let status = inspected.state.as_ref().and_then(|state| state.status); + if status != Some(ContainerStateStatusEnum::CREATED) { + return Err(BackendError::Denied(format!( + "Docker create/attach prototype requires a non-running container, found {status:?}" + ))); + } + Ok(()) +} + +fn validate_container_labels( + inspected: &bollard::models::ContainerInspectResponse, + sandbox_id: &str, + launch_generation: &str, + plan_fingerprint: &str, +) -> Result<(), BackendError> { + let labels = inspected + .config + .as_ref() + .and_then(|config| config.labels.as_ref()) + .ok_or_else(|| BackendError::Denied("Docker boundary has no trusted labels".to_string()))?; + if labels.get(LABEL_SANDBOX_ID).map(String::as_str) != Some(sandbox_id) + || labels.get(LABEL_LAUNCH_GENERATION).map(String::as_str) != Some(launch_generation) + || labels.get(LABEL_EXPERIMENTAL).map(String::as_str) != Some("docker-create") + || labels.get(LABEL_PLAN_FINGERPRINT).map(String::as_str) != Some(plan_fingerprint) + { + return Err(BackendError::Denied( + "existing Docker boundary does not match sandbox launch generation".to_string(), + )); + } + Ok(()) +} + +async fn remove_container(docker: &Docker, container_id: &str) -> Result<(), BackendError> { + if let Err(error) = docker + .remove_container( + container_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await + && !matches!( + error, + BollardError::DockerResponseServerError { + status_code: 404, + .. + } + ) + { + return Err(docker_error("remove Docker boundary", error)); + } + Ok(()) +} + +fn remove_listener_artifacts(listener_path: &Path) -> Result<(), BackendError> { + for path in [ + listener_path.to_path_buf(), + listener_path.with_extension("lock"), + ] { + if let Err(error) = std::fs::remove_file(&path) + && error.kind() != std::io::ErrorKind::NotFound + { + return Err(BackendError::Attach(format!( + "remove Docker listener artifact {}: {error}", + path.display() + ))); + } + } + Ok(()) +} + +fn bind_private_listener( + path: &Path, +) -> Result<(UnixListener, Arc), BackendError> { + let parent = path.parent().ok_or_else(|| { + BackendError::Descriptor(format!( + "Docker listener path has no parent: {}", + path.display() + )) + })?; + std::fs::create_dir_all(parent).map_err(|error| { + BackendError::Attach(format!("create Docker listener directory: {error}")) + })?; + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)).map_err(|error| { + BackendError::Attach(format!("protect Docker listener directory: {error}")) + })?; + if path.as_os_str().as_encoded_bytes().len() >= 104 { + return Err(BackendError::Descriptor(format!( + "Docker listener path is too long for AF_UNIX: {}", + path.display() + ))); + } + let lock_path = path.with_extension("lock"); + let lock = acquire_listener_lock(&lock_path)?; + if path.exists() { + std::fs::remove_file(path).map_err(|error| { + BackendError::Attach(format!("remove stale Docker seccomp listener: {error}")) + })?; + } + let listener = UnixListener::bind(path) + .map_err(|error| BackendError::Attach(format!("bind Docker seccomp listener: {error}")))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).map_err(|error| { + BackendError::Attach(format!("protect Docker seccomp listener: {error}")) + })?; + Ok(( + listener, + Arc::new(ListenerPathGuard { + socket_path: path.to_path_buf(), + lock_path, + _lock: lock, + }), + )) +} + +fn acquire_listener_lock(path: &Path) -> Result { + let lock = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(path) + .map_err(|error| BackendError::Attach(format!("open Docker listener lock: {error}")))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|error| BackendError::Attach(format!("protect Docker listener lock: {error}")))?; + // SAFETY: flock operates on the live lock-file descriptor and does not + // access memory. The open file is retained by ListenerPathGuard. + let locked = unsafe { libc::flock(lock.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if locked != 0 { + return Err(BackendError::Denied(format!( + "Docker seccomp listener is already owned by an active supervisor: {}", + path.display() + ))); + } + Ok(lock) +} + +struct ListenerPathGuard { + socket_path: PathBuf, + lock_path: PathBuf, + _lock: File, +} + +impl Drop for ListenerPathGuard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.socket_path); + let _ = std::fs::remove_file(&self.lock_path); + } +} + +struct DockerBound { + docker: Arc, + topology: DockerTopology, + listener: UnixListener, + listener_guard: Arc, + mediation: Arc, +} + +#[async_trait] +impl BoundBoundary for DockerBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + fn dns_mediation_source(&self) -> Option> { + Some(self.mediation.clone()) + } + + async fn confirm(self: Box) -> Result, BackendError> { + let inspected = self + .docker + .inspect_container(&self.topology.container_id, None) + .await + .map_err(|error| docker_error("confirm Docker boundary", error))?; + validate_existing_container( + &inspected, + &self.topology.sandbox_id, + &self.topology.launch_generation, + &self.topology.plan_fingerprint, + )?; + Ok(Box::new(DockerReady { + docker: self.docker, + topology: self.topology, + listener: self.listener, + listener_guard: self.listener_guard, + mediation: self.mediation, + })) + } +} + +struct DockerReady { + docker: Arc, + topology: DockerTopology, + listener: UnixListener, + listener_guard: Arc, + mediation: Arc, +} + +#[async_trait] +impl ReadyBoundary for DockerReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let Self { + docker, + topology, + listener, + listener_guard, + mediation, + } = *self; + let expected_metadata = topology.listener_metadata.clone(); + let expected_container = topology.container_id.clone(); + let accept_task = tokio::task::spawn_blocking(move || { + let (stream, _) = listener.accept().map_err(|error| { + BackendError::Process(format!("accept Docker seccomp listener: {error}")) + })?; + receive_listener_fd(stream, &expected_metadata, &expected_container) + }); + + if let Err(error) = docker.start_container(&topology.container_id, None).await { + accept_task.abort(); + return Err(docker_error("start Docker boundary", error)); + } + + let listener_fd = tokio::time::timeout(LISTENER_ACCEPT_TIMEOUT, accept_task) + .await + .map_err(|_| { + BackendError::Process( + "timed out waiting for Docker seccomp listener FD".to_string(), + ) + })? + .map_err(|error| { + BackendError::Process(format!("Docker seccomp listener task failed: {error}")) + })??; + let mediation_for_worker = mediation.clone(); + tokio::task::spawn_blocking(move || { + run_notification_worker(listener_fd, mediation_for_worker); + }); + + let process = Arc::new(DockerProcess { + docker: docker.clone(), + container_id: topology.container_id.clone(), + exit: Mutex::new(None), + }); + Ok(Box::new(DockerRunning { + process, + exec: Arc::new(DockerExec { + docker, + container_id: topology.container_id, + }), + port_forward: Arc::new(UnsupportedDockerPortForward), + _listener_guard: listener_guard, + })) + } +} + +struct DockerRunning { + process: Arc, + exec: Arc, + port_forward: Arc, + _listener_guard: Arc, +} + +impl RunningBoundary for DockerRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct NetworkItem { + stream: std::net::TcpStream, + binary_identity: Result, + destination: SocketAddr, +} + +struct DockerMediation { + network_tx: mpsc::UnboundedSender, + network_rx: Mutex>, + dns_tx: mpsc::UnboundedSender, + dns_rx: Mutex>, +} + +impl DockerMediation { + fn new() -> Self { + let (network_tx, network_rx) = mpsc::unbounded_channel(); + let (dns_tx, dns_rx) = mpsc::unbounded_channel(); + Self { + network_tx, + network_rx: Mutex::new(network_rx), + dns_tx, + dns_rx: Mutex::new(dns_rx), + } + } +} + +#[async_trait] +impl NetworkMediationSource for DockerMediation { + async fn accept(&self) -> Result { + let item = self.network_rx.lock().await.recv().await.ok_or_else(|| { + BackendError::Unavailable("Docker network mediation stopped".to_string()) + })?; + item.stream + .set_nonblocking(true) + .map_err(|error| BackendError::Process(format!("prepare mediated socket: {error}")))?; + let stream = tokio::net::TcpStream::from_std(item.stream) + .map_err(|error| BackendError::Process(format!("adopt mediated socket: {error}")))?; + Ok(MediatedConnection { + stream: Box::new(stream), + binary_identity: item.binary_identity, + destination: Some(item.destination), + }) + } +} + +#[async_trait] +impl DnsMediationSource for DockerMediation { + async fn accept(&self) -> Result { + self.dns_rx + .lock() + .await + .recv() + .await + .ok_or_else(|| BackendError::Unavailable("Docker DNS mediation stopped".to_string())) + } +} + +struct DockerProcess { + docker: Arc, + container_id: String, + exit: Mutex>, +} + +#[async_trait] +impl BoundaryProcess for DockerProcess { + async fn wait(&self) -> Result { + let mut exit = self.exit.lock().await; + if let Some(status) = *exit { + return Ok(status); + } + let next = self + .docker + .wait_container(&self.container_id, None) + .next() + .await; + let status = match next { + Some(Ok(ContainerWaitResponse { status_code, .. })) => { + BoundaryExitStatus::Exited(i32::try_from(status_code).unwrap_or(i32::MAX)) + } + Some(Err(BollardError::DockerContainerWaitError { code, .. })) => { + BoundaryExitStatus::Exited(i32::try_from(code).unwrap_or(i32::MAX)) + } + Some(Err(error)) => return Err(docker_error("wait for Docker boundary", error)), + None => { + return Err(BackendError::Terminated( + "Docker wait stream ended without an exit status".to_string(), + )); + } + }; + *exit = Some(status); + Ok(status) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let signal = match signal { + BoundarySignal::Term => "SIGTERM", + BoundarySignal::Kill => "SIGKILL", + BoundarySignal::Int => "SIGINT", + BoundarySignal::Hup => "SIGHUP", + }; + self.docker + .kill_container( + &self.container_id, + Some( + KillContainerOptionsBuilder::default() + .signal(signal) + .build(), + ), + ) + .await + .map_err(|error| docker_error("signal Docker boundary", error)) + } + + async fn terminate(&self) -> Result<(), BackendError> { + remove_container(&self.docker, &self.container_id).await + } +} + +struct DockerExec { + docker: Arc, + container_id: String, +} + +#[async_trait] +impl BoundaryExec for DockerExec { + async fn exec(&self, spec: ExecSpec) -> Result { + let mut command = Vec::with_capacity(spec.args.len() + 1); + command.push(spec.program); + command.extend(spec.args); + let created = self + .docker + .create_exec( + &self.container_id, + CreateExecOptions { + attach_stdin: Some(true), + attach_stdout: Some(true), + attach_stderr: Some(true), + tty: Some(spec.pty), + env: Some( + spec.env + .into_iter() + .map(|(name, value)| format!("{name}={value}")) + .collect(), + ), + cmd: Some(command), + working_dir: spec.workdir, + privileged: Some(false), + ..Default::default() + }, + ) + .await + .map_err(|error| docker_error("create Docker exec", error))?; + let started = self + .docker + .start_exec( + &created.id, + Some(StartExecOptions { + detach: false, + tty: spec.pty, + output_capacity: Some(64 * 1024), + }), + ) + .await + .map_err(|error| docker_error("start Docker exec", error))?; + let StartExecResults::Attached { output, input } = started else { + return Err(BackendError::Process( + "Docker exec unexpectedly started detached".to_string(), + )); + }; + + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + tokio::spawn(pump_docker_exec_input(stdin_pump, input)); + tokio::spawn(pump_docker_exec_output(output, stdout_pump, stderr_pump)); + + let process: Arc = Arc::new(DockerExecProcess { + docker: self.docker.clone(), + exec_id: created.id.clone(), + exit: Mutex::new(None), + }); + let terminal: Option> = if spec.pty { + let terminal: Arc = Arc::new(DockerTerminal { + docker: self.docker.clone(), + exec_id: created.id, + }); + Some(terminal) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if spec.pty { + None + } else { + let stderr: BoundaryOutput = Box::new(stderr); + Some(stderr) + }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) + } +} + +struct DockerExecProcess { + docker: Arc, + exec_id: String, + exit: Mutex>, +} + +#[async_trait] +impl BoundaryProcess for DockerExecProcess { + async fn wait(&self) -> Result { + let mut exit = self.exit.lock().await; + if let Some(status) = *exit { + return Ok(status); + } + loop { + let inspected = self + .docker + .inspect_exec(&self.exec_id) + .await + .map_err(|error| docker_error("inspect Docker exec", error))?; + if inspected.running == Some(false) { + let code = inspected + .exit_code + .and_then(|code| i32::try_from(code).ok()) + .unwrap_or(i32::MAX); + let status = BoundaryExitStatus::Exited(code); + *exit = Some(status); + return Ok(status); + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + } + + async fn signal(&self, _signal: BoundarySignal) -> Result<(), BackendError> { + Err(BackendError::Unsupported( + "Docker exec signaling is not implemented".to_string(), + )) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct DockerTerminal { + docker: Arc, + exec_id: String, +} + +#[async_trait] +impl BoundaryTerminal for DockerTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + self.docker + .resize_exec( + &self.exec_id, + ResizeExecOptions { + width: cols, + height: rows, + }, + ) + .await + .map_err(|error| docker_error("resize Docker exec terminal", error)) + } +} + +async fn pump_docker_exec_input( + mut source: tokio::io::DuplexStream, + mut destination: std::pin::Pin>, +) { + let _ = tokio::io::copy(&mut source, &mut destination).await; + let _ = destination.shutdown().await; +} + +async fn pump_docker_exec_output( + mut output: std::pin::Pin< + Box> + Send>, + >, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, +) { + while let Some(item) = output.next().await { + let Ok(item) = item else { + break; + }; + match item { + LogOutput::StdErr { message } => { + if stderr.write_all(&message).await.is_err() { + break; + } + } + LogOutput::StdOut { message } + | LogOutput::StdIn { message } + | LogOutput::Console { message } => { + if stdout.write_all(&message).await.is_err() { + break; + } + } + } + } + let _ = stdout.shutdown().await; + let _ = stderr.shutdown().await; +} + +struct UnsupportedDockerPortForward; + +#[async_trait] +impl BoundaryPortForward for UnsupportedDockerPortForward { + async fn connect(&self, _target: LoopbackTarget) -> Result { + Err(BackendError::Unsupported( + "Docker create proof does not implement port forwarding".to_string(), + )) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct OciContainerProcessState { + fds: Vec, + pid: i32, + metadata: String, + state: OciState, +} + +#[derive(Deserialize)] +struct OciState { + id: String, +} + +fn receive_listener_fd( + mut stream: UnixStream, + expected_metadata: &str, + expected_container: &str, +) -> Result { + let mut first = vec![0_u8; MAX_OCI_STATE_BYTES]; + let mut iov = [IoSliceMut::new(&mut first)]; + let mut control = cmsg_space!([RawFd; 1]); + let message = recvmsg::<()>( + stream.as_raw_fd(), + &mut iov, + Some(&mut control), + MsgFlags::empty(), + ) + .map_err(|error| BackendError::Process(format!("receive OCI seccomp state: {error}")))?; + let bytes = message.bytes; + if message.flags.contains(MsgFlags::MSG_CTRUNC) { + return Err(BackendError::Process( + "OCI seccomp ancillary data was truncated".to_string(), + )); + } + let mut received_fds = Vec::new(); + for control_message in message.cmsgs().map_err(|error| { + BackendError::Process(format!("decode OCI seccomp control message: {error}")) + })? { + if let ControlMessageOwned::ScmRights(fds) = control_message { + received_fds.extend(fds.into_iter().map(|fd| { + // SAFETY: each SCM_RIGHTS entry is a new descriptor owned by + // this process and has not been wrapped or closed elsewhere. + unsafe { OwnedFd::from_raw_fd(fd) } + })); + } + } + first.truncate(bytes); + stream + .read_to_end(&mut first) + .map_err(|error| BackendError::Process(format!("read OCI seccomp state: {error}")))?; + if first.len() > MAX_OCI_STATE_BYTES { + return Err(BackendError::Process( + "OCI seccomp process state exceeds size limit".to_string(), + )); + } + + let state: OciContainerProcessState = serde_json::from_slice(&first).map_err(|error| { + BackendError::Process(format!("decode OCI seccomp process state: {error}")) + })?; + if state.metadata != expected_metadata + || state.state.id != expected_container + || state.fds.as_slice() != ["seccompFd"] + || state.pid <= 0 + || received_fds.len() != 1 + { + return Err(BackendError::Denied( + "OCI seccomp listener identity or FD shape did not match the created boundary" + .to_string(), + )); + } + + Ok(received_fds.pop().expect("descriptor count checked")) +} + +#[repr(C)] +#[derive(Default)] +struct SeccompData { + nr: i32, + arch: u32, + instruction_pointer: u64, + args: [u64; 6], +} + +#[repr(C)] +#[derive(Default)] +struct SeccompNotif { + id: u64, + pid: u32, + flags: u32, + data: SeccompData, +} + +#[repr(C)] +#[derive(Default)] +struct SeccompNotifResp { + id: u64, + val: i64, + error: i32, + flags: u32, +} + +#[repr(C)] +#[derive(Default)] +struct SeccompNotifAddfd { + id: u64, + flags: u32, + srcfd: u32, + newfd: u32, + newfd_flags: u32, +} + +const IOC_NRBITS: u32 = 8; +const IOC_TYPEBITS: u32 = 8; +const IOC_SIZEBITS: u32 = 14; +const IOC_NRSHIFT: u32 = 0; +const IOC_TYPESHIFT: u32 = IOC_NRSHIFT + IOC_NRBITS; +const IOC_SIZESHIFT: u32 = IOC_TYPESHIFT + IOC_TYPEBITS; +const IOC_DIRSHIFT: u32 = IOC_SIZESHIFT + IOC_SIZEBITS; +const IOC_WRITE: u32 = 1; +const IOC_READ: u32 = 2; + +// Linux reserves IOC_SIZEBITS for the payload size. Both fixed seccomp +// notification structs are far smaller than u32::MAX on supported targets. +#[allow(clippy::cast_possible_truncation)] +const fn ioctl_read_write(kind: u8, number: u8) -> libc::c_ulong { + ((IOC_READ | IOC_WRITE) << IOC_DIRSHIFT + | (kind as u32) << IOC_TYPESHIFT + | (number as u32) << IOC_NRSHIFT + | (size_of::() as u32) << IOC_SIZESHIFT) as libc::c_ulong +} + +#[allow(clippy::cast_possible_truncation)] +const fn ioctl_write(kind: u8, number: u8) -> libc::c_ulong { + (IOC_WRITE << IOC_DIRSHIFT + | (kind as u32) << IOC_TYPESHIFT + | (number as u32) << IOC_NRSHIFT + | (size_of::() as u32) << IOC_SIZESHIFT) as libc::c_ulong +} + +const SECCOMP_IOCTL_NOTIF_RECV: libc::c_ulong = ioctl_read_write::(b'!', 0); +const SECCOMP_IOCTL_NOTIF_SEND: libc::c_ulong = ioctl_read_write::(b'!', 1); +const SECCOMP_IOCTL_NOTIF_ADDFD: libc::c_ulong = ioctl_write::(b'!', 3); +const SECCOMP_USER_NOTIF_FLAG_CONTINUE: u32 = 1; +const SECCOMP_ADDFD_FLAG_SEND: u32 = 2; + +enum PendingSocket { + Tcp(std::net::TcpStream), + Dns { + injected: UnixDatagram, + peer: Option, + }, +} + +fn run_notification_worker(listener_fd: OwnedFd, mediation: Arc) { + let mut pending = HashMap::<(u32, i32), PendingSocket>::new(); + loop { + let mut notification = SeccompNotif::default(); + // SAFETY: the request code and C layout match linux/seccomp.h, and the + // kernel writes only into the live notification value. + let received = unsafe { + libc::ioctl( + listener_fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_RECV, + &mut notification, + ) + }; + if received != 0 { + let error = std::io::Error::last_os_error(); + if matches!(error.raw_os_error(), Some(libc::EINTR | libc::ENOENT)) { + continue; + } + break; + } + let result = match i64::from(notification.data.nr) { + libc::SYS_socket => { + handle_socket_notification(&listener_fd, ¬ification, &mut pending) + } + libc::SYS_connect => { + handle_connect_notification(&listener_fd, ¬ification, &mut pending, &mediation) + } + libc::SYS_sendto => { + handle_sendto_notification(&listener_fd, ¬ification, &mut pending, &mediation) + } + libc::SYS_bpf => { + send_notification_response(&listener_fd, notification.id, 0, libc::EPERM, 0) + } + _ => send_notification_response( + &listener_fd, + notification.id, + 0, + 0, + SECCOMP_USER_NOTIF_FLAG_CONTINUE, + ), + }; + if result.is_err() { + let _ = send_notification_response(&listener_fd, notification.id, 0, libc::EPERM, 0); + } + } +} + +fn handle_socket_notification( + listener_fd: &OwnedFd, + notification: &SeccompNotif, + pending: &mut HashMap<(u32, i32), PendingSocket>, +) -> Result<(), BackendError> { + let domain = i32::try_from(notification.data.args[0]).unwrap_or_default(); + let socket_type = i32::try_from(notification.data.args[1]).unwrap_or_default(); + let base_type = socket_type & 0xf; + if !matches!(domain, libc::AF_INET | libc::AF_INET6) + || !matches!(base_type, libc::SOCK_STREAM | libc::SOCK_DGRAM) + { + return send_notification_response( + listener_fd, + notification.id, + 0, + 0, + SECCOMP_USER_NOTIF_FLAG_CONTINUE, + ); + } + + let cloexec = if socket_type & libc::SOCK_CLOEXEC != 0 { + libc::O_CLOEXEC + } else { + 0 + }; + let (remote_fd, socket) = if base_type == libc::SOCK_STREAM { + let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).map_err(|error| { + BackendError::Process(format!("create mediated TCP listener: {error}")) + })?; + let address = listener.local_addr().map_err(|error| { + BackendError::Process(format!("read mediated TCP address: {error}")) + })?; + let injected = std::net::TcpStream::connect(address).map_err(|error| { + BackendError::Process(format!("create mediated TCP endpoint: {error}")) + })?; + let (peer, _) = listener.accept().map_err(|error| { + BackendError::Process(format!("accept mediated TCP endpoint: {error}")) + })?; + if socket_type & libc::SOCK_NONBLOCK != 0 { + injected.set_nonblocking(true).map_err(|error| { + BackendError::Process(format!("configure mediated TCP endpoint: {error}")) + })?; + } + let remote_fd = + inject_fd_and_respond(listener_fd, notification.id, injected.as_raw_fd(), cloexec)?; + (remote_fd, PendingSocket::Tcp(peer)) + } else { + let (injected, peer) = UnixDatagram::pair().map_err(|error| { + BackendError::Process(format!("create mediated DNS datagram pair: {error}")) + })?; + let retained = injected.try_clone().map_err(|error| { + BackendError::Process(format!("retain mediated DNS endpoint: {error}")) + })?; + if socket_type & libc::SOCK_NONBLOCK != 0 { + injected.set_nonblocking(true).map_err(|error| { + BackendError::Process(format!("configure mediated DNS endpoint: {error}")) + })?; + } + let remote_fd = + inject_fd_and_respond(listener_fd, notification.id, injected.as_raw_fd(), cloexec)?; + ( + remote_fd, + PendingSocket::Dns { + injected: retained, + peer: Some(peer), + }, + ) + }; + pending.insert((notification.pid, remote_fd), socket); + Ok(()) +} + +fn handle_connect_notification( + listener_fd: &OwnedFd, + notification: &SeccompNotif, + pending: &mut HashMap<(u32, i32), PendingSocket>, + mediation: &DockerMediation, +) -> Result<(), BackendError> { + let remote_fd = i32::try_from(notification.data.args[0]).unwrap_or(-1); + let key = (notification.pid, remote_fd); + let Some(socket) = pending.get_mut(&key) else { + return send_notification_response( + listener_fd, + notification.id, + 0, + 0, + SECCOMP_USER_NOTIF_FLAG_CONTINUE, + ); + }; + let destination = read_remote_sockaddr( + notification.pid, + notification.data.args[1], + notification.data.args[2], + )?; + let identity = resolve_process_identity(notification.pid); + match socket { + PendingSocket::Tcp(_) => { + let PendingSocket::Tcp(stream) = pending.remove(&key).expect("pending socket exists") + else { + unreachable!(); + }; + if destination.port() == 53 { + spawn_dns_tcp_session(stream, identity, mediation.dns_tx.clone()); + } else { + mediation + .network_tx + .send(NetworkItem { + stream, + binary_identity: identity, + destination, + }) + .map_err(|_| { + BackendError::Unavailable("Docker network source closed".to_string()) + })?; + } + } + PendingSocket::Dns { peer, .. } => { + if destination.port() != 53 { + return send_notification_response(listener_fd, notification.id, 0, libc::EPERM, 0); + } + if let Some(peer) = peer.take() { + spawn_dns_udp_session(peer, identity, mediation.dns_tx.clone()); + } + } + } + send_notification_response(listener_fd, notification.id, 0, 0, 0) +} + +fn handle_sendto_notification( + listener_fd: &OwnedFd, + notification: &SeccompNotif, + pending: &mut HashMap<(u32, i32), PendingSocket>, + mediation: &DockerMediation, +) -> Result<(), BackendError> { + let remote_fd = i32::try_from(notification.data.args[0]).unwrap_or(-1); + let Some(PendingSocket::Dns { injected, peer }) = + pending.get_mut(&(notification.pid, remote_fd)) + else { + return send_notification_response( + listener_fd, + notification.id, + 0, + 0, + SECCOMP_USER_NOTIF_FLAG_CONTINUE, + ); + }; + if notification.data.args[4] != 0 && notification.data.args[5] != 0 { + let destination = read_remote_sockaddr( + notification.pid, + notification.data.args[4], + notification.data.args[5], + )?; + if destination.port() != 53 { + return send_notification_response(listener_fd, notification.id, 0, libc::EPERM, 0); + } + } else if peer.is_some() { + return send_notification_response(listener_fd, notification.id, 0, libc::ENOTCONN, 0); + } + if let Some(peer) = peer.take() { + spawn_dns_udp_session( + peer, + resolve_process_identity(notification.pid), + mediation.dns_tx.clone(), + ); + } + let length = usize::try_from(notification.data.args[2]) + .unwrap_or(usize::MAX) + .min(8 * 1024); + let request = read_remote_bytes(notification.pid, notification.data.args[1], length)?; + let written = injected + .send(&request) + .map_err(|error| BackendError::Process(format!("submit mediated DNS datagram: {error}")))?; + send_notification_response( + listener_fd, + notification.id, + i64::try_from(written).unwrap_or(i64::MAX), + 0, + 0, + ) +} + +fn inject_fd_and_respond( + listener_fd: &OwnedFd, + notification_id: u64, + source_fd: RawFd, + newfd_flags: i32, +) -> Result { + let mut request = SeccompNotifAddfd { + id: notification_id, + flags: SECCOMP_ADDFD_FLAG_SEND, + srcfd: u32::try_from(source_fd).map_err(|_| { + BackendError::Process("mediated source descriptor was negative".to_string()) + })?, + newfd: 0, + newfd_flags: u32::try_from(newfd_flags).unwrap_or_default(), + }; + // SAFETY: request layout and ioctl number match linux/seccomp.h; srcfd is + // live for the duration of the call and the kernel copies the descriptor. + let remote_fd = unsafe { + libc::ioctl( + listener_fd.as_raw_fd(), + SECCOMP_IOCTL_NOTIF_ADDFD, + &mut request, + ) + }; + if remote_fd < 0 { + return Err(BackendError::Process(format!( + "inject mediated socket: {}", + std::io::Error::last_os_error() + ))); + } + Ok(remote_fd) +} + +fn send_notification_response( + listener_fd: &OwnedFd, + id: u64, + value: i64, + errno: i32, + flags: u32, +) -> Result<(), BackendError> { + let response = SeccompNotifResp { + id, + val: value, + error: -errno, + flags, + }; + // SAFETY: response layout and ioctl number match linux/seccomp.h. + let sent = unsafe { libc::ioctl(listener_fd.as_raw_fd(), SECCOMP_IOCTL_NOTIF_SEND, &response) }; + if sent != 0 && std::io::Error::last_os_error().raw_os_error() != Some(libc::ENOENT) { + return Err(BackendError::Process(format!( + "respond to seccomp notification: {}", + std::io::Error::last_os_error() + ))); + } + Ok(()) +} + +fn read_remote_bytes(pid: u32, address: u64, length: usize) -> Result, BackendError> { + let mut bytes = vec![0_u8; length]; + let local = libc::iovec { + iov_base: bytes.as_mut_ptr().cast(), + iov_len: bytes.len(), + }; + let remote = libc::iovec { + iov_base: usize::try_from(address).unwrap_or_default() as *mut libc::c_void, + iov_len: bytes.len(), + }; + // SAFETY: process_vm_readv copies from the stopped notification task into + // the owned byte buffer. Both iovec arrays live through the call. + let read = unsafe { + libc::process_vm_readv( + i32::try_from(pid).unwrap_or(i32::MAX), + &raw const local, + 1, + &raw const remote, + 1, + 0, + ) + }; + if read < 0 || usize::try_from(read).ok() != Some(length) { + return Err(BackendError::Denied(format!( + "read mediated syscall arguments for pid {pid}: {}", + std::io::Error::last_os_error() + ))); + } + Ok(bytes) +} + +fn read_remote_sockaddr(pid: u32, address: u64, length: u64) -> Result { + let length = usize::try_from(length).unwrap_or(usize::MAX).min(128); + let bytes = read_remote_bytes(pid, address, length)?; + if bytes.len() < 2 { + return Err(BackendError::Denied( + "mediated socket address is truncated".to_string(), + )); + } + let family = u16::from_ne_bytes([bytes[0], bytes[1]]); + let port = bytes + .get(2..4) + .map(|value| u16::from_be_bytes([value[0], value[1]])) + .ok_or_else(|| BackendError::Denied("mediated socket port is truncated".to_string()))?; + match i32::from(family) { + libc::AF_INET if bytes.len() >= 8 => Ok(SocketAddr::new( + Ipv4Addr::new(bytes[4], bytes[5], bytes[6], bytes[7]).into(), + port, + )), + libc::AF_INET6 if bytes.len() >= 28 => { + let mut address = [0_u8; 16]; + address.copy_from_slice(&bytes[8..24]); + let scope_id = u32::from_ne_bytes(bytes[24..28].try_into().expect("length checked")); + Ok(SocketAddr::V6(std::net::SocketAddrV6::new( + Ipv6Addr::from(address), + port, + 0, + scope_id, + ))) + } + _ => Err(BackendError::Denied(format!( + "unsupported mediated socket family {family}" + ))), + } +} + +fn resolve_process_identity(pid: u32) -> Result { + let exe = PathBuf::from(format!("/proc/{pid}/exe")); + let binary_path = std::fs::read_link(&exe) + .map_err(|error| ResolveError::Failed(format!("read {}: {error}", exe.display())))?; + let binary = std::fs::read(&exe) + .map_err(|error| ResolveError::Failed(format!("hash {}: {error}", exe.display())))?; + let binary_digest = format!("{:x}", Sha256::digest(binary)) + .parse() + .map_err(|error: ResolveError| error)?; + let mut ancestors = Vec::new(); + let mut parent = process_parent_pid(pid); + for _ in 0..32 { + let Some(parent_pid) = parent.filter(|parent_pid| *parent_pid > 1) else { + break; + }; + let Ok(path) = std::fs::read_link(format!("/proc/{parent_pid}/exe")) else { + break; + }; + ancestors.push(path); + parent = process_parent_pid(parent_pid); + } + let cmdline_paths = std::fs::read(format!("/proc/{pid}/cmdline")) + .unwrap_or_default() + .split(|byte| *byte == 0) + .filter_map(|argument| std::str::from_utf8(argument).ok()) + .filter(|argument| argument.starts_with('/')) + .map(PathBuf::from) + .collect(); + Ok(BinaryIdentity { + binary_path, + binary_digest: Some(binary_digest), + ancestors, + cmdline_paths, + }) +} + +fn process_parent_pid(pid: u32) -> Option { + std::fs::read_to_string(format!("/proc/{pid}/status")) + .ok()? + .lines() + .find_map(|line| line.strip_prefix("PPid:"))? + .trim() + .parse() + .ok() +} + +fn clone_identity( + identity: &Result, +) -> Result { + match identity { + Ok(identity) => Ok(identity.clone()), + Err(ResolveError::NotFound) => Err(ResolveError::NotFound), + Err(ResolveError::Failed(message)) => Err(ResolveError::Failed(message.clone())), + } +} + +fn spawn_dns_udp_session( + socket: UnixDatagram, + identity: Result, + sender: mpsc::UnboundedSender, +) { + std::thread::spawn(move || { + let mut request = vec![0_u8; 8 * 1024]; + while let Ok(length) = socket.recv(&mut request) { + let (response_tx, response_rx) = oneshot::channel(); + if sender + .send(MediatedDnsQuery { + request: request[..length].to_vec(), + transport: DnsTransport::Udp, + binary_identity: clone_identity(&identity), + response: response_tx, + }) + .is_err() + { + return; + } + let Ok(Ok(response)) = response_rx.blocking_recv() else { + return; + }; + if socket.send(&response).is_err() { + return; + } + } + }); +} + +fn spawn_dns_tcp_session( + mut stream: std::net::TcpStream, + identity: Result, + sender: mpsc::UnboundedSender, +) { + std::thread::spawn(move || { + loop { + let mut prefix = [0_u8; 2]; + if stream.read_exact(&mut prefix).is_err() { + return; + } + let length = usize::from(u16::from_be_bytes(prefix)); + if length > 8 * 1024 { + return; + } + let mut request = vec![0_u8; length + 2]; + request[..2].copy_from_slice(&prefix); + if stream.read_exact(&mut request[2..]).is_err() { + return; + } + let (response_tx, response_rx) = oneshot::channel(); + if sender + .send(MediatedDnsQuery { + request, + transport: DnsTransport::Tcp, + binary_identity: clone_identity(&identity), + response: response_tx, + }) + .is_err() + { + return; + } + let Ok(Ok(response)) = response_rx.blocking_recv() else { + return; + }; + if stream.write_all(&response).is_err() { + return; + } + } + }); +} + +fn docker_error(operation: &str, error: BollardError) -> BackendError { + BackendError::Attach(format!("{operation}: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::SandboxPolicy; + use openshell_isolation::AgentSpec; + use openshell_isolation::contract::{BackendRegistry, BoundaryOrigin, BoundaryProvisioning}; + use uuid::Uuid; + + fn context() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: openshell_core::policy::FilesystemPolicy::default(), + network: openshell_core::policy::NetworkPolicy::default(), + landlock: openshell_core::policy::LandlockPolicy::default(), + process: openshell_core::policy::ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/usr/bin/uname".to_string(), + args: vec!["-a".to_string()], + workdir: Some("/workspace".to_string()), + timeout_secs: 10, + interactive: false, + }, + } + } + + fn plan() -> DockerBoundaryCreatePlan { + DockerBoundaryCreatePlan { + image: "example@sha256:deadbeef".to_string(), + launch_generation: "generation-1".to_string(), + listener_token: "0".repeat(MIN_LISTENER_TOKEN_BYTES), + container_name: "openshell-sandbox-1".to_string(), + labels: HashMap::new(), + env: vec!["A=B".to_string()], + user: Some("1000:1000".to_string()), + } + } + + #[tokio::test] + async fn mediation_sources_deliver_destination_and_dns_response_channels() { + let mediation = Arc::new(DockerMediation::new()); + let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let client = std::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + let (server, _) = listener.accept().unwrap(); + let destination: SocketAddr = "198.18.0.8:443".parse().unwrap(); + mediation + .network_tx + .send(NetworkItem { + stream: server, + binary_identity: Err(ResolveError::NotFound), + destination, + }) + .unwrap(); + let connection = NetworkMediationSource::accept(mediation.as_ref()) + .await + .unwrap(); + assert_eq!(connection.destination, Some(destination)); + drop(client); + + let (response_tx, response_rx) = oneshot::channel(); + mediation + .dns_tx + .send(MediatedDnsQuery { + request: vec![1, 2, 3], + transport: DnsTransport::Udp, + binary_identity: Err(ResolveError::NotFound), + response: response_tx, + }) + .unwrap(); + let query = DnsMediationSource::accept(mediation.as_ref()) + .await + .unwrap(); + assert_eq!(query.request, vec![1, 2, 3]); + query.response.send(Ok(vec![4, 5])).unwrap(); + assert_eq!(response_rx.await.unwrap().unwrap(), vec![4, 5]); + } + + #[test] + fn create_body_runs_agent_without_an_in_container_supervisor() { + let body = build_create_body( + &plan(), + &context(), + Path::new("/run/openshell/seccomp/test.sock"), + "authenticated-metadata", + HashMap::new(), + None, + ) + .expect("create body"); + assert_eq!(body.entrypoint, Some(vec!["/usr/bin/uname".to_string()])); + assert_eq!(body.cmd, Some(vec!["-a".to_string()])); + assert_eq!(body.network_disabled, Some(true)); + let host = body.host_config.expect("host config"); + assert_eq!(host.network_mode.as_deref(), Some("none")); + assert_eq!(host.cap_drop, Some(vec!["ALL".to_string()])); + } + + #[test] + fn create_body_carries_oci_notify_listener_fields() { + let body = build_create_body( + &plan(), + &context(), + Path::new("/run/openshell/seccomp/test.sock"), + "authenticated-metadata", + HashMap::new(), + None, + ) + .expect("create body"); + let seccomp = body + .host_config + .expect("host config") + .security_opt + .expect("security opts") + .into_iter() + .find_map(|value| value.strip_prefix("seccomp=").map(str::to_string)) + .expect("seccomp profile"); + let profile: serde_json::Value = serde_json::from_str(&seccomp).expect("profile JSON"); + assert_eq!(profile["listenerPath"], "/run/openshell/seccomp/test.sock"); + assert_eq!(profile["listenerMetadata"], "authenticated-metadata"); + assert_eq!(profile["syscalls"][0]["action"], "SCMP_ACT_NOTIFY"); + assert_eq!(profile["syscalls"][0]["names"][0], "socket"); + } + + #[test] + fn attach_only_contract_plan_names_the_experimental_backend_exactly() { + let envelope = plan().into_boundary_plan().expect("encode plan"); + assert_eq!(envelope.backend_name, BACKEND_NAME); + assert_eq!(envelope.version, INTERFACE_VERSION); + } + + #[test] + fn listener_lock_rejects_an_active_owner_and_recovers_after_drop() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join("seccomp.lock"); + let guard = acquire_listener_lock(&path).expect("first listener lock"); + let second = acquire_listener_lock(&path).expect_err("active listener must be denied"); + assert!(matches!(second, BackendError::Denied(_))); + drop(guard); + let _recovered_guard = + acquire_listener_lock(&path).expect("released listener lock may be recovered"); + } + + #[test] + fn create_plan_fingerprint_detects_changed_prepared_inputs() { + let original = plan_fingerprint(&plan()).expect("fingerprint"); + let mut changed = plan(); + changed.image = "different@sha256:cafebabe".to_string(); + assert_ne!( + original, + plan_fingerprint(&changed).expect("changed fingerprint") + ); + } + + #[test] + fn topology_identity_pins_cleanup_to_the_backend_listener_directory() { + let listener_dir = tempfile::tempdir().expect("listener tempdir"); + let key = resource_key("sandbox-1", "generation-1"); + let topology = DockerTopology { + sandbox_id: "sandbox-1".to_string(), + launch_generation: "generation-1".to_string(), + container_id: "container-id".to_string(), + container_name: format!("openshell-boundary-{key}"), + listener_path: listener_dir.path().join(format!("{key}.sock")), + listener_metadata: format!( + "openshell:sandbox-1:generation-1:{}", + "0".repeat(MIN_LISTENER_TOKEN_BYTES) + ), + plan_fingerprint: "fingerprint".to_string(), + }; + validate_topology_identity(&topology, "sandbox-1", listener_dir.path()) + .expect("matching topology"); + + let mut escaped = topology; + escaped.listener_path = listener_dir.path().join("../outside.sock"); + assert!(validate_topology_identity(&escaped, "sandbox-1", listener_dir.path()).is_err()); + } + + #[tokio::test] + #[ignore = "requires a local Linux Docker daemon, runc seccomp-notify support, and a pre-pulled image"] + async fn local_docker_runs_without_an_in_container_supervisor() { + let docker = + Arc::new(Docker::connect_with_unix_defaults().expect("connect to local Docker daemon")); + let image = std::env::var("OPENSHELL_DOCKER_TEST_IMAGE") + .unwrap_or_else(|_| "alpine:latest".to_string()); + let uname = std::env::var("OPENSHELL_DOCKER_TEST_UNAME") + .unwrap_or_else(|_| "/bin/uname".to_string()); + docker + .inspect_image(&image) + .await + .expect("proof image must already be present"); + let listener_dir = tempfile::tempdir().expect("listener tempdir"); + let backend = Arc::new(DockerIsolationBackend::new( + docker, + listener_dir.path().to_path_buf(), + )); + let mut registry = BackendRegistry::new(); + registry.register(backend).expect("register Docker backend"); + + let mut create = plan(); + create.image = image; + create.launch_generation = Uuid::new_v4().to_string(); + let envelope = create.into_boundary_plan().expect("encode create plan"); + let mut sandbox = context(); + sandbox.agent.program = uname; + let provisioned = registry + .provision( + BoundaryProvisioning::Create(envelope), + BACKEND_NAME, + sandbox, + ) + .await + .expect("create stopped Docker boundary"); + let (descriptor, origin, bound) = provisioned.into_parts(); + assert_eq!(descriptor.backend_name, BACKEND_NAME); + assert_eq!(origin, BoundaryOrigin::SupervisorCreated); + let ready = bound.confirm().await.expect("confirm Docker boundary"); + let running = ready.start_agent().await.expect("start Docker boundary"); + let process = running.agent(); + let status = tokio::time::timeout(Duration::from_secs(15), process.wait()) + .await + .expect("uname should not remain blocked") + .expect("observe uname exit"); + assert_eq!(status, BoundaryExitStatus::Exited(0)); + drop(process); + drop(running); + registry + .destroy_created(descriptor, origin, BACKEND_NAME, "sandbox-1") + .await + .expect("destroy supervisor-created Docker boundary"); + } +} diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b1859f54cc..7115d4757d 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -5,15 +5,16 @@ #![allow(clippy::result_large_err)] +#[cfg(target_os = "linux")] +pub mod isolation; pub mod otel_tracing; +use base64::Engine as _; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::{ ContainerCreateBody, ContainerState, ContainerStateStatusEnum, ContainerSummary, - ContainerSummaryStateEnum, CreateImageInfo, DeviceRequest, EndpointSettings, HostConfig, Mount, - MountTmpfsOptions, MountTypeEnum, MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, - ProgressDetail, SystemInfo, + ContainerSummaryStateEnum, CreateImageInfo, ProgressDetail, SystemInfo, }; use bollard::query_parameters::{ CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, @@ -21,9 +22,7 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, @@ -43,14 +42,13 @@ use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, DriverSandboxTemplate, EnsureWorkspaceRequest, - EnsureWorkspaceResponse, GatewayListenerRequirement, GetCapabilitiesRequest, - GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, - GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, - GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, StartSandboxRequest, - StartSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, gateway_listener_requirement::Selector, + EnsureWorkspaceResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, + GetGatewayListenerRequirementsRequest, GetGatewayListenerRequirementsResponse, + GetSandboxRequest, GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, + ListSandboxesResponse, StartSandboxRequest, StartSandboxResponse, StopSandboxRequest, + StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, + WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, + WatchSandboxesRequest, WatchSandboxesSandboxEvent, compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use openshell_core::proto_struct::{ @@ -60,12 +58,13 @@ use openshell_core::{Config, Error, Result as CoreResult}; use opentelemetry::trace::TraceContextExt as _; use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::pin::Pin; +use std::process::Stdio; use std::sync::Arc; use std::task::{Context, Poll}; use std::time::Duration; +use tokio::process::{Child, Command}; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio::task::JoinHandle; use tokio_stream::wrappers::ReceiverStream; @@ -78,15 +77,8 @@ const WATCH_BUFFER: usize = 128; const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; -const DOCKER_NETWORK_DRIVER: &str = "bridge"; fn provisioning_span( parent: &opentelemetry::Context, @@ -131,7 +123,7 @@ pub struct DockerComputeConfig { /// Gateway gRPC endpoint the sandbox connects back to. pub grpc_endpoint: String, - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. + /// Optional native Linux `openshell-sandbox` host-supervisor binary. pub supervisor_bin: Option, /// Optional image used to extract the Linux `openshell-sandbox` binary. @@ -139,33 +131,14 @@ pub struct DockerComputeConfig { /// the full resolution order. pub supervisor_image: Option, - /// Host-side CA certificate for Docker sandbox mTLS. + /// Host-supervisor CA certificate for gateway mTLS. pub guest_tls_ca: Option, - /// Host-side client certificate for Docker sandbox mTLS. + /// Host-supervisor client certificate for gateway mTLS. pub guest_tls_cert: Option, - /// Host-side private key for Docker sandbox mTLS. + /// Host-supervisor private key for gateway mTLS. pub guest_tls_key: Option, - - /// Docker bridge network that sandbox containers join. - pub network_name: String, - - /// Host gateway IP used for sandbox host aliases. - pub host_gateway_ip: String, - - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - - /// Container cgroup PID limit for Docker-managed sandboxes. - /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, - - /// Allow sandbox requests to attach host bind mounts through - /// `template.driver_config`. - #[serde(default)] - pub enable_bind_mounts: bool, } impl Default for DockerComputeConfig { @@ -181,11 +154,6 @@ impl Default for DockerComputeConfig { guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - host_gateway_ip: String::new(), - ssh_socket_path: openshell_core::container_paths::SSH_SOCKET_PATH.to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, - enable_bind_mounts: false, } } } @@ -199,14 +167,11 @@ pub(crate) struct DockerGuestTlsPaths { #[derive(Debug, Clone)] struct DockerDriverRuntimeConfig { + socket_path: PathBuf, default_image: String, image_pull_policy: String, sandbox_namespace: String, grpc_endpoint: String, - network_name: String, - gateway_route: DockerGatewayRoute, - gateway_callback_bind_address: Option, - ssh_socket_path: String, stop_timeout_secs: u32, log_level: String, supervisor_bin: PathBuf, @@ -214,17 +179,6 @@ struct DockerDriverRuntimeConfig { daemon_version: String, supports_gpu: bool, allow_all_default_gpu: bool, - sandbox_pids_limit: i64, - enable_bind_mounts: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, - HostGateway, } #[derive(Clone)] @@ -235,6 +189,7 @@ pub struct DockerComputeDriver { pending: Arc>>, gpu_selector: Arc, lifecycle_event_fences: DockerLifecycleEventFences, + host_supervisors: Arc>>, } /// Per-sandbox container exit timestamps that fence snapshots from an earlier run. @@ -348,7 +303,6 @@ struct DockerSandboxDriverConfig { deserialize_with = "deserialize_optional_non_empty_string_list" )] cdi_devices: Option>, - mounts: Vec, } struct ValidatedDockerSandbox<'a> { @@ -368,50 +322,6 @@ impl DockerSandboxDriverConfig { } } -use openshell_core::driver_mounts::SelinuxLabel; - -#[derive(Debug, Clone, serde::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum DockerDriverMountConfig { - Bind { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - selinux_label: Option, - }, - Volume { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, - Tmpfs { - target: String, - #[serde(default)] - options: Vec, - #[serde(default)] - size_bytes: Option, - #[serde(default)] - mode: Option, - }, - Image { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, -} - -fn default_true() -> bool { - true -} - type WatchStream = Pin> + Send + 'static>>; @@ -558,20 +468,12 @@ impl DockerComputeDriver { .is_some_and(|dirs| !dirs.is_empty()); let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; let gateway_port = config.bind_address.port(); if gateway_port == 0 { return Err(Error::config( "docker compute driver requires a fixed non-zero gateway bind port", )); } - let network_name = docker_network_name(docker_config); - let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; - let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; - let gateway_route = - docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); - let gateway_callback_bind_address = - docker_gateway_callback_bind_address(&gateway_route, config.bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { let scheme = if docker_guest_tls_configured(&docker_config) { @@ -582,11 +484,7 @@ impl DockerComputeDriver { docker_config.grpc_endpoint = format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); + let grpc_endpoint = docker_host_openshell_endpoint(&docker_config.grpc_endpoint); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; let guest_tls = docker_guest_tls_paths(&docker_config)?; @@ -594,14 +492,11 @@ impl DockerComputeDriver { let driver = Self { docker: Arc::new(docker), config: DockerDriverRuntimeConfig { + socket_path, default_image: docker_config.default_image.clone(), image_pull_policy: docker_config.image_pull_policy.clone(), sandbox_namespace: docker_config.sandbox_namespace.clone(), grpc_endpoint, - network_name, - gateway_route, - gateway_callback_bind_address, - ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: config.log_level.clone(), supervisor_bin, @@ -609,8 +504,6 @@ impl DockerComputeDriver { daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, allow_all_default_gpu, - sandbox_pids_limit: docker_config.sandbox_pids_limit, - enable_bind_mounts: docker_config.enable_bind_mounts, }, events: broadcast::channel(WATCH_BUFFER).0, pending: Arc::new(Mutex::new(HashMap::new())), @@ -619,6 +512,7 @@ impl DockerComputeDriver { allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + host_supervisors: Arc::new(Mutex::new(HashMap::new())), }; let poll_driver = driver.clone(); @@ -629,6 +523,145 @@ impl DockerComputeDriver { Ok(driver) } + fn spawn_host_supervisor( + &self, + sandbox: &DriverSandbox, + image: &DockerImageMetadata, + _workspace_root: &str, + ) -> Result { + let token = sandbox + .spec + .as_ref() + .map(|spec| spec.sandbox_token.as_str()) + .filter(|token| !token.is_empty()) + .ok_or_else(|| { + Status::failed_precondition("Docker sandbox gateway token is required") + })?; + let token_path = sandbox_token_host_path(sandbox, &self.config)?; + let state_dir = token_path + .parent() + .ok_or_else(|| Status::internal("Docker sandbox token path has no parent directory"))?; + openshell_core::paths::create_dir_restricted(state_dir).map_err(|error| { + Status::internal(format!( + "create Docker host-supervisor state directory: {error}" + )) + })?; + + let template = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; + let mut labels = template.labels.clone(); + labels.insert( + LABEL_MANAGED_BY.to_string(), + LABEL_MANAGED_BY_VALUE.to_string(), + ); + labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); + labels.insert( + LABEL_SANDBOX_WORKSPACE.to_string(), + sandbox.workspace.clone(), + ); + labels.insert( + LABEL_SANDBOX_NAMESPACE.to_string(), + self.config.sandbox_namespace.clone(), + ); + + let mut workload_env = template.environment.clone(); + if let Some(spec) = sandbox.spec.as_ref() { + workload_env.extend(spec.environment.clone()); + } + let plan = isolation::DockerBoundaryCreatePlan { + image: image.id.clone(), + launch_generation: uuid::Uuid::new_v4().to_string(), + listener_token: uuid::Uuid::new_v4().to_string(), + container_name: container_name_for_sandbox(sandbox), + labels, + env: workload_env + .into_iter() + .map(|(name, value)| format!("{name}={value}")) + .collect(), + user: Some(current_process_user().map_err(|error| { + Status::failed_precondition(format!( + "Docker host-supervisor mode requires a host user identity: {error}" + )) + })?), + } + .into_boundary_plan() + .map_err(|error| Status::internal(error.to_string()))?; + + let supervisor_binary = &self.config.supervisor_bin; + let ssh_socket = state_dir.join("ssh.sock"); + let proxy_tls_dir = state_dir.join("proxy-tls"); + let listener_dir = state_dir.join("seccomp"); + for directory in [&proxy_tls_dir, &listener_dir] { + openshell_core::paths::create_dir_restricted(directory).map_err(|error| { + Status::internal(format!( + "create Docker host-supervisor directory '{}': {error}", + directory.display() + )) + })?; + } + let mut command = Command::new(supervisor_binary); + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::from( + std::fs::File::create(state_dir.join("supervisor.log")).map_err(|error| { + Status::internal(format!("create Docker supervisor log: {error}")) + })?, + )) + .stderr(Stdio::from( + std::fs::File::create(state_dir.join("supervisor.err.log")).map_err(|error| { + Status::internal(format!("create Docker supervisor error log: {error}")) + })?, + )) + .arg("--boundary-create-backend-name=docker") + .arg(format!("--boundary-create-version={}", plan.version)) + .arg(format!( + "--boundary-create-payload-base64={}", + base64::engine::general_purpose::STANDARD.encode(plan.payload) + )) + .arg("--workdir") + .arg("/") + .arg("--") + .args(["/bin/sh", "-lc", "while :; do sleep 3600; done"]) + .env( + openshell_core::sandbox_env::ENDPOINT, + &self.config.grpc_endpoint, + ) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, &token_path) + .env(openshell_core::sandbox_env::SSH_SOCKET_PATH, &ssh_socket) + .env(openshell_core::sandbox_env::PROXY_TLS_DIR, &proxy_tls_dir) + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, &image.user) + .env( + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level), + ) + .env( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ) + .env(isolation::DOCKER_SOCKET_ENV, &self.config.socket_path) + .env(isolation::DOCKER_LISTENER_DIR_ENV, &listener_dir); + if let Some(tls) = &self.config.guest_tls { + command + .env(openshell_core::sandbox_env::TLS_CA, &tls.ca) + .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert) + .env(openshell_core::sandbox_env::TLS_KEY, &tls.key); + } + let _ = token; + command.spawn().map_err(|error| { + Status::internal(format!( + "start Docker host supervisor '{}': {error}", + supervisor_binary.display() + )) + }) + } + fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { driver_name: "docker".to_string(), @@ -664,7 +697,6 @@ impl DockerComputeDriver { let _ = docker_resource_limits(template)?; let driver_config = DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; Ok(ValidatedDockerSandbox { @@ -737,35 +769,6 @@ impl DockerComputeDriver { Ok(()) } - async fn validate_user_volume_mounts_available( - &self, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - for mount in &driver_config.mounts { - if let DockerDriverMountConfig::Volume { source, .. } = mount { - match self.docker.inspect_volume(source).await { - Ok(volume) => { - if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) - { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" - ))); - } - } - Err(err) if is_not_found_error(&err) => { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' does not exist" - ))); - } - Err(err) => { - return Err(internal_status("inspect docker volume", err)); - } - } - } - } - Ok(()) - } - async fn refresh_gpu_inventory(&self) -> Result<(), Status> { let info = self .docker @@ -845,8 +848,6 @@ impl DockerComputeDriver { async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { let validated = Self::validated_sandbox(sandbox, &self.config)?; Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; let _ = self .resolve_gpu_cdi_devices( validated.gpu_requirements, @@ -967,92 +968,80 @@ impl DockerComputeDriver { } DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - let create_body = build_container_create_body_for_image( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - &image, - ) - .map_err(|status| { + if gpu_devices.is_some() { if token_file_created { cleanup_sandbox_token_file(sandbox, &self.config); } + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + "the host-supervisor Docker backend does not yet support GPU devices", + )); + } + let limits = docker_resource_limits(template).map_err(|status| { DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) })?; - async { - openshell_otel::record_error_result( - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, - ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - }), - ) + if limits != DockerResourceLimits::default() { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + return Err(DockerProvisioningFailure::new( + "ContainerCreateFailed", + "the host-supervisor Docker backend does not yet support resource limits", + )); } - .instrument(tracing::info_span!( - "docker.create_container", - otel.name = "docker.create_container", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - container.name = %container_name, - )) - .await?; - self.publish_docker_progress( - &sandbox.id, - "Created", - format!("Created Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name.clone())]), - ); + let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) + .map_err(|error| DockerProvisioningFailure::new("ContainerCreateFailed", error))?; + let child = self + .spawn_host_supervisor(sandbox, &image, &workspace_root) + .map_err(|status| { + if token_file_created { + cleanup_sandbox_token_file(sandbox, &self.config); + } + DockerProvisioningFailure::new("ContainerStartFailed", status.message()) + })?; + self.host_supervisors + .lock() + .await + .insert(sandbox.id.clone(), child); - let start_result = async { - openshell_otel::record_error_result( - self.docker.start_container(&container_name, None).await, - ) - } - .instrument(tracing::info_span!( - "docker.start_container", - otel.name = "docker.start_container", - otel.status_code = tracing::field::Empty, - sandbox.id = %sandbox.id, - container.name = %container_name, - )) - .await; - if let Err(err) = start_result { - let cleanup = self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - if let Err(cleanup_err) = cleanup { - warn!( - sandbox_id = %sandbox.id, - container_name, - error = %cleanup_err, - "Failed to clean up Docker container after start failure" - ); + let mut running = false; + for _ in 0..300 { + match self.docker.inspect_container(&container_name, None).await { + Ok(container) + if container.state.as_ref().and_then(|state| state.running) == Some(true) => + { + running = true; + break; + } + Ok(_) + | Err(BollardError::DockerResponseServerError { + status_code: 404, .. + }) => {} + Err(error) => { + return Err(DockerProvisioningFailure::from_status( + "ContainerStartFailed", + internal_status("inspect host-supervisor Docker container", error), + )); + } } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); + let exited = { + let mut supervisors = self.host_supervisors.lock().await; + supervisors + .get_mut(&sandbox.id) + .and_then(|child| child.try_wait().ok().flatten()) + }; + if let Some(status) = exited { + return Err(DockerProvisioningFailure::new( + "ContainerStartFailed", + format!("Docker host supervisor exited before readiness: {status}"), + )); } - return Err(DockerProvisioningFailure::from_status( + tokio::time::sleep(Duration::from_millis(100)).await; + } + if !running { + return Err(DockerProvisioningFailure::new( "ContainerStartFailed", - create_status_from_docker_error("start docker sandbox container", err), + "timed out waiting for the Docker host supervisor to start the boundary", )); } self.publish_docker_progress( @@ -1080,6 +1069,12 @@ impl DockerComputeDriver { sandbox_id: &str, sandbox_name: &str, ) -> Result { + if !sandbox_id.is_empty() + && let Some(mut supervisor) = self.host_supervisors.lock().await.remove(sandbox_id) + { + let _ = supervisor.kill().await; + let _ = supervisor.wait().await; + } let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; if let Some(record) = pending.as_ref() && let Some(task) = record.task.as_ref() @@ -1326,6 +1321,11 @@ impl DockerComputeDriver { sandbox: &DriverSandbox, failure: &DockerProvisioningFailure, ) { + let supervisor = self.host_supervisors.lock().await.remove(&sandbox.id); + if let Some(mut supervisor) = supervisor { + let _ = supervisor.kill().await; + let _ = supervisor.wait().await; + } cleanup_sandbox_token_file(sandbox, &self.config); let snapshot = pending_sandbox_snapshot( sandbox, @@ -1886,21 +1886,8 @@ impl ComputeDriver for DockerComputeDriver { &self, _request: Request, ) -> Result, Status> { - let requirements = - self.config - .gateway_callback_bind_address - .map_or_else(Vec::new, |bind_address| { - vec![GatewayListenerRequirement { - reason: match self.config.gateway_route { - DockerGatewayRoute::Bridge { .. } => "docker managed bridge gateway", - DockerGatewayRoute::HostGateway => "docker host-gateway IPv4 loopback", - } - .to_string(), - selector: Some(Selector::ExactBindAddress(bind_address.to_string())), - }] - }); Ok(Response::new(GetGatewayListenerRequirementsResponse { - requirements, + requirements: Vec::new(), })) } @@ -1913,8 +1900,6 @@ impl ComputeDriver for DockerComputeDriver { .sandbox .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; let _ = self .resolve_gpu_cdi_devices( validated.gpu_requirements, @@ -2326,317 +2311,6 @@ fn attach_docker_progress_metadata( } } -#[cfg(test)] -fn docker_driver_config( - template: &DriverSandboxTemplate, - enable_bind_mounts: bool, -) -> Result { - let config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; - Ok(config) -} - -/// Collect user-supplied bind mounts as string-format binds. -/// -/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the -/// structured `Mount` API because the Docker Engine Mount object does not -/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. -fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| match m { - DockerDriverMountConfig::Bind { - source, - target, - read_only, - selinux_label, - } => Some(docker_bind_string( - source, - target, - *read_only, - *selinux_label, - )), - _ => None, - }) - .collect() -} - -fn docker_bind_string( - source: &str, - target: &str, - read_only: bool, - selinux_label: Option, -) -> Result { - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - // Legacy `-v` binds silently create missing source directories as empty, - // root-owned paths. The structured `--mount` API that was used before this - // change rejected missing sources at container-create time. Preserve that - // fail-fast behaviour with an explicit existence check. - if !Path::new(source).exists() { - return Err(Status::failed_precondition(format!( - "bind source path does not exist: {source}" - ))); - } - driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - - let mut opts = Vec::new(); - if read_only { - opts.push("ro"); - } - match selinux_label { - Some(SelinuxLabel::Shared) => opts.push("z"), - Some(SelinuxLabel::Private) => opts.push("Z"), - None => {} - } - - if opts.is_empty() { - Ok(format!("{source}:{normalized_target}")) - } else { - Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) - } -} - -/// Collect user-supplied non-bind mounts as structured `Mount` objects. -fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| docker_mount_from_config(m).transpose()) - .collect() -} - -fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { - match config { - DockerDriverMountConfig::Bind { .. } => { - // Bind mounts are handled via docker_driver_bind_strings. - Ok(None) - } - DockerDriverMountConfig::Volume { - source, - target, - read_only, - subpath, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::VOLUME), - source: Some(source.clone()), - target: Some(target.clone()), - read_only: Some(*read_only), - volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { - subpath: Some(subpath.clone()), - ..Default::default() - }), - ..Default::default() - })), - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::TMPFS), - target: Some(target.clone()), - tmpfs_options: Some(MountTmpfsOptions { - size_bytes: validate_optional_positive_integral_i64( - *size_bytes, - "tmpfs size_bytes", - )?, - mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, - options: (!options.is_empty()) - .then(|| { - options - .iter() - .map(|option| docker_tmpfs_option(option)) - .collect::, _>>() - }) - .transpose()?, - }), - ..Default::default() - })), - DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )), - } -} - -fn validate_docker_driver_mounts( - mounts: &[DockerDriverMountConfig], - enable_bind_mounts: bool, -) -> Result<(), Status> { - let mut targets = HashSet::new(); - for mount in mounts { - let target = match mount { - DockerDriverMountConfig::Bind { source, target, .. } => { - if !enable_bind_mounts { - return Err(Status::failed_precondition( - "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", - )); - } - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - target - } - DockerDriverMountConfig::Volume { - source, - target, - subpath, - .. - } => { - driver_mounts::validate_mount_source(source, "volume source") - .map_err(Status::failed_precondition)?; - if let Some(subpath) = subpath { - driver_mounts::validate_mount_subpath(subpath) - .map_err(Status::failed_precondition)?; - } - target - } - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => { - validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; - validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; - for option in options { - docker_tmpfs_option(option)?; - } - target - } - DockerDriverMountConfig::Image { - source, - target, - read_only, - subpath, - } => { - let _ = (source, target, read_only, subpath); - return Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )); - } - }; - driver_mounts::validate_container_mount_target(target) - .map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - if !targets.insert(normalized_target.clone()) { - return Err(Status::failed_precondition(format!( - "duplicate docker driver_config mount target '{normalized_target}'" - ))); - } - } - Ok(()) -} - -fn validate_optional_positive_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value <= 0 { - return Err(Status::failed_precondition(format!( - "{field} must be positive" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_nonnegative_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value < 0 { - return Err(Status::failed_precondition(format!( - "{field} must be zero or greater" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { - let Some(value) = value else { - return Ok(None); - }; - if !value.is_finite() || value.fract() != 0.0 { - return Err(Status::failed_precondition(format!( - "{field} must be an integer" - ))); - } - value.to_string().parse::().map(Some).map_err(|_| { - Status::failed_precondition(format!("{field} must be representable as an i64")) - }) -} - -fn docker_tmpfs_option(option: &str) -> Result, Status> { - let option = option.trim(); - if option.is_empty() { - return Err(Status::failed_precondition( - "tmpfs options must not contain empty values", - )); - } - if let Some((key, value)) = option.split_once('=') { - let key = key.trim(); - let value = value.trim(); - if key.is_empty() || value.is_empty() { - return Err(Status::failed_precondition( - "tmpfs key=value options must include both key and value", - )); - } - Ok(vec![key.to_string(), value.to_string()]) - } else { - Ok(vec![option.to_string()]) - } -} - -fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { - volume.driver == "local" - && volume.options.get("o").is_some_and(|options| { - options.split(',').any(|option| { - let option = option.trim(); - option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") - }) - }) -} - -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) -} - fn sandbox_token_host_path( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, @@ -2731,131 +2405,6 @@ fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRunti } } -#[cfg(test)] -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - build_environment_for_oci_user(sandbox, config, "") -} - -fn build_environment_for_oci_user( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - oci_user: &str, -) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), - ), - ]); - - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec(sandbox.spec.as_ref()) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - environment.insert( - openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), - oci_user.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_UID.to_string(), - String::new(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_GID.to_string(), - String::new(), - ); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() -} - fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { CdiGpuInventory::new( info.discovered_devices @@ -2886,198 +2435,6 @@ fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { Status::failed_precondition(err.to_string()) } -#[cfg(test)] -fn build_container_create_body( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let template = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; - let gpu_requirements = sandbox - .spec - .as_ref() - .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); - let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - Some(cdi_devices.as_slice()) - } else { - None - }; - build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) -} - -#[cfg(test)] -fn build_container_create_body_with_gpu_devices( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - driver_config: &DockerSandboxDriverConfig, - gpu_device_ids: Option<&[String]>, -) -> Result { - let template = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - build_container_create_body_for_image( - sandbox, - config, - driver_config, - gpu_device_ids, - &DockerImageMetadata { - id: template.image.clone(), - user: String::new(), - working_dir: String::new(), - volumes: Vec::new(), - }, - ) -} - -fn build_container_create_body_for_image( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - driver_config: &DockerSandboxDriverConfig, - gpu_device_ids: Option<&[String]>, - image: &DockerImageMetadata, -) -> Result { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let resource_limits = docker_resource_limits(template)?; - let workspace_root = driver_mounts::resolve_oci_workspace_root(&image.working_dir) - .map_err(Status::failed_precondition)?; - driver_mounts::validate_workspace_control_path(&workspace_root, &config.ssh_socket_path) - .map_err(Status::failed_precondition)?; - for volume in &image.volumes { - driver_mounts::validate_container_mount_target(volume).map_err(|error| { - Status::failed_precondition(format!( - "invalid image-declared volume '{volume}': {error}" - )) - })?; - driver_mounts::validate_workspace_mount_target(volume, &workspace_root).map_err(|_| { - Status::failed_precondition(format!( - "image-declared volume '{volume}' masks OCI WorkingDir '{workspace_root}' before workspace validation" - )) - })?; - driver_mounts::validate_mount_control_path(volume, &config.ssh_socket_path) - .map_err(Status::failed_precondition)?; - } - for mount in &driver_config.mounts { - let target = match mount { - DockerDriverMountConfig::Bind { target, .. } - | DockerDriverMountConfig::Volume { target, .. } - | DockerDriverMountConfig::Tmpfs { target, .. } - | DockerDriverMountConfig::Image { target, .. } => target, - }; - driver_mounts::validate_workspace_mount_target(target, &workspace_root) - .map_err(Status::failed_precondition)?; - driver_mounts::validate_mount_control_path(target, &config.ssh_socket_path) - .map_err(Status::failed_precondition)?; - } - let user_mounts = docker_driver_mounts(driver_config)?; - let user_bind_strings = docker_driver_bind_strings(driver_config)?; - let device_requests = gpu_device_ids.map(|device_ids| { - vec![DeviceRequest { - driver: Some("cdi".to_string()), - device_ids: Some(device_ids.to_vec()), - ..Default::default() - }] - }); - let mut labels = template.labels.clone(); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - labels.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use - // the same value here. `DriverSandbox.namespace` is unset on the request - // path (the gateway elides it), and using it would produce containers - // that the driver itself cannot find afterwards. - labels.insert( - LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), - ); - - Ok(ContainerCreateBody { - image: Some(image.id.clone()), - user: Some("0".to_string()), - // The image workspace may need to be created or rejected by the - // supervisor, so do not let the OCI runtime chdir there first. - working_dir: Some("/".to_string()), - env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), - labels: Some(labels), - host_config: Some(HostConfig { - nano_cpus: resource_limits.nano_cpus, - memory: resource_limits.memory_bytes, - pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, - device_requests, - binds: { - let mut binds = build_binds(sandbox, config)?; - binds.extend(user_bind_strings); - Some(binds) - }, - mounts: Some(user_mounts), - // Canonical main-process exit is terminal. Runtime restart would - // silently create a new process generation behind the gateway. - restart_policy: None, - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), - ..Default::default() - }) -} - -/// Reject driver requests that arrive with neither a sandbox id nor a -/// sandbox name. Without this guard, downstream label filters degenerate -/// to "match every managed container in the namespace", which would let -/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary -/// sandbox out of the set the driver manages. fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { if sandbox_id.is_empty() && sandbox_name.is_empty() { return Err(Status::invalid_argument( @@ -3087,236 +2444,45 @@ fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<() Ok(()) } -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); +#[cfg(target_os = "linux")] +fn current_process_user() -> Result { + let status = std::fs::read_to_string("/proc/self/status")?; + let effective = |name: &str| { + status + .lines() + .find_map(|line| line.strip_prefix(name)) + .and_then(|values| values.split_whitespace().nth(1)) + .map(str::to_string) }; - - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); - } - - endpoint.to_string() -} - -fn docker_network_name(config: &DockerComputeConfig) -> String { - let name = config.network_name.trim(); - if name.is_empty() { - return DEFAULT_DOCKER_NETWORK_NAME.to_string(); - } - name.to_string() -} - -fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - trimmed - .parse() - .map(Some) - .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) -} - -fn docker_gateway_route( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, -) -> DockerGatewayRoute { - docker_gateway_route_for_host( - info, - bridge_gateway_ip, - port, - host_gateway_ip, - host_runtime_requires_host_gateway_alias(), - ) -} - -fn docker_gateway_route_for_host( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, - host_requires_host_gateway_alias: bool, -) -> DockerGatewayRoute { - if let Some(host_alias_ip) = host_gateway_ip { - return DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, - }; - } - - if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { - DockerGatewayRoute::HostGateway - } else { - DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, - } - } -} - -fn docker_gateway_callback_bind_address( - route: &DockerGatewayRoute, - primary_bind_address: SocketAddr, -) -> Option { - match route { - DockerGatewayRoute::Bridge { bind_address, .. } => Some(*bind_address), - DockerGatewayRoute::HostGateway => match primary_bind_address.ip() { - IpAddr::V4(ip) if ip.is_unspecified() || ip == Ipv4Addr::LOCALHOST => None, - _ => Some(SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - primary_bind_address.port(), - )), - }, - } -} - -fn host_runtime_requires_host_gateway_alias() -> bool { - cfg!(target_os = "macos") -} - -/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, -/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing -/// constraint: the bridge gateway IP is reachable from inside containers but -/// not from the `OpenShell` server process running on the host, so callbacks -/// must traverse `host-gateway`. -/// -/// Each runtime is detected via the daemon's reported OS string or hostname, -/// supplemented by labels where the runtime publishes them. -fn uses_host_gateway_alias(info: &SystemInfo) -> bool { - let operating_system = info - .operating_system - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if operating_system.contains("docker desktop") { - return true; - } - - let name = info - .name - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if name.starts_with("colima") - || name.starts_with("lima-") - || name.starts_with("rancher-desktop") - || name.starts_with("orbstack") - { - return true; - } - - info.labels.as_ref().is_some_and(|labels| { - labels.iter().any(|label| { - label.starts_with("com.docker.desktop.") - || label.starts_with("dev.rancherdesktop.") - || label.starts_with("dev.orbstack.") - }) - }) -} - -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - -async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { - match docker.inspect_network(network_name, None).await { - Ok(network) => return validate_bridge_network(network_name, &network), - Err(err) if !is_not_found_error(&err) => { - return Err(Error::execution(format!( - "failed to inspect Docker network '{network_name}': {err}" - ))); - } - Err(_) => {} - } - - docker - .create_network(NetworkCreateRequest { - name: network_name.to_string(), - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - attachable: Some(true), - labels: Some(HashMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }) - .await - .map(|_| ()) - .or_else(|err| { - if is_conflict_error(&err) { - Ok(()) - } else { - Err(Error::execution(format!( - "failed to create Docker network '{network_name}': {err}" - ))) - } - })?; - - let network = docker - .inspect_network(network_name, None) - .await - .map_err(|err| { - Error::execution(format!( - "failed to inspect Docker network '{network_name}' after create: {err}" - )) - })?; - validate_bridge_network(network_name, &network) + let uid = effective("Uid:").ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "missing effective uid") + })?; + let gid = effective("Gid:").ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "missing effective gid") + })?; + Ok(format!("{uid}:{gid}")) } -fn validate_bridge_network( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { - return Err(Error::config(format!( - "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", - network.driver.as_deref().unwrap_or("unknown") - ))); - } - - docker_bridge_gateway_ip(network_name, network) +#[cfg(not(target_os = "linux"))] +fn current_process_user() -> Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Docker host-supervisor mode requires Linux", + )) } -fn docker_bridge_gateway_ip( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { - return Err(Error::config(format!( - "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" - ))); +fn docker_host_openshell_endpoint(endpoint: &str) -> String { + let Ok(mut url) = Url::parse(endpoint) else { + return endpoint.to_string(); }; - - for config in configs { - let Some(gateway) = config.gateway.as_deref() else { - continue; - }; - let ip = gateway.parse::().map_err(|err| { - Error::config(format!( - "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" - )) - })?; - if matches!(ip, IpAddr::V4(_)) { - return Ok(ip); - } + if matches!( + url.host_str(), + Some(HOST_OPENSHELL_INTERNAL | HOST_DOCKER_INTERNAL) + ) && url.set_host(Some("127.0.0.1")).is_ok() + { + return url.to_string(); } - - Err(Error::config(format!( - "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" - ))) + endpoint.to_string() } fn docker_resource_limits( @@ -3343,28 +2509,6 @@ fn docker_resource_limits( }) } -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { - return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", - )); - } - Ok(()) -} - -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", - )); - } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) - } -} - #[allow(clippy::cast_possible_truncation)] fn parse_cpu_limit(value: &str) -> Result, Status> { let value = value.trim(); @@ -4023,16 +3167,6 @@ fn is_not_found_error(err: &BollardError) -> bool { ) } -fn is_conflict_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) -} - fn is_not_modified_error(err: &BollardError) -> bool { matches!( err, @@ -4043,20 +3177,6 @@ fn is_not_modified_error(err: &BollardError) -> bool { ) } -fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { - if matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) { - Status::already_exists("sandbox already exists") - } else { - internal_status(operation, err) - } -} - fn internal_status(operation: &str, err: BollardError) -> Status { Status::internal(format!("{operation} failed: {err}")) } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index b52cb87836..50f95ffcb3 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 use super::*; -use openshell_core::config::DEFAULT_SERVER_PORT; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, LABEL_SANDBOX_NAMESPACE, supervisor_cache_path_with_base, @@ -13,18 +12,14 @@ use openshell_core::progress::{ PROGRESS_STEP_STARTING_SANDBOX, }; use openshell_core::proto::compute::v1::{ - DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, - GetGatewayListenerRequirementsRequest, GpuResourceRequirements, ResourceRequirements, - gateway_listener_requirement::Selector, + DriverResourceRequirements, DriverSandboxSpec, DriverSandboxTemplate, GpuResourceRequirements, + ResourceRequirements, }; use std::fs; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::path::PathBuf; +use std::sync::Arc; use tempfile::TempDir; -const TLS_MOUNT_DIR: &str = "/etc/openshell/tls/client"; -static ENV_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); - fn test_sandbox() -> DriverSandbox { // Mirrors the gateway-supplied request: the public `Sandbox` API no // longer carries `namespace`, so the gateway elides the field and the @@ -92,23 +87,11 @@ fn gpu_resources(count: Option) -> ResourceRequirements { fn runtime_config() -> DockerDriverRuntimeConfig { DockerDriverRuntimeConfig { + socket_path: PathBuf::from("/var/run/docker.sock"), default_image: "image:latest".to_string(), image_pull_policy: String::new(), sandbox_namespace: "default".to_string(), grpc_endpoint: "https://localhost:8443".to_string(), - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - gateway_route: DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - ), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - }, - gateway_callback_bind_address: Some(SocketAddr::new( - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - )), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), @@ -120,31 +103,6 @@ fn runtime_config() -> DockerDriverRuntimeConfig { daemon_version: "28.0.0".to_string(), supports_gpu: false, allow_all_default_gpu: false, - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, - enable_bind_mounts: false, - } -} - -fn json_struct(value: serde_json::Value) -> prost_types::Struct { - let serde_json::Value::Object(object) = value else { - panic!("expected JSON object"); - }; - openshell_core::proto_struct::json_object_to_struct(object) - .expect("test JSON must convert to a protobuf Struct") -} - -fn inspected_volume(driver: &str, options: HashMap) -> bollard::models::Volume { - bollard::models::Volume { - name: "openshell-test-volume".to_string(), - driver: driver.to_string(), - mountpoint: "/var/lib/docker/volumes/openshell-test-volume/_data".to_string(), - created_at: None, - status: None, - labels: HashMap::new(), - scope: None, - cluster_volume: None, - options, - usage_data: None, } } @@ -157,12 +115,13 @@ fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDr ), config, events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + pending: Arc::new(Mutex::new(HashMap::new())), gpu_selector: Arc::new(CdiGpuDefaultSelector::new( CdiGpuInventory::default(), allow_all_default_gpu, )), lifecycle_event_fences: DockerLifecycleEventFences::default(), + host_supervisors: Arc::new(Mutex::new(HashMap::new())), } } @@ -622,374 +581,6 @@ async fn tracing_in_process_stream_records_cancelled_when_dropped() { provider.shutdown().unwrap(); } -#[tokio::test] -async fn gateway_listener_requirements_report_managed_bridge_address() { - let config = runtime_config(); - let expected_address = match config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => bind_address, - DockerGatewayRoute::HostGateway => panic!("test config must use a managed bridge"), - }; - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.requirements.len(), 1); - assert_eq!( - response.requirements[0].selector, - Some(Selector::ExactBindAddress(expected_address.to_string())) - ); -} - -#[tokio::test] -async fn gateway_listener_requirements_are_empty_for_host_gateway_route() { - let mut config = runtime_config(); - config.gateway_route = DockerGatewayRoute::HostGateway; - config.gateway_callback_bind_address = None; - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert!(response.requirements.is_empty()); -} - -#[tokio::test] -async fn host_gateway_route_reports_ipv4_loopback_callback_listener() { - let mut config = runtime_config(); - config.gateway_route = DockerGatewayRoute::HostGateway; - config.gateway_callback_bind_address = Some("127.0.0.1:17670".parse().unwrap()); - let driver = test_driver_with_config(config); - - let response = driver - .get_gateway_listener_requirements(Request::new(GetGatewayListenerRequirementsRequest {})) - .await - .unwrap() - .into_inner(); - - assert_eq!(response.requirements.len(), 1); - assert_eq!( - response.requirements[0].selector, - Some(Selector::ExactBindAddress("127.0.0.1:17670".to_string())) - ); -} - -#[test] -fn container_visible_endpoint_rewrites_loopback_hosts() { - assert_eq!( - docker_container_openshell_endpoint( - "https://localhost:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "http://127.0.0.1:8080", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "http://host.openshell.internal:17670/" - ); - assert_eq!( - docker_container_openshell_endpoint( - "https://gateway.internal:8443", - HOST_OPENSHELL_INTERNAL, - DEFAULT_SERVER_PORT, - ), - "https://host.openshell.internal:17670/" - ); -} - -#[test] -fn docker_bridge_gateway_ip_requires_ipv4_gateway() { - let network = bollard::models::NetworkInspect { - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - ipam: Some(bollard::models::Ipam { - config: Some(vec![ - bollard::models::IpamConfig { - gateway: Some("fd00::1".to_string()), - ..Default::default() - }, - bollard::models::IpamConfig { - gateway: Some("172.18.0.1".to_string()), - ..Default::default() - }, - ]), - ..Default::default() - }), - ..Default::default() - }; - - assert_eq!( - docker_bridge_gateway_ip(DEFAULT_DOCKER_NETWORK_NAME, &network).unwrap(), - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)) - ); - - let ipv6_only_network = bollard::models::NetworkInspect { - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - ipam: Some(bollard::models::Ipam { - config: Some(vec![bollard::models::IpamConfig { - gateway: Some("fd00::1".to_string()), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - }; - - assert!( - docker_bridge_gateway_ip(DEFAULT_DOCKER_NETWORK_NAME, &ipv6_only_network) - .unwrap_err() - .to_string() - .contains("IPv4 IPAM gateway") - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_docker_desktop() { - let info = SystemInfo { - operating_system: Some("Docker Desktop".to_string()), - labels: Some(vec![ - "com.docker.desktop.address=unix:///tmp/docker.sock".to_string(), - ]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); -} - -#[test] -fn host_gateway_route_requests_ipv4_loopback_for_ipv6_primary() { - assert_eq!( - docker_gateway_callback_bind_address( - &DockerGatewayRoute::HostGateway, - "[::1]:17670".parse().unwrap(), - ), - Some("127.0.0.1:17670".parse().unwrap()) - ); -} - -#[test] -fn host_gateway_route_reuses_ipv4_primary_when_it_covers_loopback() { - for primary in ["127.0.0.1:17670", "0.0.0.0:17670"] { - assert_eq!( - docker_gateway_callback_bind_address( - &DockerGatewayRoute::HostGateway, - primary.parse().unwrap(), - ), - None, - "{primary} already covers the IPv4 loopback callback" - ); - } -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_colima() { - let info = SystemInfo { - name: Some("colima".to_string()), - operating_system: Some("Ubuntu 24.04.4 LTS".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 20, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); - assert_eq!( - docker_extra_hosts(&DockerGatewayRoute::HostGateway), - vec![ - "host.docker.internal:host-gateway".to_string(), - "host.openshell.internal:host-gateway".to_string() - ] - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_colima_named_profile() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - // `colima start --profile ` sets the daemon hostname to - // `colima-`; the prefix match still catches it. - name: Some("colima-default".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_rancher_desktop() { - let info = SystemInfo { - operating_system: Some("Alpine Linux v3.20".to_string()), - name: Some("lima-rancher-desktop".to_string()), - labels: Some(vec![ - "dev.rancherdesktop.profile=Rancher Desktop".to_string(), - ]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_for_orbstack() { - let info = SystemInfo { - operating_system: Some("OrbStack".to_string()), - name: Some("orbstack".to_string()), - labels: Some(vec!["dev.orbstack.machine_type=docker".to_string()]), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_uses_bridge_gateway_for_linux_docker() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - let route = docker_gateway_route_for_host( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - None, - false, - ); - - assert_eq!( - route, - DockerGatewayRoute::Bridge { - bind_address: "172.18.0.1:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - } - ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ] - ); -} - -#[test] -fn docker_gateway_route_uses_host_gateway_when_host_runtime_requires_it() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - assert_eq!( - docker_gateway_route_for_host( - &info, - IpAddr::V4(Ipv4Addr::new(10, 89, 10, 1)), - DEFAULT_SERVER_PORT, - None, - true, - ), - DockerGatewayRoute::HostGateway - ); -} - -#[test] -fn docker_gateway_route_prefers_configured_host_gateway_ip() { - let info = SystemInfo { - operating_system: Some("Ubuntu 24.04 LTS".to_string()), - ..Default::default() - }; - - let route = docker_gateway_route( - &info, - IpAddr::V4(Ipv4Addr::new(172, 18, 0, 1)), - DEFAULT_SERVER_PORT, - Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))), - ); - - assert_eq!( - route, - DockerGatewayRoute::Bridge { - bind_address: "172.20.0.4:17670".parse().unwrap(), - host_alias_ip: IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4)), - } - ); - assert_eq!( - docker_extra_hosts(&route), - vec![ - "host.docker.internal:172.20.0.4".to_string(), - "host.openshell.internal:172.20.0.4".to_string() - ] - ); -} - -#[test] -fn parse_optional_host_gateway_ip_rejects_invalid_values() { - assert_eq!(parse_optional_host_gateway_ip("").unwrap(), None); - assert_eq!( - parse_optional_host_gateway_ip("172.20.0.4").unwrap(), - Some(IpAddr::V4(Ipv4Addr::new(172, 20, 0, 4))) - ); - assert!( - parse_optional_host_gateway_ip("not-an-ip") - .unwrap_err() - .to_string() - .contains("host_gateway_ip") - ); -} - #[test] fn parse_cpu_limit_supports_cores_and_millicores() { assert_eq!(parse_cpu_limit("250m").unwrap(), Some(250_000_000)); @@ -1046,1084 +637,81 @@ fn docker_resource_limits_applies_cpu_and_memory_limits() { } #[test] -fn docker_pids_limit_uses_driver_default_and_allows_runtime_inherit() { - assert_eq!( - docker_pids_limit(DEFAULT_SANDBOX_PIDS_LIMIT).unwrap(), - Some(DEFAULT_SANDBOX_PIDS_LIMIT) - ); - assert_eq!(docker_pids_limit(0).unwrap(), None); - assert!(docker_pids_limit(-1).is_err()); -} +fn managed_container_label_filters_include_gateway_namespace() { + let filters = + managed_container_label_filters("tenant-a", [format!("{LABEL_SANDBOX_ID}=sbx-123")]); + let labels = filters.get("label").unwrap(); -#[test] -fn docker_compute_config_disables_bind_mounts_by_default() { - let cfg = DockerComputeConfig::default(); - assert!(!cfg.enable_bind_mounts); + assert!(labels.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"))); + assert!(labels.contains(&format!("{LABEL_SANDBOX_NAMESPACE}=tenant-a"))); + assert!(labels.contains(&format!("{LABEL_SANDBOX_ID}=sbx-123"))); } #[test] -fn container_create_body_sets_driver_owned_pids_limit() { - let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); - let host_config = body.host_config.expect("host config"); - assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); -} +fn validate_sandbox_rejects_gpu_when_cdi_unavailable() { + let config = runtime_config(); + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); -#[test] -fn build_environment_sets_docker_tls_paths() { - let env = build_environment(&test_sandbox(), &runtime_config()); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={TLS_CA_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={TLS_CERT_MOUNT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={TLS_KEY_MOUNT_PATH}"))); - assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); - assert!(env.contains(&"SPEC_ENV=spec".to_string())); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - let encoded = env - .iter() - .find_map(|entry| { - entry - .strip_prefix("OPENSHELL_MAIN_PROCESS_SPEC=") - .map(str::to_string) - }) - .expect("main-process transport"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(&encoded).unwrap(); - assert_eq!(main.command, vec!["/bin/bash", "-l"]); - assert!(main.tty); + let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("Docker CDI")); } #[test] -fn build_environment_keeps_network_capabilities_driver_controlled() { +fn validate_sandbox_rejects_missing_gpu_support_before_request_shape() { + let config = runtime_config(); let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - "spoofed".to_string(), - ); - let env = build_environment(&sandbox, &runtime_config()); - assert!(env.contains(&format!( - "{}={}", - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY - ))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); + let spec = sandbox.spec.as_mut().unwrap(); + spec.resource_requirements = Some(gpu_resources(Some(2))); + spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); + + let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!(err.message().contains("Docker CDI")); } #[test] -fn build_environment_protects_oci_identity_metadata() { +fn validate_sandbox_rejects_invalid_cdi_devices_before_gpu_capability() { + let config = runtime_config(); let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - for (key, value) in [ - (openshell_core::sandbox_env::OCI_IMAGE_USER, "spoofed"), - (openshell_core::sandbox_env::SANDBOX_UID, "9999"), - (openshell_core::sandbox_env::SANDBOX_GID, "9999"), - ] { - spec.environment.insert(key.to_string(), value.to_string()); - } + spec.resource_requirements = Some(gpu_resources(None)); + spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&[])); - let env = build_environment_for_oci_user(&sandbox, &runtime_config(), "app:staff"); + let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - assert!(env.contains(&format!( - "{}=app:staff", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_UID))); - assert!(env.contains(&format!("{}=", openshell_core::sandbox_env::SANDBOX_GID))); - assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); - assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert!(err.message().contains("invalid docker driver_config")); + assert!(err.message().contains("non-empty list")); } #[test] -fn build_environment_strips_gateway_tls_server_name() { +fn validate_sandbox_rejects_unknown_driver_config_fields() { + let config = runtime_config(); let mut sandbox = test_sandbox(); let spec = sandbox.spec.as_mut().unwrap(); - spec.environment.insert( - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), - "evil.attacker.example.com".to_string(), - ); + spec.resource_requirements = Some(gpu_resources(None)); + spec.template.as_mut().unwrap().driver_config = + Some(cdi_device_typo_config(&["nvidia.com/gpu=0"])); - let env = build_environment(&sandbox, &runtime_config()); + let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - assert!( - !env.iter().any(|entry| entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME - ))), - "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" - ); + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert!(err.message().contains("unknown field")); } #[test] -fn container_creation_uses_inspected_immutable_image() { - let sandbox = test_sandbox(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace/project".to_string(), - volumes: Vec::new(), - }; - let body = build_container_create_body_for_image( - &sandbox, - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap(); +fn validate_sandbox_accepts_gpu_count_request_shape() { + let mut config = runtime_config(); + config.supports_gpu = true; + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(Some(2))); - assert_eq!(body.image.as_deref(), Some("sha256:immutable")); - assert_eq!(body.user.as_deref(), Some("0")); - assert_eq!(body.working_dir.as_deref(), Some("/")); - assert_eq!( - body.cmd.as_deref(), - Some(&["--workdir".to_string(), "/workspace/project".to_string()][..]) - ); - assert!(body.env.unwrap().contains(&format!( - "{}=1234:1235", - openshell_core::sandbox_env::OCI_IMAGE_USER - ))); -} - -#[test] -fn container_creation_rejects_invalid_oci_working_dir() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "relative/workspace".to_string(), - volumes: Vec::new(), - }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("must be an absolute container path")); -} - -#[test] -fn container_creation_rejects_openshell_control_path_working_dir() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/opt/openshell/bin/project".to_string(), - volumes: Vec::new(), - }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("OpenShell control path")); -} - -#[test] -fn container_creation_rejects_image_volume_that_masks_working_dir() { - let sandbox = test_sandbox(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace/project".to_string(), - volumes: vec!["/workspace".to_string()], - }; - - let error = build_container_create_body_for_image( - &sandbox, - &runtime_config(), - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - - assert!( - error - .message() - .contains("masks OCI WorkingDir '/workspace/project'") - ); -} - -#[test] -fn container_creation_rejects_image_volume_over_configured_ssh_socket() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: vec!["/custom-runtime".to_string()], - }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); - - let error = build_container_create_body_for_image( - &test_sandbox(), - &config, - &DockerSandboxDriverConfig::default(), - None, - &metadata, - ) - .unwrap_err(); - - assert!(error.message().contains("OpenShell control path")); -} - -#[test] -fn container_creation_reserves_resolved_workspace_root_but_allows_nested_mounts() { - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: Vec::new(), - }; - let root_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace"}] - })) - .unwrap(); - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &root_mount, - None, - &metadata, - ) - .unwrap_err(); - assert!( - err.message() - .contains("reserved for the OpenShell workspace") - ); - - let ancestor_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace"}] - })) - .unwrap(); - let nested_metadata = DockerImageMetadata { - working_dir: "/workspace/project".to_string(), - volumes: Vec::new(), - ..metadata.clone() - }; - let err = build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &ancestor_mount, - None, - &nested_metadata, - ) - .unwrap_err(); - assert!( - err.message() - .contains("reserved for the OpenShell workspace") - ); - - let nested_mount: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/workspace/cache"}] - })) - .unwrap(); - build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &nested_mount, - None, - &metadata, - ) - .expect("nested workspace mounts remain supported"); - - let compatibility_path_mount: DockerSandboxDriverConfig = - serde_json::from_value(serde_json::json!({ - "mounts": [{"type": "tmpfs", "target": "/sandbox"}] - })) - .unwrap(); - build_container_create_body_for_image( - &test_sandbox(), - &runtime_config(), - &compatibility_path_mount, - None, - &metadata, - ) - .expect("/sandbox remains mountable when the inspected workspace is elsewhere"); -} - -#[test] -fn build_environment_keeps_path_driver_controlled() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.environment - .insert("PATH".to_string(), "/malicious/spec/bin".to_string()); - spec.template - .as_mut() - .unwrap() - .environment - .insert("PATH".to_string(), "/malicious/template/bin".to_string()); - - let env = build_environment(&sandbox, &runtime_config()); - let path_entries = env - .iter() - .filter(|entry| entry.starts_with("PATH=")) - .collect::>(); - - let expected_path = format!("PATH={SUPERVISOR_PATH}"); - assert_eq!(path_entries.len(), 1); - assert_eq!(path_entries[0], &expected_path); -} - -#[test] -fn build_environment_keeps_telemetry_toggle_driver_controlled() { - let _guard = ENV_LOCK.lock().unwrap(); - temp_env::with_vars( - [( - openshell_core::sandbox_env::TELEMETRY_ENABLED, - Some("false"), - )], - || { - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - "true".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - let telemetry_entries = env - .iter() - .filter(|entry| { - entry.starts_with(&format!( - "{}=", - openshell_core::sandbox_env::TELEMETRY_ENABLED - )) - }) - .collect::>(); - - assert_eq!(telemetry_entries.len(), 1); - assert_eq!( - telemetry_entries[0], - &format!("{}=false", openshell_core::sandbox_env::TELEMETRY_ENABLED) - ); - }, - ); -} - -#[test] -fn build_binds_uses_docker_tls_directory() { - let binds = build_binds(&test_sandbox(), &runtime_config()).unwrap(); - let targets = binds - .iter() - .filter_map(|bind| bind.split(':').nth(1).map(String::from)) - .collect::>(); - assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); - assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); - assert!( - targets - .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) - ); -} - -#[test] -fn build_container_create_body_includes_driver_config_mounts() { - let mut sandbox = test_sandbox(); - let template = sandbox.spec.as_mut().unwrap().template.as_mut().unwrap(); - template.driver_config = Some(json_struct(serde_json::json!({ - "mounts": [ - { - "type": "volume", - "source": "work-nfs", - "target": "/sandbox/work", - "read_only": true, - "subpath": "project-a" - }, - { - "type": "tmpfs", - "target": "/sandbox/cache", - "options": ["nosuid", "size=1048576"], - "size_bytes": 1_048_576, - "mode": 511 - } - ] - }))); - - let body = build_container_create_body(&sandbox, &runtime_config()).unwrap(); - let mounts = body - .host_config - .unwrap() - .mounts - .expect("driver config mounts should be set"); - - assert_eq!(mounts.len(), 2); - assert_eq!(mounts[0].typ, Some(MountTypeEnum::VOLUME)); - assert_eq!(mounts[0].source.as_deref(), Some("work-nfs")); - assert_eq!(mounts[0].target.as_deref(), Some("/sandbox/work")); - assert_eq!(mounts[0].read_only, Some(true)); - assert_eq!( - mounts[0] - .volume_options - .as_ref() - .and_then(|options| options.subpath.as_deref()), - Some("project-a") - ); - assert_eq!(mounts[1].typ, Some(MountTypeEnum::TMPFS)); - assert_eq!(mounts[1].target.as_deref(), Some("/sandbox/cache")); - assert_eq!( - mounts[1] - .tmpfs_options - .as_ref() - .and_then(|options| options.size_bytes), - Some(1_048_576) - ); -} - -#[test] -fn driver_config_defaults_volume_mounts_to_read_only() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "volume", - "source": "work-nfs", - "target": "/sandbox/work" - }] - }))); - - let body = build_container_create_body(&sandbox, &runtime_config()).unwrap(); - let mounts = body - .host_config - .unwrap() - .mounts - .expect("driver config mounts should be set"); - - assert_eq!(mounts[0].read_only, Some(true)); -} - -#[test] -fn driver_config_allows_explicit_writable_volume_mounts() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "volume", - "source": "work-nfs", - "target": "/sandbox/work", - "read_only": false - }] - }))); - - let body = build_container_create_body(&sandbox, &runtime_config()).unwrap(); - let mounts = body - .host_config - .unwrap() - .mounts - .expect("driver config mounts should be set"); - - assert_eq!(mounts[0].read_only, Some(false)); -} - -#[test] -fn driver_config_rejects_duplicate_mount_targets() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [ - { - "type": "volume", - "source": "work-nfs", - "target": "/sandbox/work" - }, - { - "type": "tmpfs", - "target": "/sandbox/work" - } - ] - }))); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!( - err.message() - .contains("duplicate docker driver_config mount target") - ); -} - -#[test] -fn driver_config_rejects_bind_mounts_unless_enabled() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": "/host/path", - "target": "/sandbox/host" - }] - }))); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("enable_bind_mounts = true")); -} - -#[test] -fn build_container_create_body_includes_bind_mounts_when_enabled() { - let bind_src = TempDir::new().unwrap(); - let src_path = bind_src.path().to_str().unwrap(); - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": src_path, - "target": "/sandbox/host", - "read_only": true - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let body = build_container_create_body(&sandbox, &config).unwrap(); - let binds = body - .host_config - .as_ref() - .unwrap() - .binds - .as_ref() - .expect("binds should be set"); - - // User bind mount appears after the system binds. - let expected = format!("{src_path}:/sandbox/host:ro"); - assert!( - binds.iter().any(|b| b == &expected), - "expected bind entry '{expected}', got {binds:?}" - ); - // Bind mounts must not appear in the structured mounts vec. - let mounts = body.host_config.unwrap().mounts.unwrap_or_default(); - assert!( - mounts.iter().all(|m| m.typ != Some(MountTypeEnum::BIND)), - "bind mounts should not appear in structured mounts" - ); -} - -#[test] -fn driver_config_defaults_enabled_bind_mounts_to_read_only() { - let bind_src = TempDir::new().unwrap(); - let src_path = bind_src.path().to_str().unwrap(); - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": src_path, - "target": "/sandbox/host" - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let body = build_container_create_body(&sandbox, &config).unwrap(); - let binds = body - .host_config - .unwrap() - .binds - .expect("binds should be set"); - - let expected = format!("{src_path}:/sandbox/host:ro"); - assert!( - binds.iter().any(|b| b == &expected), - "default bind mount should be read-only, got {binds:?}" - ); -} - -#[test] -fn bind_mount_selinux_shared_label() { - let bind_src = TempDir::new().unwrap(); - let src_path = bind_src.path().to_str().unwrap(); - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": src_path, - "target": "/sandbox/data", - "read_only": true, - "selinux_label": "shared" - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let body = build_container_create_body(&sandbox, &config).unwrap(); - let binds = body - .host_config - .unwrap() - .binds - .expect("binds should be set"); - - let expected = format!("{src_path}:/sandbox/data:ro,z"); - assert!( - binds.iter().any(|b| b == &expected), - "expected ':ro,z' label, got {binds:?}" - ); -} - -#[test] -fn bind_mount_selinux_private_label() { - let bind_src = TempDir::new().unwrap(); - let src_path = bind_src.path().to_str().unwrap(); - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": src_path, - "target": "/sandbox/data", - "read_only": false, - "selinux_label": "private" - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let body = build_container_create_body(&sandbox, &config).unwrap(); - let binds = body - .host_config - .unwrap() - .binds - .expect("binds should be set"); - - let expected = format!("{src_path}:/sandbox/data:Z"); - assert!( - binds.iter().any(|b| b == &expected), - "expected ':Z' label, got {binds:?}" - ); -} - -#[test] -fn bind_mount_without_selinux_label() { - let bind_src = TempDir::new().unwrap(); - let src_path = bind_src.path().to_str().unwrap(); - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": src_path, - "target": "/sandbox/host", - "read_only": false - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let body = build_container_create_body(&sandbox, &config).unwrap(); - let binds = body - .host_config - .unwrap() - .binds - .expect("binds should be set"); - - let expected = format!("{src_path}:/sandbox/host"); - assert!( - binds.iter().any(|b| b == &expected), - "expected no options suffix, got {binds:?}" - ); -} - -#[test] -fn driver_config_rejects_missing_bind_source() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": "/no/such/path", - "target": "/sandbox/data" - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let err = build_container_create_body(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!( - err.message().contains("bind source path does not exist"), - "expected missing-source error, got: {}", - err.message() - ); -} - -#[test] -fn driver_config_rejects_relative_bind_sources_when_enabled() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "bind", - "source": "relative/path", - "target": "/sandbox/host" - }] - }))); - let mut config = runtime_config(); - config.enable_bind_mounts = true; - - let err = build_container_create_body(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!( - err.message() - .contains("bind source must be an absolute host path") - ); -} - -#[test] -fn driver_config_rejects_image_mounts() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "image", - "source": "ghcr.io/acme/tools:latest", - "target": "/opt/tools" - }] - }))); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("invalid docker driver_config")); -} - -#[test] -fn driver_config_rejects_reserved_mount_targets() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(json_struct(serde_json::json!({ - "mounts": [{ - "type": "volume", - "source": "work-nfs", - "target": "/etc/openshell/auth" - }] - }))); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("reserved OpenShell path")); -} - -#[test] -fn driver_config_rejects_mount_over_configured_ssh_socket() { - let mount_config: DockerSandboxDriverConfig = serde_json::from_value(serde_json::json!({ - "mounts": [{ - "type": "tmpfs", - "target": "/custom-runtime" - }] - })) - .unwrap(); - let metadata = DockerImageMetadata { - id: "sha256:immutable".to_string(), - user: "1234:1235".to_string(), - working_dir: "/workspace".to_string(), - volumes: Vec::new(), - }; - let mut config = runtime_config(); - config.ssh_socket_path = "/custom-runtime/ssh.sock".to_string(); - - let error = build_container_create_body_for_image( - &test_sandbox(), - &config, - &mount_config, - None, - &metadata, - ) - .unwrap_err(); - - assert!(error.message().contains("OpenShell control path")); -} - -#[test] -fn docker_local_volume_with_bind_option_is_bind_backed() { - let volume = inspected_volume( - "local", - HashMap::from([ - ("type".to_string(), "none".to_string()), - ("o".to_string(), "rw,bind".to_string()), - ("device".to_string(), "/tmp/openshell".to_string()), - ]), - ); - - assert!(docker_volume_is_bind_backed(&volume)); -} - -#[test] -fn docker_local_volume_with_rbind_option_is_bind_backed() { - let volume = inspected_volume( - "local", - HashMap::from([ - ("type".to_string(), "none".to_string()), - ("o".to_string(), "rw,rbind".to_string()), - ("device".to_string(), "/tmp/openshell".to_string()), - ]), - ); - - assert!(docker_volume_is_bind_backed(&volume)); -} - -#[test] -fn docker_local_volume_without_bind_option_is_not_bind_backed() { - let volume = inspected_volume( - "local", - HashMap::from([ - ("type".to_string(), "nfs".to_string()), - ("o".to_string(), "addr=127.0.0.1,rw".to_string()), - ("device".to_string(), ":/exports/openshell".to_string()), - ]), - ); - - assert!(!docker_volume_is_bind_backed(&volume)); -} - -#[test] -fn docker_nonlocal_volume_with_bind_option_is_not_bind_backed() { - let volume = inspected_volume( - "custom", - HashMap::from([("o".to_string(), "bind".to_string())]), - ); - - assert!(!docker_volume_is_bind_backed(&volume)); -} - -#[test] -fn build_environment_uses_token_file_without_raw_token_env() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.sandbox_token = "secret.jwt.value".to_string(); - spec.environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN.to_string(), - "user-provided-token".to_string(), - ); - - let env = build_environment(&sandbox, &runtime_config()); - - assert!(!env.iter().any(|entry| { - entry.starts_with(&format!("{}=", openshell_core::sandbox_env::SANDBOX_TOKEN)) - })); - assert!(env.contains(&format!( - "{}={SANDBOX_TOKEN_MOUNT_PATH}", - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); -} - -#[test] -fn managed_container_label_filters_include_gateway_namespace() { - let filters = - managed_container_label_filters("tenant-a", [format!("{LABEL_SANDBOX_ID}=sbx-123")]); - let labels = filters.get("label").unwrap(); - - assert!(labels.contains(&format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"))); - assert!(labels.contains(&format!("{LABEL_SANDBOX_NAMESPACE}=tenant-a"))); - assert!(labels.contains(&format!("{LABEL_SANDBOX_ID}=sbx-123"))); -} - -#[test] -fn build_container_create_body_replaces_inherited_cmd_with_workspace_arg() { - let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); - - assert_eq!( - create_body.entrypoint, - Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]) - ); - assert_eq!( - create_body.cmd, - Some(vec!["--workdir".to_string(), "/sandbox".to_string()]) - ); - assert_eq!( - create_body - .labels - .as_ref() - .and_then(|labels| labels.get(LABEL_SANDBOX_NAMESPACE)), - Some(&"default".to_string()) - ); - let host_config = create_body.host_config.as_ref().unwrap(); - assert!( - host_config.device_requests.as_ref().is_none(), - "non-GPU containers should not request Docker devices" - ); - assert_eq!( - host_config.security_opt.as_ref(), - Some(&vec!["apparmor=unconfined".to_string()]) - ); - assert_eq!( - host_config.network_mode.as_deref(), - Some(DEFAULT_DOCKER_NETWORK_NAME) - ); - assert_eq!( - host_config.extra_hosts.as_ref(), - Some(&vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]) - ); - assert_eq!( - create_body - .networking_config - .as_ref() - .and_then(|config| config.endpoints_config.as_ref()) - .and_then(|endpoints| endpoints.get(DEFAULT_DOCKER_NETWORK_NAME)), - Some(&EndpointSettings::default()) - ); -} - -#[test] -fn validate_sandbox_rejects_gpu_when_cdi_unavailable() { - let config = runtime_config(); - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); - - let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("Docker CDI")); -} - -#[test] -fn validate_sandbox_rejects_missing_gpu_support_before_request_shape() { - let config = runtime_config(); - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(Some(2))); - spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); - - let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::FailedPrecondition); - assert!(err.message().contains("Docker CDI")); -} - -#[test] -fn validate_sandbox_rejects_invalid_cdi_devices_before_gpu_capability() { - let config = runtime_config(); - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(None)); - spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&[])); - - let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!(err.message().contains("invalid docker driver_config")); - assert!(err.message().contains("non-empty list")); -} - -#[test] -fn validate_sandbox_rejects_unknown_driver_config_fields() { - let config = runtime_config(); - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(None)); - spec.template.as_mut().unwrap().driver_config = - Some(cdi_device_typo_config(&["nvidia.com/gpu=0"])); - - let err = DockerComputeDriver::validate_sandbox(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!(err.message().contains("unknown field")); -} - -#[test] -fn validate_sandbox_accepts_gpu_count_request_shape() { - let mut config = runtime_config(); - config.supports_gpu = true; - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(Some(2))); - - DockerComputeDriver::validate_sandbox(&sandbox, &config) - .expect("default GPU count shape should be accepted before inventory selection"); + DockerComputeDriver::validate_sandbox(&sandbox, &config) + .expect("default GPU count shape should be accepted before inventory selection"); } #[test] @@ -2252,195 +840,6 @@ fn validate_sandbox_auth_accepts_gateway_token() { DockerComputeDriver::validate_sandbox_auth(&sandbox).unwrap(); } -#[test] -fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() { - let mut config = runtime_config(); - config.supports_gpu = true; - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); - - let driver_config = DockerSandboxDriverConfig::default(); - let gpu_devices = vec!["nvidia.com/gpu=1".to_string()]; - let create_body = build_container_create_body_with_gpu_devices( - &sandbox, - &config, - &driver_config, - Some(&gpu_devices), - ) - .unwrap(); - let request = create_body - .host_config - .as_ref() - .and_then(|host_config| host_config.device_requests.as_ref()) - .and_then(|requests| requests.first()) - .expect("GPU request should add a Docker device request"); - - assert_eq!(request.driver.as_deref(), Some("cdi")); - assert_eq!( - request.device_ids.as_ref().unwrap(), - &vec!["nvidia.com/gpu=1".to_string()] - ); -} - -#[test] -fn build_container_create_body_omits_devices_without_resolved_default_cdi_devices() { - let mut config = runtime_config(); - config.supports_gpu = true; - let mut sandbox = test_sandbox(); - sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); - - let create_body = build_container_create_body(&sandbox, &config).unwrap(); - - assert!( - create_body - .host_config - .as_ref() - .and_then(|host_config| host_config.device_requests.as_ref()) - .is_none() - ); -} - -#[test] -fn build_container_create_body_passes_explicit_cdi_device_id_through() { - let mut config = runtime_config(); - config.supports_gpu = true; - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(None)); - spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); - - let create_body = build_container_create_body(&sandbox, &config).unwrap(); - let request = create_body - .host_config - .as_ref() - .and_then(|host_config| host_config.device_requests.as_ref()) - .and_then(|requests| requests.first()) - .expect("GPU request should add a Docker device request"); - - assert_eq!(request.driver.as_deref(), Some("cdi")); - assert_eq!( - request.device_ids.as_ref().unwrap(), - &vec!["nvidia.com/gpu=0".to_string()] - ); -} - -#[test] -fn build_container_create_body_rejects_gpu_count_mismatched_cdi_devices() { - let mut config = runtime_config(); - config.supports_gpu = true; - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(Some(2))); - spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); - - let err = build_container_create_body(&sandbox, &config).unwrap_err(); - - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!( - err.message() - .contains("gpu count (2) must match driver_config.cdi_devices length (1)") - ); -} - -#[test] -fn build_container_create_body_rejects_cdi_devices_without_gpu_request() { - let mut sandbox = test_sandbox(); - sandbox - .spec - .as_mut() - .unwrap() - .template - .as_mut() - .unwrap() - .driver_config = Some(cdi_devices_config(&["nvidia.com/gpu=0"])); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!(err.message().contains("requires a gpu request")); -} - -#[test] -fn build_container_create_body_rejects_empty_cdi_devices() { - let mut sandbox = test_sandbox(); - let spec = sandbox.spec.as_mut().unwrap(); - spec.resource_requirements = Some(gpu_resources(None)); - spec.template.as_mut().unwrap().driver_config = Some(cdi_devices_config(&[])); - - let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); - assert_eq!(err.code(), tonic::Code::InvalidArgument); - assert!(err.message().contains("non-empty list")); -} - -#[test] -fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() { - let mut config = runtime_config(); - config.supports_gpu = true; - let driver = test_driver_with_config(config); - driver.gpu_selector.refresh( - CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]), - false, - ); - let mut first_sandbox = test_sandbox(); - first_sandbox.id = "sbx-first".to_string(); - first_sandbox.name = "first".to_string(); - first_sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); - let mut second_sandbox = test_sandbox(); - second_sandbox.id = "sbx-second".to_string(); - second_sandbox.name = "second".to_string(); - second_sandbox.spec.as_mut().unwrap().resource_requirements = Some(gpu_resources(None)); - - DockerComputeDriver::validate_sandbox(&first_sandbox, &driver.config).unwrap(); - assert_eq!( - driver.gpu_selector.peek_device_ids(1).unwrap(), - vec!["nvidia.com/gpu=0".to_string()] - ); - let first_devices = driver.gpu_selector.next_device_ids(1).unwrap(); - let driver_config = DockerSandboxDriverConfig::default(); - let first_create_body = build_container_create_body_with_gpu_devices( - &first_sandbox, - &driver.config, - &driver_config, - Some(&first_devices), - ) - .unwrap(); - - DockerComputeDriver::validate_sandbox(&second_sandbox, &driver.config).unwrap(); - assert_eq!( - driver.gpu_selector.peek_device_ids(1).unwrap(), - vec!["nvidia.com/gpu=1".to_string()] - ); - let second_devices = driver.gpu_selector.next_device_ids(1).unwrap(); - let second_create_body = build_container_create_body_with_gpu_devices( - &second_sandbox, - &driver.config, - &driver_config, - Some(&second_devices), - ) - .unwrap(); - - let first_request = first_create_body - .host_config - .as_ref() - .and_then(|host_config| host_config.device_requests.as_ref()) - .and_then(|requests| requests.first()) - .expect("first default GPU request should add a Docker device request"); - let second_request = second_create_body - .host_config - .as_ref() - .and_then(|host_config| host_config.device_requests.as_ref()) - .and_then(|requests| requests.first()) - .expect("second default GPU request should add a Docker device request"); - - assert_eq!( - first_request.device_ids.as_ref().unwrap(), - &vec!["nvidia.com/gpu=0".to_string()] - ); - assert_eq!( - second_request.device_ids.as_ref().unwrap(), - &vec!["nvidia.com/gpu=1".to_string()] - ); -} - #[test] fn docker_info_reports_wsl2_from_kernel_version() { let info = SystemInfo { @@ -2504,49 +903,6 @@ fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() { require_sandbox_identifier("sbx-1", "demo").expect("id and name is accepted"); } -#[test] -fn build_container_create_body_uses_bridge_network() { - let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); - let host_config = create_body.host_config.expect("host_config is populated"); - - assert_eq!( - host_config.network_mode, - Some(DEFAULT_DOCKER_NETWORK_NAME.to_string()), - "sandbox should join the driver-managed bridge network" - ); - assert_eq!( - host_config.extra_hosts, - Some(vec![ - "host.docker.internal:172.18.0.1".to_string(), - "host.openshell.internal:172.18.0.1".to_string() - ]), - "sandbox should expose stable host aliases for gateway callbacks" - ); -} - -#[test] -fn build_container_create_body_uses_runtime_namespace_label() { - // Regression test: the namespace label must come from the driver's - // runtime config, not from `DriverSandbox.namespace`. The gateway - // does not populate `DriverSandbox.namespace`, so a container created - // with that empty value would not match subsequent list/get/find - // queries (which filter on `config.sandbox_namespace`), leaking - // sandboxes that the driver itself cannot observe. - let mut config = runtime_config(); - config.sandbox_namespace = "tenant-a".to_string(); - let mut sandbox = test_sandbox(); - sandbox.namespace = "ignored-by-driver".to_string(); - - let create_body = build_container_create_body(&sandbox, &config).unwrap(); - let labels = create_body.labels.expect("labels are populated"); - - assert_eq!( - labels.get(LABEL_SANDBOX_NAMESPACE), - Some(&"tenant-a".to_string()), - "namespace label must reflect the driver's runtime config" - ); -} - #[test] fn driver_status_keeps_running_sandboxes_provisioning_with_stable_message() { let running = ContainerSummary { diff --git a/crates/openshell-isolation-vm/src/backend.rs b/crates/openshell-isolation-vm/src/backend.rs index d424b11a81..2bf2fc876a 100644 --- a/crates/openshell-isolation-vm/src/backend.rs +++ b/crates/openshell-isolation-vm/src/backend.rs @@ -594,6 +594,7 @@ impl NetworkMediationSource for VmNetworkMediation { Ok(MediatedConnection { stream, binary_identity: identity.into_result(), + destination: None, }) } } diff --git a/crates/openshell-isolation/src/contract.rs b/crates/openshell-isolation/src/contract.rs index f94212ca9e..ab9f6ece87 100644 --- a/crates/openshell-isolation/src/contract.rs +++ b/crates/openshell-isolation/src/contract.rs @@ -10,8 +10,10 @@ //! chain of boxed states: //! //! ```text +//! create plan + sandbox context -> Bound -> confirm -> Ready +//! -> start_agent -> Running //! attach topology + sandbox context -> Bound -> confirm -> Ready -//! -> start_agent -> Running +//! -> start_agent -> Running //! ``` //! //! Each transition consumes the prior state by value (`self: Box`), and no @@ -20,10 +22,11 @@ //! registry is the only lookup by `backend_name`, and everything past it is a //! `Box` / `Arc`. //! -//! `attach` is atomic from the caller's perspective: it returns `Bound` or fails -//! closed, and it never binds a resource that is already bound to an active -//! boundary. Binary identity travels on every [`MediatedConnection`], resolved -//! by the backend for that exact connection; an unresolved identity denies the +//! `create` and `attach` are atomic from the caller's perspective: either route +//! returns `Bound` or fails closed. Creation is optional and never falls back; +//! attachment never binds a resource already bound to an active boundary. +//! Binary identity travels on every [`MediatedConnection`], resolved by the +//! backend for that exact connection; an unresolved identity denies the //! connection and never authorizes anything. //! //! The contract is transport-neutral. Concrete topology implementations keep @@ -31,19 +34,20 @@ use std::collections::HashMap; use std::fmt; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::oneshot; pub use openshell_core::policy::SandboxPolicy; /// The Isolation Backend contract version. The descriptor and the resolved /// backend must both equal the supervisor-supported version exactly. -pub const INTERFACE_VERSION: u32 = 1; +pub const INTERFACE_VERSION: u32 = 2; // ============================================================================ // Errors @@ -62,6 +66,8 @@ pub enum BackendError { Denied(String), /// Boundary temporarily unavailable. Unavailable(String), + /// The selected backend does not implement an optional contract operation. + Unsupported(String), /// Attachment-phase failure (establishment or mediation bring-up). Attach(String), /// Readiness confirmation failed (do not start workload code). @@ -98,7 +104,7 @@ impl BackendError { match self { Self::Descriptor(_) | Self::NotRegistered(_) => BackendErrorKind::Invalid, Self::Denied(_) => BackendErrorKind::Denied, - Self::Unavailable(_) => BackendErrorKind::Unavailable, + Self::Unavailable(_) | Self::Unsupported(_) => BackendErrorKind::Unavailable, Self::Attach(_) | Self::Confirm(_) | Self::Process(_) => BackendErrorKind::Failed, Self::Terminated(_) => BackendErrorKind::Terminated, } @@ -112,6 +118,7 @@ impl fmt::Display for BackendError { Self::NotRegistered(m) => write!(f, "backend not registered: {m}"), Self::Denied(m) => write!(f, "attachment denied: {m}"), Self::Unavailable(m) => write!(f, "boundary unavailable: {m}"), + Self::Unsupported(m) => write!(f, "operation unsupported: {m}"), Self::Attach(m) => write!(f, "attachment failed: {m}"), Self::Confirm(m) => write!(f, "confirmation failed: {m}"), Self::Process(m) => write!(f, "process error: {m}"), @@ -163,6 +170,52 @@ pub struct TopologyDescriptor { pub payload: Vec, } +/// Trusted, backend-specific inputs for creating a fresh isolation boundary. +/// +/// Unlike [`TopologyDescriptor`], this envelope describes a resource to create, +/// not one that already exists. Trusted deployment code selects the backend and +/// protects the opaque payload from workload modification. +#[derive(Debug, Clone)] +pub struct BoundaryCreatePlan { + /// The Isolation Backend contract version this plan targets. + pub version: u32, + /// The backend the supervisor must ask to create the boundary. + pub backend_name: String, + /// Backend-specific prepared creation inputs. + pub payload: Vec, +} + +/// Trusted provisioning input selected before the supervisor drives a backend. +/// +/// The variants are explicit and never used as fallback alternatives. +#[derive(Debug, Clone)] +pub enum BoundaryProvisioning { + /// Ask a create-capable backend to create a fresh resource. + Create(BoundaryCreatePlan), + /// Attach to a resource created by a compute driver or orchestrator. + Attach(TopologyDescriptor), +} + +/// Durable resource-lifecycle ownership selected by the provisioning route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BoundaryOrigin { + /// The Isolation Backend created the resource at the supervisor's request. + SupervisorCreated, + /// A compute driver or external orchestrator created the resource. + ExternallyCreated, +} + +impl BoundaryProvisioning { + /// Backend admitted for this provisioning route. + #[must_use] + pub fn backend_name(&self) -> &str { + match self { + Self::Create(plan) => &plan.backend_name, + Self::Attach(descriptor) => &descriptor.backend_name, + } + } +} + /// A descriptor whose common envelope has passed registry verification. /// /// Minted only by [`BackendRegistry::resolve`]; no public constructor, so an @@ -173,6 +226,33 @@ pub struct VerifiedTopologyDescriptor { descriptor: TopologyDescriptor, } +/// A create plan whose common envelope has passed registry verification. +/// +/// The selected backend still validates the opaque payload during `create`. +pub struct VerifiedBoundaryCreatePlan { + plan: BoundaryCreatePlan, +} + +impl VerifiedBoundaryCreatePlan { + /// The verified backend name. + #[must_use] + pub fn backend_name(&self) -> &str { + &self.plan.backend_name + } + + /// The backend-specific creation payload. + #[must_use] + pub fn payload(&self) -> &[u8] { + &self.plan.payload + } + + /// The interface version. + #[must_use] + pub fn version(&self) -> u32 { + self.plan.version + } +} + impl VerifiedTopologyDescriptor { /// The verified backend name. #[must_use] @@ -301,6 +381,178 @@ impl BackendRegistry { } Ok((backend, VerifiedTopologyDescriptor { descriptor })) } + + /// Verify a fresh-boundary create plan and resolve its backend. + /// + /// Creation is selected explicitly by trusted orchestration. Resolution + /// never falls back to attachment or to another backend when creation is + /// unsupported. + /// + /// # Errors + /// + /// Returns [`BackendError::Descriptor`] for a version or admission + /// mismatch, and [`BackendError::NotRegistered`] when no backend is + /// registered for the admitted name. + pub fn resolve_create( + &self, + plan: BoundaryCreatePlan, + admitted_backend_name: &str, + ) -> Result<(Arc, VerifiedBoundaryCreatePlan), BackendError> { + if plan.version != INTERFACE_VERSION { + return Err(BackendError::Descriptor(format!( + "create plan interface version {} unsupported (expected {INTERFACE_VERSION})", + plan.version + ))); + } + if plan.backend_name != admitted_backend_name { + return Err(BackendError::Descriptor(format!( + "create plan backend {:?} does not match admitted backend {admitted_backend_name:?}", + plan.backend_name + ))); + } + let backend = self + .backends + .get(&plan.backend_name) + .ok_or_else(|| BackendError::NotRegistered(plan.backend_name.clone()))? + .clone(); + if backend.backend_name() != plan.backend_name { + return Err(BackendError::Descriptor(format!( + "registry returned backend {:?} for name {:?}", + backend.backend_name(), + plan.backend_name + ))); + } + if backend.version() != INTERFACE_VERSION { + return Err(BackendError::Descriptor(format!( + "backend {:?} speaks interface version {}, supervisor requires {INTERFACE_VERSION}", + plan.backend_name, + backend.version() + ))); + } + Ok((backend, VerifiedBoundaryCreatePlan { plan })) + } + + /// Drive the explicitly selected create or attach entry operation. + /// + /// Both routes return the same bound state plus the descriptor needed for + /// recovery and an origin that trusted durable state uses to assign cleanup + /// ownership. This method never falls back between routes. + /// + /// # Errors + /// + /// Returns the selected route's verification or backend error without + /// advancing to [`BoundBoundary`] on failure. + pub async fn provision( + &self, + provisioning: BoundaryProvisioning, + admitted_backend_name: &str, + sandbox: SandboxContext, + ) -> Result { + match provisioning { + BoundaryProvisioning::Create(plan) => { + let (backend, verified) = self.resolve_create(plan, admitted_backend_name)?; + let created = backend.create(verified, sandbox).await?; + let (descriptor, boundary) = created.into_parts(); + Ok(ProvisionedBoundary::new( + descriptor, + BoundaryOrigin::SupervisorCreated, + boundary, + )) + } + BoundaryProvisioning::Attach(descriptor) => { + let recovery_descriptor = descriptor.clone(); + let (backend, verified) = self.resolve(descriptor, admitted_backend_name)?; + let boundary = backend.attach(verified, sandbox).await?; + Ok(ProvisionedBoundary::new( + recovery_descriptor, + BoundaryOrigin::ExternallyCreated, + boundary, + )) + } + } + } + + /// Destroy a resource created by the supervisor through this registry. + /// + /// Trusted durable state supplies `origin`; externally created resources + /// remain owned by their compute driver or orchestrator and are rejected. + /// Destruction is idempotent at the backend-specific resource layer. + /// + /// # Errors + /// + /// Returns [`BackendError::Denied`] when `origin` is not + /// [`BoundaryOrigin::SupervisorCreated`], an envelope resolution error, or + /// the selected backend's destruction error. + pub async fn destroy_created( + &self, + descriptor: TopologyDescriptor, + origin: BoundaryOrigin, + admitted_backend_name: &str, + sandbox_id: &str, + ) -> Result<(), BackendError> { + if origin != BoundaryOrigin::SupervisorCreated { + return Err(BackendError::Denied( + "the supervisor cannot destroy an externally created isolation resource" + .to_string(), + )); + } + let (backend, verified) = self.resolve(descriptor, admitted_backend_name)?; + backend.destroy(verified, sandbox_id).await + } +} + +/// A fresh supervisor-created resource already bound to its admitted sandbox. +/// +/// The descriptor is persisted for recovery through [`IsolationBackend::attach`]. +/// The boundary enters the same typestate chain as an externally created +/// resource. +pub struct CreatedBoundary { + descriptor: TopologyDescriptor, + boundary: Box, +} + +impl CreatedBoundary { + /// Construct a successful create result. + #[must_use] + pub fn new(descriptor: TopologyDescriptor, boundary: Box) -> Self { + Self { + descriptor, + boundary, + } + } + + /// Split the durable recovery descriptor from the bound runtime state. + #[must_use] + pub fn into_parts(self) -> (TopologyDescriptor, Box) { + (self.descriptor, self.boundary) + } +} + +/// The common result after either provisioning route reaches `Bound`. +pub struct ProvisionedBoundary { + descriptor: TopologyDescriptor, + origin: BoundaryOrigin, + boundary: Box, +} + +impl ProvisionedBoundary { + fn new( + descriptor: TopologyDescriptor, + origin: BoundaryOrigin, + boundary: Box, + ) -> Self { + Self { + descriptor, + origin, + boundary, + } + } + + /// Split the recovery data, lifecycle owner, and bound runtime state. + #[must_use] + pub fn into_parts(self) -> (TopologyDescriptor, BoundaryOrigin, Box) { + (self.descriptor, self.origin, self.boundary) + } } /// Establishes and operates boundaries for one admitted backend implementation. @@ -313,6 +565,40 @@ pub trait IsolationBackend: Send + Sync { /// exactly against [`INTERFACE_VERSION`]; there is no capability negotiation. fn version(&self) -> u32; + /// Create a fresh resource and atomically bind it to the trusted sandbox + /// context. This operation is optional so distributed or externally + /// provisioned topologies can remain attach-only. + /// + /// Trusted orchestration selects creation explicitly. An unsupported + /// implementation fails without falling back to [`Self::attach`] or to a + /// different backend. + async fn create( + &self, + _plan: VerifiedBoundaryCreatePlan, + _sandbox: SandboxContext, + ) -> Result { + Err(BackendError::Unsupported(format!( + "backend {:?} does not support supervisor-owned creation", + self.backend_name() + ))) + } + + /// Destroy a resource previously returned by [`Self::create`]. + /// + /// This operation is optional for attach-only backends. Implementations + /// must validate the descriptor against `sandbox_id` and make retries + /// idempotent so recovery can finish cleanup after a supervisor crash. + async fn destroy( + &self, + _descriptor: VerifiedTopologyDescriptor, + _sandbox_id: &str, + ) -> Result<(), BackendError> { + Err(BackendError::Unsupported(format!( + "backend {:?} does not support supervisor-owned destruction", + self.backend_name() + ))) + } + /// Validate the opaque payload and atomically bind it to the trusted /// sandbox context: returns `Bound` or fails closed. Never binds a resource /// that is already bound to an active boundary. @@ -336,6 +622,13 @@ pub trait BoundBoundary: Send { /// Retained by the supervisor before consuming `Bound`. fn network_mediation_source(&self) -> Arc; + /// Optional transport for workload DNS exchanges. Backends that expose + /// this source keep DNS inside the supervisor-owned policy path rather + /// than granting the workload access to a resolver socket. + fn dns_mediation_source(&self) -> Option> { + None + } + /// Confirm standing enforcement. How a backend establishes confidence is /// private to that backend; confirmation fails closed. async fn confirm(self: Box) -> Result, BackendError>; @@ -588,6 +881,9 @@ pub struct MediatedConnection { pub stream: BoundaryDuplexStream, /// Executable identity, resolved by the backend for this connection. pub binary_identity: Result, + /// Original destination captured by the backend. Explicit-proxy + /// transports leave this absent; transparent transports must supply it. + pub destination: Option, } /// A logical per-boundary stream of workload connections, consumed by the @@ -605,5 +901,34 @@ pub trait NetworkMediationSource: Send + Sync { async fn accept(&self) -> Result; } +/// DNS transport used by one workload exchange. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DnsTransport { + /// One DNS wire datagram without a TCP length prefix. + Udp, + /// One two-byte-length-prefixed DNS message. + Tcp, +} + +/// One workload DNS request and its fail-closed response channel. +pub struct MediatedDnsQuery { + /// DNS request bytes in the framing selected by [`Self::transport`]. + pub request: Vec, + /// Workload DNS transport. + pub transport: DnsTransport, + /// Identity of the process that issued the DNS request. + pub binary_identity: Result, + /// Single-use response channel owned by the backend adapter. + pub response: oneshot::Sender, BackendError>>, +} + +/// Logical per-boundary stream of DNS exchanges. The backend handles syscall, +/// packet, or guest-agent transport details; the supervisor owns policy DNS. +#[async_trait] +pub trait DnsMediationSource: Send + Sync { + /// Await the next DNS query from this boundary. + async fn accept(&self) -> Result; +} + #[cfg(test)] mod tests; diff --git a/crates/openshell-isolation/src/contract/tests.rs b/crates/openshell-isolation/src/contract/tests.rs index 645bb7edee..20fef7bf9a 100644 --- a/crates/openshell-isolation/src/contract/tests.rs +++ b/crates/openshell-isolation/src/contract/tests.rs @@ -97,6 +97,7 @@ impl NetworkMediationSource for MockSource { ancestors: vec![], cmdline_paths: vec![], }), + destination: None, }) } } @@ -113,6 +114,7 @@ impl NetworkMediationSource for UnattributedSource { Ok(MediatedConnection { stream: Box::new(near), binary_identity: Err(ResolveError::Failed("hash unavailable".to_string())), + destination: None, }) } } @@ -284,6 +286,65 @@ impl IsolationBackend for WrongVersionBackend { } } +/// A backend that opts into supervisor-owned creation. The ordinary mock +/// backends intentionally rely on the trait's attach-only default. +struct CreatingBackend { + destroyed: Arc, +} + +#[async_trait] +impl IsolationBackend for CreatingBackend { + fn backend_name(&self) -> &'static str { + "mock-creating" + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn create( + &self, + plan: VerifiedBoundaryCreatePlan, + sandbox: SandboxContext, + ) -> Result { + assert_eq!(plan.backend_name(), self.backend_name()); + assert_eq!(plan.payload(), b"prepared-inputs"); + let descriptor = TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: self.backend_name().to_string(), + payload: sandbox.sandbox_id.into_bytes(), + }; + Ok(CreatedBoundary::new( + descriptor, + Box::new(MockBound:: { + source: Arc::new(MockSource(PhantomData)), + }), + )) + } + + async fn destroy( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox_id: &str, + ) -> Result<(), BackendError> { + assert_eq!(descriptor.backend_name(), self.backend_name()); + assert_eq!(descriptor.payload(), sandbox_id.as_bytes()); + self.destroyed.store(true, Ordering::SeqCst); + Ok(()) + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + _sandbox: SandboxContext, + ) -> Result, BackendError> { + assert_eq!(descriptor.backend_name(), self.backend_name()); + Ok(Box::new(MockBound:: { + source: Arc::new(MockSource(PhantomData)), + })) + } +} + // --------------------------------------------------------------------------- // Helpers. // --------------------------------------------------------------------------- @@ -305,6 +366,14 @@ fn descriptor(backend_name: &str) -> TopologyDescriptor { } } +fn create_plan(backend_name: &str) -> BoundaryCreatePlan { + BoundaryCreatePlan { + version: INTERFACE_VERSION, + backend_name: backend_name.to_string(), + payload: b"prepared-inputs".to_vec(), + } +} + fn sandbox_ctx() -> SandboxContext { SandboxContext { sandbox_id: "sb-1".to_string(), @@ -332,8 +401,15 @@ async fn drive( descriptor: TopologyDescriptor, admitted: &str, ) -> Result, BackendError> { - let (backend, verified) = reg.resolve(descriptor, admitted)?; - let bound = backend.attach(verified, sandbox_ctx()).await?; + let provisioned = reg + .provision( + BoundaryProvisioning::Attach(descriptor), + admitted, + sandbox_ctx(), + ) + .await?; + let (_descriptor, origin, bound) = provisioned.into_parts(); + assert_eq!(origin, BoundaryOrigin::ExternallyCreated); // The mediation source is retained before consuming `Bound` and stays // usable across the confirm/start transitions. let _ingress = bound.network_mediation_source(); @@ -341,6 +417,22 @@ async fn drive( ready.start_agent().await } +async fn drive_create( + reg: &BackendRegistry, + plan: BoundaryCreatePlan, + admitted: &str, +) -> Result<(TopologyDescriptor, Box), BackendError> { + let provisioned = reg + .provision(BoundaryProvisioning::Create(plan), admitted, sandbox_ctx()) + .await?; + let (descriptor, origin, bound) = provisioned.into_parts(); + assert_eq!(origin, BoundaryOrigin::SupervisorCreated); + let _ingress = bound.network_mediation_source(); + let ready = bound.confirm().await?; + let running = ready.start_agent().await?; + Ok((descriptor, running)) +} + // --------------------------------------------------------------------------- // Registry and descriptor. // --------------------------------------------------------------------------- @@ -384,6 +476,81 @@ fn registry_rejects_unknown_backend() { assert!(matches!(err, BackendError::NotRegistered(_))); } +#[test] +fn registry_rejects_create_plan_for_wrong_admitted_backend() { + let reg = registry(); + let err = reg + .resolve_create(create_plan("mock-primary"), "mock-secondary") + .err() + .expect("mismatched create plan must fail"); + assert!(matches!(err, BackendError::Descriptor(_))); +} + +#[tokio::test] +async fn attach_only_backend_may_omit_create() { + let reg = registry(); + let (backend, verified) = reg + .resolve_create(create_plan("mock-primary"), "mock-primary") + .expect("resolve attach-only backend"); + let err = backend + .create(verified, sandbox_ctx()) + .await + .err() + .expect("create must be unsupported"); + assert!(matches!(err, BackendError::Unsupported(_))); +} + +#[tokio::test] +async fn create_and_attach_converge_on_the_same_lifecycle() { + let mut reg = BackendRegistry::new(); + let destroyed = Arc::new(AtomicBool::new(false)); + reg.register(Arc::new(CreatingBackend { + destroyed: destroyed.clone(), + })) + .expect("register creating backend"); + + let (descriptor, created) = drive_create(®, create_plan("mock-creating"), "mock-creating") + .await + .expect("create lifecycle"); + assert_eq!( + created.agent().wait().await.expect("created agent wait"), + BoundaryExitStatus::Exited(0) + ); + + let attached = drive(®, descriptor.clone(), "mock-creating") + .await + .expect("attach lifecycle"); + assert_eq!( + attached.agent().wait().await.expect("attached agent wait"), + BoundaryExitStatus::Exited(0) + ); + + reg.destroy_created( + descriptor, + BoundaryOrigin::SupervisorCreated, + "mock-creating", + "sb-1", + ) + .await + .expect("destroy supervisor-created resource"); + assert!(destroyed.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn registry_never_destroys_an_externally_created_resource() { + let reg = registry(); + let error = reg + .destroy_created( + descriptor("mock-primary"), + BoundaryOrigin::ExternallyCreated, + "mock-primary", + "sb-1", + ) + .await + .expect_err("external lifecycle owner must retain destruction authority"); + assert!(matches!(error, BackendError::Denied(_))); +} + #[test] fn registry_rejects_descriptor_admission_mismatch_without_fallback() { let reg = registry(); @@ -701,6 +868,10 @@ fn error_kinds_map_to_supervisor_status_classes() { BackendError::Unavailable("x".into()).kind(), BackendErrorKind::Unavailable ); + assert_eq!( + BackendError::Unsupported("x".into()).kind(), + BackendErrorKind::Unavailable + ); assert_eq!( BackendError::Attach("x".into()).kind(), BackendErrorKind::Failed diff --git a/crates/openshell-isolation/src/lib.rs b/crates/openshell-isolation/src/lib.rs index d7f4e32183..64c15f26f1 100644 --- a/crates/openshell-isolation/src/lib.rs +++ b/crates/openshell-isolation/src/lib.rs @@ -18,12 +18,13 @@ //! //! # Ordering is a security property //! -//! The lifecycle states run in order: attach -> Bound -> confirm -> Ready -> -//! `start_agent` -> Running. Nothing untrusted runs inside the boundary until it -//! is confirmed ready. This is enforced *by construction*: each transition -//! consumes the prior state by value, and no state type has a public -//! constructor, so the supervisor cannot skip a stage or run a workload before -//! [`contract::ReadyBoundary`] exists. +//! The lifecycle states run in order: either create or attach -> Bound -> +//! confirm -> Ready -> `start_agent` -> Running. Creation is optional; a +//! distributed topology can remain compute-driver-provisioned and attach-only. +//! Nothing untrusted runs inside the boundary until it is confirmed ready. This +//! is enforced *by construction*: each transition consumes the prior state by +//! value, and no state type has a public constructor, so the supervisor cannot +//! skip a stage or run a workload before [`contract::ReadyBoundary`] exists. //! //! [`AgentSpec`] is shared between the workload definition the supervisor //! submits and the [`contract::SandboxContext`] that `attach` binds to a diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 869a8b4765..b83ccd025e 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -18,6 +18,7 @@ path = "src/main.rs" openshell-core = { path = "../openshell-core", default-features = false } openshell-isolation = { path = "../openshell-isolation" } openshell-isolation-vm = { path = "../openshell-isolation-vm" } +openshell-driver-docker = { path = "../openshell-driver-docker" } base64 = { workspace = true } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 2f82397536..b086f69c28 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -104,7 +104,7 @@ pub async fn run_sandbox( network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, - topology_descriptor: Option, + boundary_provisioning: Option, ) -> Result { let (program, args) = command .split_first() @@ -341,35 +341,39 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - if let Some(descriptor) = topology_descriptor { + if let Some(provisioning) = boundary_provisioning { if sidecar_network_enforcement || !network_enabled || !process_enabled { return Err(miette::miette!( - "the VM isolation backend requires combined network,process mode" - )); - } - if descriptor.backend_name != "vm" { - return Err(miette::miette!( - "unsupported prototype isolation backend {:?}; expected \"vm\"", - descriptor.backend_name + "isolation backends require combined network,process mode" )); } let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); let proxy_bind_ip = Arc::new(std::sync::Mutex::new(None)); - let admitted_backend_name = descriptor.backend_name.clone(); - let backend: Arc = - Arc::new(openshell_isolation_vm::VmHostBackend::new( + let admitted_backend_name = provisioning.backend_name().to_string(); + let backend: Arc = match admitted_backend_name.as_str() { + "vm" => Arc::new(openshell_isolation_vm::VmHostBackend::new( "vm", ca_file_paths.clone(), provider_env.clone(), - )); + )), + #[cfg(target_os = "linux")] + "docker" => Arc::new( + openshell_driver_docker::isolation::DockerIsolationBackend::from_host_environment( + provider_env.clone(), + ) + .map_err(|error| miette::miette!(error.to_string()))?, + ), + other => { + return Err(miette::miette!( + "unsupported isolation backend {other:?}" + )); + } + }; let mut registry = openshell_isolation::contract::BackendRegistry::new(); registry .register(backend) .map_err(|error| miette::miette!(error.to_string()))?; - let (backend, verified) = registry - .resolve(descriptor, &admitted_backend_name) - .map_err(|error| miette::miette!(error.to_string()))?; let context = openshell_isolation::contract::SandboxContext { sandbox_id: sandbox_id.clone().unwrap_or_default(), policy: policy.clone(), @@ -381,12 +385,14 @@ pub async fn run_sandbox( interactive, }, }; - let bound = backend - .attach(verified, context) + let provisioned = registry + .provision(provisioning, &admitted_backend_name, context) .await .map_err(|error| miette::miette!(error.to_string()))?; - info!(backend = %admitted_backend_name, "Isolation boundary attached"); + let (recovery_descriptor, origin, bound) = provisioned.into_parts(); + info!(backend = %admitted_backend_name, ?origin, "Isolation boundary provisioned"); let network_mediation_source = bound.network_mediation_source(); + let dns_mediation_source = bound.dns_mediation_source(); let mediation_bind_ip = *proxy_bind_ip.lock().expect("proxy bind IP lock"); let networking = openshell_supervisor_network::run::run_networking( &policy, @@ -406,6 +412,7 @@ pub async fn run_sandbox( workspace_rx.clone(), &upstream_proxy_args, Some(network_mediation_source), + dns_mediation_source, ) .await?; info!( @@ -565,6 +572,18 @@ pub async fn run_sandbox( drop(boundary_access); drop(networking); + if origin == openshell_isolation::contract::BoundaryOrigin::SupervisorCreated { + registry + .destroy_created( + recovery_descriptor, + origin, + &admitted_backend_name, + sandbox_id.as_deref().unwrap_or_default(), + ) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + } + return result; } @@ -604,6 +623,7 @@ pub async fn run_sandbox( workspace_rx.clone(), &upstream_proxy_args, None, + None, ) .await?, ) diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index bc38433165..033e7a6f39 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -242,6 +242,18 @@ struct Args { /// Base64-encoded opaque topology-descriptor payload. #[arg(long)] topology_payload_base64: Option, + + /// Backend asked to create a fresh isolation boundary. + #[arg(long)] + boundary_create_backend_name: Option, + + /// Isolation Backend interface version for a fresh boundary plan. + #[arg(long)] + boundary_create_version: Option, + + /// Base64-encoded backend-private fresh boundary plan. + #[arg(long)] + boundary_create_payload_base64: Option, } /// Copy the running executable to `dest`, creating parent directories as @@ -660,6 +672,39 @@ fn main() -> Result<()> { )); } }; + let boundary_create_plan = match ( + args.boundary_create_backend_name, + args.boundary_create_version, + args.boundary_create_payload_base64, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => { + use base64::Engine as _; + Some(openshell_isolation::contract::BoundaryCreatePlan { + backend_name, + version, + payload: base64::engine::general_purpose::STANDARD + .decode(payload) + .into_diagnostic()?, + }) + } + _ => { + return Err(miette::miette!( + "boundary create plan requires backend name, version, and payload" + )); + } + }; + if topology_descriptor.is_some() && boundary_create_plan.is_some() { + return Err(miette::miette!( + "topology attachment and boundary creation are mutually exclusive" + )); + } + let boundary_provisioning = topology_descriptor + .map(openshell_isolation::contract::BoundaryProvisioning::Attach) + .or_else(|| { + boundary_create_plan + .map(openshell_isolation::contract::BoundaryProvisioning::Create) + }); run_sandbox( command, @@ -679,7 +724,7 @@ fn main() -> Result<()> { args.mode.network, args.mode.process, upstream_proxy_args, - topology_descriptor, + boundary_provisioning, ) .await })?; diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index dea867237d..f3540a628f 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -163,21 +163,6 @@ enable_bind_mounts = true assert!(cfg.enable_bind_mounts); } - #[test] - fn docker_config_reads_bind_mount_opt_in_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.docker] -enable_bind_mounts = true -", - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert!(cfg.enable_bind_mounts); - } - #[test] fn docker_config_reads_socket_path_from_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index e81ed02d50..19871ea259 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -13,6 +13,7 @@ pub mod identity_source; pub mod inference_routes; pub mod l7; pub mod opa; +mod policy_dns; pub mod policy_local; pub mod procfs; pub mod proxy; diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index ad6095efa9..b2d003928a 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -9,6 +9,7 @@ use super::{PolicyDnsService, SocketTrustedResolver, wire}; use crate::opa::OpaEngine; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_isolation::contract::{DnsMediationSource, DnsTransport}; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; use std::sync::Arc; @@ -63,6 +64,75 @@ pub(crate) struct PolicyDnsRuntime { } impl PolicyDnsRuntime { + /// Start policy DNS over an isolation-backend exchange source. No UDP or + /// TCP listener is bound in the supervisor namespace. + pub(crate) fn start_mediated( + policy: Arc, + source: Arc, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + mut engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let task = tokio::spawn(async move { + if engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + loop { + let Ok(query) = source.accept().await else { + return; + }; + let service = service.clone(); + tokio::spawn(async move { + let response = match query.transport { + DnsTransport::Udp => { + wire::handle_udp_query_with_ipv6(&service, &query.request, false).await + } + DnsTransport::Tcp => { + wire::handle_tcp_query_with_ipv6(&service, &query.request, false).await + } + } + .map_err(|error| { + openshell_isolation::contract::BackendError::Process(format!( + "policy DNS response failed: {error}" + )) + }); + let _ = query.response.send(response); + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message("Policy DNS connected to isolation boundary") + .build() + ); + Ok(Self { + store, + tasks: vec![task, expiry_task], + }) + } + pub(crate) fn start( policy: Arc, udp: tokio::net::UdpSocket, diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index ea2cd7c0f8..439cdcfa36 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -73,6 +73,17 @@ pub(crate) struct MappingLookup { } impl MappingLookup { + pub(crate) fn pinned_addresses(&self) -> Vec { + let mut seen = HashSet::new(); + self.record + .contracts + .iter() + .filter(|contract| contract.port == self.port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .filter(|address| seen.insert(*address)) + .collect() + } + pub(crate) fn endpoint_ids(&self) -> impl Iterator { self.record .contracts diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 3025e1b0d2..d0b674b6ab 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -48,8 +48,8 @@ use tokio::task::JoinHandle; use tracing::{debug, warn}; use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, - validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_pinned_validation_plan, + build_validation_plan, validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -214,6 +214,7 @@ impl ProxyHandle { engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, network_mediation_source: Option>, + policy_dns_store: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -229,10 +230,11 @@ impl ProxyHandle { )); } - let listener = if network_mediation_source.is_none() { - Some(TcpListener::bind(http_addr).await.into_diagnostic()?) - } else { + let source_backed = network_mediation_source.is_some(); + let listener = if source_backed { None + } else { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) }; let local_addr = match listener.as_ref() { Some(listener) => listener.local_addr().into_diagnostic()?, @@ -244,7 +246,11 @@ impl ProxyHandle { .severity(SeverityId::Informational) .status(StatusId::Success) .dst_endpoint(Endpoint::from_ip(local_addr.ip(), local_addr.port())) - .message(format!("Proxy listening on {local_addr}")) + .message(if source_backed { + "Proxy consuming isolation-boundary streams".to_string() + } else { + format!("Proxy listening on {local_addr}") + }) .build(); ocsf_emit!(event); } @@ -337,7 +343,12 @@ impl ProxyHandle { .accept() .await .map(|connection| { - (connection.stream, Some(connection.binary_identity), None) + ( + connection.stream, + Some(connection.binary_identity), + None, + connection.destination, + ) }) .map_err(ProxyAcceptError::Source) } else { @@ -351,12 +362,12 @@ impl ProxyHandle { let workload_addr = stream.peer_addr().ok(); let proxy_addr = stream.local_addr().ok(); let stream: BoundaryDuplexStream = Box::new(stream); - (stream, None, workload_addr.zip(proxy_addr)) + (stream, None, workload_addr.zip(proxy_addr), None) }) .map_err(ProxyAcceptError::Listener) }; match accepted { - Ok((stream, supplied_identity, socket_addrs)) => { + Ok((stream, supplied_identity, socket_addrs, transparent_destination)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; let opa = opa_engine.clone(); @@ -368,6 +379,7 @@ impl ProxyHandle { let proposals = agent_proposals.clone(); let gw = trusted_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); + let dns_store = policy_dns_store.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); @@ -384,6 +396,8 @@ impl ProxyHandle { tokio::io::BufReader::new(stream), supplied_identity, socket_addrs, + transparent_destination, + dns_store, opa, cache, spid, @@ -446,7 +460,7 @@ impl ProxyHandle { }); Ok(Self { - http_addr: Some(local_addr), + http_addr: (!source_backed).then_some(local_addr), join, source_failure, }) @@ -1188,6 +1202,8 @@ async fn handle_tcp_connection( tokio::io::BufReader::new(stream), None, socket_addrs, + None, + None, opa_engine, identity_cache, entrypoint_pid, @@ -1205,11 +1221,52 @@ async fn handle_tcp_connection( .await } +/// Adapt a transparent application stream to the existing CONNECT pipeline. +/// The synthetic CONNECT request is supervisor-owned and its successful 200 +/// response is consumed before bytes are returned to the workload. +fn virtual_connect_stream( + workload: BoundaryDuplexStream, + authority: String, +) -> BoundaryDuplexStream { + let (handler, bridge) = tokio::io::duplex(64 * 1024); + let (mut bridge_read, mut bridge_write) = tokio::io::split(bridge); + let (mut workload_read, mut workload_write) = tokio::io::split(workload); + tokio::spawn(async move { + let request = format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n"); + if bridge_write.write_all(request.as_bytes()).await.is_ok() { + let _ = tokio::io::copy(&mut workload_read, &mut bridge_write).await; + } + let _ = bridge_write.shutdown().await; + }); + tokio::spawn(async move { + let mut header = Vec::with_capacity(256); + let mut byte = [0_u8; 1]; + while header.len() < MAX_HEADER_BYTES { + match bridge_read.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => header.push(byte[0]), + } + if header.ends_with(b"\r\n\r\n") { + break; + } + } + if !header.starts_with(b"HTTP/1.1 200 ") && !header.starts_with(b"HTTP/1.0 200 ") { + let _ = workload_write.shutdown().await; + return; + } + let _ = tokio::io::copy(&mut bridge_read, &mut workload_write).await; + let _ = workload_write.shutdown().await; + }); + Box::new(handler) +} + #[allow(clippy::too_many_arguments)] async fn handle_mediated_connection( mut client: ProxyClient, supplied_identity: Option>, socket_addrs: Option<(SocketAddr, SocketAddr)>, + transparent_destination: Option, + policy_dns_store: Option>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1230,6 +1287,28 @@ async fn handle_mediated_connection( denial_tx: Option>, activity_tx: Option, ) -> Result<()> { + let transparent_mapping = if let Some(destination) = transparent_destination { + let store = policy_dns_store + .as_ref() + .ok_or_else(|| miette::miette!("transparent connection arrived without policy DNS"))?; + let mapping = store + .lookup( + destination.ip(), + destination.port(), + opa_engine.current_generation(), + std::time::Instant::now(), + ) + .map_err(|error| miette::miette!("transparent destination denied: {error}"))?; + let authority = format!( + "{}:{}", + mapping.record.normalized_name.as_str(), + destination.port() + ); + client = tokio::io::BufReader::new(virtual_connect_stream(client.into_inner(), authority)); + Some(mapping) + } else { + None + }; let mut buf = vec![0u8; MAX_HEADER_BYTES]; let mut used = 0usize; @@ -1485,6 +1564,13 @@ async fn handle_mediated_connection( return Ok(()); } } + if let Some(mapping) = transparent_mapping.as_ref() { + decision.endpoint.destination = Some( + build_pinned_validation_plan(mapping.pinned_addresses()).map_err(|denial| { + miette::miette!("transparent destination mapping denied: {}", denial.reason) + })?, + ); + } let destination_plan = decision .endpoint .destination @@ -5409,6 +5495,26 @@ mod tests { struct FailedMediationSource; + #[tokio::test] + async fn virtual_connect_is_portless_and_hides_the_synthetic_handshake() { + let (workload, mut workload_peer) = tokio::io::duplex(1024); + let mut handler = virtual_connect_stream(Box::new(workload), "api.example.com:443".into()); + + workload_peer.write_all(b"client-tls").await.unwrap(); + let mut request = vec![0_u8; 128]; + let length = handler.read(&mut request).await.unwrap(); + let request = &request[..length]; + assert!(request.starts_with(b"CONNECT api.example.com:443 HTTP/1.1\r\n")); + + handler + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\nserver-tls") + .await + .unwrap(); + let mut response = [0_u8; 10]; + workload_peer.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"server-tls"); + } + #[async_trait::async_trait] impl NetworkMediationSource for FailedMediationSource { async fn accept( @@ -5493,6 +5599,7 @@ network_policies: {} ready_rx, &upstream_proxy::UpstreamProxyArgs::default(), Some(Arc::new(FailedMediationSource)), + None, ) .await .expect("proxy starts before source accept"); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 106c6b1b1c..526a2fceed 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,7 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; -use openshell_isolation::contract::NetworkMediationSource; +use openshell_isolation::contract::{DnsMediationSource, NetworkMediationSource}; /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. @@ -54,6 +54,7 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + _policy_dns: Option, } /// Set up the networking stack: ephemeral CA + TLS state, proxy server, @@ -92,6 +93,7 @@ pub async fn run_networking( workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, network_mediation_source: Option>, + dns_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -273,6 +275,21 @@ pub async fn run_networking( (None, None) }; + let policy_dns = if let Some(source) = dns_mediation_source { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("Mediated DNS requires an OPA engine"))?; + Some(crate::policy_dns::PolicyDnsRuntime::start_mediated( + engine, + source, + None, + crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(0)?, + engine_ready_rx.clone(), + )?) + } else { + None + }; + let proxy_handle = if matches!(policy.network.mode, NetworkMode::Proxy) { let proxy_policy = policy.network.proxy.as_ref().ok_or_else(|| { miette::miette!("Network mode is set to proxy but no proxy configuration was provided") @@ -319,6 +336,7 @@ pub async fn run_networking( engine_ready_rx, upstream_proxy_args, network_mediation_source, + policy_dns.as_ref().map(|runtime| runtime.store.clone()), ) .await?; Some(proxy_handle) @@ -330,5 +348,6 @@ pub async fn run_networking( proxy: proxy_handle, ca_file_paths, policy_local_ctx, + _policy_dns: policy_dns, }) } diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 655851e156..9f0484f0f8 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -563,7 +563,12 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Docker uses a native host supervisor and a network-disabled workload container. +OCI seccomp notification carries policy DNS and transparent TCP directly to the +supervisor, so the container has no OpenShell binary or proxy port. Workloads +run as the gateway user's numeric UID and GID; images that require root are not +supported by this mode. Gateway mTLS material remains on the host, while only +the generated public proxy CA is mounted read-only into the workload. ```toml [openshell] @@ -580,9 +585,9 @@ default_image = "ghcr.io/nvidia/openshell/sandbox:latest" # Docker vocabulary: Always | IfNotPresent | Never. Empty behaves like IfNotPresent. image_pull_policy = "IfNotPresent" sandbox_namespace = "docker-dev" -# Empty auto-detects https://host.openshell.internal: when guest TLS is set. -grpc_endpoint = "https://host.openshell.internal:17670" -# Skip the image-pull-and-extract step by pointing at a locally built binary. +# Empty auto-detects the native supervisor's gateway endpoint. +grpc_endpoint = "https://127.0.0.1:17670" +# Skip image extraction by pointing at a locally built host binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. # Defaults to the gateway version; override to pin a specific build. @@ -590,15 +595,6 @@ supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" -network_name = "openshell-docker" -host_gateway_ip = "172.17.0.1" -ssh_socket_path = "/run/openshell/ssh.sock" -# Unsafe operator override. Host bind mounts, including Docker local-driver -# bind-backed volumes, expose gateway-host paths inside sandboxes and can -# negate OpenShell isolation and filesystem controls. -enable_bind_mounts = false -# Set to 0 to leave Docker's runtime default unchanged. -sandbox_pids_limit = 2048 ``` ### Podman diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 958ea1b0eb..9e880326ec 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -115,9 +115,6 @@ DRIVER_LOG="${WORKDIR}/docker-driver.log" DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" DRIVER_CONFIG="${WORKDIR}/docker-driver.toml" E2E_NAMESPACE="" -DOCKER_NETWORK_NAME="" -DOCKER_NETWORK_CONNECTED_CONTAINER="" -DOCKER_NETWORK_MANAGED=0 GPU_MODE="${OPENSHELL_E2E_DOCKER_GPU:-0}" OIDC_MODE="${OPENSHELL_E2E_OIDC_GATEWAY:-0}" OIDC_ISSUER="${OPENSHELL_E2E_OIDC_ISSUER:-}" @@ -173,20 +170,6 @@ cleanup() { fi fi - if [ -n "${DOCKER_NETWORK_CONNECTED_CONTAINER}" ] \ - && [ -n "${DOCKER_NETWORK_NAME}" ] \ - && command -v docker >/dev/null 2>&1; then - docker network disconnect -f \ - "${DOCKER_NETWORK_NAME}" \ - "${DOCKER_NETWORK_CONNECTED_CONTAINER}" >/dev/null 2>&1 || true - fi - - if [ "${DOCKER_NETWORK_MANAGED}" = "1" ] \ - && [ -n "${DOCKER_NETWORK_NAME}" ] \ - && command -v docker >/dev/null 2>&1; then - docker network rm "${DOCKER_NETWORK_NAME}" >/dev/null 2>&1 || true - fi - e2e_print_gateway_log_on_failure "${exit_code}" "${GATEWAY_LOG}" if [ "${exit_code}" -ne 0 ] && [ -f "${DRIVER_LOG}" ]; then echo "=== external Docker compute driver log ===" @@ -198,70 +181,6 @@ cleanup() { } trap cleanup EXIT -ensure_e2e_docker_network() { - local network=$1 - - if docker network inspect "${network}" >/dev/null 2>&1; then - return 0 - fi - - docker network create \ - --driver bridge \ - --attachable \ - --label openshell.ai/managed-by=openshell \ - --label "openshell.ai/sandbox-namespace=${E2E_NAMESPACE}" \ - "${network}" >/dev/null - DOCKER_NETWORK_MANAGED=1 -} - -github_actions_container_id() { - if [ "${GITHUB_ACTIONS:-}" != "true" ] || [ ! -f /.dockerenv ]; then - return 1 - fi - - local container - container="$(hostname)" - if docker inspect "${container}" >/dev/null 2>&1; then - printf '%s\n' "${container}" - return 0 - fi - - return 1 -} - -connect_current_container_to_docker_network() { - local network=$1 - local container - - if ! container="$(github_actions_container_id)"; then - return 1 - fi - - local connect_err="${WORKDIR}/docker-network-connect.err" - if ! docker network connect \ - --alias host.openshell.internal \ - "${network}" \ - "${container}" 2>"${connect_err}"; then - if ! grep -qi "already exists" "${connect_err}"; then - cat "${connect_err}" >&2 - return 1 - fi - fi - - DOCKER_NETWORK_CONNECTED_CONTAINER="${container}" - - local container_ip - container_ip="$(docker inspect \ - --format "{{with index .NetworkSettings.Networks \"${network}\"}}{{.IPAddress}}{{end}}" \ - "${container}")" - if [ -z "${container_ip}" ]; then - echo "ERROR: failed to resolve current job container IP on Docker network ${network}" >&2 - return 1 - fi - - GATEWAY_HOST_ALIAS_IP="${container_ip}" -} - if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then case "${OPENSHELL_GATEWAY_ENDPOINT}" in http://*) ;; @@ -467,19 +386,8 @@ JWT_DIR="${STATE_DIR}/jwt" GATEWAY_ENDPOINT="https://host.openshell.internal:${HOST_PORT}" E2E_NAMESPACE="e2e-docker-$$-${HOST_PORT}" -DOCKER_NETWORK_NAME="${E2E_NAMESPACE}" -GATEWAY_HOST_ALIAS_IP="" - -ensure_e2e_docker_network "${DOCKER_NETWORK_NAME}" -export OPENSHELL_E2E_DOCKER_NETWORK_NAME="${DOCKER_NETWORK_NAME}" -export OPENSHELL_E2E_NETWORK_NAME="${DOCKER_NETWORK_NAME}" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${E2E_NAMESPACE}" export OPENSHELL_E2E_DRIVER="docker" -if connect_current_container_to_docker_network "${DOCKER_NETWORK_NAME}"; then - echo "Connected CI job container to Docker network ${DOCKER_NETWORK_NAME} (${GATEWAY_HOST_ALIAS_IP})." -else - GATEWAY_HOST_ALIAS_IP="" -fi echo "Starting openshell-gateway on port ${HOST_PORT} (namespace: ${E2E_NAMESPACE})..." echo "Using sandbox image: ${SANDBOX_IMAGE} (pull policy: ${SANDBOX_IMAGE_PULL_POLICY})" @@ -514,36 +422,26 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" else printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" - fi fi } > "${GATEWAY_CONFIG}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then { printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - printf 'enable_bind_mounts = true\n' printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" - fi } >"${DRIVER_CONFIG}" "${DRIVER_BIN}" \ --bind-socket "${DRIVER_SOCKET}" \ diff --git a/mise.toml b/mise.toml index ec643ed08e..a33b5779c2 100644 --- a/mise.toml +++ b/mise.toml @@ -61,10 +61,10 @@ _.file = [".env"] KUBECONFIG = "{{config_root}}/kubeconfig" UV_CACHE_DIR = "{{config_root}}/.cache/uv" -# Enable sccache for faster Rust builds. Preserve an SCCACHE_DIR selected by the -# caller so workstations and CI can configure their cache locations separately. +# Enable sccache for faster Rust builds. Use one user-level OpenShell cache +# shared by every worktree while preserving an override from the caller. RUSTC_WRAPPER = "sccache" -SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=config_root ~ '/.cache/sccache') }}" +SCCACHE_DIR = "{{ get_env(name='SCCACHE_DIR', default=xdg_cache_home ~ '/openshell/sccache') }}" # Shared build constants (overridable via environment). # DOCKER_BUILDKIT enables BuildKit for Docker; ignored by podman. diff --git a/rfc/0012-isolation-backend/README.md b/rfc/0012-isolation-backend/README.md index 07bbc4dbab..02efedb0e2 100644 --- a/rfc/0012-isolation-backend/README.md +++ b/rfc/0012-isolation-backend/README.md @@ -19,7 +19,7 @@ links: Today the supervisor both builds the workload's isolation boundary and applies its network policy. Because the supervisor runs inside the agent container, the privilege needed to build that boundary sits beside the code it confines. This RFC moves boundary construction and process operations behind a pluggable **Isolation Backend**. The supervisor continues to apply approved network policy through network mediation. -The compute driver provisions the workload and trusted components. The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. The same lifecycle supports today's in-pod implementation and future delegated implementations without topology-specific supervisor paths. +The logical supervisor is the trusted bridge between the gateway and the workload: it maintains the gateway connection, handles authorized requests, and drives the backend. A topology may let its compute driver or external orchestrator provision the boundary and have the supervisor attach, or it may supply a create-capable backend and let the supervisor create the boundary itself. The backend establishes the isolation controls, manages workload processes, and routes egress to network mediation. Both provisioning routes converge on the same lifecycle without topology-specific supervisor paths. ## Motivation @@ -33,7 +33,7 @@ All three come from coupling boundary construction to boundary operation. A comm ## Non-goals -- **Implementing a delegated backend.** Each topology requires its own design and implementation. +- **Standardizing a topology's resource API.** Docker, Kubernetes, VM, and other creation mechanisms remain backend-specific. - **Changing authorization.** [RFC 0001](../0001-core-architecture/README.md) owns control-plane and sandbox identity. A delegated backend must still authenticate callers and scope them to one boundary. - **Standardizing backend-internal component coordination.** A backend may coordinate helper, sidecar, or interception processes behind one lifecycle; how those components cooperate is backend-specific, not contract surface. - **Changing gateway lifecycle or public status.** This RFC adds no gateway activation operation, public phase, or status API, and it does not define how a boundary's effective isolation model is surfaced to operators. @@ -42,7 +42,7 @@ All three come from coupling boundary construction to boundary operation. A comm The mental model has three roles: -- The **compute driver** provisions the sandbox instance according to the selected placement of the workload and trusted isolation components. That placement is the **topology**. +- The **compute driver** prepares placement and creation inputs. Depending on the topology, it either provisions the sandbox instance itself or supplies a create-capable backend to the host supervisor. That placement is the **topology**. - The **Isolation Backend** establishes and operates the topology-specific controls around the workload. It also routes workload egress to network mediation and provides process operations. - The **logical supervisor** is the trusted control-plane bridge between the gateway and the workload. It drives the backend, handles authorized gateway requests, and applies approved network policy through network mediation. @@ -52,13 +52,16 @@ Each active boundary has at most one logical supervisor, which may span multiple [RFC 0001](../0001-core-architecture/README.md) continues to own sandbox authentication and authorization. In this contract, sandbox identity means binding the authenticated sandbox context to the isolation boundary. -Admission selects the sandbox's topology and determines its trusted context. The compute driver sets up the topology and gives the logical supervisor a `TopologyDescriptor` describing what it provisioned. The supervisor uses the descriptor to attach the matching Isolation Backend. The backend prepares the required controls before the agent starts. +Admission selects the sandbox's topology, provisioning route, and trusted context. An externally provisioned topology gives the logical supervisor a `TopologyDescriptor` describing the concrete resource to attach. A supervisor-provisioned topology gives it a `BoundaryCreatePlan` for the matching backend. The backend prepares the required controls before the agent starts. ```mermaid flowchart TB Gateway["Gateway"] -->|"create sandbox"| Driver["Compute driver"] - subgraph Topology["Driver-provisioned topology (placement varies)"] + Driver -->|"external provisioning"| Existing["Existing boundary + TopologyDescriptor"] + Driver -->|"prepared inputs"| Plan["BoundaryCreatePlan"] + + subgraph Topology["Admitted topology (placement varies)"] Supervisor["Supervisor"] Backend["Isolation Backend (may coordinate components)"] subgraph Boundary["Isolation boundary"] @@ -68,7 +71,9 @@ flowchart TB end end - Supervisor -->|"drives contract"| Backend + Existing -->|"attach"| Supervisor + Plan -->|"create"| Supervisor + Supervisor -->|"drives selected route"| Backend Backend -->|"establishes and confirms"| Boundary Backend -.->|"routes all workload egress to"| Mediator Supervisor -.->|"applies network policy through"| Mediator @@ -76,13 +81,12 @@ flowchart TB Workload ==>|"only egress"| Mediator end - Driver -->|"resources + TopologyDescriptor"| Supervisor Mediator -->|"allowed egress"| Egress["Egress"] ``` In the in-pod topology, the supervisor drives a backend implemented in the same process. Other topologies may delegate backend operations without changing the supervisor lifecycle. -A boundary is active from successful `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend owns the active-boundary binding; the compute driver owns the sandbox instance and topology lifecycle. +A boundary is active from successful `create` or `attach` until normal backend cleanup releases the binding or the topology's trusted cleanup path invalidates it. A backend may coordinate multiple trusted helper or interception processes for that boundary. The backend always owns the active-boundary binding. Resource lifecycle ownership follows creation: the backend owns a supervisor-created resource, while the compute driver or external orchestrator owns an externally created resource. Durable trusted state records that origin separately from the opaque descriptor. ### Contract invariants @@ -91,7 +95,7 @@ Six invariants hold for every boundary: 1. Workload egress is denied except through network mediation for the boundary's lifetime. 2. No untrusted instruction executes until every admitted control applicable to that process is in force. 3. An operation is authorized only when the complete effective policy permits it; network operations are decided through network mediation. There is no silent weakening. -4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the compute driver's provisioned execution environment. +4. Agent startup, `exec`, and forwarding occur only through the active backend, and every workload process remains in the admitted execution environment created by the selected lifecycle owner. 5. Shared infrastructure preserves strict per-boundary lifecycle, policy, identity, enforcement, and cleanup isolation. 6. If the logical supervisor is lost, the boundary remains under its last confirmed enforcement state while supervisor-dependent operations fail closed. Loss of required enforcement ends `Running` and terminates all workload processes within a documented bound; detection and termination may be performed by a trusted node or control-plane actor. Network-mediation unavailability denies outbound connections and never enables direct egress. @@ -99,11 +103,12 @@ Each backend states its termination bound in its implementation documentation. L ### Provisioning -Provisioning runs on the control plane, and three rules hold in every topology: +Provisioning is selected by trusted control-plane configuration, and four rules hold in every topology: -1. **Admission selects the topology** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. The `TopologyDescriptor` supplied by the compute driver must name that backend, and resolution never falls back to another backend. -2. **The compute driver provisions the topology** and anything the selected backend needs. -3. **The backend establishes standing enforcement before untrusted code runs**, during provisioning or `attach`, depending on the backend. +1. **Admission selects the topology and provisioning route** from trusted deployment configuration, not `SandboxPolicy`, and records its required backend. A `TopologyDescriptor` or `BoundaryCreatePlan` must name that backend, and resolution never falls back to another backend or route. +2. **External provisioning remains first-class.** A compute driver or orchestrator may create the resource and supply a descriptor for `attach`, as in a Kubernetes controller-driven topology. +3. **Supervisor-owned creation is optional.** A compute driver may instead supply prepared inputs and a create-capable backend. The supervisor invokes `create`; an attach-only backend reports the operation as unsupported. +4. **The backend establishes standing enforcement before untrusted code runs**, during `create`, `attach`, or `confirm`, depending on the backend. If a topology depends on cluster-scoped coverage or registration, admission verifies that the prerequisite covers the boundary's placement before untrusted code runs. @@ -111,9 +116,27 @@ Every topology provides a trusted cleanup path that does not depend on logical-s A compute driver may provision a resource and `TopologyDescriptor` before the control plane assigns it to a sandbox. No untrusted workload runs while the resource is unassigned. After claim or assignment produces a trusted `SandboxContext`, the supervisor calls `attach`; the backend either binds that context to the prepared resource and returns `Bound`, or rejects it as incompatible. Pool creation, claim, reset, release, and recycling remain outside this contract. +For supervisor-owned creation, the compute driver prepares a `BoundaryCreatePlan` without creating a runnable boundary. After assignment produces the trusted `SandboxContext`, the supervisor calls `create`. The backend creates and binds a non-runnable resource and returns `CreatedBoundary`, containing `Bound` plus a `TopologyDescriptor` for durable recovery. Creation is idempotent on trusted sandbox identity and launch generation. Partial resources are removed or remain labeled for trusted reconciliation. + +### The boundary create plan + +The create-plan envelope has the same backend-name and interface-version fields as a topology descriptor, but its opaque payload describes prepared inputs rather than an existing resource. + +```rust +struct BoundaryCreatePlan { + backend_name: String, + version: u32, + payload: Vec, +} +``` + +The supervisor validates the common fields and produces `VerifiedBoundaryCreatePlan`. The backend validates the payload during `create`. A plan may contain resolved image or disk identities, normalized runtime settings, placement results, or protected references to prepared artifacts. Workload-controlled input cannot select or modify the backend envelope. + +Creation is an additive operation, not capability negotiation. Backends are attach-only by default. Trusted orchestration explicitly selects `create` or `attach`; an unsupported `create` fails and never falls back to attachment, another backend, or compute-driver provisioning. + ### The topology descriptor -The driver supplies a descriptor for every topology admitted to this contract, including in-pod and resources prepared before assignment. The common envelope names the backend and carries an opaque payload. +The creation owner supplies a descriptor for every concrete topology admitted to this contract. An external provisioner supplies it before `attach`; a successful backend `create` returns it for persistence and recovery. The common envelope names the backend and carries an opaque payload. ```rust struct TopologyDescriptor { @@ -139,9 +162,10 @@ The supervisor validates the descriptor's common fields and produces a `Verified The contract does not prescribe enforcement mechanisms; it standardizes how the supervisor drives whichever backend a deployment admits. -A backend registers under a `backend_name` and version. The supervisor attaches to the admitted topology and drives the boundary through a fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. +A backend registers under a `backend_name` and version. The supervisor follows the admitted create or attach route and then drives the boundary through one fixed sequence of states. Each transition consumes the prior state, so the supervisor cannot skip a stage or invoke a later transition through an earlier handle. The Rust names are illustrative; the states and their semantics are normative. ```text +create plan + sandbox context -----> Bound -> confirm -> Ready -> start_agent -> Running attach topology + sandbox context -> Bound -> confirm -> Ready -> start_agent -> Running ``` @@ -151,6 +175,22 @@ trait IsolationBackend: Send + Sync { fn backend_name(&self) -> &str; fn version(&self) -> u32; + async fn create( + &self, + plan: VerifiedBoundaryCreatePlan, + sandbox: SandboxContext, + ) -> Result { + Err(BackendError::Unsupported(...)) + } + + async fn destroy( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox_id: &SandboxId, + ) -> Result<(), BackendError> { + Err(BackendError::Unsupported(...)) + } + async fn attach( &self, descriptor: VerifiedTopologyDescriptor, @@ -158,6 +198,11 @@ trait IsolationBackend: Send + Sync { ) -> Result, BackendError>; } +struct CreatedBoundary { + descriptor: TopologyDescriptor, + boundary: Box, +} + struct SandboxContext { sandbox_id: SandboxId, policy: SandboxPolicy, @@ -167,6 +212,7 @@ struct SandboxContext { #[async_trait] trait BoundBoundary: Send { fn network_mediation_source(&self) -> Arc; + fn dns_mediation_source(&self) -> Option>; async fn confirm( self: Box, @@ -194,13 +240,13 @@ trait RunningBoundary: Send + Sync { The states have normative meanings: -- **Bound:** the topology descriptor and trusted sandbox context are bound to the same resource, and the network-mediation source is available. No untrusted workload code is running. +- **Bound:** the trusted sandbox context is bound to the created or attached resource, and the network-mediation source is available. No untrusted workload code is running. - **Ready:** the backend has confirmed standing enforcement for this concrete boundary and is prepared to apply the admitted launch-time controls before untrusted execution. - **Running:** `start_agent` has made the admitted agent runnable and returned `RunningBoundary`. Every applicable launch-time control was in force before the first untrusted instruction. Whether the backend creates the agent process or releases a held, driver-provisioned execution object is backend-specific; the contract fixes the ordering, not the mechanism. `confirm` is the pre-launch commit point. The supervisor calls it only after connecting the boundary's network-mediation source to network mediation. The backend confirms standing enforcement for the concrete boundary and may rely on a trusted provisioning-time or out-of-pod signal tied to that boundary's placement, but not on general placement health alone. -`attach` rejects a resource already bound to an active boundary. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `attach` or `confirm`, or the supervisor fails network-mediation initialization. +`attach` rejects a resource already bound to an active boundary. `create` rejects a conflicting existing launch generation and returns the same compatible inactive resource on an idempotent retry. A boundary that cannot enforce the complete admitted policy does not reach `Ready`: the backend fails `create`, `attach`, or `confirm`, or the supervisor fails network-mediation initialization. **Standing enforcement** is established independently of a workload process. **Launch-time controls** must be in force before a process executes its first untrusted instruction. Both `start_agent` and `BoundaryExec::exec` enforce this ordering and preserve the provisioned execution environment. @@ -257,11 +303,32 @@ trait NetworkMediationSource: Send + Sync { struct MediatedConnection { stream: BoundaryDuplexStream, binary_identity: Result, + destination: Option, +} + +#[async_trait] +trait DnsMediationSource: Send + Sync { + async fn accept(&self) -> Result; +} + +struct MediatedDnsQuery { + request: Vec, + transport: DnsTransport, + binary_identity: Result, + response: oneshot::Sender, BackendError>>, } ``` `NetworkMediationSource` supplies outbound connections from one boundary to supervisor-owned network mediation. The backend routes all workload egress through that source and authoritatively associates each connection with the boundary without relying solely on workload-provided data. Capture, transport, placement, and coordination are backend-private. +Explicit-proxy transports leave `destination` absent. Transparent transports +capture the original socket destination and supply it before the supervisor +consumes workload bytes. `DnsMediationSource` carries portless DNS exchanges to +the supervisor-owned policy DNS service. It is optional because explicit-proxy +topologies resolve destinations in the supervisor and do not expose workload +DNS. A backend that advertises transparent networking supplies both sources; +DNS or connection-source failure closes that boundary's egress. + Every topology may use the same supervisor-owned mediation libraries or services; the source does not require a backend-specific policy engine. Shared implementations isolate each boundary's state and enforcement. Failure or teardown of one boundary cannot weaken another. Network-mediation unavailability never enables direct egress. @@ -291,11 +358,11 @@ Binary identity is mandatory conformance: RFC 0002 makes it part of the outbound The logical supervisor resolves `backend_name` and version through a trusted implementation registry. Adding a backend adds an implementation and registration, not branches in lifecycle, proxy, SSH, or session code. Delegated transport and coordination remain backend-private. -The supervisor runs the same sequence for every backend: +The supervisor runs one of two admitted entry sequences and the same lifecycle for every backend: -1. Obtain the `TopologyDescriptor` and trusted `SandboxContext`. -2. Verify the descriptor and resolve its `backend_name` and version without fallback. -3. Call `attach` to obtain `Bound`. +1. Obtain trusted `SandboxContext` plus either a `TopologyDescriptor` or `BoundaryCreatePlan` selected by admission. +2. Verify the envelope and resolve its `backend_name` and version without fallback. +3. For a descriptor, call `attach`. For a create plan, call optional `create`, persist its returned recovery descriptor and supervisor-owned origin, and retain its `Bound` state. 4. Connect the boundary's `NetworkMediationSource` to network mediation. 5. Call `confirm` to obtain `Ready`, then `start_agent` to obtain `Running`. 6. Use the returned runtime handles for agent wait, `exec`, and port forwarding while network mediation consumes outbound connections. @@ -310,20 +377,22 @@ Every failure carries a machine-readable kind for supervisor status mapping: enum BackendErrorKind { Invalid, Denied, Unavailable, Failed, Terminated } ``` -`Invalid` covers descriptor, version, and backend mismatches; `Denied` covers authenticated attachment rejection; `Unavailable` covers transient inability to serve an operation; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend selection never falls back. +`Invalid` covers plan, descriptor, version, and backend mismatches; `Denied` covers authenticated creation or attachment rejection; `Unavailable` covers transient inability to serve an operation and an optional operation the selected backend does not implement; `Failed` covers other backend faults; and `Terminated` reports boundary or workload termination, or an operation against an inactive boundary. An error never advances the lifecycle or authorizes an operation, and backend or provisioning-route selection never falls back. -A backend may retry backend-private work within one `attach` call. The supervisor calls `attach` at most once per provisioned topology. If it does not return `Bound`, the topology is reclaimed rather than reused. +A backend may retry backend-private work within one `create` or `attach` call. The supervisor calls the admitted entry operation at most once per orchestration attempt. Creation retries use sandbox identity and launch generation as an idempotency key. If the operation does not return `Bound`, its lifecycle owner reclaims the topology rather than reusing an ambiguous resource. Failures resolve as follows: -- an `attach` or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the driver to reclaim the topology; -- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the driver reclaims the topology; +- a `create`, `attach`, or `confirm` failure, or network-mediation initialization failure while `Bound`, prevents untrusted workload execution and causes the creation owner to reclaim the topology; +- if `start_agent` does not return `Running`, no untrusted process from that attempt remains, and the creation owner reclaims the topology; - if `exec` or port-forward `connect` fails, the backend terminates any process or closes any connection created by that attempt while the boundary otherwise remains active; - after `Running`, supervisor or enforcement loss follows invariant 6; when enforcement loss ends the agent, `BoundaryProcess::wait` fails with `BackendErrorKind::Terminated` where process-exit observation survives; - network-mediation errors yield no authorized connection and do not by themselves end `Running`; and - retained runtime handles and the network-mediation source reject new operations whenever the boundary ends, except `BoundaryProcess::wait` where the backend can still return its stable result. -Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the compute driver reclaims or deprovisions the topology. If normal backend cleanup is unavailable, the compute driver uses the topology's trusted cleanup path to terminate the execution environment and invalidate the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. +Whenever a boundary ends, the backend terminates remaining workload processes and releases the active-boundary binding before the lifecycle owner reclaims or deprovisions the topology. A supervisor-created resource is destroyed through the backend; an externally created resource is destroyed by its compute driver or orchestrator. If normal cleanup is unavailable, the topology's trusted reconciler terminates the execution environment and invalidates the binding before reclaim or reuse. On normal agent exit, `BoundaryProcess::wait` returns the stable exit status. A retained `wait` result may outlive teardown. + +`BackendRegistry::destroy_created` verifies the recovery descriptor, requires the trusted durable origin to be `SupervisorCreated`, and invokes the selected backend's idempotent `destroy` operation. It rejects `ExternallyCreated` resources without calling the backend. The backend revalidates the descriptor against the admitted sandbox identity before deleting anything. Attach-only backends may leave `destroy` unsupported because their compute driver or orchestrator retains deletion authority. ### Topologies @@ -331,11 +400,12 @@ The contract fixes the roles; a topology fixes their placement. Components may b ## Implementation plan -This RFC defines the contract; implementation lands in three phases: +This RFC defines the contract; implementation lands in four phases: -1. **Contract.** Add the common types, descriptor handling, registry, and explicit backend selection from deployment configuration. -2. **Co-located backend.** Implement the co-located backend behind a deployment flag and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through it without changing behavior. -3. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus descriptor verification, lifecycle ordering, runtime operations, and failure semantics. Make the co-located backend the default after parity validation. Parity covers the agent, binary identity, SSH, `exec`, and forwarding paths; enablement also closes the in-pod egress gaps pinned in [codebase-grounding.md](./codebase-grounding.md), which parity alone would preserve. +1. **Contract.** Add the common types, descriptor handling, optional creation and destruction, registry, and explicit provisioning-route selection from deployment configuration. +2. **Co-located and attachment backends.** Implement attachment for externally provisioned topologies and route agent launch, egress interception, the network-mediation source, SSH, `exec`, and forwarding through the common lifecycle. +3. **Supervisor-created proof.** Extract reusable host-supervisor orchestration, let a compute-driver binary inject a create-capable backend, and validate Docker create-before-start plus OCI seccomp listener-FD delivery. The proof remains non-conformant until it implements every mandatory runtime surface. +4. **Conformance and enablement.** Require every topology admitted to the RFC 0012 lifecycle to pass tests for the six contract invariants plus envelope verification, lifecycle ordering, runtime operations, ownership-aware cleanup, and failure semantics. Existing placements remain outside this contract until their backend is implemented and admitted; they do not claim conformance. Delegated backends remain separate design and implementation work. @@ -343,7 +413,9 @@ Existing placements remain outside this contract until their backend is implemen | Risk | Mitigation | |---|---| -| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep the responsibility boundary explicit: the compute driver owns, provisions, and deprovisions the topology; the backend binds and operates the active boundary. The same component may implement both roles. | +| The Isolation Backend could duplicate compute-driver responsibilities or allow topology-specific behavior to leak back into the supervisor. | Keep placement and preparation in the compute driver and enforcement sequencing in the supervisor. Creation ownership is explicit per resource, and concrete backends are injected through the registry rather than imported by generic supervisor code. | +| Optional creation could produce two subtly different lifecycle implementations. | Both entry operations must return the same `BoundBoundary`; no later lifecycle operation branches on origin. Conformance runs from `Bound` for both routes. | +| A crash between resource creation and descriptor persistence could orphan a boundary. | Make creation idempotent by sandbox launch generation, label partial resources for reconciliation, and do not start untrusted code before the recovery descriptor and ownership origin are durable. | | Contract conformance could be mistaken for equivalent isolation across topologies. | Treat conformance as behavioral, not as a security-strength rating. Document and validate each topology's actual containment and reject policy it cannot enforce. | | Shared backend or network-mediation components concentrate privilege and failure impact. | Isolate state, connection attribution, enforcement, and control authority per boundary. Failure of one boundary must not weaken another or enable direct egress. | | The mandatory contract may exclude otherwise useful but incomplete backends. | Keep the network-mediation source, binary identity, process control, `exec`, and port forwarding mandatory. An incomplete backend does not claim conformance or silently degrade. | @@ -358,11 +430,11 @@ OpenShell could keep the current in-pod design and add topology-specific supervi Doing nothing avoids a new interface, but retains privileged boundary construction beside the workload. Implementing each delegated topology as a one-off supervisor change moves that privilege for one placement but accretes topology-specific supervisor behavior. The proposed contract instead keeps one supervisor lifecycle while allowing the topology to change. -### Extend the compute-driver contract +### Keep all creation in the compute-driver contract -The compute driver could own both provisioning and active-boundary operation. +The compute driver could always create the resource and use only `attach`, even for local Docker and VM topologies. -This is natural for topologies such as MXC, and the same component may implement both responsibilities. The interfaces remain distinct because they serve different callers and lifecycles: the gateway uses the compute driver to provision and deprovision resources, while the supervisor uses the Isolation Backend to operate an active boundary. Combining them would couple runtime policy, identity, network mediation, and process operations to the gateway-facing driver API. +This remains the chosen route for controller-driven systems such as Kubernetes. Requiring it everywhere prevents the supervisor from establishing host listeners and enforcement state before a local runtime creates the workload. Optional `create` preserves external provisioning while allowing security-sensitive ordering to move under the supervisor where the topology benefits. ### Start with a remote backend service @@ -381,10 +453,12 @@ That would make known deployments explicit, but it would also encode current top - **Driver-backed subsystems (CRI/CNI/CSI).** Kubernetes factors runtime, networking, and storage into pluggable driver contracts so the orchestrator drives one interface while implementations vary. RFC 0001 describes OpenShell's other subsystems the same way; this RFC specifies the one it left open: isolation. - **Istio privilege placement.** Init-sidecar and node-agent modes demonstrate that network setup can move without changing the policy data path. OpenShell keeps its identity-aware proxy. - **CRI exec/attach/port-forward.** `exec` and `connect` follow CRI's `Exec` and `PortForward` shape; lifecycle and network mediation remain OpenShell-specific. +- **[OCI seccomp listener handoff](https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#seccomp).** The runtime specification lets a runtime send a seccomp notification FD and process state to a host Unix listener. It demonstrates why some local enforcement must exist before workload creation and informs the optional supervisor-owned `create` route. ## Open questions -None. +- Which creation inputs should become typed common fields instead of remaining in the backend-private payload? +- Where should the durable resource-origin record be committed so a crash cannot lose supervisor-owned cleanup responsibility? ## Appendix: codebase grounding