From 91dd3cd65a651c77d032a2bef737138381511a6f Mon Sep 17 00:00:00 2001 From: Drew Newberry <385+drew@users.noreply.github.com> Date: Wed, 26 Aug 2026 04:56:26 +0000 Subject: [PATCH] feat(isolation): enable the co-located RFC 0012 backend Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com> --- .agents/skills/helm-dev-environment/SKILL.md | 3 + .github/workflows/branch-checks.yml | 55 + .github/workflows/driver-vm-linux.yml | 14 +- .github/workflows/driver-vm-macos.yml | 19 +- Cargo.lock | 17 + architecture/sandbox.md | 28 + crates/openshell-core/src/driver_mounts.rs | 18 + crates/openshell-driver-docker/Cargo.toml | 5 +- crates/openshell-driver-docker/src/lib.rs | 194 +- crates/openshell-driver-docker/src/tests.rs | 23 +- crates/openshell-driver-kubernetes/Cargo.toml | 1 + .../openshell-driver-kubernetes/src/driver.rs | 263 ++- crates/openshell-driver-podman/Cargo.toml | 1 + .../openshell-driver-podman/src/container.rs | 33 +- crates/openshell-driver-vm/Cargo.toml | 1 + crates/openshell-driver-vm/README.md | 6 +- crates/openshell-driver-vm/build.rs | 16 +- crates/openshell-driver-vm/runtime/README.md | 2 +- .../scripts/openshell-vm-sandbox-init.sh | 32 +- crates/openshell-driver-vm/src/driver.rs | 6 +- crates/openshell-driver-vm/src/rootfs.rs | 146 +- crates/openshell-sandbox/Cargo.toml | 5 +- crates/openshell-sandbox/src/inpod.rs | 1622 +++++++++++++++++ crates/openshell-sandbox/src/lib.rs | 408 +++-- crates/openshell-sandbox/src/main.rs | 35 + .../openshell-supervisor-network/src/proxy.rs | 444 ++++- .../openshell-supervisor-network/src/run.rs | 3 + .../src/netns/mod.rs | 142 ++ .../src/process.rs | 10 + .../openshell-supervisor-process/src/run.rs | 339 +++- deploy/docker/Dockerfile.supervisor | 23 +- docs/reference/gateway-config.mdx | 13 +- tasks/scripts/gateway-vm.sh | 3 +- tasks/scripts/vm/build-supervisor-bundle.sh | 46 + 34 files changed, 3630 insertions(+), 346 deletions(-) create mode 100644 crates/openshell-sandbox/src/inpod.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 2dad568c79..c4579ba523 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -65,6 +65,9 @@ mise run helm:skaffold:run mise run helm:skaffold:run:sidecar ``` +Combined topology selects RFC 0012's in-pod backend and supplies its descriptor; +sidecar topology remains on its separate lifecycle. + **Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running): ```bash mise run helm:skaffold:run:sidecar-mtls diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index bece5c8825..425108d913 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -161,6 +161,61 @@ jobs: fi exit 0 + isolation-conformance: + name: Isolation conformance (privileged Linux) + needs: pr_metadata + if: needs.pr_metadata.outputs.should_run == 'true' + runs-on: linux-amd64-cpu8 + timeout-minutes: 20 + container: + image: ghcr.io/nvidia/openshell/ci:latest + options: --privileged + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install tools and network helpers + run: | + apt-get update + apt-get install -y --no-install-recommends iproute2 nftables iptables util-linux + mise install --locked + + - name: Materialize the Alpine trusted helper runtime fixture + run: | + alpine_root="${RUNNER_TEMP}/openshell-alpine-root" + runtime="${RUNNER_TEMP}/openshell-runtime" + archive="${RUNNER_TEMP}/alpine-minirootfs.tar.gz" + curl -fsSL \ + https://dl-cdn.alpinelinux.org/alpine/v3.22/releases/x86_64/alpine-minirootfs-3.22.5-x86_64.tar.gz \ + -o "$archive" + echo "4b4daa9fe2fc696c4919c4412a4c3d3e770d8fb70292a004a2c72f5096175282 $archive" \ + | sha256sum -c - + mkdir -p "$alpine_root" "$runtime" + tar -xzf "$archive" -C "$alpine_root" + cp /etc/resolv.conf "$alpine_root/etc/resolv.conf" + chroot "$alpine_root" /sbin/apk add --no-cache \ + iproute2 nftables iptables iptables-legacy + for path in /bin /sbin /lib /lib64 /usr/bin /usr/sbin /usr/lib /usr/lib64 /etc/iproute2 /usr/share/nftables; do + if [ -e "$alpine_root$path" ]; then + (cd "$alpine_root" && cp -aL --parents ".$path" "$runtime") + fi + done + chmod -R go-w "$runtime" + + - name: Exercise the live default-deny ceiling + run: | + cargo test -p openshell-isolation -p openshell-supervisor-process \ + -p openshell-supervisor-network -p openshell-sandbox + OPENSHELL_TEST_TRUSTED_RUNTIME_ROOT="${RUNNER_TEMP}/openshell-runtime" \ + cargo test -p openshell-supervisor-process \ + installed_egress_ceiling_ -- \ + --ignored --nocapture --test-threads=1 + cargo test -p openshell-sandbox \ + pid_one_exit_kills_unregistered_setsid_descendant_within_bound -- \ + --ignored --nocapture --test-threads=1 + rust-macos: name: Rust lint (macOS) needs: pr_metadata diff --git a/.github/workflows/driver-vm-linux.yml b/.github/workflows/driver-vm-linux.yml index 942cacdfbd..355253c148 100644 --- a/.github/workflows/driver-vm-linux.yml +++ b/.github/workflows/driver-vm-linux.yml @@ -116,6 +116,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -141,8 +143,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Download kernel runtime tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -168,14 +174,14 @@ jobs: - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed/${file}" done - name: Scope workspace to driver-vm crates run: | set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml + sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core", "crates/openshell-isolation"]|' Cargo.toml - name: Patch workspace version if: ${{ inputs['cargo-version'] != '' }} diff --git a/.github/workflows/driver-vm-macos.yml b/.github/workflows/driver-vm-macos.yml index a97ade9cbb..41636eaad1 100644 --- a/.github/workflows/driver-vm-macos.yml +++ b/.github/workflows/driver-vm-macos.yml @@ -75,6 +75,8 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -100,8 +102,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Build bundled supervisor run: | @@ -116,7 +122,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: driver-vm-supervisor-arm64 - path: target/vm-runtime-compressed/openshell-sandbox.zst + path: | + target/vm-runtime-compressed/openshell-sandbox.zst + target/vm-runtime-compressed/openshell-runtime.tar.zst retention-days: 1 build-driver-vm-macos: @@ -180,12 +188,13 @@ jobs: run: | set -euo pipefail test -f target/vm-runtime-compressed-macos/openshell-sandbox.zst - ls -lh target/vm-runtime-compressed-macos/openshell-sandbox.zst + test -f target/vm-runtime-compressed-macos/openshell-runtime.tar.zst + ls -lh target/vm-runtime-compressed-macos/openshell-{sandbox,runtime.tar}.zst - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed-macos/${file}" done diff --git a/Cargo.lock b/Cargo.lock index b9d4b7a32f..347d41f55f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3401,6 +3401,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.3" @@ -3573,6 +3582,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -3948,6 +3958,7 @@ dependencies = [ "futures", "miette", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -3980,6 +3991,7 @@ dependencies = [ "miette", "notify", "openshell-core", + "openshell-isolation", "openshell-policy", "prost", "prost-types", @@ -4027,6 +4039,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "opentelemetry", @@ -4083,6 +4096,7 @@ dependencies = [ "nix 0.29.0", "oci-client", "openshell-core", + "openshell-isolation", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -4257,12 +4271,15 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "async-trait", + "base64 0.22.1", "clap", "futures", "miette", "nix 0.29.0", "openshell-core", "openshell-extension-core", + "openshell-isolation", "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 786ed5194d..3473974920 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -50,6 +50,34 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: The supervisor may enrich baseline filesystem allowances for runtime-required paths, such as proxy support files or GPU device paths when a GPU is present. +## Isolation Backend + +[RFC 0012](../rfc/0012-isolation-backend/README.md) defines the Isolation +Backend contract for topology-specific boundary construction and process +operations. The contract uses consuming +lifecycle states (`attach` → `Bound` → `confirm` → `Ready` → `start_agent` → +`Running`) so untrusted workload execution cannot begin before standing +enforcement is confirmed. + +The logical supervisor remains the trusted bridge between the gateway and the +workload. It drives the backend and applies approved network policy through +supervisor-owned mediation; the backend routes workload egress to that +mediation. The trusted Kubernetes driver selects the co-located backend for +combined topology and supplies its topology descriptor to the supervisor. +Sidecar topology remains on its pre-RFC lifecycle; a conforming backend for +that placement requires separate design and implementation. Docker, Podman, +and VM drivers provision the same co-located topology and supply its descriptor +by default. The co-located backend requires the +supervisor to own the execution environment's PID namespace so boundary +teardown can terminate every remaining workload process. + +For proxy-mode boundaries, the co-located backend verifies its default-deny +kernel egress ceiling before exposing any workload execution surface and then +rechecks it every 250 milliseconds. Each check has a two-second deadline. +Verification failure or timeout ends the boundary and triggers process cleanup; +the PID-1 supervisor exits so the kernel terminates the complete workload PID +namespace. The topology's detection-and-termination bound is five seconds. + ## Network and Inference See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index b1a3049882..235f1f21f3 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -343,6 +343,24 @@ mod tests { assert!(err.contains("/etc/openshell")); } + #[test] + fn container_target_rejects_parents_that_shadow_reserved_trees() { + for target in ["/opt", "/etc", "/run"] { + let err = validate_container_mount_target(target).unwrap_err(); + assert!( + err.contains("reserved OpenShell path"), + "expected {target} to be rejected: {err}" + ); + } + } + + #[test] + fn container_target_rejects_proc_shadowing() { + for target in ["/proc", "/proc/self", "/"] { + assert!(validate_container_mount_target(target).is_err()); + } + } + #[test] fn container_target_does_not_prefix_match_unrelated_paths() { validate_container_mount_target("/etc/openshell-tools").unwrap(); diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 1c9e675f77..23c0b97f37 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } opentelemetry = { workspace = true } @@ -35,15 +36,15 @@ url = { workspace = true } clap = { workspace = true } miette = { workspace = true } toml = { workspace = true } +tar = "0.4" +tempfile = "3" [dev-dependencies] openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } -tar = "0.4" temp-env = "0.3" -tempfile = "3" tracing-subscriber = { workspace = true } [lints] diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b1859f54cc..50ea3d3315 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -79,6 +79,8 @@ const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const SUPERVISOR_RUNTIME_MOUNT_PATH: &str = "/opt/openshell/bin/openshell-runtime"; +const SUPERVISOR_IMAGE_RUNTIME_PATH: &str = "/openshell-runtime"; const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; @@ -134,9 +136,9 @@ pub struct DockerComputeConfig { /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. pub supervisor_bin: Option, - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. + /// Optional image used to extract the Linux `openshell-sandbox` binary and + /// its trusted helper runtime. With `supervisor_bin`, the image remains the + /// runtime source unless the binary has a valid sibling runtime directory. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -210,6 +212,7 @@ struct DockerDriverRuntimeConfig { stop_timeout_secs: u32, log_level: String, supervisor_bin: PathBuf, + supervisor_runtime: PathBuf, guest_tls: Option, daemon_version: String, supports_gpu: bool, @@ -589,6 +592,8 @@ impl DockerComputeDriver { ); let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; + let supervisor_runtime = + resolve_supervisor_runtime(&docker, &docker_config, &supervisor_bin).await?; let guest_tls = docker_guest_tls_paths(&docker_config)?; let driver = Self { @@ -605,6 +610,7 @@ impl DockerComputeDriver { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: config.log_level.clone(), supervisor_bin, + supervisor_runtime, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), supports_gpu, @@ -2609,11 +2615,18 @@ fn build_binds( sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; + let mut binds = vec![ + format!( + "{}:{}:ro,z", + config.supervisor_bin.display(), + SUPERVISOR_MOUNT_PATH + ), + format!( + "{}:{}:ro,z", + config.supervisor_runtime.display(), + SUPERVISOR_RUNTIME_MOUNT_PATH + ), + ]; if let Some(tls) = &config.guest_tls { binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); binds.push(format!( @@ -3025,8 +3038,18 @@ fn build_container_create_body_for_image( env: Some(build_environment_for_oci_user(sandbox, config, &image.user)), entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), // Replace the image CMD with the supervisor's resolved workspace - // argument so Docker cannot append inherited image arguments. - cmd: Some(vec!["--workdir".to_string(), workspace_root]), + // and admitted topology arguments so Docker cannot append inherited + // image arguments or select the security boundary from image state. + cmd: Some(vec![ + "--workdir".to_string(), + workspace_root, + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + ]), labels: Some(labels), host_config: Some(HostConfig { nano_cpus: resource_limits.nano_cpus, @@ -3764,6 +3787,32 @@ pub(crate) async fn resolve_supervisor_bin( } } +async fn resolve_supervisor_runtime( + docker: &Docker, + docker_config: &DockerComputeConfig, + supervisor_bin: &Path, +) -> CoreResult { + if let Some(runtime) = supervisor_bin + .parent() + .map(|parent| parent.join("openshell-runtime")) + && validate_supervisor_runtime(&runtime).is_ok() + { + return Ok(runtime); + } + + let image = docker_config + .supervisor_image + .clone() + .unwrap_or_else(openshell_core::config::default_supervisor_image); + let extracted_bin = extract_supervisor_bin_from_image(docker, &image).await?; + let runtime = extracted_bin + .parent() + .expect("cache path has a parent") + .join("openshell-runtime"); + validate_supervisor_runtime(&runtime)?; + Ok(runtime) +} + fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { match daemon_arch { "arm64" => vec![PathBuf::from( @@ -3836,6 +3885,8 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core .map_err(Error::config)?; if cache_path.is_file() { validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; return Ok(cache_path); } @@ -3849,9 +3900,96 @@ async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> Core let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; write_cache_binary_atomic(&cache_path, &binary_bytes).map_err(Error::config)?; validate_linux_elf_binary(&cache_path).map_err(Error::config)?; + ensure_cached_supervisor_runtime(docker, image, cache_path.parent().expect("cache parent")) + .await?; Ok(cache_path) } +fn validate_supervisor_runtime(runtime: &Path) -> CoreResult<()> { + if !runtime.is_dir() { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is missing", + runtime.display() + ))); + } + let has_ip = ["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["usr/sbin/nft", "sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = std::fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if !has_ip || !has_nft || !has_loader { + return Err(Error::config(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + ))); + } + Ok(()) +} + +async fn ensure_cached_supervisor_runtime( + docker: &Docker, + image: &str, + cache_dir: &Path, +) -> CoreResult<()> { + let runtime = cache_dir.join("openshell-runtime"); + if validate_supervisor_runtime(&runtime).is_ok() { + return Ok(()); + } + + let archive = extract_supervisor_runtime_archive(docker, image).await?; + let staging = tempfile::Builder::new() + .prefix(".openshell-runtime-") + .tempdir_in(cache_dir) + .map_err(|err| Error::config(format!("create runtime staging directory: {err}")))?; + let mut tar = tar::Archive::new(std::io::Cursor::new(archive)); + for entry in tar + .entries() + .map_err(|err| Error::config(format!("open supervisor runtime archive: {err}")))? + { + let mut entry = entry + .map_err(|err| Error::config(format!("read supervisor runtime archive: {err}")))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err(Error::config( + "supervisor runtime archive contains a non-materialized link or special file", + )); + } + if !entry + .unpack_in(staging.path()) + .map_err(|err| Error::config(format!("extract supervisor runtime archive: {err}")))? + { + return Err(Error::config( + "supervisor runtime archive contains a path outside its root", + )); + } + } + let extracted = staging.path().join("openshell-runtime"); + validate_supervisor_runtime(&extracted)?; + match std::fs::rename(&extracted, &runtime) { + Ok(()) => {} + Err(_) if validate_supervisor_runtime(&runtime).is_ok() => {} + Err(err) => { + return Err(Error::config(format!( + "install trusted supervisor runtime '{}': {err}", + runtime.display() + ))); + } + } + validate_supervisor_runtime(&runtime) +} + async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { let mut stream = docker.create_image( Some(CreateImageOptions { @@ -3875,6 +4013,19 @@ async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { /// binary as a tar archive, and return the untarred file bytes. The /// container is always removed, even on error paths. async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_BINARY_PATH, true).await +} + +async fn extract_supervisor_runtime_archive(docker: &Docker, image: &str) -> CoreResult> { + extract_supervisor_path_archive(docker, image, SUPERVISOR_IMAGE_RUNTIME_PATH, false).await +} + +async fn extract_supervisor_path_archive( + docker: &Docker, + image: &str, + path: &str, + extract_single_file: bool, +) -> CoreResult> { let container_name = temp_extract_container_name(); docker .create_container( @@ -3898,7 +4049,8 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe })?; // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; + let result = + download_path_from_container(docker, &container_name, path, extract_single_file).await; if let Err(remove_err) = docker .remove_container( &container_name, @@ -3915,12 +4067,14 @@ async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreRe result } -async fn download_binary_from_container( +async fn download_path_from_container( docker: &Docker, container_name: &str, + path: &str, + extract_single_file: bool, ) -> CoreResult> { let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) + .path(path) .build(); let mut stream = docker.download_from_container(container_name, Some(options)); @@ -3934,11 +4088,15 @@ async fn download_binary_from_container( tar_bytes.extend_from_slice(&chunk); } - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) + if extract_single_file { + extract_first_tar_entry(&tar_bytes).map_err(|err| { + Error::config(format!( + "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", + )) + }) + } else { + Ok(tar_bytes) + } } fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index b52cb87836..500dcfbd66 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -112,6 +112,7 @@ fn runtime_config() -> DockerDriverRuntimeConfig { stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, log_level: "info".to_string(), supervisor_bin: PathBuf::from("/tmp/openshell-sandbox"), + supervisor_runtime: PathBuf::from("/tmp/openshell-runtime"), guest_tls: Some(DockerGuestTlsPaths { ca: PathBuf::from("/tmp/ca.crt"), cert: PathBuf::from("/tmp/tls.crt"), @@ -1068,6 +1069,17 @@ fn container_create_body_sets_driver_owned_pids_limit() { assert_eq!(host_config.pids_limit, Some(DEFAULT_SANDBOX_PIDS_LIMIT)); } +#[test] +fn admitted_container_does_not_restart() { + let body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); + let restart_policy = body + .host_config + .expect("host config") + .restart_policy + .expect("restart policy"); + assert_eq!(restart_policy.name, Some(RestartPolicyNameEnum::NO)); +} + #[test] fn build_environment_sets_docker_tls_paths() { let env = build_environment(&test_sandbox(), &runtime_config()); @@ -1417,14 +1429,15 @@ fn build_binds_uses_docker_tls_directory() { .filter_map(|bind| bind.split(':').nth(1).map(String::from)) .collect::>(); assert!(targets.contains(&SUPERVISOR_MOUNT_PATH.to_string())); + assert!(targets.contains(&SUPERVISOR_RUNTIME_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_CA_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_CERT_MOUNT_PATH.to_string())); assert!(targets.contains(&TLS_KEY_MOUNT_PATH.to_string())); - assert!( - targets - .iter() - .all(|target| target.starts_with(TLS_MOUNT_DIR) || target == SUPERVISOR_MOUNT_PATH) - ); + assert!(targets.iter().all(|target| { + target.starts_with(TLS_MOUNT_DIR) + || target == SUPERVISOR_MOUNT_PATH + || target == SUPERVISOR_RUNTIME_MOUNT_PATH + })); } #[test] diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index 714b7d05c9..ab2a9015dd 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } openshell-policy = { path = "../openshell-policy" } tokio = { workspace = true } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 84d7029de4..b0ebdfa660 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -284,7 +284,10 @@ const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ WORKSPACE_VOLUME_NAME, ]; -const KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS: &[&str] = &[SERVICE_ACCOUNT_TOKEN_MOUNT_PATH]; +const KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS: &[&str] = &[ + SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, + openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR, +]; fn validate_kubernetes_driver_volumes( volumes: &[KubernetesDriverVolumeConfig], @@ -2335,6 +2338,7 @@ fn extract_image_size(message: &str) -> Option { /// Path where the supervisor binary is mounted inside the agent container. const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_DIR; +const IN_POD_ISOLATION_BACKEND_NAME: &str = "in-pod"; /// Name of the volume used to side-load the supervisor binary. const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; @@ -2368,7 +2372,7 @@ const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; const SIDECAR_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = openshell_core::container_paths::SIDECAR_CLIENT_TLS_DIR; -/// Build the emptyDir volume that holds the supervisor binary. +/// Build the emptyDir volume that holds the trusted supervisor runtime. /// /// The init container writes the binary here; the agent container reads it. fn supervisor_volume() -> serde_json::Value { @@ -2378,7 +2382,7 @@ fn supervisor_volume() -> serde_json::Value { }) } -/// Build the read-only volume mount for the supervisor binary in the agent container. +/// Build the read-only volume mount for the trusted supervisor runtime. fn supervisor_volume_mount() -> serde_json::Value { serde_json::json!({ "name": SUPERVISOR_VOLUME_NAME, @@ -2390,8 +2394,8 @@ fn supervisor_volume_mount() -> serde_json::Value { /// Build an image volume that mounts the supervisor OCI image directly. /// /// Requires Kubernetes >= v1.33 (`ImageVolume` beta) or >= v1.36 (GA). -/// The entire image filesystem is mounted read-only, making the binary -/// available at `{SUPERVISOR_MOUNT_PATH}/openshell-sandbox`. +/// The entire image filesystem is mounted read-only, making the supervisor and +/// its network-setup helpers available from one driver-controlled artifact. fn supervisor_image_volume( supervisor_image: &str, supervisor_image_pull_policy: &str, @@ -2408,26 +2412,23 @@ fn supervisor_image_volume( }) } -/// Build the init container that copies the supervisor binary into the emptyDir. +/// Build the init container that copies the trusted supervisor runtime into the emptyDir. /// /// The supervisor image contains the supervisor binary at `/openshell-sandbox`. -/// We invoke that binary with the `copy-self` subcommand so it copies itself -/// into the shared emptyDir volume, where the agent container then executes it -/// from a fixed, writable path. This pattern (binary self-copy) avoids requiring -/// `sh`/`cp` in the supervisor image and mirrors the approach used by argoexec's -/// emissary executor. +/// We invoke the supervisor's built-in installer so the binary, network helper +/// binaries, and their libraries all come from the supervisor image. The agent +/// container mounts the resulting volume read-only. fn supervisor_init_container( supervisor_image: &str, supervisor_image_pull_policy: &str, ) -> serde_json::Value { - let installed_path = format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"); let mut spec = serde_json::json!({ "name": SUPERVISOR_INIT_CONTAINER_NAME, "image": supervisor_image, "command": [ SUPERVISOR_IMAGE_BINARY_PATH, - "copy-self", - installed_path, + "copy-runtime", + SUPERVISOR_MOUNT_PATH, ], "securityContext": {"runAsUser": 0}, "volumeMounts": [{ @@ -2644,6 +2645,67 @@ fn has_upstream_proxy_credentials(params: &SandboxPodParams<'_>) -> bool { params.proxy_auth_secret_name.is_some() && params.proxy_auth_secret_key.is_some() } +fn apply_topology_descriptor(pod_template: &mut serde_json::Value) -> bool { + let Some(containers) = pod_template + .pointer_mut("/spec/containers") + .and_then(serde_json::Value::as_array_mut) + else { + return false; + }; + let index = containers + .iter() + .position(|container| container.get("name").and_then(|name| name.as_str()) == Some("agent")) + .unwrap_or(0); + let Some(command) = containers + .get_mut(index) + .and_then(|container| container.get_mut("command")) + .and_then(serde_json::Value::as_array_mut) + else { + return false; + }; + remove_protected_topology_arguments(command); + command.extend([ + serde_json::json!(format!( + "--topology-backend-name={IN_POD_ISOLATION_BACKEND_NAME}" + )), + serde_json::json!(format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + )), + serde_json::json!("--topology-payload-base64="), + ]); + true +} + +fn remove_protected_topology_arguments(arguments: &mut Vec) { + const PROTECTED: &[&str] = &[ + "--isolation-backend", + "--topology-backend-name", + "--topology-version", + "--topology-payload-base64", + ]; + let mut remove_value = false; + arguments.retain(|argument| { + if remove_value { + remove_value = false; + return false; + } + let Some(argument) = argument.as_str() else { + return true; + }; + for protected in PROTECTED { + if argument == *protected { + remove_value = true; + return false; + } + if argument.starts_with(&format!("{protected}=")) { + return false; + } + } + true + }); +} + fn sidecar_state_volume_mount() -> serde_json::Value { serde_json::json!({ "name": SIDECAR_STATE_VOLUME_NAME, @@ -3341,7 +3403,7 @@ fn sandbox_to_k8s_spec( &driver_config, inject_workspace, params, - ), + )?, ); if !template.agent_socket_path.is_empty() { root.insert( @@ -3375,10 +3437,48 @@ fn sandbox_to_k8s_spec( &driver_config, inject_workspace, params, - ), + )?, ); } + if params.topology == SupervisorTopology::Combined { + let selected = root + .get("podTemplate") + .and_then(|template| template.pointer("/spec/containers")) + .and_then(serde_json::Value::as_array) + .and_then(|containers| { + containers.iter().find(|container| { + container.get("name").and_then(|name| name.as_str()) == Some("agent") + }) + }) + .and_then(|container| container.get("command")) + .and_then(serde_json::Value::as_array) + .is_some_and(|command| { + [ + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + ] + .iter() + .all(|expected| { + command + .iter() + .filter(|argument| argument.as_str() == Some(expected.as_str())) + .count() + == 1 + }) + }); + if !selected { + return Err( + "failed to apply the admitted in-pod isolation backend to the supervisor command" + .to_string(), + ); + } + } + Ok(serde_json::Value::Object( std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), )) @@ -3404,6 +3504,7 @@ fn sandbox_template_to_k8s( inject_workspace, params, ) + .expect("test pod template should accept the selected isolation backend") } #[cfg(test)] @@ -3425,6 +3526,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( inject_workspace, params, ) + .expect("test pod template should accept the selected isolation backend") } fn sandbox_template_to_k8s_with_validated_config( @@ -3435,7 +3537,7 @@ fn sandbox_template_to_k8s_with_validated_config( driver_config: &KubernetesSandboxDriverConfig, inject_workspace: bool, params: &SandboxPodParams<'_>, -) -> serde_json::Value { +) -> Result { let mut metadata = serde_json::Map::new(); let mut pod_labels = template .labels @@ -3746,7 +3848,16 @@ fn sandbox_template_to_k8s_with_validated_config( match params.topology { SupervisorTopology::Combined => { + // A bound topology is one-shot. A new supervisor must receive a + // newly admitted descriptor instead of restarting an old binding. + result["spec"]["restartPolicy"] = serde_json::json!("Never"); apply_supervisor_sideload_with_params(&mut result, params); + if !apply_topology_descriptor(&mut result) { + return Err( + "failed to apply the admitted in-pod isolation backend to the supervisor command" + .to_string(), + ); + } } SupervisorTopology::Sidecar => { apply_supervisor_sidecar_topology( @@ -3770,7 +3881,7 @@ fn sandbox_template_to_k8s_with_validated_config( ); } - result + Ok(result) } fn apply_pod_driver_config( @@ -5125,6 +5236,42 @@ mod tests { assert!(err.contains("/var/run/secrets/openshell")); } + #[test] + fn driver_config_reserves_the_complete_supervisor_runtime_mount() { + for mount_path in [ + "/opt/openshell", + SUPERVISOR_MOUNT_PATH, + "/opt/openshell/bin/openshell-runtime/lib", + ] { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": mount_path + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + assert!( + err.contains("reserved OpenShell path"), + "expected {mount_path:?} to conflict with the supervisor runtime: {err}" + ); + } + } + #[test] fn driver_config_allows_spiffe_workload_path_without_provider_spiffe() { let spec = SandboxSpec { @@ -5447,6 +5594,72 @@ mod tests { ); } + #[test] + fn trusted_driver_selects_in_pod_isolation_backend() { + let mut pod_template = serde_json::json!({ + "spec": { "containers": [{ + "name": "agent", + "command": [ + "/openshell/bin/openshell-sandbox", + "--isolation-backend=legacy", + "--topology-version=999" + ], + "args": [ + "/bin/tool", + "--isolation-backend=legacy", + "--topology-backend-name", "attacker", + "--topology-payload-base64=Zm9yZ2Vk" + ] + }] } + }); + + assert!(apply_topology_descriptor(&mut pod_template)); + + assert_eq!( + pod_template["spec"]["containers"][0]["command"], + serde_json::json!([ + "/openshell/bin/openshell-sandbox", + "--topology-backend-name=in-pod", + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=", + "--" + ]) + ); + assert_eq!( + pod_template["spec"]["containers"][0]["args"], + serde_json::json!([ + "/bin/tool", + "--isolation-backend=legacy", + "--topology-backend-name", + "attacker", + "--topology-payload-base64=Zm9yZ2Vk" + ]), + "arguments after the trusted delimiter belong to the workload" + ); + } + + #[test] + fn admitted_pod_does_not_restart() { + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + true, + &SandboxPodParams::default(), + ); + + assert_eq!(pod_template["spec"]["restartPolicy"], "Never"); + } + + #[test] + fn in_pod_selection_fails_when_no_agent_command_exists() { + let mut pod_template = serde_json::json!({ "spec": { "containers": [] } }); + assert!(!apply_topology_descriptor(&mut pod_template)); + } + #[test] fn supervisor_sideload_replaces_spoofed_identity_environment() { let mut pod_template = serde_json::json!({ @@ -5568,17 +5781,19 @@ mod tests { assert_eq!(init_containers[0]["imagePullPolicy"], "IfNotPresent"); // The init container must invoke the binary directly with - // `copy-self ` rather than depending on shell utilities. + // `copy-runtime ` rather than depending on shell utilities or + // helpers from the workload image. let init_command = init_containers[0]["command"] .as_array() .expect("init container command should be set"); - assert_eq!(init_command.len(), 3, "expected [binary, copy-self, dest]"); - assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); - assert_eq!(init_command[1], "copy-self"); assert_eq!( - init_command[2].as_str().unwrap(), - format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox") + init_command.len(), + 3, + "expected [binary, copy-runtime, dest]" ); + assert_eq!(init_command[0], SUPERVISOR_IMAGE_BINARY_PATH); + assert_eq!(init_command[1], "copy-runtime"); + assert_eq!(init_command[2].as_str().unwrap(), SUPERVISOR_MOUNT_PATH); assert!( !init_command.iter().any(|v| v == "sh"), "init container must not depend on a shell" diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 8b3e014e8c..6146b07ba6 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } tokio = { workspace = true } diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index abb9d69dd2..e5eb22d49f 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -468,6 +468,18 @@ fn upstream_proxy_cli_args(config: &PodmanComputeConfig) -> Vec { args } +fn in_pod_topology_descriptor_args() -> Vec { + vec![ + "--topology-backend-name=in-pod".to_string(), + format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ), + "--topology-payload-base64=".to_string(), + "--".to_string(), + ] +} + fn build_env( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1072,6 +1084,7 @@ pub fn build_container_spec_for_image( driver_mounts::DEFAULT_WORKSPACE_ROOT.to_string(), ]; command.extend(upstream_proxy_cli_args(config)); + command.extend(in_pod_topology_descriptor_args()); let container_spec = ContainerSpec { name, @@ -1094,9 +1107,9 @@ pub fn build_container_spec_for_image( // the supervisor path as an argument instead of executing it directly. entrypoint: vec![SUPERVISOR_BINARY_PATH.into()], // Keep Podman's existing /sandbox workspace contract explicit while - // the supervisor supports driver-selected workdirs. Operator-owned - // corporate proxy flags follow it; the workload command comes from - // the reserved environment variable. + // the supervisor supports driver-selected workdirs. Trusted operator + // proxy and topology arguments follow it; workload argv remains in + // the reserved environment transport. command, // Force the supervisor to run as root (UID 0). Sandbox images may // set a non-root USER directive (e.g. `USER sandbox`), but the @@ -2098,6 +2111,20 @@ mod tests { .collect() } + #[test] + fn container_spec_admits_the_in_pod_backend_by_default() { + let spec = build_container_spec(&test_sandbox("test-id", "test-name"), &test_config()); + let command = spec_command(&spec); + + assert!(command.contains(&"--topology-backend-name=in-pod".to_string())); + assert!(command.contains(&format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ))); + assert!(command.contains(&"--topology-payload-base64=".to_string())); + assert_eq!(command.last().map(String::as_str), Some("--")); + } + #[test] fn container_spec_passes_operator_proxy_on_supervisor_argv() { let sandbox = test_sandbox("test-id", "test-name"); diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index ebcb9d2bc2..4a8c4d0425 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -20,6 +20,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } openshell-otel = { path = "../openshell-otel" } openshell-policy = { path = "../openshell-policy" } openshell-vfio = { path = "../openshell-vfio" } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..05b9933459 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -10,7 +10,7 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) fo flowchart LR subgraph host["Host process"] gateway["openshell-server
(compute::vm::spawn)"] - driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] + driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
├── openshell-sandbox.zst
└── openshell-runtime.tar.zst"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver end @@ -35,7 +35,7 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th mise run gateway:vm ``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor plus its trusted network-helper runtime. The latter uses Docker Buildx to materialize the same runtime shipped in the supervisor image. Subsequent runs are cached. By default `mise run gateway:vm`: @@ -94,7 +94,7 @@ If you want to drive the launch yourself instead of using `mise run gateway:vm` ```shell # 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/ mise run vm:setup -mise run vm:supervisor # if openshell-sandbox.zst is not already present +mise run vm:supervisor # builds openshell-sandbox.zst and its trusted helper runtime # 2. Build both binaries with the staged artifacts embedded OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs index 1763590545..17f1f00bf8 100644 --- a/crates/openshell-driver-vm/build.rs +++ b/crates/openshell-driver-vm/build.rs @@ -21,6 +21,7 @@ fn main() { "libkrunfw.5.dylib.zst", "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ] { println!("cargo:rerun-if-changed={dir}/{name}"); @@ -38,7 +39,13 @@ fn main() { println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}"); generate_stub_resources( &out_dir, - &["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"], + &[ + "libkrun", + "libkrunfw", + "openshell-sandbox.zst", + "openshell-runtime.tar.zst", + "umoci.zst", + ], ); return; } @@ -56,6 +63,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -75,6 +83,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -92,6 +101,10 @@ fn main() { "openshell-sandbox.zst".to_string(), "openshell-sandbox.zst".to_string(), ), + ( + "openshell-runtime.tar.zst".to_string(), + "openshell-runtime.tar.zst".to_string(), + ), ("umoci.zst".to_string(), "umoci.zst".to_string()), ]; @@ -135,6 +148,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index 11aab67f43..13bd5cac62 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -36,7 +36,7 @@ VM sandboxes can run the same supervisor enforcement path as other backends. # Download the current pre-built runtime and stage compressed artifacts mise run vm:setup -# Build the bundled guest supervisor +# Build the bundled guest supervisor and trusted helper runtime (requires Docker Buildx) mise run vm:supervisor # Build the gateway and VM driver with embedded runtime artifacts diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..f411972d6a 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -117,6 +117,10 @@ ensure_target_runtime() { cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox" chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox" fi + if [ -d /opt/openshell/bin/openshell-runtime ]; then + rm -rf "$image_root/opt/openshell/bin/openshell-runtime" + cp -a /opt/openshell/bin/openshell-runtime "$image_root/opt/openshell/bin/openshell-runtime" + fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then @@ -214,14 +218,29 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- fi done @@ -837,7 +856,12 @@ ts "starting openshell-sandbox supervisor" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then exec_supervisor_in_newroot fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox \ + --workdir /sandbox \ + --topology-backend-name=in-pod \ + --topology-version=@ISOLATION_INTERFACE_VERSION@ \ + --topology-payload-base64= \ + -- } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 13e57f546d..05ba97b9dc 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -4453,10 +4453,12 @@ fn build_guest_environment( || guest_visible_openshell_endpoint(&config.openshell_endpoint), String::from, ); - // 1. User-supplied environment (lowest priority). + // User-supplied values travel only through the serialized child-environment + // channel. They must not become guest-init or supervisor environment + // variables: guest init runs as root and sources only driver-owned keys, + // while the supervisor applies this map when it launches workload code. let user_env = merged_environment(sandbox); let mut environment: HashMap = HashMap::new(); - environment.extend(user_env.clone()); if !user_env.is_empty() && let Ok(json) = serde_json::to_string(&user_env) { diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..ced6bff9f8 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -11,10 +11,13 @@ use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst")); +const SUPERVISOR_RUNTIME: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/openshell-runtime.tar.zst")); const UMOCI: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/umoci.zst")); const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; +const SANDBOX_SUPERVISOR_RUNTIME_PATH: &str = "/opt/openshell/bin/openshell-runtime"; const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; const SANDBOX_OWNER_NORMALIZED_MARKER: &str = openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; @@ -362,11 +365,12 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> if let Some(parent) = init_path.parent() { fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; } - fs::write( - &init_path, - include_str!("../scripts/openshell-vm-sandbox-init.sh"), - ) - .map_err(|e| format!("write {}: {e}", init_path.display()))?; + let init_script = include_str!("../scripts/openshell-vm-sandbox-init.sh").replace( + "@ISOLATION_INTERFACE_VERSION@", + &openshell_isolation::contract::INTERFACE_VERSION.to_string(), + ); + fs::write(&init_path, init_script) + .map_err(|e| format!("write {}: {e}", init_path.display()))?; #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; @@ -376,6 +380,7 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> } ensure_supervisor_binary(rootfs)?; + ensure_supervisor_runtime(rootfs)?; ensure_umoci_binary(rootfs)?; let opt_dir = rootfs.join("opt/openshell"); @@ -395,6 +400,7 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> { require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?; require_rootfs_path(rootfs, SANDBOX_SUPERVISOR_PATH)?; + validate_supervisor_runtime(rootfs)?; require_rootfs_path(rootfs, SANDBOX_UMOCI_PATH)?; require_any_rootfs_path(rootfs, &["/bin/bash"])?; require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?; @@ -871,6 +877,78 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { Ok(()) } +fn ensure_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + if validate_supervisor_runtime(rootfs).is_ok() { + return Ok(()); + } + if SUPERVISOR_RUNTIME.is_empty() { + return Err( + "trusted supervisor helper runtime not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:supervisor` first" + .to_string(), + ); + } + + install_supervisor_runtime_archive(rootfs, SUPERVISOR_RUNTIME) +} + +fn install_supervisor_runtime_archive(rootfs: &Path, archive_bytes: &[u8]) -> Result<(), String> { + let destination = rootfs.join("opt/openshell/bin"); + fs::create_dir_all(&destination) + .map_err(|e| format!("create {}: {e}", destination.display()))?; + let decoder = zstd::Decoder::new(Cursor::new(archive_bytes)) + .map_err(|e| format!("decompress supervisor runtime: {e}"))?; + let mut archive = tar::Archive::new(decoder); + for entry in archive + .entries() + .map_err(|e| format!("open supervisor runtime archive: {e}"))? + { + let mut entry = entry.map_err(|e| format!("read supervisor runtime archive: {e}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err( + "supervisor runtime archive contains a non-materialized link or special file" + .to_string(), + ); + } + if !entry + .unpack_in(&destination) + .map_err(|e| format!("extract supervisor runtime archive: {e}"))? + { + return Err("supervisor runtime archive contains a path outside its root".to_string()); + } + } + validate_supervisor_runtime(rootfs) +} + +fn validate_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/')); + let has_ip = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["sbin/nft", "usr/sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if has_ip && has_nft && has_loader { + Ok(()) + } else { + Err(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + )) + } +} + fn ensure_umoci_binary(rootfs: &Path) -> Result<(), String> { let path = rootfs.join(SANDBOX_UMOCI_PATH.trim_start_matches('/')); if UMOCI.is_empty() { @@ -979,6 +1057,19 @@ mod tests { assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); assert!(rootfs.join("opt/openshell/bin/umoci").is_file()); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + let init_script = fs::read_to_string(rootfs.join("srv/openshell-vm-sandbox-init.sh")) + .expect("read guest init"); + assert!(init_script.contains("--topology-backend-name=in-pod")); + assert!(init_script.contains(&format!( + "--topology-version={}", + openshell_isolation::contract::INTERFACE_VERSION + ))); + assert!(!init_script.contains("@ISOLATION_INTERFACE_VERSION@")); assert!(rootfs.join("sandbox").is_dir()); assert!(rootfs.join("image-cache").is_dir()); assert!(rootfs.join("lower").is_dir()); @@ -1010,6 +1101,44 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn supervisor_runtime_archive_materializes_below_the_trusted_path() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + let mut tar_bytes = Vec::new(); + { + let mut archive = tar::Builder::new(&mut tar_bytes); + for (path, bytes, mode) in [ + ("openshell-runtime/usr/sbin/ip", b"ip".as_slice(), 0o755), + ("openshell-runtime/usr/sbin/nft", b"nft".as_slice(), 0o755), + ( + "openshell-runtime/lib/ld-musl-test.so.1", + b"loader".as_slice(), + 0o755, + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes) + .expect("append runtime entry"); + } + archive.finish().expect("finish runtime archive"); + } + let compressed = zstd::encode_all(Cursor::new(tar_bytes), 1).expect("compress runtime"); + + install_supervisor_runtime_archive(&rootfs, &compressed).expect("install runtime"); + validate_supervisor_runtime(&rootfs).expect("validate runtime"); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + } + #[test] fn prepare_sandbox_rootfs_preserves_image_workdir_contents_in_rootfs() { let dir = unique_temp_dir(); @@ -1230,6 +1359,13 @@ mod tests { } fn write_fake_runtime_binaries(rootfs: &Path) { + let helper_runtime = rootfs.join("opt/openshell/bin/openshell-runtime"); + fs::create_dir_all(helper_runtime.join("usr/sbin")).expect("create helper bin directory"); + fs::create_dir_all(helper_runtime.join("lib")).expect("create helper lib directory"); + fs::write(helper_runtime.join("usr/sbin/ip"), b"ip").expect("write ip helper"); + fs::write(helper_runtime.join("usr/sbin/nft"), b"nft").expect("write nft helper"); + fs::write(helper_runtime.join("lib/ld-musl-test.so.1"), b"loader") + .expect("write helper loader"); fs::write( rootfs.join("opt/openshell/bin/openshell-sandbox"), b"sandbox", diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index c653db84dd..5f12245ee8 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation = { path = "../openshell-isolation" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } @@ -25,6 +26,7 @@ openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-mid openshell-supervisor-process = { path = "../openshell-supervisor-process" } # Async runtime +async-trait = "0.1" tokio = { workspace = true } # gRPC (tonic::Status downcast in error mapping) @@ -38,12 +40,13 @@ clap = { workspace = true } miette = { workspace = true } # Unix ownership for Kubernetes sidecar init setup -nix = { workspace = true } +nix = { workspace = true, features = ["socket"] } # TLS crypto provider install (main.rs) rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +base64 = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } prost = { workspace = true } diff --git a/crates/openshell-sandbox/src/inpod.rs b/crates/openshell-sandbox/src/inpod.rs new file mode 100644 index 0000000000..ede172f6b5 --- /dev/null +++ b/crates/openshell-sandbox/src/inpod.rs @@ -0,0 +1,1622 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The in-pod isolation backend (RFC 0012 runtime-selectable contract). +//! +//! This is the co-located placement: the supervisor process hosts the +//! supervisor role, the mediation service, and the backend in the agent's +//! container. It implements the object-safe boxed state chain +//! (`attach -> Bound -> confirm -> Ready -> start_agent -> Running`) over the +//! existing supervisor primitives without changing their behavior: +//! `create_netns_for_proxy` (network), supervisor-owned proxy mediation, +//! the pre-exec ceiling in `spawn_workload` (filesystem/Landlock + +//! syscall/seccomp), and procfs (binary identity). +//! +//! `attach` validates the (empty) in-pod payload and atomically binds the +//! trusted [`SandboxContext`] to the boundary it establishes: the workload +//! network namespace and backend-owned connection source come up inside +//! `attach`. The supervisor connects that source to mediation before `confirm`, so +//! `Bound` means what the RFC says it means — descriptor and context bound to +//! the same resource, mediation source available, no untrusted workload code +//! running. Each transition consumes the prior state by value, so the call +//! order, and thus "no untrusted instruction before the boundary is ready", is +//! enforced by construction. +//! +//! Execution-domain note: the in-pod backend relies on container-runtime +//! inheritance — the supervisor and every child it spawns run in the pod's +//! cgroup with the device set the CRI granted the container — so every workload +//! descendant remains in the compute driver's provisioned execution +//! environment by construction. + +use std::collections::HashMap; +#[cfg(target_os = "linux")] +use std::future::Future; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use async_trait::async_trait; + +use openshell_core::activity::ActivitySender; +use openshell_core::denial::DenialEvent; +use openshell_core::policy::NetworkMode; +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation::contract::{ + BackendError, BoundBoundary, BoundaryExec, BoundaryExitStatus, BoundaryPortForward, + BoundaryProcess, BoundarySignal, INTERFACE_VERSION, IsolationBackend, MediatedConnection, + NetworkMediationSource, ReadyBoundary, RunningBoundary, SandboxContext, + VerifiedTopologyDescriptor, +}; +use openshell_supervisor_network::identity_source::ProcfsIdentityResolver; +use openshell_supervisor_process::process::ProcessEnforcementMode; +use openshell_supervisor_process::process::ResolvedProcessIdentity; +use openshell_supervisor_process::process::ResolvedWorkspace; +use openshell_supervisor_process::run::{AgentSignaler, SpawnedAgent, spawn_workload}; +use tokio::sync::mpsc::UnboundedSender; + +#[cfg(target_os = "linux")] +use openshell_supervisor_process::netns::{NetworkNamespace, create_conformant_netns_for_proxy}; + +/// Stable name of the co-located backend implementation. +pub const IN_POD_BACKEND_NAME: &str = "in-pod"; + +// ============================================================================ +// Config and backend +// ============================================================================ + +/// Runtime collaborators the in-pod lifecycle calls need, captured once when the +/// backend is built. Move-once values (the event senders) are held behind a +/// `Mutex>` so the `&self` backend/state methods can take them exactly +/// when the matching transition fires. Policy, workload, and sandbox identity +/// are *not* here; they arrive in the trusted [`SandboxContext`] at `attach`. +pub struct InPodConfig { + /// Require the supervisor to own the execution environment's PID namespace. + pub require_exclusive_pid_namespace: bool, + pub network_enabled: bool, + pub process_enabled: bool, + pub entrypoint_pid: Arc, + pub provider_credentials: ProviderCredentialState, + /// Child environment for the agent, resolved at startup. Mutated in place by + /// `attach` if the GCE metadata loopback server fails to come up. + pub provider_env: Mutex>, + /// Process launch-time enforcement level (full privileged setup vs. + /// network-sidecar reduced mode), resolved by the supervisor at startup. + pub process_enforcement_mode: ProcessEnforcementMode, + pub resolved_process_identity: ResolvedProcessIdentity, + /// Workspace resolution already normalized against the active driver. + pub workspace: ResolvedWorkspace, + pub agent_proposals: AgentProposals, + pub openshell_endpoint: Option, + pub ssh_socket_path: Option, + /// Bypass-monitor denial / activity senders (consumed by `start_agent`). + #[cfg(target_os = "linux")] + pub bypass_denial_tx: Mutex>>, + #[cfg(target_os = "linux")] + pub bypass_activity_tx: Mutex>, + /// Co-located coordination set by supervisor-owned mediation after it has + /// connected the source and before `confirm`. + pub mediation_ready: Arc, + pub ca_file_paths: Arc>>, + pub proxy_bind_ip: Arc>>, +} + +/// The backend for the in-pod backend. Holds the per-sandbox [`InPodConfig`] and +/// hands it to the boundary on the single `attach`. +pub struct InPodBackend { + config: Mutex>, + /// Whether a prior `attach` consumed the config and then failed during + /// establishment. The one-shot event senders are consumed with it, so the + /// in-pod resource cannot be re-attached; this keeps the error truthful + /// ("attempt failed", not "already bound"). + attach_failed: AtomicBool, +} + +impl InPodBackend { + /// Build the backend from its per-sandbox runtime collaborators. + #[must_use] + pub fn new(config: InPodConfig) -> Self { + Self { + config: Mutex::new(Some(config)), + attach_failed: AtomicBool::new(false), + } + } +} + +#[async_trait] +impl IsolationBackend for InPodBackend { + fn backend_name(&self) -> &'static str { + IN_POD_BACKEND_NAME + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + // Validate the in-pod payload: the supervisor process *is* the + // resource, so the payload carries nothing. + if !descriptor.payload().is_empty() { + return Err(BackendError::Descriptor( + "in-pod descriptor payload must be empty".to_string(), + )); + } + // `attach` never binds a resource that is already bound to an active + // boundary: the in-pod resource is this process, bindable exactly once. + let config = self + .config + .lock() + .expect("in-pod config lock") + .take() + .ok_or_else(|| { + if self.attach_failed.load(Ordering::SeqCst) { + BackendError::Attach( + "a previous in-pod attach failed during establishment; \ + the in-pod resource cannot be re-attached" + .to_string(), + ) + } else { + BackendError::Denied( + "in-pod resource is already bound to an active boundary".to_string(), + ) + } + })?; + + match establish(config, sandbox).await { + Ok(bound) => Ok(Box::new(bound)), + Err(e) => { + self.attach_failed.store(true, Ordering::SeqCst); + Err(e) + } + } + } +} + +/// Establish the in-pod boundary: standing enforcement (the workload network +/// namespace) and the mediation service (the in-pod proxy), bound atomically to +/// the trusted sandbox context. Consumes the one-shot config; failure fails +/// closed with partial state released by RAII. +async fn establish( + config: InPodConfig, + sandbox: SandboxContext, +) -> Result { + #[cfg(not(target_os = "linux"))] + return Err(BackendError::Attach( + "the in-pod RFC 0012 backend requires Linux enforcement primitives".to_string(), + )); + + if matches!(sandbox.policy.network.mode, NetworkMode::Allow) { + return Err(BackendError::Denied( + "the in-pod RFC 0012 topology does not admit unrestricted network mode; workload egress must be mediated or blocked" + .to_string(), + )); + } + + // Establish the network dimension of standing enforcement: create the + // workload's network namespace and install the bypass-detection rules. + // Filesystem and syscall are launch-time controls applied per process; + // binary identity is resolved per accepted connection. + #[cfg(target_os = "linux")] + let netns = if config.network_enabled { + create_conformant_netns_for_proxy(&sandbox.policy) + .map_err(|e| BackendError::Attach(e.to_string()))? + } else { + None + }; + + #[cfg(target_os = "linux")] + let proxy_bind_ip = netns.as_ref().map(NetworkNamespace::host_ip); + #[cfg(not(target_os = "linux"))] + let proxy_bind_ip: Option = None; + *config.proxy_bind_ip.lock().expect("proxy bind IP lock") = proxy_bind_ip; + + if config.require_exclusive_pid_namespace && std::process::id() != 1 { + return Err(BackendError::Attach( + "the in-pod topology requires the supervisor to be PID 1 in its execution environment" + .to_string(), + )); + } + let runtime = if config.require_exclusive_pid_namespace { + openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new_exclusive_pid_namespace( + ) + } else { + openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new() + }; + let network_mediation_source: Arc = if config.network_enabled + && matches!(sandbox.policy.network.mode, NetworkMode::Proxy) + { + let proxy_policy = sandbox.policy.network.proxy.as_ref().ok_or_else(|| { + BackendError::Attach("proxy mode requires a proxy configuration".to_string()) + })?; + let default_ip = proxy_bind_ip.unwrap_or_else(|| std::net::IpAddr::from([127, 0, 0, 1])); + let port = proxy_policy.http_addr.map_or(3128, |addr| addr.port()); + let listener = tokio::net::TcpListener::bind((default_ip, port)) + .await + .map_err(|error| BackendError::Attach(error.to_string()))?; + Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: config.entrypoint_pid.clone(), + }, + runtime: runtime.clone(), + }) + } else { + Arc::new(InactiveNetworkMediationSource { + runtime: runtime.clone(), + }) + }; + + // Start the GCE metadata loopback server inside the namespace so Go's + // metadata client (which bypasses HTTP_PROXY) can reach it via direct + // TCP. Must come up before start_agent; on failure the GCE env vars are + // stripped so the SDK falls back cleanly. + #[cfg(target_os = "linux")] + if let Some(ns) = netns.as_ref() { + ensure_gce_metadata_server(&config, ns).await; + } + + Ok(InPodBound { + config, + sandbox, + #[cfg(target_os = "linux")] + netns, + network_mediation_source, + runtime, + }) +} + +// ============================================================================ +// Lifecycle states +// ============================================================================ + +/// Bound: the descriptor and trusted sandbox context are bound to this process's +/// boundary, and the mediation source is available. No untrusted workload code +/// is running. +struct InPodBound { + config: InPodConfig, + sandbox: SandboxContext, + #[cfg(target_os = "linux")] + netns: Option, + network_mediation_source: Arc, + runtime: Arc, +} + +#[async_trait] +impl BoundBoundary for InPodBound { + fn network_mediation_source(&self) -> Arc { + self.network_mediation_source.clone() + } + + async fn confirm(self: Box) -> Result, BackendError> { + if !self.config.process_enabled { + return Err(BackendError::Confirm( + "the co-located backend requires the process supervisor leaf".to_string(), + )); + } + // Structural-mediation check (fail closed). The proxy listener must be + // connected and the live default-deny ceiling must still be present + // before the backend advances its lifecycle. + if self.config.network_enabled + && matches!(self.sandbox.policy.network.mode, NetworkMode::Proxy) + { + #[cfg(target_os = "linux")] + if self.netns.is_none() { + return Err(BackendError::Confirm( + "proxy mode requires a workload network namespace; none established" + .to_string(), + )); + } + #[cfg(target_os = "linux")] + if let Some(netns) = self.netns.as_ref() { + let proxy_port = self + .sandbox + .policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + netns + .egress_ceiling_verifier() + .verify_bounded(proxy_port, std::time::Duration::from_secs(2)) + .await + .map_err(|error| BackendError::Confirm(error.to_string()))?; + } + if !self.config.mediation_ready.load(Ordering::Acquire) { + return Err(BackendError::Confirm( + "the supervisor has not connected network mediation to the boundary source" + .to_string(), + )); + } + } + + Ok(Box::new(InPodReady { + config: self.config, + sandbox: self.sandbox, + #[cfg(target_os = "linux")] + netns: self.netns, + runtime: self.runtime, + })) + } +} + +/// Ready: standing enforcement and mediation are confirmed. Only agent +/// activation is possible. +struct InPodReady { + config: InPodConfig, + sandbox: SandboxContext, + #[cfg(target_os = "linux")] + netns: Option, + runtime: Arc, +} + +#[async_trait] +impl ReadyBoundary for InPodReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let this = *self; + let config = this.config; + let sandbox = this.sandbox; + let runtime = this.runtime; + #[cfg(target_os = "linux")] + let netns = this.netns; + + #[cfg(target_os = "linux")] + let enforcement_monitor = if let Some(netns) = netns.as_ref() + && matches!(sandbox.policy.network.mode, NetworkMode::Proxy) + { + let proxy_port = sandbox + .policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + Some( + start_egress_ceiling_monitor( + netns.egress_ceiling_verifier(), + proxy_port, + runtime.clone(), + config.require_exclusive_pid_namespace, + ) + .await?, + ) + } else { + None + }; + + // The in-pod backend creates the agent process itself; the launch-time + // controls (Landlock, seccomp, privilege drop) are applied inside + // `spawn_workload`'s pre-exec ceiling, before the first untrusted + // instruction. + let (agent, exec, port_forward): ( + Arc, + Arc, + Arc, + ) = { + let spec = &sandbox.agent; + let ca_file_paths = config.ca_file_paths.lock().expect("ca paths lock").clone(); + let provider_env = config + .provider_env + .lock() + .expect("provider_env lock") + .clone(); + + #[cfg(target_os = "linux")] + let bypass_denial_tx = config + .bypass_denial_tx + .lock() + .expect("bypass_denial_tx lock") + .take(); + #[cfg(target_os = "linux")] + let bypass_activity_tx = config + .bypass_activity_tx + .lock() + .expect("bypass_activity_tx lock") + .take(); + + let spawned = spawn_workload( + &spec.program, + &spec.args, + config.workspace.clone(), + spec.timeout_secs, + spec.interactive, + Some(sandbox.sandbox_id.as_str()), + config.openshell_endpoint.as_deref(), + config.ssh_socket_path.clone(), + // In-pod co-locates the SSH socket with the workload; it is not + // shared with a separate network-sidecar container. + false, + None, + &sandbox.policy, + config.resolved_process_identity, + config.process_enforcement_mode, + config.entrypoint_pid.clone(), + // No sidecar control channel awaits the in-pod entrypoint PID. + None, + None, + config.provider_credentials.clone(), + provider_env, + ca_file_paths, + config.agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + Some(runtime.clone()), + ) + .await + .map_err(|error| { + runtime.deactivate(); + BackendError::Process(error.to_string()) + })?; + + let exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + ( + Arc::new(InPodAgentProcess::running(spawned)), + exec, + port_forward, + ) + }; + + Ok(Box::new(InPodRunning { + agent, + exec, + port_forward, + runtime, + #[cfg(target_os = "linux")] + _enforcement_monitor: enforcement_monitor, + #[cfg(target_os = "linux")] + _netns: netns, + })) + } +} + +/// Running: the agent is runnable behind the boundary. Exec, forwarding, wait, +/// and signal are available. The mediation source was retained by the +/// supervisor from the `Bound` state. +struct InPodRunning { + agent: Arc, + exec: Arc, + port_forward: Arc, + runtime: Arc, + #[cfg(target_os = "linux")] + _enforcement_monitor: Option, + /// Held to keep the network namespace alive for the boundary's life; + /// dropping the running state tears it down (RAII), which is the + /// backend reclaiming backend-private state — the contract defines no public + /// cleanup transition. + #[cfg(target_os = "linux")] + _netns: Option, +} + +#[cfg(target_os = "linux")] +async fn start_egress_ceiling_monitor( + verifier: openshell_supervisor_process::netns::EgressCeilingVerifier, + proxy_port: u16, + runtime: Arc, + exit_execution_environment_on_loss: bool, +) -> Result { + let verify: EnforcementCheck = Arc::new(move || { + let verifier = verifier.clone(); + Box::pin(async move { + verifier + .verify_bounded(proxy_port, std::time::Duration::from_secs(2)) + .await + .map_err(|error| error.to_string()) + }) + }); + start_enforcement_monitor( + runtime, + std::time::Duration::from_millis(250), + verify, + exit_execution_environment_on_loss, + ) + .await +} + +#[cfg(target_os = "linux")] +type EnforcementCheck = Arc< + dyn Fn() -> std::pin::Pin> + Send + 'static>> + + Send + + Sync, +>; + +#[cfg(target_os = "linux")] +async fn start_enforcement_monitor( + runtime: Arc, + period: std::time::Duration, + verify: EnforcementCheck, + exit_execution_environment_on_loss: bool, +) -> Result { + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + if let Err(error) = verify().await { + let _ = ready_tx.send(Err(error.clone())); + if runtime.deactivate_for_enforcement_loss() { + report_enforcement_loss(&error); + exit_execution_environment(exit_execution_environment_on_loss); + } + return; + } + if ready_tx.send(Ok(())).is_err() { + runtime.deactivate(); + return; + } + let mut interval = tokio::time::interval(period); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + interval.tick().await; + while runtime.is_active() { + interval.tick().await; + if let Err(error) = verify().await { + if runtime.deactivate_for_enforcement_loss() { + report_enforcement_loss(&error); + exit_execution_environment(exit_execution_environment_on_loss); + } + break; + } + } + }); + ready_rx + .await + .map_err(|_| BackendError::Confirm("egress monitor failed to start".to_string()))? + .map_err(BackendError::Confirm)?; + Ok(EnforcementMonitorGuard { task }) +} + +#[cfg(target_os = "linux")] +fn exit_execution_environment(enabled: bool) { + if enabled { + // This backend admits only an exclusive workload PID namespace with + // the supervisor as PID 1. Exiting its init process makes the kernel + // terminate every remaining process in that execution environment, + // including descendants that changed process group or session. + std::process::exit(125); + } +} + +#[cfg(target_os = "linux")] +struct EnforcementMonitorGuard { + task: tokio::task::JoinHandle<()>, +} + +#[cfg(target_os = "linux")] +impl Drop for EnforcementMonitorGuard { + fn drop(&mut self) { + self.task.abort(); + } +} + +#[cfg(target_os = "linux")] +fn report_enforcement_loss(error: &str) { + let message = format!( + "Isolation boundary lost its default-deny egress ceiling; terminating workloads [error:{error}]" + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(crate::ocsf_ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "enforcement_lost") + .message(message.clone()) + .build() + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::DetectionFindingBuilder::new(crate::ocsf_ctx()) + .activity(openshell_ocsf::ActivityId::Open) + .action(openshell_ocsf::ActionId::Denied) + .disposition(openshell_ocsf::DispositionId::Blocked) + .severity(openshell_ocsf::SeverityId::High) + .is_alert(true) + .finding_info(openshell_ocsf::FindingInfo::new( + "isolation-egress-enforcement-lost", + "Isolation egress enforcement lost", + )) + .message(message) + .build() + ); +} + +impl Drop for InPodRunning { + fn drop(&mut self) { + self.runtime.deactivate(); + } +} + +impl RunningBoundary for InPodRunning { + fn agent(&self) -> Arc { + self.agent.clone() + } + fn exec(&self) -> Arc { + self.exec.clone() + } + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +// ============================================================================ +// Agent process handle +// ============================================================================ + +/// The agent process running inside the in-pod boundary. `wait` returns a stable +/// terminal status across repeated calls; signals go through the lock-free +/// pid-based [`AgentSignaler`] so they never contend with an in-flight `wait`. +struct InPodAgentProcess { + signaler: Option, + result: Arc>>>, + exited: Arc, + terminal: Arc, + runtime: Arc, +} + +#[derive(Clone)] +enum StableWaitError { + Process(String), + EnforcementLost, +} + +impl InPodAgentProcess { + fn running(spawned: SpawnedAgent) -> Self { + let signaler = spawned.signaler(); + let runtime = spawned.boundary_runtime(); + let result = Arc::new(Mutex::new(None)); + let exited = Arc::new(tokio::sync::Notify::new()); + let result_for_wait = result.clone(); + let exited_for_wait = exited.clone(); + let terminal = Arc::new(AtomicBool::new(false)); + let terminal_for_wait = terminal.clone(); + let runtime_for_wait = runtime.clone(); + tokio::spawn(async move { + let mut spawned = spawned; + let waited = spawned + .wait() + .await + .map_err(|error| StableWaitError::Process(error.to_string())) + .map(|process_status| { + process_status.signal().map_or_else( + || BoundaryExitStatus::Exited(process_status.code()), + BoundaryExitStatus::Signaled, + ) + }); + let waited = if runtime_for_wait.enforcement_was_lost() { + Err(StableWaitError::EnforcementLost) + } else { + waited + }; + terminal_for_wait.store(true, Ordering::Release); + if let Ok(mut slot) = result_for_wait.lock() { + *slot = Some(waited); + } + exited_for_wait.notify_waiters(); + }); + Self { + signaler: Some(signaler), + result, + exited, + terminal, + runtime, + } + } +} + +#[async_trait] +impl BoundaryProcess for InPodAgentProcess { + async fn wait(&self) -> Result { + loop { + let notified = self.exited.notified(); + let result = self + .result + .lock() + .map_err(|_| BackendError::Process("agent result lock poisoned".to_string()))? + .clone(); + if let Some(result) = result { + self.terminal.store(true, Ordering::Release); + return result.map_err(|error| match error { + StableWaitError::Process(message) => BackendError::Process(message), + StableWaitError::EnforcementLost => BackendError::Terminated( + "required isolation enforcement was lost".to_string(), + ), + }); + } + notified.await; + } + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + if self.terminal.load(Ordering::Acquire) { + return Err(BackendError::Terminated("agent has exited".to_string())); + } + let Some(signaler) = self.signaler.as_ref() else { + // Network-only hold-open: no workload process to signal. + return Ok(()); + }; + let result = match signal { + BoundarySignal::Term => signaler.term(), + BoundarySignal::Kill => signaler.kill(), + BoundarySignal::Int => signaler.interrupt(), + BoundarySignal::Hup => signaler.hangup(), + }; + result.map_err(|e| BackendError::Process(e.to_string())) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.runtime.ensure_active()?; + if self.terminal.load(Ordering::Acquire) { + return Err(BackendError::Terminated("agent has exited".to_string())); + } + let Some(signaler) = self.signaler.as_ref() else { + return Ok(()); + }; + signaler + .kill() + .map_err(|e| BackendError::Process(e.to_string())) + } +} + +// ============================================================================ +/// In-pod mediation source: owns the listener and resolves trusted procfs +/// identity for each accepted TCP connection before handing it to mediation. +/// A stronger backend may use another resolution mechanism without changing +/// the mediation contract. +struct InPodNetworkMediationSource { + listener: tokio::net::TcpListener, + identity: ProcfsIdentityResolver, + runtime: Arc, +} + +#[async_trait] +impl NetworkMediationSource for InPodNetworkMediationSource { + async fn accept(&self) -> Result { + self.runtime.ensure_active()?; + let (stream, workload_addr) = self + .listener + .accept() + .await + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + self.runtime.ensure_active()?; + let proxy_addr = stream + .local_addr() + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + let resolver = self.identity.clone(); + let binary_identity = tokio::task::spawn_blocking(move || { + resolver.resolve_connection(workload_addr, proxy_addr) + }) + .await + .map_err(|error| BackendError::Unavailable(error.to_string()))?; + self.runtime.ensure_active()?; + Ok(MediatedConnection { + stream: Box::new(stream), + binary_identity, + }) + } +} + +/// A topology with no proxy listener has no mediated connections. Calling +/// `accept` is an orchestration error, so fail closed rather than fabricating a +/// connection. +struct InactiveNetworkMediationSource { + runtime: Arc, +} + +#[async_trait] +impl NetworkMediationSource for InactiveNetworkMediationSource { + async fn accept(&self) -> Result { + self.runtime.ensure_active()?; + Err(BackendError::Unavailable( + "network mediation is inactive for this admitted network mode".to_string(), + )) + } +} + +// ============================================================================ +// GCE metadata loopback server +// ============================================================================ + +/// Bring up the GCE metadata loopback server inside the network namespace, +/// stripping the GCE env vars from the agent's environment if it fails so the +/// Go SDK falls back cleanly. +#[cfg(target_os = "linux")] +async fn ensure_gce_metadata_server(config: &InPodConfig, ns: &NetworkNamespace) { + use std::time::Duration; + use tokio::time::timeout; + use tracing::{info, warn}; + + if !config + .provider_credentials + .snapshot() + .child_env + .contains_key("GCE_METADATA_HOST") + { + return; + } + + let ctx = + crate::google_cloud_metadata::MetadataContext::new(config.provider_credentials.clone()); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + match ns + .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) + .await + { + Ok(listener) => { + tokio::spawn(crate::metadata_server::run(listener, ctx, ready_tx)); + if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { + info!(addr = %addr, "GCE metadata loopback server ready"); + } else { + warn!("GCE metadata server failed to become ready, removing metadata env vars"); + strip_gce_env(config); + } + } + Err(e) => { + warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); + strip_gce_env(config); + } + } +} + +/// Remove the GCE metadata env vars from both the agent's child env and the +/// provider credential state. +#[cfg(target_os = "linux")] +fn strip_gce_env(config: &InPodConfig) { + let mut env = config.provider_env.lock().expect("provider_env lock"); + env.remove("GCE_METADATA_HOST"); + env.remove("GCE_METADATA_IP"); + env.remove("METADATA_SERVER_DETECTION"); + drop(env); + config + .provider_credentials + .remove_env_key("GCE_METADATA_HOST"); +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_isolation::AgentSpec; + use openshell_isolation::contract::{BackendRegistry, TopologyDescriptor}; + + /// A minimal in-pod config with networking disabled. The process leaf is + /// declared available so `confirm` can certify launch readiness, but tests + /// that use this fixture do not call `start_agent`. + fn minimal_config() -> InPodConfig { + InPodConfig { + require_exclusive_pid_namespace: false, + network_enabled: false, + process_enabled: true, + entrypoint_pid: Arc::new(AtomicU32::new(0)), + provider_credentials: ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + HashMap::new(), + ), + provider_env: Mutex::new(HashMap::new()), + process_enforcement_mode: ProcessEnforcementMode::Full, + resolved_process_identity: ResolvedProcessIdentity::new( + Some(nix::unistd::geteuid().as_raw()), + Some(nix::unistd::getegid().as_raw()), + ), + workspace: ResolvedWorkspace::default(), + agent_proposals: AgentProposals::new(false), + openshell_endpoint: None, + ssh_socket_path: None, + #[cfg(target_os = "linux")] + bypass_denial_tx: Mutex::new(None), + #[cfg(target_os = "linux")] + bypass_activity_tx: Mutex::new(None), + mediation_ready: Arc::new(AtomicBool::new(true)), + ca_file_paths: Arc::new(Mutex::new(None)), + proxy_bind_ip: Arc::new(Mutex::new(None)), + } + } + + fn block_mode_policy() -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Block, + proxy: None, + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + fn descriptor() -> TopologyDescriptor { + TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: IN_POD_BACKEND_NAME.to_string(), + payload: Vec::new(), + } + } + + fn sandbox_context() -> SandboxContext { + SandboxContext { + sandbox_id: "test-sandbox".to_string(), + policy: block_mode_policy(), + agent: AgentSpec { + program: "true".to_string(), + args: vec![], + workdir: None, + timeout_secs: 0, + interactive: false, + }, + } + } + + // ----- Backend and registry ----- + + #[test] + fn backend_speaks_the_version() { + let backend = InPodBackend::new(minimal_config()); + assert_eq!(backend.backend_name(), IN_POD_BACKEND_NAME); + assert_eq!(backend.version(), INTERFACE_VERSION); + } + + #[test] + fn registry_selects_in_pod_backend() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, _verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + assert_eq!(backend.backend_name(), IN_POD_BACKEND_NAME); + } + + #[test] + fn registry_rejects_duplicate_in_pod() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("first register"); + assert!( + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .is_err() + ); + } + + #[test] + fn registry_rejects_admission_mismatch() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + // The descriptor names in-pod, but admission expects a different backend. + assert!( + registry + .resolve(descriptor(), "some-other-backend") + .map(|_| ()) + .is_err() + ); + } + + // ----- Lifecycle (no root / no netns) ----- + + /// Drive the real in-pod chain attach -> Bound -> confirm -> Ready and prove + /// the retained mediation source survives the consuming transitions. + #[tokio::test] + async fn lifecycle_reaches_ready_and_retains_source() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let source = bound.network_mediation_source(); + let _ready = bound.confirm().await.expect("confirm"); + // Block mode has no connection source, so a retained accept fails + // closed after `Bound` is consumed. + assert!(source.accept().await.is_err()); + } + + #[tokio::test] + async fn live_source_accepts_stream_and_carries_fail_closed_identity() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let source = Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }, + runtime: openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(), + }); + let client = tokio::spawn(async move { + let mut stream = tokio::net::TcpStream::connect(address).await.unwrap(); + stream.write_all(b"ping").await.unwrap(); + }); + let mut connection = source.accept().await.expect("accept"); + assert!(connection.binary_identity.is_err()); + let mut bytes = [0_u8; 4]; + connection.stream.read_exact(&mut bytes).await.unwrap(); + assert_eq!(&bytes, b"ping"); + client.await.unwrap(); + } + + #[tokio::test] + async fn pending_source_accept_rejects_connection_after_boundary_end() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let source = Arc::new(InPodNetworkMediationSource { + listener, + identity: ProcfsIdentityResolver { + entrypoint_pid: Arc::new(AtomicU32::new(0)), + }, + runtime: runtime.clone(), + }); + let pending = tokio::spawn({ + let source = source.clone(); + async move { source.accept().await } + }); + tokio::task::yield_now().await; + runtime.deactivate(); + let _client = tokio::net::TcpStream::connect(address).await.unwrap(); + assert!(matches!( + pending.await.unwrap(), + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn mediation_source_failure_is_fail_static_without_ending_the_boundary() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let source = InactiveNetworkMediationSource { + runtime: runtime.clone(), + }; + + assert!(matches!( + source.accept().await, + Err(BackendError::Unavailable(_)) + )); + runtime + .ensure_active() + .expect("network failure must leave Running active"); + assert!(matches!( + source.accept().await, + Err(BackendError::Unavailable(_)) + )); + } + + /// `attach` is atomic and never binds a resource that is already bound to an + /// active boundary: the in-pod resource binds exactly once. + #[tokio::test] + async fn second_attach_is_denied() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let _bound = backend + .attach(verified, sandbox_context()) + .await + .expect("first attach"); + + let (_backend2, verified2) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("re-resolve"); + let err = backend + .attach(verified2, sandbox_context()) + .await + .map(|_| ()) + .expect_err("second attach must fail"); + assert!(matches!(err, BackendError::Denied(_))); + } + + /// The in-pod payload is empty by construction; a non-empty payload is a + /// descriptor error, validated by the backend at `attach`. + #[tokio::test] + async fn non_empty_payload_is_rejected() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let bad = TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: IN_POD_BACKEND_NAME.to_string(), + payload: vec![1, 2, 3], + }; + let (backend, verified) = registry.resolve(bad, IN_POD_BACKEND_NAME).expect("resolve"); + let err = backend + .attach(verified, sandbox_context()) + .await + .map(|_| ()) + .expect_err("payload must be rejected"); + assert!(matches!(err, BackendError::Descriptor(_))); + } + + #[tokio::test] + async fn unrestricted_network_mode_is_not_admitted() { + let backend = Arc::new(InPodBackend::new(minimal_config())); + let mut registry = BackendRegistry::new(); + registry.register(backend.clone()).expect("register"); + let (_resolved, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let mut sandbox = sandbox_context(); + sandbox.policy.network.mode = NetworkMode::Allow; + let error = backend + .attach(verified, sandbox) + .await + .map(|_| ()) + .expect_err("unrestricted egress cannot conform"); + assert_eq!( + error.kind(), + openshell_isolation::contract::BackendErrorKind::Denied + ); + } + + #[tokio::test] + async fn confirm_rejects_missing_launch_control_leaf() { + let mut config = minimal_config(); + config.process_enabled = false; + let backend = Arc::new(InPodBackend::new(config)); + let mut registry = BackendRegistry::new(); + registry.register(backend.clone()).expect("register"); + let (_resolved, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let error = bound + .confirm() + .await + .map(|_| ()) + .expect_err("Ready requires launch controls"); + assert!(matches!(error, BackendError::Confirm(_))); + } + + #[tokio::test] + async fn failed_agent_start_ends_boundary_and_invalidates_retained_source() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let mut sandbox = sandbox_context(); + sandbox.agent.program = "/definitely/missing/openshell-agent".to_string(); + let bound = backend.attach(verified, sandbox).await.expect("attach"); + let source = bound.network_mediation_source(); + let ready = bound.confirm().await.expect("confirm"); + assert!(matches!( + ready.start_agent().await, + Err(BackendError::Process(_)) + )); + assert!(matches!( + source.accept().await, + Err(BackendError::Terminated(_)) + )); + } + + #[tokio::test] + async fn normal_agent_exit_invalidates_runtime_interfaces() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(InPodBackend::new(minimal_config()))) + .expect("register"); + let (backend, verified) = registry + .resolve(descriptor(), IN_POD_BACKEND_NAME) + .expect("resolve"); + let bound = backend + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let source = bound.network_mediation_source(); + let ready = bound.confirm().await.expect("confirm"); + let running = ready.start_agent().await.expect("start agent"); + let agent = running.agent(); + let exec = running.exec(); + let forward = running.port_forward(); + + assert_eq!( + agent.wait().await.expect("normal exit"), + BoundaryExitStatus::Exited(0) + ); + assert!(matches!( + exec.exec(openshell_isolation::contract::ExecSpec { + program: "/bin/true".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }) + .await, + Err(BackendError::Terminated(_)) + )); + let target = openshell_isolation::contract::LoopbackTarget::new( + std::net::Ipv4Addr::LOCALHOST.into(), + 1, + ) + .expect("loopback target"); + assert!(matches!( + forward.connect(target).await, + Err(BackendError::Terminated(_)) + )); + assert!(matches!( + source.accept().await, + Err(BackendError::Terminated(_)) + )); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn enforcement_monitor_terminates_boundary_after_verification_loss() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + if healthy.load(Ordering::Acquire) { + Ok(()) + } else { + Err("test enforcement loss".to_string()) + } + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(5), + verify, + false, + ) + .await + .expect("initial enforcement verification"); + tokio::time::sleep(std::time::Duration::from_millis(15)).await; + runtime.ensure_active().expect("healthy enforcement"); + + healthy.store(false, Ordering::Release); + tokio::time::timeout(std::time::Duration::from_millis(100), async { + while runtime.is_active() { + tokio::task::yield_now().await; + } + }) + .await + .expect("monitor must terminate within its bound"); + assert!(runtime.ensure_active().is_err()); + assert!(runtime.enforcement_was_lost()); + } + + #[cfg(unix)] + #[tokio::test] + async fn enforcement_loss_kills_registered_workload_within_bound() { + use std::os::unix::process::CommandExt as _; + + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let mut command = std::process::Command::new("/bin/sleep"); + command.arg("30").process_group(0); + let mut child = command.spawn().expect("spawn workload process"); + runtime + .register_process_group( + child.id(), + Arc::new(AtomicBool::new(false)), + Arc::new(Mutex::new(())), + ) + .expect("register workload process group"); + + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + healthy + .load(Ordering::Acquire) + .then_some(()) + .ok_or_else(|| "test enforcement loss".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(5), + verify, + false, + ) + .await + .expect("initial verification"); + healthy.store(false, Ordering::Release); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) => {} + Err(error) if error.raw_os_error() == Some(nix::libc::ECHILD) => break, + Err(error) => panic!("wait workload: {error}"), + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("workload must terminate within the topology bound"); + assert!(runtime.enforcement_was_lost()); + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "requires privileged PID namespace creation"] + #[allow(unsafe_code)] + #[allow( + clippy::zombie_processes, + reason = "PID 1 exits to make the kernel reap this deliberately unregistered descendant" + )] + fn pid_namespace_exit_helper() { + use std::io::Write as _; + use std::os::unix::process::CommandExt as _; + + let Some(ready_path) = std::env::var_os("OPENSHELL_PIDNS_TEST_READY") else { + return; + }; + let trigger_path = std::env::var_os("OPENSHELL_PIDNS_TEST_TRIGGER") + .expect("PID namespace helper trigger path"); + let identity_socket = std::env::var_os("OPENSHELL_PIDNS_TEST_SOCKET") + .expect("PID namespace helper identity socket"); + assert_eq!(std::process::id(), 1, "helper must be PID 1"); + std::os::unix::net::UnixStream::connect(&identity_socket) + .expect("connect PID 1 identity socket") + .write_all(b"pid1") + .expect("publish PID 1 identity"); + let mut command = + std::process::Command::new(std::env::current_exe().expect("current test executable")); + command + .args([ + "--ignored", + "--exact", + "inpod::tests::pid_namespace_descendant_helper", + "--nocapture", + ]) + .env("OPENSHELL_PIDNS_TEST_SOCKET", &identity_socket); + // SAFETY: `setsid` is async-signal-safe and has no captured state. + unsafe { + command.pre_exec(|| { + if nix::libc::setsid() >= 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + let _unregistered_descendant = command.spawn().expect("spawn setsid descendant"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .build() + .expect("build monitor runtime"); + runtime.block_on(async move { + let boundary = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let healthy = Arc::new(AtomicBool::new(true)); + let verify: EnforcementCheck = { + let healthy = healthy.clone(); + Arc::new(move || { + let healthy = healthy.clone(); + Box::pin(async move { + healthy + .load(Ordering::Acquire) + .then_some(()) + .ok_or_else(|| "privileged test enforcement loss".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + boundary, + std::time::Duration::from_millis(5), + verify, + true, + ) + .await + .expect("start enforcement monitor"); + std::fs::write(&ready_path, b"ready").expect("publish helper readiness"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !std::path::Path::new(&trigger_path).exists() { + assert!( + std::time::Instant::now() < deadline, + "parent did not trigger enforcement loss" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + healthy.store(false, Ordering::Release); + std::future::pending::<()>().await; + }); + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "helper for privileged PID namespace test"] + fn pid_namespace_descendant_helper() { + use std::io::Write as _; + + let Some(socket_path) = std::env::var_os("OPENSHELL_PIDNS_TEST_SOCKET") else { + return; + }; + std::os::unix::net::UnixStream::connect(socket_path) + .expect("connect descendant identity socket") + .write_all(b"descendant") + .expect("publish descendant identity"); + loop { + std::thread::park(); + } + } + + #[cfg(target_os = "linux")] + #[test] + #[ignore = "requires privileged PID namespace creation"] + fn pid_one_exit_kills_unregistered_setsid_descendant_within_bound() { + fn process_is_running(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + let Some((_, fields)) = stat.rsplit_once(") ") else { + return false; + }; + !matches!(fields.as_bytes().first(), Some(b'Z' | b'X')) + } + + let tempdir = tempfile::tempdir().expect("tempdir"); + let ready = tempdir.path().join("ready"); + let trigger = tempdir.path().join("trigger"); + let identity_socket = tempdir.path().join("identity.sock"); + let listener = + std::os::unix::net::UnixListener::bind(&identity_socket).expect("bind identity socket"); + listener + .set_nonblocking(true) + .expect("set identity socket nonblocking"); + let current_exe = std::env::current_exe().expect("current test executable"); + let mut namespace = std::process::Command::new("unshare") + .args(["--mount", "--pid", "--fork", "--kill-child", "--mount-proc"]) + .arg(current_exe) + .args([ + "--ignored", + "--exact", + "inpod::tests::pid_namespace_exit_helper", + "--nocapture", + ]) + .env("OPENSHELL_PIDNS_TEST_READY", &ready) + .env("OPENSHELL_PIDNS_TEST_TRIGGER", &trigger) + .env("OPENSHELL_PIDNS_TEST_SOCKET", &identity_socket) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("start isolated PID namespace"); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut isolated = Vec::new(); + while !ready.exists() || isolated.len() < 2 { + match listener.accept() { + Ok((stream, _)) => { + use nix::sys::socket::{getsockopt, sockopt::PeerCredentials}; + + let credentials = getsockopt(&stream, PeerCredentials) + .expect("read namespaced process credentials"); + isolated.push(u32::try_from(credentials.pid()).expect("positive peer PID")); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {} + Err(error) => panic!("accept identity connection: {error}"), + } + if let Some(status) = namespace.try_wait().expect("poll namespace") { + let stderr = namespace + .stderr + .take() + .and_then(|mut stderr| { + use std::io::Read as _; + let mut output = String::new(); + stderr.read_to_string(&mut output).ok()?; + Some(output) + }) + .unwrap_or_default(); + panic!("PID namespace helper exited early ({status}): {stderr}"); + } + assert!( + std::time::Instant::now() < deadline, + "helper readiness timeout" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + + assert_eq!( + isolated.len(), + 2, + "expected PID 1 and its setsid descendant" + ); + std::fs::write(&trigger, b"exit").expect("trigger PID 1 exit"); + let status = namespace.wait().expect("wait for namespace exit"); + assert_eq!(status.code(), Some(125)); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while isolated.iter().copied().any(process_is_running) { + assert!( + std::time::Instant::now() < deadline, + "namespace descendant survived the documented termination bound" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + + #[tokio::test] + async fn wait_reports_enforcement_loss_as_terminated() { + let process = InPodAgentProcess { + signaler: None, + result: Arc::new(Mutex::new(Some(Err(StableWaitError::EnforcementLost)))), + exited: Arc::new(tokio::sync::Notify::new()), + terminal: Arc::new(AtomicBool::new(true)), + runtime: openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(), + }; + + let error = process + .wait() + .await + .expect_err("enforcement loss is abnormal"); + assert_eq!( + error.kind(), + openshell_isolation::contract::BackendErrorKind::Terminated + ); + } + + #[tokio::test] + async fn normal_teardown_is_not_reclassified_by_inflight_verification() { + let runtime = openshell_supervisor_process::boundary_io::BoundaryRuntimeState::new(); + let calls = Arc::new(AtomicU32::new(0)); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let verify: EnforcementCheck = { + let calls = calls.clone(); + let entered = entered.clone(); + let release = release.clone(); + Arc::new(move || { + let calls = calls.clone(); + let entered = entered.clone(); + let release = release.clone(); + Box::pin(async move { + if calls.fetch_add(1, Ordering::AcqRel) == 0 { + return Ok(()); + } + entered.notify_one(); + release.notified().await; + Err("verification completed after teardown".to_string()) + }) + }) + }; + let _monitor = start_enforcement_monitor( + runtime.clone(), + std::time::Duration::from_millis(1), + verify, + false, + ) + .await + .expect("initial verification"); + entered.notified().await; + runtime.deactivate(); + release.notify_one(); + tokio::task::yield_now().await; + + assert!(!runtime.enforcement_was_lost()); + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b1c226cebd..520720deb0 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -9,6 +9,7 @@ mod activity_aggregator; mod denial_aggregator; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod google_cloud_metadata; +mod inpod; mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; @@ -116,6 +117,7 @@ pub async fn run_sandbox( network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: Option, ) -> Result { let (program, args) = command .split_first() @@ -145,7 +147,12 @@ pub async fn run_sandbox( } let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); + let admitted_topology = topology_descriptor.is_some(); + let process_enforcement_mode = if admitted_topology { + ProcessEnforcementMode::Full + } else { + process_enforcement_mode() + }; let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; let mut process_control_connection = None; @@ -390,7 +397,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { + let netns = if network_enabled && !sidecar_network_enforcement && !admitted_topology { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -403,6 +410,12 @@ pub async fn run_sandbox( .transpose()? .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); #[cfg(target_os = "linux")] + if admitted_topology && transparent_tcp_requested { + return Err(miette::miette!( + "the RFC 0012 in-pod prototype does not yet compose its strict egress ceiling with policy DNS and transparent TCP" + )); + } + #[cfg(target_os = "linux")] let runtime_capabilities = std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); #[cfg(target_os = "linux")] @@ -516,7 +529,98 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - let mut networking = if network_enabled { + let mut admitted_ready: Option> = None; + let mut networking = if let Some(descriptor) = topology_descriptor { + if sidecar_network_enforcement || !network_enabled || !process_enabled { + return Err(miette::miette!( + "an admitted isolation backend requires the co-located network,process topology" + )); + } + let mediation_ready = Arc::new(AtomicBool::new(false)); + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let proxy_bind_ip = Arc::new(std::sync::Mutex::new(None)); + let backend = Arc::new(inpod::InPodBackend::new(inpod::InPodConfig { + require_exclusive_pid_namespace: true, + network_enabled, + process_enabled, + entrypoint_pid: entrypoint_pid.clone(), + provider_credentials: provider_credentials.clone(), + provider_env: std::sync::Mutex::new(provider_env.clone()), + process_enforcement_mode, + resolved_process_identity, + workspace: workspace.clone(), + agent_proposals: agent_proposals.clone(), + openshell_endpoint: openshell_endpoint_for_proxy.clone(), + ssh_socket_path: ssh_socket_path.clone(), + #[cfg(target_os = "linux")] + bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx.clone()), + #[cfg(target_os = "linux")] + bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx.clone()), + mediation_ready: mediation_ready.clone(), + ca_file_paths: ca_file_paths.clone(), + proxy_bind_ip: proxy_bind_ip.clone(), + })); + let mut registry = openshell_isolation::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(descriptor, inpod::IN_POD_BACKEND_NAME) + .map_err(|error| miette::miette!(error.to_string()))?; + let bound = backend + .attach( + verified, + openshell_isolation::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workdir.clone(), + timeout_secs, + interactive, + }, + }, + ) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let source = bound.network_mediation_source(); + let bind_ip = *proxy_bind_ip.lock().expect("proxy bind IP lock"); + let networking = openshell_supervisor_network::run::run_networking( + &policy, + bind_ip, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + process_enabled, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + Some(source), + #[cfg(target_os = "linux")] + None, + ) + .await?; + ca_file_paths + .lock() + .expect("ca paths lock") + .clone_from(&networking.ca_file_paths); + mediation_ready.store(true, Ordering::Release); + admitted_ready = Some( + bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?, + ); + Some(networking) + } else if network_enabled { #[cfg(target_os = "linux")] let proxy_bind_ip = netns .as_ref() @@ -542,6 +646,7 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, + None, #[cfg(target_os = "linux")] transparent_runtime, ) @@ -839,36 +944,57 @@ pub async fn run_sandbox( tokio::pin!(proxy_exited); let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None + if let Some(ready) = admitted_ready.take() { + let running = ready + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + let agent = running.agent(); + let exit = tokio::select! { + result = agent.wait() => result.map_err(|error| miette::miette!(error.to_string()))?, + () = &mut proxy_exited => { + return Err(miette::miette!("RFC boundary mediation exited unexpectedly")); } - }); - - let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx { - Box::pin(async { - let _ = rx.await; - }) + }; + match exit { + openshell_isolation::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + } } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(ssh_exited); + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } + }); - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + (Some(tx), Some(rx)) + } else { + (None, None) + }; + let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx + { + Box::pin(async { + let _ = rx.await; + }) + } else { + Box::pin(std::future::pending()) + }; + tokio::pin!(ssh_exited); + + let entrypoint_started_tx = if process_uses_sidecar_control + && let Some(writer) = process_control_writer.clone() + { let (tx, rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { match rx.await { @@ -889,8 +1015,9 @@ pub async fn run_sandbox( } else { None }; - let sidecar_exit_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let sidecar_exit_tx = if process_uses_sidecar_control + && let Some(writer) = process_control_writer.clone() + { let exit_ack = Arc::clone(&process_exit_ack); let (tx, mut rx) = tokio::sync::mpsc::channel::< openshell_supervisor_process::run::SidecarExitReport, @@ -919,116 +1046,117 @@ pub async fn run_sandbox( None }; - let process = openshell_supervisor_process::run::run_process( - program, - args, - workspace, - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - ssh_exit_tx, - &process_policy, - resolved_process_identity, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - sidecar_exit_tx, - provider_credentials, - main_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); + let process = openshell_supervisor_process::run::run_process( + program, + args, + workspace, + timeout_secs, + interactive, + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path, + sidecar_network_enforcement, + ssh_exit_tx, + &process_policy, + resolved_process_identity, + process_enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + sidecar_exit_tx, + provider_credentials, + main_env, + ca_file_paths, + agent_proposals.clone(), + #[cfg(target_os = "linux")] + netns.as_ref(), + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + ); - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } - } else { - tokio::select! { - result = process => result?, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); + if let Some(control_closed) = process_control_closed.as_mut() { + tokio::select! { + result = process => result?, + _ = control_closed => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating process container" + ) + .build() + ); + return Err(miette::miette!( + "authoritative network-sidecar control channel closed" + )); + } + () = &mut proxy_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Proxy accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "proxy accept loop exited unexpectedly" + )); + } + () = &mut ssh_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "SSH accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "SSH accept loop exited unexpectedly" + )); + } } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); + } else { + tokio::select! { + result = process => result?, + () = &mut proxy_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Proxy accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "proxy accept loop exited unexpectedly" + )); + } + () = &mut ssh_exited => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "SSH accept loop exited unexpectedly; terminating sandbox" + ) + .build() + ); + return Err(miette::miette!( + "SSH accept loop exited unexpectedly" + )); + } } } } diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 6d244fb6bc..99f69301fe 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -235,6 +235,18 @@ struct Args { /// re-signed upstream certificates and the sandbox trust bundle. #[arg(long)] upstream_proxy_ca_bundle: Option, + + /// Backend selected by the compute driver's admitted topology descriptor. + #[arg(long)] + topology_backend_name: Option, + + /// Isolation Backend interface version. + #[arg(long)] + topology_version: Option, + + /// Base64-encoded opaque backend payload. + #[arg(long)] + topology_payload_base64: Option, } /// Internal one-shot command used by the privileged supervisor to validate an @@ -683,6 +695,28 @@ fn main() -> Result<()> { proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, proxy_ca_bundle: args.upstream_proxy_ca_bundle, }; + let topology_descriptor = match ( + args.topology_backend_name, + args.topology_version, + args.topology_payload_base64, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => { + use base64::Engine as _; + Some(openshell_isolation::contract::TopologyDescriptor { + backend_name, + version, + payload: base64::engine::general_purpose::STANDARD + .decode(payload) + .into_diagnostic()?, + }) + } + _ => { + return Err(miette::miette!( + "topology descriptor requires backend name, version, and payload" + )); + } + }; run_sandbox( command, @@ -702,6 +736,7 @@ fn main() -> Result<()> { args.mode.network, args.mode.process, upstream_proxy_args, + topology_descriptor, ) .await })?; diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index dc2736a4ea..56c0703ed6 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -24,6 +24,10 @@ use openshell_core::net::{ use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + ResolveError, +}; use openshell_ocsf::{ ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, @@ -36,9 +40,17 @@ use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; use tokio::net::{TcpListener, TcpStream}; + +type ProxyClient = tokio::io::BufReader; + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation::contract::BackendError), +} use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; @@ -233,6 +245,7 @@ pub struct ProxyHandle { http_addr: Option, join: JoinHandle<()>, exited_rx: Option>, + source_failure: tokio::sync::watch::Receiver>, } impl ProxyHandle { @@ -255,6 +268,7 @@ impl ProxyHandle { activity_tx: Option, engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, + network_mediation_source: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -270,8 +284,15 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let listener = if network_mediation_source.is_none() { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + } else { + None + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) @@ -338,6 +359,7 @@ impl ProxyHandle { } let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); + let (source_failure_tx, source_failure) = tokio::sync::watch::channel(None); let join = tokio::spawn(async move { // Hold the sender for the lifetime of this task — when the task // exits (panic, abort, or loop break), the sender drops and the @@ -371,11 +393,34 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + source + .accept() + .await + .map(|connection| { + (connection.stream, Some(connection.binary_identity), None) + }) + .map_err(ProxyAcceptError::Source) + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + set_tcp_nodelay_best_effort(&stream); + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr)) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -398,8 +443,10 @@ impl ProxyHandle { let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, opa, cache, spid, @@ -427,7 +474,18 @@ impl ProxyHandle { } }); } - Err(err) => { + Err(ProxyAcceptError::Source(err)) => { + let _ = source_failure_tx.send(Some(err.to_string())); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!("Network-mediation source failed: {err}")) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(err)) => { match classify_accept_error( &err, &mut consecutive_resource_errors, @@ -468,6 +526,7 @@ impl ProxyHandle { http_addr: Some(local_addr), join, exited_rx: Some(exited_rx), + source_failure, }) } @@ -479,6 +538,20 @@ impl ProxyHandle { pub fn take_exit_receiver(&mut self) -> Option> { self.exited_rx.take() } + + /// Wait until a backend-provided mediation source fails terminally. + pub async fn wait_for_source_failure(&self) -> String { + let mut receiver = self.source_failure.clone(); + loop { + let current = receiver.borrow().clone(); + if let Some(error) = current { + return error; + } + if receiver.changed().await.is_err() { + return "network mediation stopped".to_string(); + } + } + } } impl Drop for ProxyHandle { @@ -1192,21 +1265,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1585,7 +1661,7 @@ fn build_forward_destination_deny_ocsf_event( #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1631,7 +1707,7 @@ async fn deny_connect_destination( #[allow(clippy::too_many_arguments)] async fn deny_forward_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1681,9 +1757,58 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + trusted_host_gateway, + upstream_proxy, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1757,6 +1882,8 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, @@ -1803,22 +1930,27 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - // Evaluate OPA policy with process-identity binding. - // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: - // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -2205,7 +2337,7 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; @@ -2780,6 +2912,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { + intent, + action, + policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2825,7 +3028,7 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. async fn handle_inference_interception( - client: TcpStream, + client: ProxyClient, host: &str, port: u16, tls_state: Option<&Arc>, @@ -3420,7 +3623,7 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette } async fn reject_stale_connect_policy( - client: &mut TcpStream, + client: &mut ProxyClient, host: &str, port: u16, activity_tx: Option<&ActivitySender>, @@ -4818,7 +5021,9 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -4922,19 +5127,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -6160,7 +6373,7 @@ fn normalize_host(raw_host: &str) -> &str { raw_host.strip_suffix('.').unwrap_or(raw_host) } -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; Ok(()) } @@ -6301,7 +6514,7 @@ const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (C /// refused (the caller must stop) and `false` when the caller should proceed to /// establish the tunnel. async fn refuse_connect_when_tls_unavailable( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), tls_state_present: bool, effective_tls_skip: bool, ) -> Result { @@ -6497,6 +6710,22 @@ mod tests { } } + struct FailedMediationSource; + + #[async_trait::async_trait] + impl NetworkMediationSource for FailedMediationSource { + async fn accept( + &self, + ) -> std::result::Result< + openshell_isolation::contract::MediatedConnection, + openshell_isolation::contract::BackendError, + > { + Err(openshell_isolation::contract::BackendError::Unavailable( + "test source unavailable".to_string(), + )) + } + } + async fn drive_raw_request_through_handler(raw: Vec) -> Vec { let policy = include_str!("../data/sandbox-policy.rego"); let data = r#" @@ -6541,6 +6770,50 @@ network_policies: {} client.await.unwrap() } + #[tokio::test] + async fn terminal_mediation_source_failure_stops_accepting_fail_static() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, + ) + .expect("engine"), + ); + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + Some(Arc::new(FailedMediationSource)), + ) + .await + .expect("proxy starts before source accept"); + + let failure = tokio::time::timeout( + std::time::Duration::from_secs(1), + handle.wait_for_source_failure(), + ) + .await + .expect("source failure must be observed"); + assert!(failure.contains("test source unavailable")); + assert!( + handle.join.is_finished(), + "accept loop must stop after source loss" + ); + } + #[tokio::test] async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { for host in ["api.example.com", "unmatched.example.com"] { @@ -7294,6 +7567,59 @@ network_policies: ); } + #[test] + fn backend_supplied_unresolved_identity_denies_an_allowed_endpoint() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, true) + .expect("identity-aware engine"); + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert!(matches!( + decision.identity, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed) + )); + } + + #[test] + fn backend_supplied_unresolved_identity_denies_in_endpoint_only_mode() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) + .expect("endpoint-only engine"); + temp_env::with_var( + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + Some("false"), + || { + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + }, + ); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, @@ -7354,13 +7680,14 @@ network_policies: let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); let (server, _) = listener.accept().await.unwrap(); + let mut server = tokio::io::BufReader::new(server); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut server) .await .expect("peek should succeed") .expect("client sent bytes"); @@ -10243,9 +10570,10 @@ network_policies: #[tokio::test] async fn test_resolve_check_allowed_ips_rejects_outside_allowlist() { - // 8.8.8.8 resolves to a public IP which is NOT in 10.0.0.0/8 + // A public IP outside 10.0.0.0/8 must be rejected. Use the literal so + // this security check does not depend on external DNS availability. let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("dns.google", 443, &nets, 0).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 443, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -11644,9 +11972,9 @@ network_policies: #[tokio::test] async fn test_forward_public_ip_allowed_without_allowed_ips() { - // Public IPs (e.g. dns.google -> 8.8.8.8) should pass through - // resolve_and_reject_internal without needing allowed_ips. - let result = resolve_and_reject_internal("dns.google", 80, 0).await; + // Public IPs should pass through resolve_and_reject_internal without + // needing allowed_ips. Use a literal to keep the test hermetic. + let result = resolve_and_reject_internal("8.8.8.8", 80, 0).await; assert!( result.is_ok(), "Public IP should be allowed without allowed_ips: {result:?}" @@ -11657,7 +11985,7 @@ network_policies: for addr in &addrs { assert!( !is_internal_ip(addr.ip()), - "dns.google should resolve to public IPs, got {}", + "expected a public IP, got {}", addr.ip() ); } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 2a71702b4b..b50b7800a6 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,6 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +use openshell_isolation::contract::NetworkMediationSource; #[cfg(target_os = "linux")] pub struct TransparentRuntimeSetup { @@ -196,6 +197,7 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, + network_mediation_source: Option>, #[cfg(target_os = "linux")] transparent_runtime: Option, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll @@ -447,6 +449,7 @@ pub async fn run_networking( activity_tx.clone(), engine_ready_rx, upstream_proxy_args, + network_mediation_source, ) .await?; Some(proxy_handle) diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..4f42f10d80 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -53,6 +53,13 @@ pub struct NetworkNamespace { ns_fd: Option, } +/// Cloneable coordinates for continuously verifying a live RFC boundary. +#[derive(Clone, Debug)] +pub struct EgressCeilingVerifier { + namespace: String, + host_ip: IpAddr, +} + impl NetworkNamespace { /// Create a new isolated network namespace with veth pair. /// @@ -249,6 +256,21 @@ impl NetworkNamespace { self.ns_fd } + /// Duplicate the namespace descriptor for a boundary-owned asynchronous + /// operation whose lifetime may outlive this borrow. + pub fn try_clone_ns_fd(&self) -> Result> { + use std::os::fd::FromRawFd; + + let Some(fd) = self.ns_fd else { + return Ok(None); + }; + let duplicated = nix::unistd::dup(fd).into_diagnostic()?; + // nix 0.29 returns a raw descriptor from dup. + Ok(Some(unsafe { + std::os::fd::OwnedFd::from_raw_fd(duplicated) + })) + } + /// Install nftables rules for bypass detection inside the namespace. /// /// Sets up OUTPUT chain rules that: @@ -323,6 +345,30 @@ impl NetworkNamespace { Ok(()) } + /// Install the mandatory RFC default-deny fence. Unlike legacy bypass + /// diagnostics, absence or failure of nftables is fatal. + pub fn install_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + let nft = find_nft().ok_or_else(|| { + miette::miette!("nft not found; cannot establish default-deny egress ceiling") + })?; + let log_prefix = format!("openshell:bypass:{}:", self.name); + enable_nf_log_all_netns(); + let commands = nft_ruleset::generate_egress_ceiling_commands( + &self.host_ip.to_string(), + proxy_port, + Some(&log_prefix), + ); + run_nft_commands_netns(&self.name, &nft, &commands) + } + + #[must_use] + pub fn egress_ceiling_verifier(&self) -> EgressCeilingVerifier { + EgressCeilingVerifier { + namespace: self.name.clone(), + host_ip: self.host_ip, + } + } + /// Replace the ordinary bypass fence with the policy-DNS and transparent /// TCP ruleset. This is fail-closed: callers must not release workload /// execution unless every required rule was installed. @@ -548,6 +594,81 @@ impl NetworkNamespace { } } +impl EgressCeilingVerifier { + /// Read the installed kernel rules under a deadline. Validation requires a + /// policy-drop output chain and explicit proxy/loopback accepts; any other + /// accept in that chain fails closed. + pub async fn verify_bounded( + &self, + proxy_port: u16, + deadline: std::time::Duration, + ) -> Result<()> { + let nft = find_nft().ok_or_else(|| miette::miette!("nft not found"))?; + let nsenter = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; + let net_flag = format!( + "--net={}", + openshell_core::container_paths::netns_path(&self.namespace).display() + ); + let mut command = tokio::process::Command::new(nsenter); + command.kill_on_drop(true).args([ + &net_flag, + "--", + &nft, + "-j", + "list", + "chain", + "inet", + "openshell_bypass", + "output", + ]); + let output = tokio::time::timeout(deadline, command.output()) + .await + .map_err(|_| miette::miette!("egress ceiling verification timed out"))? + .into_diagnostic()?; + if !output.status.success() { + return Err(miette::miette!( + "could not read back egress ceiling: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + verify_egress_ceiling_json(&output.stdout, &self.host_ip.to_string(), proxy_port) + } +} + +fn verify_egress_ceiling_json(json: &[u8], host_ip: &str, proxy_port: u16) -> Result<()> { + let document: serde_json::Value = serde_json::from_slice(json).into_diagnostic()?; + let objects = document + .get("nftables") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("nft response has no object list"))?; + let default_deny = objects.iter().any(|object| { + object.get("chain").is_some_and(|chain| { + chain.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && chain.get("table").and_then(serde_json::Value::as_str) + == Some("openshell_bypass") + && chain.get("name").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("hook").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("policy").and_then(serde_json::Value::as_str) == Some("drop") + }) + }); + if !default_deny { + return Err(miette::miette!( + "egress ceiling output chain is not policy drop" + )); + } + let rendered = serde_json::to_string(&document).into_diagnostic()?; + if !rendered.contains(host_ip) + || !rendered.contains(&proxy_port.to_string()) + || !rendered.contains("oifname") + || !rendered.contains("lo") + { + return Err(miette::miette!( + "egress ceiling is missing the proxy or loopback allow" + )); + } + Ok(()) +} + impl Drop for NetworkNamespace { fn drop(&mut self) { debug!(namespace = %self.name, "Cleaning up network namespace"); @@ -638,6 +759,27 @@ pub fn create_netns_for_proxy( } } +/// Create the RFC in-pod namespace with mandatory standing egress +/// enforcement. The legacy helper remains best-effort for compatibility. +pub fn create_conformant_netns_for_proxy( + policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + use openshell_core::policy::NetworkMode; + + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Ok(None); + } + let namespace = NetworkNamespace::create()?; + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + namespace.install_egress_ceiling(proxy_port)?; + Ok(Some(namespace)) +} + /// Install pod-network bypass enforcement for Kubernetes sidecar topology. /// /// This runs in the current network namespace, not in a per-workload netns. diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 52557493cd..9b815ebbe9 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -2370,6 +2370,16 @@ pub struct ProcessStatus { } impl ProcessStatus { + /// Construct a synthetic normal exit status for supervisor-generated + /// terminal outcomes such as a policy timeout. + #[must_use] + pub const fn exited(code: i32) -> Self { + Self { + code: Some(code), + signal: None, + } + } + /// Get the conventional exit code when the process exited normally. #[must_use] pub const fn exit_code(&self) -> Option { diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index f0e792306d..517167f762 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -57,7 +57,7 @@ fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { /// Returns an error if SSH server startup fails, if the entrypoint child /// fails to spawn, or if waiting for the child returns an OS error. #[allow(clippy::too_many_arguments, clippy::implicit_hasher)] -pub async fn run_process( +pub async fn spawn_workload( program: &str, args: &[String], workspace: ResolvedWorkspace, @@ -83,7 +83,8 @@ pub async fn run_process( tokio::sync::mpsc::UnboundedSender, >, #[cfg(target_os = "linux")] bypass_activity_tx: Option, -) -> Result { + boundary_runtime: Option>, +) -> Result { // Platform drivers with a resolved numeric UID/GID retain the legacy // account-file update. OCI-image identity leaves those environment values // empty, so the image's account files remain unchanged. @@ -266,6 +267,27 @@ pub async fn run_process( let main_session = crate::main_session::MainSession::new(handle.take_io(), main_pid); let main_instance_id = uuid::Uuid::new_v4().to_string(); + #[cfg(target_os = "linux")] + let boundary_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); + #[cfg(not(target_os = "linux"))] + let boundary_netns_fd = None; + let boundary_runtime = + boundary_runtime.unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + boundary_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + // SSH-spawned shells get http_proxy=http://: exported into // their env so cooperative tools (curl, npm, Node) route through the // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back @@ -275,6 +297,20 @@ pub async fn run_process( #[cfg(not(target_os = "linux"))] let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workspace.owned_root(), + boundary_netns_fd, + ssh_proxy_url.clone(), + ca_file_paths.clone().map(Arc::new), + provider_credentials.clone(), + user_environment, + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { let policy_clone = policy.clone(); @@ -382,6 +418,11 @@ pub async fn run_process( // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); + let terminal = Arc::new(AtomicBool::new(false)); + let signal_lock = Arc::new(std::sync::Mutex::new(())); + boundary_runtime + .register_process_group(handle.pid(), terminal.clone(), signal_lock.clone()) + .map_err(|error| miette::miette!(error.to_string()))?; if early_exit.is_none() && let Some(tx) = entrypoint_started_tx { @@ -400,70 +441,250 @@ pub async fn run_process( .build() ); - let outcome = if let Some(status) = early_exit { - ProcessWaitOutcome::Exited(status) - } else { - wait_for_process_exit_or_shutdown(&mut handle, timeout_secs, &supervisor_terminating) - .await? - }; + Ok(SpawnedAgent { + handle, + early_exit, + timeout_secs, + supervisor_terminating, + supervisor_session_task, + main_session, + main_instance_id, + sidecar_exit_tx, + openshell_endpoint: openshell_endpoint.map(str::to_string), + sandbox_id: sandbox_id.map(str::to_string), + boundary_exec, + port_forward, + boundary_runtime, + terminal, + signal_lock, + }) +} - let rendered_code = match outcome { - ProcessWaitOutcome::Exited(status) => status.code(), - ProcessWaitOutcome::TimedOut => { - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message("Process timed out, killing") - .build() - ); - 124 - } - ProcessWaitOutcome::ShutdownSignal { signal, status } => { - info!( - signal, - exit_code = status.code(), - "Entrypoint exited after supervisor shutdown signal" - ); - status.code() +/// Run a command through the legacy orchestration entry point. +/// +/// This is intentionally a thin adapter over the owned RFC lifecycle. Existing +/// callers still receive an exit code, while runtime-selectable backends retain +/// the running process and its boundary capabilities. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn run_process( + program: &str, + args: &[String], + workspace: ResolvedWorkspace, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option, + shared_ssh_socket: bool, + ssh_exit_tx: Option>, + policy: &SandboxPolicy, + resolved_process_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + sidecar_exit_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + agent_proposals: AgentProposals, + #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, + #[cfg(target_os = "linux")] bypass_denial_tx: Option< + tokio::sync::mpsc::UnboundedSender, + >, + #[cfg(target_os = "linux")] bypass_activity_tx: Option, +) -> Result { + let mut spawned = spawn_workload( + program, + args, + workspace, + timeout_secs, + interactive, + sandbox_id, + openshell_endpoint, + ssh_socket_path, + shared_ssh_socket, + ssh_exit_tx, + policy, + resolved_process_identity, + enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + sidecar_exit_tx, + provider_credentials, + provider_env, + ca_file_paths, + agent_proposals, + #[cfg(target_os = "linux")] + netns, + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + None, + ) + .await?; + Ok(spawned.wait().await?.code()) +} + +/// Owned canonical workload plus the boundary capabilities tied to it. +pub struct SpawnedAgent { + handle: ProcessHandle, + early_exit: Option, + timeout_secs: u64, + supervisor_terminating: Arc, + supervisor_session_task: Option>, + main_session: Arc, + main_instance_id: String, + sidecar_exit_tx: Option>, + openshell_endpoint: Option, + sandbox_id: Option, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, + terminal: Arc, + signal_lock: Arc>, +} + +impl SpawnedAgent { + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + AgentSignaler { + pid: self.handle.pid(), + terminal: self.terminal.clone(), + signal_lock: self.signal_lock.clone(), } - }; - supervisor_terminating.store(true, Ordering::Release); - main_session.finish(rendered_code).await; + } - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .exit_code(rendered_code) - .message(format!("Process exited with code {rendered_code}")) - .build() - ); + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } - if let Some(task) = supervisor_session_task { - task.abort(); + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() } - if let Some(tx) = sidecar_exit_tx { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send((main_instance_id.clone(), rendered_code, ack_tx)) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error))?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + + #[must_use] + pub fn boundary_runtime(&self) -> Arc { + self.boundary_runtime.clone() } - Ok(rendered_code) + /// Wait for the canonical process and finalize every existing reporting + /// and retained-I/O obligation before publishing the boundary exit. + pub async fn wait(&mut self) -> Result { + let pid = self.handle.pid(); + let outcome = if let Some(status) = self.early_exit.take() { + ProcessWaitOutcome::Exited(status) + } else { + wait_for_process_exit_or_shutdown( + &mut self.handle, + self.timeout_secs, + &self.supervisor_terminating, + ) + .await? + }; + let status = match outcome { + ProcessWaitOutcome::Exited(status) => status, + ProcessWaitOutcome::TimedOut => { + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Close) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message("Process timed out, killing") + .build() + ); + ProcessStatus::exited(124) + } + ProcessWaitOutcome::ShutdownSignal { signal, status } => { + info!( + signal, + exit_code = status.code(), + "Entrypoint exited after supervisor shutdown signal" + ); + status + } + }; + let rendered_code = status.code(); + self.terminal.store(true, Ordering::Release); + self.boundary_runtime + .unregister_process_group(pid, &self.terminal); + self.boundary_runtime.deactivate(); + self.supervisor_terminating.store(true, Ordering::Release); + self.main_session.finish(rendered_code).await; + + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Close) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .exit_code(rendered_code) + .message(format!("Process exited with code {rendered_code}")) + .build() + ); + if let Some(task) = self.supervisor_session_task.take() { + task.abort(); + } + if let Some(tx) = self.sidecar_exit_tx.take() { + let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + tx.send((self.main_instance_id.clone(), rendered_code, ack_tx)) + .await + .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; + ack_rx + .await + .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? + .map_err(|error| miette::miette!(error))?; + } else if let (Some(endpoint), Some(id)) = ( + self.openshell_endpoint.as_deref(), + self.sandbox_id.as_deref(), + ) { + report_main_process_exit_until_ack(endpoint, id, &self.main_instance_id, rendered_code) + .await; + info!(instance_id = %self.main_instance_id, "main-process exit acknowledged"); + } + Ok(status) + } +} + +/// Concurrent signal handle for a running canonical agent process group. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, +} + +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, signal: nix::sys::signal::Signal) -> Result<()> { + let _guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); + } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(nix::unistd::Pid::from_raw(pid), signal).into_diagnostic() + } + + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) + } } async fn report_main_process_exit_until_ack( diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c77c5c0aff..80c431f6d9 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -6,9 +6,8 @@ # Supervisor image build. # # The final image carries the static `openshell-sandbox` binary used by Docker -# extraction, Podman image volumes, and the Kubernetes init container copy-self -# path. It also includes nftables so the Kubernetes supervisor sidecar can -# install pod-namespace egress enforcement rules. +# extraction, Podman image volumes, and Kubernetes side-loading. It also carries +# a materialized helper runtime used for trusted network setup and enforcement. # # The Rust binary is built natively before this image build runs and staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox @@ -26,7 +25,23 @@ FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 nftables iptables iptables-legacy \ + && mkdir -p /openshell-runtime/usr /openshell-runtime/etc \ + && cp -aL /bin /sbin /lib /openshell-runtime/ \ + && cp -aL /usr/bin /usr/sbin /usr/lib /openshell-runtime/usr/ \ + && if [ -d /etc/iproute2 ]; then \ + cp -aL /etc/iproute2 /openshell-runtime/etc/; \ + fi \ + && if [ -d /usr/share/nftables ]; then \ + mkdir -p /openshell-runtime/usr/share; \ + cp -aL /usr/share/nftables /openshell-runtime/usr/share/; \ + fi \ + && loader="$(find /openshell-runtime/lib -maxdepth 1 -type f -name 'ld-musl-*.so.1' | head -n 1)" \ + && test -n "$loader" \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/sbin/ip -Version \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/usr/sbin/nft --version # --chmod=0555 restores execute bits after the actions/upload-artifact + # download-artifact roundtrip strips them. Ownership stays root (0:0) for diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d1fed9ae44..3f1fa3da9f 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -563,7 +563,7 @@ the SPIRE OIDC discovery endpoint or its TLS CA. ### Docker -Sandboxes run as containers on a local bridge network. The supervisor binary is bind-mounted from the host (no in-cluster image pull required); guest mTLS material is supplied as host paths. +Sandboxes run as containers on a local bridge network. The driver supplies RFC 0012's co-located backend descriptor by default. The supervisor binary and its trusted network-helper runtime are bind-mounted read-only from driver-controlled sources; guest mTLS material is supplied as host paths. ```toml [openshell] @@ -582,10 +582,11 @@ image_pull_policy = "IfNotPresent" sandbox_namespace = "docker-dev" # Empty auto-detects https://host.openshell.internal: when guest TLS is set. grpc_endpoint = "https://host.openshell.internal:17670" -# Skip the image-pull-and-extract step by pointing at a locally built binary. +# Use a locally built supervisor binary. If it has no sibling +# openshell-runtime directory, the driver still extracts that runtime from supervisor_image. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" -# When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. -# Defaults to the gateway version; override to pin a specific build. +# Source for /openshell-sandbox and its trusted helper runtime. Defaults to the +# gateway version; override to pin a specific build. # supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" @@ -603,7 +604,7 @@ sandbox_pids_limit = 2048 ### Podman -Sandboxes run as Podman containers on a user-mode bridge network. The supervisor image is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. +Sandboxes run as Podman containers on a user-mode bridge network. The driver supplies RFC 0012's co-located backend descriptor by default. The supervisor image, including its trusted network-helper runtime, is mounted read-only via Podman's `type=image` mount; guest mTLS material is supplied as host paths. ```toml [openshell] @@ -737,7 +738,7 @@ health_check_interval_secs = 10 ### MicroVM -Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. +Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. The driver supplies RFC 0012's co-located backend descriptor by default and embeds its trusted network-helper runtime in the guest bootstrap. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 22ba1b039f..a43d36268f 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -281,7 +281,8 @@ if [ ! -d "${COMPRESSED_DIR}" ] \ mise run vm:setup fi -if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then +if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ] \ + || [ ! -f "${COMPRESSED_DIR}/openshell-runtime.tar.zst" ]; then check_supervisor_cross_toolchain echo "==> Building bundled VM supervisor (mise run vm:supervisor)" mise run vm:supervisor diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index 0085c0619d..a31e999603 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,6 +60,7 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" +SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -123,6 +124,51 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" +case "${GUEST_ARCH}" in + aarch64|arm64) DOCKER_ARCH="arm64" ;; + x86_64|amd64) DOCKER_ARCH="amd64" ;; +esac + +echo "==> Building trusted supervisor helper runtime" +STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" +RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" +mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" +cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" + +case "$(uname -m)" in + aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; + x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; + *) HOST_DOCKER_ARCH="" ;; +esac + +if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then + docker build \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +else + docker buildx build \ + --load \ + --platform "linux/${DOCKER_ARCH}" \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +fi + +RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" +cleanup_runtime_image() { + docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true + docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true +} +trap cleanup_runtime_image EXIT +docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ + | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" +cleanup_runtime_image +trap - EXIT + echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" +echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)"