diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 3fc53df904..78fc40e8a7 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -577,9 +577,32 @@ kubectl -n logs -c openshell-supervisor-networ Use the VM driver logs and host diagnostics available in the user's environment. Verify: - The VM driver process is running and reachable by the gateway. -- The runtime rootfs exists and matches the expected architecture. -- Host virtualization support is enabled. -- The sandbox supervisor can establish its callback connection to the gateway. +- The custom kernel and runtime rootfs exist and match the expected architecture. +- Host virtualization support is enabled and the active process can read and write `/dev/kvm`. +- The native host supervisor can establish its callback connection to the gateway. +- The guest process leaf accepts the authenticated virtio-vsock control channel. + +For the managed libkrun driver, inspect the per-sandbox host-supervisor and +guest-console logs separately: + +```bash +rg -n 'vm|grpc_endpoint|guest_tls|state_dir' .cache/gateway-vm/gateway.toml +stat /dev/kvm +id +find /tmp/openshell-vm-driver-*/sandboxes -name 'supervisor*.log' -o -name 'rootfs-console.log' +tail -n 200 /tmp/openshell-vm-driver-*/sandboxes/*/supervisor.err.log +tail -n 200 /tmp/openshell-vm-driver-*/sandboxes/*/rootfs-console.log +``` + +The historical `guest_tls_*` configuration fields are host-supervisor mTLS +paths and must never appear in the guest image. `grpc_endpoint` is also +host-reachable; loopback is valid. A successful guest boot logs +`VM process supervisor leaf listening on vsock port ...`. libkrun needs KVM but +does not need `CAP_NET_ADMIN`. QEMU/VFIO uses TAP and host nftables and still +requires the corresponding networking and device privileges. If the account is +listed in the group that owns `/dev/kvm` but `id` does not show that group, use +`mise run gateway:vm` or the VM e2e runner; they re-exec through the configured +group without sudo. Then run: diff --git a/.github/workflows/driver-vm-linux.yml b/.github/workflows/driver-vm-linux.yml index 942cacdfbd..0567a999ff 100644 --- a/.github/workflows/driver-vm-linux.yml +++ b/.github/workflows/driver-vm-linux.yml @@ -116,6 +116,8 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -141,8 +143,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Download kernel runtime tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -168,15 +174,10 @@ jobs: - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed/${file}" done - - name: Scope workspace to driver-vm crates - run: | - set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml - - name: Patch workspace version if: ${{ inputs['cargo-version'] != '' }} run: | diff --git a/.github/workflows/driver-vm-macos.yml b/.github/workflows/driver-vm-macos.yml index a97ade9cbb..ef81bf76f3 100644 --- a/.github/workflows/driver-vm-macos.yml +++ b/.github/workflows/driver-vm-macos.yml @@ -75,6 +75,8 @@ jobs: credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + volumes: + - /var/run/docker.sock:/var/run/docker.sock env: MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} @@ -100,8 +102,12 @@ jobs: cache-directories: .cache/sccache cache-targets: "true" - - name: Install zstd - run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + - name: Install zstd and verify Docker + run: | + apt-get update + apt-get install -y --no-install-recommends zstd + rm -rf /var/lib/apt/lists/* + docker info - name: Build bundled supervisor run: | @@ -116,7 +122,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: driver-vm-supervisor-arm64 - path: target/vm-runtime-compressed/openshell-sandbox.zst + path: | + target/vm-runtime-compressed/openshell-sandbox.zst + target/vm-runtime-compressed/openshell-runtime.tar.zst retention-days: 1 build-driver-vm-macos: @@ -180,12 +188,13 @@ jobs: run: | set -euo pipefail test -f target/vm-runtime-compressed-macos/openshell-sandbox.zst - ls -lh target/vm-runtime-compressed-macos/openshell-sandbox.zst + test -f target/vm-runtime-compressed-macos/openshell-runtime.tar.zst + ls -lh target/vm-runtime-compressed-macos/openshell-{sandbox,runtime.tar}.zst - name: Verify embedded driver inputs run: | set -euo pipefail - for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst; do + for file in libkrun.dylib.zst libkrunfw.5.dylib.zst gvproxy.zst umoci.zst openshell-sandbox.zst openshell-runtime.tar.zst; do test -s "target/vm-runtime-compressed-macos/${file}" done @@ -203,14 +212,16 @@ jobs: . - name: Verify packaged binary shape - run: test -x out/openshell-driver-vm + run: | + test -x out/openshell-driver-vm + test -x out/openshell-sandbox - name: Package binary run: | set -euo pipefail mkdir -p artifacts tar -czf artifacts/openshell-driver-vm-aarch64-apple-darwin.tar.gz \ - -C out openshell-driver-vm + -C out openshell-driver-vm openshell-sandbox ls -lh artifacts/ - name: Upload artifact diff --git a/AGENTS.md b/AGENTS.md index 604395353c..c31a78a1e2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary | | `crates/openshell-sandbox/` | Sandbox runtime | Container supervision, policy-enforced egress routing | | `crates/openshell-isolation/` | Isolation backend contract | RFC 0012 `IsolationBackend` trait + types; the supervisor-facing runtime contract for the boundary | +| `crates/openshell-isolation-vm/` | VM isolation transport | Shared authenticated host/guest boundary transport and portable process leaf for VM drivers | | `crates/openshell-policy/` | Policy engine | Filesystem, network, process, and inference constraints | | `crates/openshell-router/` | Privacy router | Privacy-aware LLM routing | | `crates/openshell-bootstrap/` | Gateway metadata | Gateway registration metadata, auth token storage, mTLS bundle storage | @@ -51,7 +52,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods | | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers | | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers | -| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | +| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` with a native host supervisor and embedded guest runtime | | `crates/openshell-prover/` | Policy prover | Policy verification and proof generation | | `crates/openshell-server-macros/` | Server macros | Compile-time helpers for gateway RPC authorization | | `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution | diff --git a/Cargo.lock b/Cargo.lock index b9d4b7a32f..00efaf157a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4072,6 +4072,7 @@ dependencies = [ name = "openshell-driver-vm" version = "0.0.0" dependencies = [ + "base64 0.22.1", "bollard", "clap", "flate2", @@ -4083,6 +4084,8 @@ dependencies = [ "nix 0.29.0", "oci-client", "openshell-core", + "openshell-isolation", + "openshell-isolation-vm", "openshell-otel", "openshell-otel-test-support", "openshell-policy", @@ -4092,6 +4095,7 @@ dependencies = [ "polling", "prost", "prost-types", + "rand 0.9.4", "rustix 1.1.4", "serde", "serde_json", @@ -4156,6 +4160,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "openshell-isolation-vm" +version = "0.0.0" +dependencies = [ + "async-trait", + "libc", + "nix 0.29.0", + "openshell-core", + "openshell-isolation", + "openshell-supervisor-network", + "openshell-supervisor-process", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "openshell-ocsf" version = "0.0.0" @@ -4257,12 +4279,14 @@ dependencies = [ name = "openshell-sandbox" version = "0.0.0" dependencies = [ + "base64 0.22.1", "clap", "futures", "miette", "nix 0.29.0", "openshell-core", - "openshell-extension-core", + "openshell-isolation", + "openshell-isolation-vm", "openshell-ocsf", "openshell-policy", "openshell-supervisor-middleware", @@ -4282,7 +4306,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "uuid", ] [[package]] diff --git a/architecture/build.md b/architecture/build.md index 7daf4a7c66..593f42a667 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -17,7 +17,7 @@ OpenShell builds these main artifacts: | Gateway container image | `deploy/docker/Dockerfile.gateway` | | Supervisor container image | `deploy/docker/Dockerfile.supervisor` | | Helm chart | `deploy/helm/openshell` | -| VM driver/runtime assets | `crates/openshell-driver-vm` | +| VM driver/runtime assets | `crates/openshell-driver-vm` plus shared transport in `crates/openshell-isolation-vm` | | Published docs site | `docs/` rendered by Fern config in `fern/` | Sandbox community images are built outside this repository. @@ -173,6 +173,16 @@ Runtime layout: enforcement. The VM driver bundles its own supervisor build (`tasks/scripts/vm/build-supervisor-bundle.sh`) and does not read `SUPERVISOR_LIBC`. + before publishing artifacts. On Linux, the driver can materialize the + same-target embedded `openshell-sandbox` as its native host supervisor. The + macOS VM-driver archive includes a native `openshell-sandbox` sibling because + the embedded guest leaf is a Linux binary. +- **Supervisor**: Alpine base with `nftables`, static musl binary at + `/openshell-sandbox`. Static linkage keeps the binary usable when the image + is mounted or extracted into sandbox environments. The VM bundle build also + packages the Linux process leaf and a driver-controlled network-helper + runtime into the guest bootstrap while the logical VM supervisor runs + natively on the host. Gateway image builds bake the corresponding supervisor image tag into the gateway binary so Docker sandboxes do not depend on `:latest` by default. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 2a36073486..eadb06ad43 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -4,6 +4,12 @@ Compute runtimes create, stop, start, delete, and watch sandbox workloads for th gateway. They do not replace sandbox policy enforcement. Every runtime starts a workload that runs the `openshell-sandbox` supervisor, and the supervisor enforces the sandbox contract locally. +Compute runtimes create, stop, delete, and watch sandbox workloads for the +gateway. They do not replace sandbox policy enforcement. Container runtimes run +the logical `openshell-sandbox` supervisor in the workload boundary. VM runtimes +may instead run it on the host and use an authenticated process leaf inside the +guest. In both placements the logical supervisor owns policy and the gateway +session. ## Driver Contract @@ -16,6 +22,9 @@ Each runtime receives a sandbox spec from the gateway and is responsible for: - Forwarding the exact canonical main-process argv and TTY mode without shell reconstruction. The sandbox-level environment and policy workspace apply to the main process. +- Injecting sandbox identity and the admitted topology descriptor. +- Supplying TLS or secret material only to the logical supervisor placement. +- Providing the logical supervisor and any boundary-local process leaf. - Reporting lifecycle and platform events back to the gateway. - Cleaning up runtime-owned resources. @@ -207,6 +216,8 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. A launch-time endpoint may use a canonical built-in name to preserve its driver-config key while replacing in-process construction. The gateway connects to an operator-provisioned UDS, snapshots `GetCapabilities`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | +| VM | Experimental host-supervised microVM isolation. | Per-sandbox libkrun VM with a portable guest process leaf. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`; the driver starts a native logical supervisor and connects it to the guest over authenticated virtio-vsock. The guest has no gateway credentials. The existing custom kernel, cached bootstrap `rootfs.ext4`, in-VM `umoci` preparation, read-only image disk, writable `overlay.ext4`, and restart persistence remain unchanged. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -255,25 +266,29 @@ Runtime-specific implementation notes belong in the driver crate README: - `crates/openshell-driver-kubernetes/README.md` - `crates/openshell-driver-vm/README.md` -The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun -executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves -that identity when it execs the supervisor after mounting and network setup. +The VM topology is delegated. libkrun executes the driver-owned guest bootstrap +as PID 1; after mounting and network setup it execs `openshell-sandbox vm-guest` +as the portable process leaf. The native host supervisor drives RFC 0012 over a +driver-private, token-authenticated virtio-vsock transport. Lifecycle, +exec/PTY, loopback forwarding, and mediated egress all cross this transport; +gateway JWT and mTLS material remain on the host. ## Supervisor Delivery -The supervisor must be available inside each sandbox workload: +The logical supervisor or its boundary-local process leaf must be available at +the placement selected by the runtime: | Runtime | Delivery model | |---|---| | Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | | Podman | Read-only OCI image volume by default; host-cached bind mount when `userns` is configured. | | Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | -| VM | Embedded in the guest rootfs bundle. | +| VM | Native host `openshell-sandbox` beside the driver; portable Linux process leaf embedded in the guest rootfs bundle. | | Extension | Defined by the out-of-tree driver. | Driver-controlled environment variables must override sandbox image or template -values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS -paths, and command metadata. +values for sandbox ID, sandbox name, relay path, and command metadata. Gateway +endpoint and TLS values go only to the logical supervisor placement. ## Process Identity @@ -285,7 +300,8 @@ driver then supplies one authoritative identity input to the supervisor: resolves the workspace from OCI `Config.WorkingDir` during that inspection. - Kubernetes passes its platform-resolved numeric UID/GID, including OpenShift SCC-derived values. -- VM keeps its existing guest identity behavior. +- VM resolves the configured numeric UID/GID on the host and transfers the pair + to the guest leaf as authenticated launch state. Explicit numeric workload identities may use any Linux UID/GID from `1` through `u32::MAX - 1`. UID/GID `0` remains prohibited as root, and diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 786ed5194d..8f40555dfb 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -50,6 +50,36 @@ OpenShell uses overlapping controls rather than a single sandbox primitive: The supervisor may enrich baseline filesystem allowances for runtime-required paths, such as proxy support files or GPU device paths when a GPU is present. +## Isolation Backend + +[RFC 0012](../rfc/0012-isolation-backend/README.md) defines the Isolation +Backend contract for topology-specific boundary construction and process +operations. The contract uses consuming +lifecycle states (`attach` → `Bound` → `confirm` → `Ready` → `start_agent` → +`Running`) so untrusted workload execution cannot begin before standing +enforcement is confirmed. + +The logical supervisor remains the trusted bridge between the gateway and the +workload. It drives the backend and applies approved network policy through +supervisor-owned mediation; the backend routes workload egress to that +mediation. Existing container placements remain on their legacy lifecycle +while the VM driver prototypes the backend contract. + +VM uses delegated placement. The logical supervisor stays on +the host and sends admitted policy, workload state, and proxy CA material over +a token-authenticated, backend-private virtio-vsock channel only after the RFC +lifecycle reaches `start_agent`. The driver uses the portable +`openshell-isolation-vm` guest leaf, which invokes the existing +`openshell-supervisor-process` implementation inside the VM. The guest resolves +workload socket/process identity from its own `/proc` and sends that evidence +with network connections; the host supervisor performs policy evaluation and +relays approved traffic. Exec/PTY and loopback forwarding use the same boundary +transport. Gateway JWT and mTLS credentials never enter the guest. + +The VM guest installs and verifies its default-deny kernel egress ceiling before +the host exposes workload execution. Boundary failure leaves the ceiling in +place and triggers guest process cleanup. + ## Network and Inference See [Sandbox Limits](sandbox-limits.md) for the current numeric safety ceilings, diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index ebcb9d2bc2..a5b40cd163 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -21,9 +21,12 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-otel = { path = "../openshell-otel" } +openshell-isolation = { path = "../openshell-isolation" } +openshell-isolation-vm = { path = "../openshell-isolation-vm" } openshell-policy = { path = "../openshell-policy" } openshell-vfio = { path = "../openshell-vfio" } +base64 = { workspace = true } bollard = { version = "0.20", features = ["ssh"] } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } @@ -41,6 +44,7 @@ opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } tracing-opentelemetry = { workspace = true } miette = { workspace = true } +rand = { workspace = true } url = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 19ac66c3f9..ab3ddbaef0 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -2,32 +2,37 @@ > Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized. -Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy), guest OCI unpacker, and sandbox supervisor are embedded directly in the binary; each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. +Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess and talks to it over the `openshell.compute.v1.ComputeDriver` Unix-socket surface. The logical `openshell-sandbox` supervisor runs as a native host process. A portable process leaf inside each microVM applies guest-local process isolation and serves the authenticated RFC 0012 lifecycle, exec, forwarding, and network-mediation transport over virtio-vsock. + +The driver still embeds libkrun, libkrunfw, gvproxy, the guest OCI unpacker, the portable guest leaf, and the existing custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. ## How it fits together ```mermaid flowchart LR - subgraph host["Host process"] + subgraph host["Host"] gateway["openshell-server
(compute::vm::spawn)"] - driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] + driver["openshell-driver-vm
libkrun + gvproxy"] + supervisor["openshell-sandbox
logical supervisor"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver + supervisor <-->|"authenticated gRPC
policy + relay"| gateway end subgraph guest["Per-sandbox microVM"] init["/srv/openshell-vm-
sandbox-init.sh"] - supervisor["/opt/openshell/bin/
openshell-sandbox
(PID 1)"] - init --> supervisor + leaf["openshell-sandbox vm-guest
process leaf (PID 1)"] + workload["sandbox workload"] + init --> leaf --> workload end driver -->|"CreateSandbox
boots via libkrun"| guest - supervisor -.->|"gRPC callback
--grpc-endpoint"| gateway + supervisor <-->|"authenticated RFC 0012
over virtio-vsock"| leaf - client["openshell-cli"] -->|"SSH proxy
127.0.0.1:<port>"| supervisor + client["openshell-cli"] -->|"connect / exec / forward"| gateway client -->|"CreateSandbox / Watch"| gateway ``` -Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside the VM. gvproxy exposes a single inbound SSH port (`host:` → `guest:2222`) and provides virtio-net egress. +The host supervisor owns gateway credentials, admitted policy, provider resolution, middleware, the network proxy, and relay registration. The guest receives no gateway JWT or mTLS key. It receives a one-time driver-authored boundary token and only the policy/environment state needed to launch the workload. The portable leaf reuses `openshell-supervisor-process`; it is transport glue, not a second supervisor model. ## Quick start (recommended) @@ -35,7 +40,7 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th mise run gateway:vm ``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy/umoci and `mise run vm:supervisor` builds the portable Linux guest leaf plus its trusted helper runtime. The development task also builds the native host supervisor. Subsequent runs are cached. By default `mise run gateway:vm`: @@ -92,13 +97,13 @@ rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev" If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`): ```shell -# 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/ +# 1. Stage runtime artifacts + guest process leaf into target/vm-runtime-compressed/ mise run vm:setup -mise run vm:supervisor # if openshell-sandbox.zst is not already present +mise run vm:supervisor # builds the Linux guest leaf and trusted helper runtime -# 2. Build both binaries with the staged artifacts embedded +# 2. Build gateway, native host supervisor, and driver OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-server -p openshell-sandbox -p openshell-driver-vm # 3. macOS only: codesign the driver for Hypervisor.framework codesign \ @@ -117,7 +122,7 @@ disable_tls = true [openshell.drivers.vm] default_image = "" -grpc_endpoint = "http://host.containers.internal:18081" +grpc_endpoint = "http://127.0.0.1:18081" driver_dir = "$PWD/target/debug" state_dir = "/tmp/openshell-vm-driver-$USER-vm-dev" EOF @@ -138,7 +143,7 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | Configuration key | Default | Purpose | |---|---|---| -| `grpc_endpoint` | empty | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | +| `grpc_endpoint` | empty | Required. URL the native host supervisor uses to reach the gateway. Host loopback such as `http://127.0.0.1:` is valid. This endpoint is never sent into the VM. | | `state_dir` | `target/openshell-vm-driver` | Per-sandbox overlay disks, console logs, image cache, and private `run/compute-driver.sock` UDS. | | `driver_dir` | unset | Override the directory searched for `openshell-driver-vm`. | | `default_image` | OpenShell base image | Sandbox image used when a create request omits one. | @@ -147,9 +152,9 @@ Select the VM driver with `--drivers vm`, `OPENSHELL_DRIVERS=vm`, or `compute_dr | `mem_mib` | `2048` | Memory per sandbox, in MiB. | | `overlay_disk_mib` | `4096` | Sparse writable overlay disk size per sandbox, in MiB. | | `krun_log_level` | `1` | libkrun verbosity (0-5). | -| `guest_tls_ca` | unset | CA cert for the guest's mTLS client bundle. Required when `grpc_endpoint` uses `https://`. | -| `guest_tls_cert` | unset | Guest client certificate. | -| `guest_tls_key` | unset | Guest client private key. | +| `guest_tls_ca` | unset | Historical key name for the host supervisor's gateway CA certificate. Required when `grpc_endpoint` uses `https://`; never copied into the guest. | +| `guest_tls_cert` | unset | Historical key name for the host supervisor's client certificate; never copied into the guest. | +| `guest_tls_key` | unset | Historical key name for the host supervisor's client private key; never copied into the guest. | See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the gateway process flag surface. @@ -221,14 +226,17 @@ RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ ``` The VM guest's serial console is appended to `//console.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions and removes same-owner stale sockets. On clean shutdown, the gateway sends the managed driver `SIGTERM`, waits up to five seconds for it to flush telemetry and exit, then force-kills it if necessary and removes the socket. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. +The VM serial console is appended to `/sandboxes//rootfs-console.log`. Host-supervisor stdout and stderr are written beside it as `supervisor.log` and `supervisor.err.log`. Sandbox IDs must match `[A-Za-z0-9._-]{1,128}` before the driver uses them in host paths. The gateway-owned compute-driver socket lives at `/run/compute-driver.sock`; OpenShell creates `run/` with owner-only permissions, removes same-owner stale sockets, and the gateway removes the socket on clean shutdown via `ManagedDriverProcess::drop`. UDS clients must match the driver UID and provide the expected gateway process PID by default. Standalone same-UID UDS mode requires the explicit `--allow-same-uid-peer` development flag. TCP mode is disabled by default because it is unauthenticated; use `--allow-unauthenticated-tcp --bind-address 127.0.0.1:50061` only for local development. ## Host-side nftables rules -The VM driver creates a per-VM nftables table on the host (`openshell_vm_vmtap_`) with three chains. These rules serve two purposes: NAT infrastructure (required for VM connectivity) and defense-in-depth host isolation. Primary security enforcement — proxy-only egress and bypass detection — is handled by the sandbox supervisor's own nftables rules inside the VM guest. +This section applies to the QEMU/VFIO path, which uses a host TAP device. The normal libkrun path uses gvproxy and needs KVM access but does not require `CAP_NET_ADMIN`. The host supervisor performs primary policy enforcement and receives guest-originated connections through the authenticated network-mediation stream. + +The QEMU path creates a per-VM nftables table on the host (`openshell_vm_vmtap_`) with three chains for NAT infrastructure and defense-in-depth host isolation. **`postrouting` (NAT):** Masquerades outbound VM traffic so it can be routed from the VM's private subnet to the external network. This chain handles forwarded traffic (VM → internet), not traffic destined for the host. -**`forward` (defense-in-depth):** Accepts all outbound traffic from the VM (security enforcement happens guest-side) and accepts established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces. +**`forward` (defense-in-depth):** Accepts outbound traffic from the VM and established/related response traffic back to the VM. Drops unsolicited inbound connections to the VM from the broader network. This chain handles forwarded traffic only — packets transiting the host between the TAP interface and other interfaces. **`input` (defense-in-depth):** Accepts traffic from the VM to the gateway port on the host. Drops all other traffic from the VM destined for the host itself. This limits what a compromised guest can reach on the host to the gateway service only. @@ -246,9 +254,9 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain - e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection -- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): +- Guest-leaf cross-compile toolchain (needed on macOS, and on Linux when host arch differs from the guest): - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. + - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the Linux `openshell-sandbox vm-guest` leaf. - [mise](https://mise.jdx.dev/) task runner - Docker or Podman socket on the local CLI/gateway host when using `openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI @@ -279,11 +287,12 @@ The RPM gateway package is configured for the Podman driver. On Apple Silicon macOS, `install.sh` stages the generated `openshell.rb` formula from the selected release in the `nvidia/openshell` Homebrew tap. -Homebrew installs `openshell`, `openshell-gateway`, and -`openshell-driver-vm`, ad-hoc signs the driver with the Hypervisor entitlement -in `post_install`, and owns the `brew services` gateway lifecycle. The service -also leaves `OPENSHELL_DRIVERS` unset so driver choice remains automatic unless -the user explicitly overrides it. +Homebrew installs `openshell`, `openshell-gateway`, `openshell-driver-vm`, and +the native `openshell-sandbox` host supervisor beside the driver. It ad-hoc +signs the driver with the Hypervisor entitlement in `post_install` and owns the +`brew services` gateway lifecycle. The service also leaves `OPENSHELL_DRIVERS` +unset so driver choice remains automatic unless the user explicitly overrides +it. ## TODOs diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs index 1763590545..17f1f00bf8 100644 --- a/crates/openshell-driver-vm/build.rs +++ b/crates/openshell-driver-vm/build.rs @@ -21,6 +21,7 @@ fn main() { "libkrunfw.5.dylib.zst", "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ] { println!("cargo:rerun-if-changed={dir}/{name}"); @@ -38,7 +39,13 @@ fn main() { println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}"); generate_stub_resources( &out_dir, - &["libkrun", "libkrunfw", "openshell-sandbox.zst", "umoci.zst"], + &[ + "libkrun", + "libkrunfw", + "openshell-sandbox.zst", + "openshell-runtime.tar.zst", + "umoci.zst", + ], ); return; } @@ -56,6 +63,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -75,6 +83,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); @@ -92,6 +101,10 @@ fn main() { "openshell-sandbox.zst".to_string(), "openshell-sandbox.zst".to_string(), ), + ( + "openshell-runtime.tar.zst".to_string(), + "openshell-runtime.tar.zst".to_string(), + ), ("umoci.zst".to_string(), "umoci.zst".to_string()), ]; @@ -135,6 +148,7 @@ fn main() { &format!("{libkrunfw_name}.zst"), "gvproxy.zst", "openshell-sandbox.zst", + "openshell-runtime.tar.zst", "umoci.zst", ], ); diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index 11aab67f43..3bd409be5c 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -12,14 +12,15 @@ runtime/ ``` `openshell-driver-vm` embeds libkrun, libkrunfw, gvproxy, umoci for guest-side -OCI image unpacking, and the bundled `openshell-sandbox` supervisor. +OCI image unpacking, and the portable `openshell-sandbox vm-guest` process leaf. ## Why The stock `libkrunfw` kernel does not include the bridge, netfilter, -conntrack, cgroup, seccomp, and Landlock features the sandbox supervisor needs -inside each microVM. `kernel/openshell.kconfig` extends the libkrunfw kernel so -VM sandboxes can run the same supervisor enforcement path as other backends. +conntrack, cgroup, seccomp, and Landlock features the process leaf needs inside +each microVM. `kernel/openshell.kconfig` extends the libkrunfw kernel so VM +sandboxes retain guest-local process and filesystem enforcement while the +logical supervisor runs on the host. ## Build Scripts @@ -36,12 +37,12 @@ VM sandboxes can run the same supervisor enforcement path as other backends. # Download the current pre-built runtime and stage compressed artifacts mise run vm:setup -# Build the bundled guest supervisor +# Build the portable Linux guest leaf and trusted helper runtime (requires Docker Buildx) mise run vm:supervisor -# Build the gateway and VM driver with embedded runtime artifacts +# Build the gateway, native host supervisor, and VM driver OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-server -p openshell-sandbox -p openshell-driver-vm ``` Use `FROM_SOURCE=1 mise run vm:setup` to build the runtime from source instead diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 14dbc0466b..f60d031181 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -4,8 +4,7 @@ # Minimal init for sandbox VMs. Runs as PID 1 inside the guest, mounts the # essential filesystems, configures networking (gvproxy DHCP or TAP static), -# optionally loads NVIDIA GPU drivers, then execs the OpenShell sandbox -# supervisor. +# optionally loads NVIDIA GPU drivers, then execs the portable VM process leaf. set -euo pipefail @@ -117,6 +116,10 @@ ensure_target_runtime() { cp /opt/openshell/bin/openshell-sandbox "$image_root/opt/openshell/bin/openshell-sandbox" chmod 0755 "$image_root/opt/openshell/bin/openshell-sandbox" fi + if [ -d /opt/openshell/bin/openshell-runtime ]; then + rm -rf "$image_root/opt/openshell/bin/openshell-runtime" + cp -a /opt/openshell/bin/openshell-runtime "$image_root/opt/openshell/bin/openshell-runtime" + fi touch "$image_root/etc/passwd" "$image_root/etc/group" "$image_root/etc/shadow" "$image_root/etc/gshadow" if ! grep -q '^sandbox:' "$image_root/etc/group" 2>/dev/null; then @@ -214,14 +217,17 @@ exec_supervisor_in_newroot() { "${bootstrap}/lib64/ld-linux-aarch64.so.1"; do if [ -x "/newroot${loader}" ]; then lib_path="${bootstrap}/lib:${bootstrap}/lib64:${bootstrap}/usr/lib:${bootstrap}/usr/lib64:${bootstrap}/lib/aarch64-linux-gnu:${bootstrap}/lib/x86_64-linux-gnu:${bootstrap}/usr/lib/aarch64-linux-gnu:${bootstrap}/usr/lib/x86_64-linux-gnu" - exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$loader" --library-path "$lib_path" "$supervisor" \ + vm-guest /etc/openshell/vm-guest.json fi done - exec "$chroot_bin" /newroot "$supervisor" --workdir /sandbox + exec "$chroot_bin" /newroot "$supervisor" \ + vm-guest /etc/openshell/vm-guest.json fi if [ -x /newroot/opt/openshell/bin/openshell-sandbox ]; then - exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox --workdir /sandbox + exec "$chroot_bin" /newroot /opt/openshell/bin/openshell-sandbox \ + vm-guest /etc/openshell/vm-guest.json fi done @@ -292,65 +298,9 @@ setup_overlay_root() { run_post_overlay_setup } -parse_endpoint() { - local endpoint="$1" - local scheme rest authority path host port - - case "$endpoint" in - *://*) - scheme="${endpoint%%://*}" - rest="${endpoint#*://}" - ;; - *) - return 1 - ;; - esac - - authority="${rest%%/*}" - path="${rest#"$authority"}" - if [ "$path" = "$rest" ]; then - path="" - fi - - if [[ "$authority" =~ ^\[([^]]+)\]:(.+)$ ]]; then - host="${BASH_REMATCH[1]}" - port="${BASH_REMATCH[2]}" - elif [[ "$authority" =~ ^\[([^]]+)\]$ ]]; then - host="${BASH_REMATCH[1]}" - port="" - elif [[ "$authority" == *:* ]]; then - host="${authority%%:*}" - port="${authority##*:}" - else - host="$authority" - port="" - fi - - if [ -z "$port" ]; then - case "$scheme" in - https) port="443" ;; - *) port="80" ;; - esac - fi - - printf '%s\n%s\n%s\n%s\n' "$scheme" "$host" "$port" "$path" -} - -tcp_probe() { - local host="$1" - local port="$2" - - if command -v timeout >/dev/null 2>&1; then - timeout 2 bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1 - else - bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1 - fi -} - ensure_host_gateway_aliases() { - # Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor - # can reach the OpenShell server even when gvproxy's built-in DNS is not - # in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8). + # Seed /etc/hosts with the well-known gvproxy hostnames for workload + # compatibility even when gvproxy's built-in DNS is not in resolv.conf. # # Critical distinction: host.* aliases point at the gvproxy *host-loopback* # IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the @@ -399,63 +349,6 @@ write_host_gateway_aliases() { rm -f "$hosts_tmp" } -rewrite_openshell_endpoint_if_needed() { - local endpoint="${OPENSHELL_ENDPOINT:-}" - [ -n "$endpoint" ] || return 0 - - local parsed - if ! parsed="$(parse_endpoint "$endpoint")"; then - ts "WARNING: could not parse OPENSHELL_ENDPOINT=$endpoint" - return 0 - fi - - local scheme host port path - scheme="$(printf '%s\n' "$parsed" | sed -n '1p')" - host="$(printf '%s\n' "$parsed" | sed -n '2p')" - port="$(printf '%s\n' "$parsed" | sed -n '3p')" - path="$(printf '%s\n' "$parsed" | sed -n '4p')" - - if tcp_probe "$host" "$port"; then - return 0 - fi - - # Probe candidates in preference order. Hostnames first for informative - # log output, then a bare IP as a final safety net. In gvproxy mode the - # bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's - # the TAP host gateway. - local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP" - if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then - fallback_ip="$GATEWAY_IP" - fi - local candidates="host.openshell.internal host.containers.internal host.docker.internal" - if [ "$scheme" != "https" ]; then - candidates="${candidates} ${fallback_ip}" - fi - - for candidate in $candidates; do - if [ "$candidate" = "$host" ]; then - continue - fi - if tcp_probe "$candidate" "$port"; then - local authority="$candidate" - if ! { [ "$scheme" = "http" ] && [ "$port" = "80" ]; } \ - && ! { [ "$scheme" = "https" ] && [ "$port" = "443" ]; }; then - authority="${authority}:${port}" - fi - export OPENSHELL_ENDPOINT="${scheme}://${authority}${path}" - ts "rewrote OPENSHELL_ENDPOINT to ${OPENSHELL_ENDPOINT}" - return 0 - fi - done - - if [ "$scheme" = "https" ]; then - ts "WARNING: could not preflight HTTPS OpenShell endpoint ${host}:${port}; preserving hostname for TLS verification" - return 0 - fi - - ts "WARNING: could not reach OpenShell endpoint ${host}:${port}" -} - create_gpu_device_nodes_mknod() { # Mode 666 is intentional: single-tenant microVM with the VM itself as the # isolation boundary. The sandbox user is the only non-root user. @@ -813,31 +706,16 @@ fi run_openshell_init_dropins -rewrite_openshell_endpoint_if_needed - -# Log supervisor connectivity state for debugging stuck-in-Provisioning issues -if [ -n "${OPENSHELL_ENDPOINT:-}" ]; then - _ep_parsed="$(parse_endpoint "$OPENSHELL_ENDPOINT" 2>/dev/null || true)" - if [ -n "$_ep_parsed" ]; then - _ep_host="$(printf '%s\n' "$_ep_parsed" | sed -n '2p')" - _ep_port="$(printf '%s\n' "$_ep_parsed" | sed -n '3p')" - if tcp_probe "$_ep_host" "$_ep_port"; then - ts "gateway reachable at ${_ep_host}:${_ep_port}" - else - ts "WARNING: gateway NOT reachable at ${_ep_host}:${_ep_port} — supervisor may fail to connect" - fi - fi - ts "OPENSHELL_ENDPOINT=${OPENSHELL_ENDPOINT}" -fi if [ -n "${OPENSHELL_SANDBOX_ID:-}" ]; then ts "OPENSHELL_SANDBOX_ID=${OPENSHELL_SANDBOX_ID}" fi -ts "starting openshell-sandbox supervisor" +ts "starting OpenShell VM process leaf" if [ "${ROOT_PREFIX:-}" = "/newroot" ]; then exec_supervisor_in_newroot fi -exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox +exec /opt/openshell/bin/openshell-sandbox \ + vm-guest /etc/openshell/vm-guest.json } if [ "${1:-}" != "--post-overlay" ]; then diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 13e57f546d..699aeb63a4 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1,19 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(unsafe_code)] + use crate::gpu::{ GpuInventory, SubnetAllocator, allocate_vsock_cid, mac_from_sandbox_id, tap_device_name, }; + use crate::lifecycle::{ BackendFeature, GuestInitDropin, LaunchAbortReason, LaunchPlan, LifecycleExtensionRegistry, RestoreContext, extension_state_dir, }; +#[cfg(target_os = "linux")] +use crate::rootfs::extract_host_supervisor; use crate::rootfs::{ clone_or_copy_sparse_file, create_ext4_image_from_dir_with_size, create_rootfs_image_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, - set_rootfs_image_file_mode, write_rootfs_image_file, + sandbox_guest_runtime_identity, set_rootfs_image_file_mode, write_rootfs_image_file, }; use crate::runtime::VmBackend; +use base64::Engine as _; use bollard::Docker; use bollard::errors::Error as BollardError; use bollard::models::ContainerCreateBody; @@ -53,11 +59,14 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; +use openshell_isolation::contract::INTERFACE_VERSION; +use openshell_isolation_vm::{GuestConfig, VmTopology, VmTransport}; use openshell_vfio::SysfsRoot; use opentelemetry::trace::TraceContextExt as _; use prost::Message; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; use std::fs; use std::io::Read; use std::net::Ipv4Addr; @@ -146,13 +155,25 @@ const OPENSHELL_HOST_GATEWAY_ALIAS: &str = "host.openshell.internal"; /// /// Both names ultimately route through the gvproxy NAT path on /// `GVPROXY_HOST_LOOPBACK_IP` — they do **not** go through the gateway IP. +#[allow(dead_code)] const GVPROXY_HOST_LOOPBACK_ALIAS: &str = OPENSHELL_HOST_GATEWAY_ALIAS; +#[allow(dead_code)] const GUEST_SSH_SOCKET_PATH: &str = openshell_core::container_paths::SSH_SOCKET_PATH; +#[allow(dead_code)] const GUEST_TLS_CA_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CA_PATH; +#[allow(dead_code)] const GUEST_TLS_CERT_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_CERT_PATH; +#[allow(dead_code)] const GUEST_TLS_KEY_PATH: &str = openshell_core::container_paths::VM_GUEST_TLS_KEY_PATH; +#[allow(dead_code)] const GUEST_SANDBOX_TOKEN_PATH: &str = openshell_core::container_paths::VM_GUEST_SANDBOX_TOKEN_PATH; const GUEST_INIT_DROPIN_DIR: &str = openshell_core::container_paths::VM_GUEST_INIT_DROPIN_DIR; +const GUEST_BOUNDARY_CONFIG_PATH: &str = "/etc/openshell/vm-guest.json"; +const HOST_SANDBOX_TOKEN_FILE: &str = "sandbox.jwt"; +#[cfg(target_os = "linux")] +const HOST_SUPERVISOR_BINARY: &str = "host-runtime/openshell-sandbox"; +const VM_CONTROL_SOCKET: &str = "control.sock"; +const VM_CONTROL_PORT: u32 = 5500; /// Guest path of the driver-authored manifest enumerating which /// `init.d` drop-ins the guest init script is allowed to execute. /// @@ -176,7 +197,7 @@ const GUEST_IMAGE_CONFIG_DIR: &str = "openshell-image"; const GUEST_IMAGE_OCI_LAYOUT_DIR: &str = "oci"; const GUEST_IMAGE_OCI_REF: &str = "openshell"; const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar"; -const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v3"; +const BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-bootstrap-rootfs-ext4-v4"; const PREPARED_IMAGE_CACHE_LAYOUT_VERSION: &str = "sandbox-prepared-rootfs-ext4-umoci-v3"; const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; @@ -320,7 +341,7 @@ impl VmDriverConfig { if provided.iter().all(Option::is_none) { return if self.requires_tls_materials() { Err( - "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so sandbox VMs can authenticate to the gateway" + "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so the host supervisor can authenticate to the gateway" .to_string(), ) } else { @@ -382,6 +403,7 @@ fn validate_openshell_endpoint(endpoint: &str) -> Result<(), String> { #[derive(Debug)] struct VmProcess { child: Child, + supervisor: Child, deleting: bool, } @@ -517,6 +539,157 @@ impl VmDriver { Ok(driver) } + async fn host_supervisor_binary(&self) -> Result { + if let Some(configured) = std::env::var_os("OPENSHELL_VM_SUPERVISOR_BIN") { + let configured = PathBuf::from(configured); + if configured.is_file() { + return Ok(configured); + } + return Err(Status::failed_precondition(format!( + "configured host supervisor does not exist: {}", + configured.display() + ))); + } + + if let Some(parent) = self.launcher_bin.parent() { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + return Ok(sibling); + } + } + + #[cfg(not(target_os = "linux"))] + { + return Err(Status::failed_precondition( + "the native host supervisor is missing; install openshell-sandbox beside openshell-driver-vm or set OPENSHELL_VM_SUPERVISOR_BIN", + )); + } + + #[cfg(target_os = "linux")] + { + let destination = self.config.state_dir.join(HOST_SUPERVISOR_BINARY); + if destination.is_file() { + return Ok(destination); + } + let _cache_guard = self.image_cache_lock.lock().await; + if destination.is_file() { + return Ok(destination); + } + let destination_for_extract = destination.clone(); + tokio::task::spawn_blocking(move || extract_host_supervisor(&destination_for_extract)) + .await + .map_err(|error| { + Status::internal(format!("host supervisor extraction panicked: {error}")) + })? + .map_err(Status::failed_precondition)?; + Ok(destination) + } + } + + async fn spawn_host_supervisor( + &self, + sandbox: &Sandbox, + state_dir: &Path, + tls_paths: Option<&VmDriverTlsPaths>, + topology: &VmTopology, + ) -> Result { + let supervisor_binary = self.host_supervisor_binary().await?; + let token = sandbox + .spec + .as_ref() + .map(|spec| spec.sandbox_token.as_str()) + .filter(|token| !token.is_empty()) + .ok_or_else(|| Status::failed_precondition("VM sandbox gateway token is required"))?; + let token_path = state_dir.join(HOST_SANDBOX_TOKEN_FILE); + tokio::fs::write(&token_path, format!("{token}\n")) + .await + .map_err(|error| Status::internal(format!("write host sandbox token: {error}")))?; + #[cfg(unix)] + tokio::fs::set_permissions(&token_path, fs::Permissions::from_mode(0o600)) + .await + .map_err(|error| Status::internal(format!("restrict host sandbox token: {error}")))?; + + let payload = topology + .encode() + .map_err(|error| Status::internal(error.to_string()))?; + let sandbox_user_id = self.config.resolve_sandbox_uid(); + let primary_group_id = self.config.resolve_sandbox_gid(sandbox_user_id); + let mut command = Command::new(&supervisor_binary); + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::from( + fs::File::create(state_dir.join("supervisor.log")) + .map_err(|error| Status::internal(format!("create supervisor log: {error}")))?, + )) + .stderr(Stdio::from( + fs::File::create(state_dir.join("supervisor.err.log")).map_err(|error| { + Status::internal(format!("create supervisor error log: {error}")) + })?, + )) + .arg("--topology-backend-name=vm") + .arg(format!("--topology-version={INTERFACE_VERSION}")) + .arg(format!( + "--topology-payload-base64={}", + base64::engine::general_purpose::STANDARD.encode(payload) + )) + .arg("--workdir") + .arg("/sandbox") + .arg("--") + .args(["/bin/sh", "-lc", "while :; do sleep 3600; done"]) + .env( + openshell_core::sandbox_env::ENDPOINT, + &self.config.openshell_endpoint, + ) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, &token_path) + .env( + openshell_core::sandbox_env::SSH_SOCKET_PATH, + state_dir.join("ssh.sock"), + ) + .env( + openshell_core::sandbox_env::PROXY_TLS_DIR, + state_dir.join("proxy-tls"), + ) + .env( + openshell_core::sandbox_env::SANDBOX_UID, + sandbox_user_id.to_string(), + ) + .env( + openshell_core::sandbox_env::SANDBOX_GID, + primary_group_id.to_string(), + ) + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "") + .env( + openshell_core::sandbox_env::LOG_LEVEL, + openshell_core::driver_utils::sandbox_log_level(sandbox, &self.config.log_level), + ) + .env( + openshell_core::sandbox_env::TELEMETRY_ENABLED, + openshell_core::telemetry::enabled_env_value(), + ); + if let Some(tls) = tls_paths { + command + .env(openshell_core::sandbox_env::TLS_CA, &tls.ca) + .env(openshell_core::sandbox_env::TLS_CERT, &tls.cert) + .env(openshell_core::sandbox_env::TLS_KEY, &tls.key); + } + #[cfg(target_os = "linux")] + unsafe { + command.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|error| std::io::Error::other(error.to_string())) + }); + } + command.spawn().map_err(|error| { + Status::internal(format!( + "start host supervisor '{}': {error}", + supervisor_binary.display() + )) + }) + } + #[must_use] pub fn capabilities(&self) -> GetCapabilitiesResponse { GetCapabilitiesResponse { @@ -745,6 +918,7 @@ impl VmDriver { let root_disk = image_plan.root_disk; let image_disk = image_plan.image_disk; let overlay_disk = disk_paths.overlay_disk; + let bootstrap_token = random_boundary_token(); self.publish_platform_event( sandbox.id.clone(), @@ -756,16 +930,7 @@ impl VmDriver { ), ); if let Err(err) = self - .prepare_runtime_overlay( - &overlay_disk, - tls_paths.as_ref(), - sandbox - .spec - .as_ref() - .map(|spec| spec.sandbox_token.as_str()) - .filter(|token| !token.is_empty()), - overlay_preparation, - ) + .prepare_runtime_overlay(&overlay_disk, overlay_preparation) .await { return Err(Status::internal(format!( @@ -774,6 +939,23 @@ impl VmDriver { } self.ensure_provisioning_active(&sandbox.id).await?; + let guest_config = GuestConfig { + boundary_id: sandbox.id.clone(), + bootstrap_token: bootstrap_token.clone(), + control_port: VM_CONTROL_PORT, + agent_uid: self.config.resolve_sandbox_uid(), + agent_gid: self + .config + .resolve_sandbox_gid(self.config.resolve_sandbox_uid()), + trusted_runtime_root: PathBuf::from( + "/.openshell-bootstrap/opt/openshell/bin/openshell-runtime", + ), + child_env: merged_environment(&sandbox), + }; + inject_guest_boundary_config(&overlay_disk, &guest_config).map_err(|error| { + Status::internal(format!("inject VM guest boundary configuration: {error}")) + })?; + if let Err(err) = write_sandbox_image_metadata(&state_dir, &image_ref, &image_identity).await { @@ -906,15 +1088,24 @@ impl VmDriver { return Err(err); } - let endpoint_override = if plan.backend == VmBackend::Qemu { - plan.host_ip.as_deref().map(|host_ip| { - guest_visible_openshell_endpoint_for_tap(&self.config.openshell_endpoint, host_ip) - }) - } else { - None - }; - let console_output = state_dir.join("rootfs-console.log"); + let control_socket = state_dir.join(VM_CONTROL_SOCKET); + let topology = VmTopology { + boundary_id: sandbox.id.clone(), + transport: if plan.backend == VmBackend::Qemu { + VmTransport::HostVsock { + guest_cid: plan.vsock_cid.ok_or_else(|| { + Status::internal("QEMU launch plan is missing a guest vsock CID") + })?, + control_port: VM_CONTROL_PORT, + } + } else { + VmTransport::MappedUnix { + socket_path: control_socket.clone(), + } + }, + bootstrap_token, + }; let mut command = Command::new(&self.launcher_bin); command.kill_on_drop(true); command.stdin(Stdio::null()); @@ -958,6 +1149,13 @@ impl VmDriver { if let Some(port) = plan.gateway_port { command.arg("--vm-gateway-port").arg(port.to_string()); } + } else { + let _ = tokio::fs::remove_file(&control_socket).await; + command + .arg("--vm-vsock-control-port") + .arg(VM_CONTROL_PORT.to_string()) + .arg("--vm-vsock-control-socket") + .arg(&control_socket); } self.ensure_provisioning_active(&sandbox.id).await?; @@ -966,7 +1164,7 @@ impl VmDriver { .arg("--vm-krun-log-level") .arg(self.config.krun_log_level.to_string()); - for env in build_guest_environment(&sandbox, &self.config, endpoint_override.as_deref()) { + for env in build_guest_environment(&sandbox, &self.config) { command.arg("--vm-env").arg(env); } for env in &plan.env { @@ -979,7 +1177,7 @@ impl VmDriver { console_output = %console_output.display(), "vm driver: spawning VM launcher" ); - let child = match spawn_vm_launcher(&mut command, &sandbox.id, &plan.backend) { + let mut child = match command.spawn() { Ok(child) => child, Err(err) => { warn!( @@ -1006,8 +1204,27 @@ impl VmDriver { launcher_pid = child.id().unwrap_or(0), "vm driver: launcher spawned" ); + let supervisor = match self + .spawn_host_supervisor(&sandbox, &state_dir, tls_paths.as_ref(), &topology) + .await + { + Ok(supervisor) => supervisor, + Err(error) => { + let _ = terminate_vm_process(&mut child).await; + self.lifecycle_extensions + .after_launch_failed( + &sandbox, + &state_dir, + LaunchAbortReason::LauncherSpawnFailed, + ) + .await; + self.release_gpu_and_subnet(&sandbox.id); + return Err(error); + } + }; let process = Arc::new(Mutex::new(VmProcess { child, + supervisor, deleting: false, })); @@ -1032,6 +1249,9 @@ impl VmDriver { { let mut process = process.lock().await; process.deleting = true; + terminate_vm_process(&mut process.supervisor) + .await + .map_err(|err| Status::internal(format!("failed to stop supervisor: {err}")))?; terminate_vm_process(&mut process.child) .await .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; @@ -1261,6 +1481,9 @@ impl VmDriver { if let Some(process) = process { let mut process = process.lock().await; process.deleting = true; + terminate_vm_process(&mut process.supervisor) + .await + .map_err(|err| Status::internal(format!("failed to stop supervisor: {err}")))?; terminate_vm_process(&mut process.child) .await .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; @@ -1711,7 +1934,7 @@ impl VmDriver { "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] )); - plan.gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); + plan.gateway_port = None; Ok(()) } @@ -1843,8 +2066,6 @@ impl VmDriver { mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] ); let tap = tap_device_name(sandbox_id); - let gateway_port = gateway_port_from_endpoint(&self.config.openshell_endpoint); - let (vcpus, mem_mib) = if is_gpu { (self.config.gpu_vcpus, self.config.gpu_mem_mib) } else { @@ -1865,7 +2086,7 @@ impl VmDriver { host_ip: Some(subnet.host_ip.to_string()), vsock_cid: Some(vsock_cid), guest_mac: Some(mac_str), - gateway_port, + gateway_port: None, guest_init_dropins: Vec::new(), env: Vec::new(), }) @@ -2041,16 +2262,9 @@ impl VmDriver { async fn prepare_runtime_overlay( &self, overlay_disk: &Path, - tls_paths: Option<&VmDriverTlsPaths>, - sandbox_token: Option<&str>, preparation: OverlayPreparation, ) -> Result<(), String> { let span_status = openshell_otel::ErrorStatusGuard::current(); - let tls_materials = match tls_paths { - Some(paths) => Some(read_guest_tls_materials(paths).await?), - None => None, - }; - let sandbox_token = sandbox_token.map(str::to_string); let overlay_disk = overlay_disk.to_path_buf(); let overlay_size_bytes = self .config @@ -2078,8 +2292,6 @@ impl VmDriver { prepare_sandbox_overlay_image( &template_path, &overlay_disk, - tls_materials.as_ref(), - sandbox_token.as_deref(), preparation, overlay_size_bytes, ) @@ -3166,39 +3378,48 @@ impl VmDriver { process.clone() }; - let exit_status = { + let poll_result = { let mut process = process.lock().await; if process.deleting { return; } match process.child.try_wait() { - Ok(status) => status, - Err(err) => { - if let Some(snapshot) = self - .set_snapshot_condition( - &sandbox_id, - error_condition("ProcessPollFailed", &err.to_string()), - false, - ) - .await - { - self.publish_snapshot(snapshot); - } - self.publish_platform_event( - sandbox_id.clone(), - platform_event( - "vm", - "Warning", - "ProcessPollFailed", - format!("Failed to poll VM helper process: {err}"), - ), - ); - return; + Ok(Some(status)) => Ok(Some(("VM", status))), + Ok(None) => process + .supervisor + .try_wait() + .map(|status| status.map(|status| ("host supervisor", status))), + Err(error) => Err(error), + } + }; + + let exit_status = match poll_result { + Ok(status) => status, + Err(err) => { + if let Some(snapshot) = self + .set_snapshot_condition( + &sandbox_id, + error_condition("ProcessPollFailed", &err.to_string()), + false, + ) + .await + { + self.publish_snapshot(snapshot); } + self.publish_platform_event( + sandbox_id.clone(), + platform_event( + "vm", + "Warning", + "ProcessPollFailed", + format!("Failed to poll VM sandbox process: {err}"), + ), + ); + return; } }; - if let Some(status) = exit_status { + if let Some((component, status)) = exit_status { let state_dir = { let registry = self.registry.lock().await; registry @@ -3218,9 +3439,17 @@ impl VmDriver { "vm driver: failed to persist canonical-process exit tombstone" ); } + { + let mut process = process.lock().await; + if component == "VM" { + let _ = terminate_vm_process(&mut process.supervisor).await; + } else { + let _ = terminate_vm_process(&mut process.child).await; + } + } let message = status.code().map_or_else( - || "VM process exited".to_string(), - |code| format!("VM process exited with status {code}"), + || format!("{component} process exited"), + |code| format!("{component} process exited with status {code}"), ); if let Some(snapshot) = self .set_snapshot_condition( @@ -4383,46 +4612,12 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { environment } -/// Rewrites loopback host references in a gateway URL to a hostname the guest -/// can reach via gvproxy. -/// -/// The driver receives the gateway endpoint from `--openshell-endpoint`, which -/// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is -/// useless inside the guest because the guest's loopback interface is its own, -/// not the host's. Inside the guest we need a name that gvproxy will translate -/// into the host's loopback address. -/// -/// We rewrite to `host.openshell.internal`, which gvproxy's embedded DNS resolves -/// to the host-loopback IP `192.168.127.254`. gvproxy installs a default NAT entry -/// rewriting that destination to the host's `127.0.0.1` and dialing out from the -/// host process, so any port the host is listening on becomes reachable. The -/// gateway IP `192.168.127.1` does **not** do this — it only listens on gvproxy's -/// own service ports (DNS, DHCP, HTTP API). The guest init script also seeds the -/// hostname in `/etc/hosts` so resolution works even if gvproxy's DNS isn't in -/// resolv.conf (e.g. when DHCP fails). -/// -/// Non-loopback URLs are returned unchanged. -fn guest_visible_openshell_endpoint(endpoint: &str) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - let should_rewrite = match url.host() { - Some(Host::Ipv4(ip)) => ip.is_loopback(), - Some(Host::Ipv6(ip)) => ip.is_loopback(), - Some(Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), - None => false, - }; - - if should_rewrite && url.set_host(Some(GVPROXY_HOST_LOOPBACK_ALIAS)).is_ok() { - return url.to_string(); +fn random_boundary_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); } - - endpoint.to_string() -} - -fn gateway_port_from_endpoint(endpoint: &str) -> Option { - Url::parse(endpoint).ok().and_then(|url| url.port()) + token } fn has_complete_qemu_network(plan: &LaunchPlan) -> bool { @@ -4433,50 +4628,17 @@ fn has_complete_qemu_network(plan: &LaunchPlan) -> bool { && plan.guest_mac.is_some() } -fn guest_visible_openshell_endpoint_for_tap(endpoint: &str, host_ip: &str) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - if url.set_host(Some(host_ip)).is_ok() { - url.to_string() - } else { - endpoint.to_string() - } -} - -fn build_guest_environment( - sandbox: &Sandbox, - config: &VmDriverConfig, - endpoint_override: Option<&str>, -) -> Vec { - let openshell_endpoint = endpoint_override.map_or_else( - || guest_visible_openshell_endpoint(&config.openshell_endpoint), - String::from, - ); - // 1. User-supplied environment (lowest priority). - let user_env = merged_environment(sandbox); +fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec { + // The guest receives only driver-owned boot metadata. Gateway credentials, + // TLS material, and logical-supervisor configuration remain on the host; + // workload environment is carried in the authenticated GuestConfig. let mut environment: HashMap = HashMap::new(); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - - // 2. Required driver vars (highest priority -- always overwrite). environment.insert("HOME".to_string(), "/root".to_string()); environment.insert( "PATH".to_string(), "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), ); environment.insert("TERM".to_string(), "xterm".to_string()); - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - openshell_endpoint, - ); environment.insert( openshell_core::sandbox_env::SANDBOX_ID.to_string(), sandbox.id.clone(), @@ -4485,68 +4647,14 @@ fn build_guest_environment( openshell_core::sandbox_env::SANDBOX.to_string(), sandbox.name.clone(), ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - GUEST_SSH_SOCKET_PATH.to_string(), - ); - // The libkrun guest environment path does not preserve spaces in values - // before guest startup. Use a whitespace-free base64url envelope so - // command arguments remain lossless. - let main_process = - openshell_core::sandbox_env::MainProcessConfig::encode_driver_spec_base64url( - sandbox.spec.as_ref(), - ) - .expect("main process config serialization cannot fail"); - environment.insert( - openshell_core::sandbox_env::MAIN_PROCESS_SPEC.to_string(), - main_process, - ); environment.insert( openshell_core::sandbox_env::LOG_LEVEL.to_string(), openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), ); - if config.requires_tls_materials() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - GUEST_TLS_CA_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - GUEST_TLS_CERT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - GUEST_TLS_KEY_PATH.to_string(), - ); - } environment.insert( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), ); - // Runtime capabilities are driver-owned. The VM driver does not yet - // provide policy DNS and transparent TCP interception. - environment.insert( - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - String::new(), - ); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - // Prevent user-supplied environment from overriding the TLS server name - // the supervisor verifies — a sandbox user who can redirect the gateway - // hostname could otherwise present a certificate for a name they control - // and intercept the sandbox JWT. - environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - GUEST_SANDBOX_TOKEN_PATH.to_string(), - ); - } - let mut pairs = environment.into_iter().collect::>(); pairs.sort_by(|left, right| left.0.cmp(&right.0)); pairs @@ -4754,8 +4862,9 @@ fn write_oci_layout_for_manifest( fn bootstrap_image_cache_identity(image_identity: &str) -> String { format!( - "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:{image_identity}", - openshell_core::VERSION + "{BOOTSTRAP_IMAGE_CACHE_LAYOUT_VERSION}:openshell-{}:guest-{}:{image_identity}", + openshell_core::VERSION, + sandbox_guest_runtime_identity() ) } @@ -4874,26 +4983,6 @@ fn validate_restored_sandbox_state( Ok(()) } -#[derive(Debug, Clone)] -struct GuestTlsMaterials { - ca: Vec, - cert: Vec, - key: Vec, -} - -async fn read_guest_tls_materials(paths: &VmDriverTlsPaths) -> Result { - let ca = tokio::fs::read(&paths.ca) - .await - .map_err(|err| format!("read {}: {err}", paths.ca.display()))?; - let cert = tokio::fs::read(&paths.cert) - .await - .map_err(|err| format!("read {}: {err}", paths.cert.display()))?; - let key = tokio::fs::read(&paths.key) - .await - .map_err(|err| format!("read {}: {err}", paths.key.display()))?; - Ok(GuestTlsMaterials { ca, cert, key }) -} - async fn overlay_template_image_ready(path: &Path, size_bytes: u64) -> Result { match tokio::fs::metadata(path).await { Ok(metadata) => Ok(metadata.is_file() && metadata.len() == size_bytes), @@ -4978,36 +5067,19 @@ fn create_empty_sandbox_overlay_image(overlay_disk: &Path, size_bytes: u64) -> R fn create_sandbox_overlay_image_from_template( template_path: &Path, overlay_disk: &Path, - tls_materials: Option<&GuestTlsMaterials>, - sandbox_token: Option<&str>, ) -> Result<(), String> { - clone_or_copy_sparse_file(template_path, overlay_disk)?; - if let Some(tls) = tls_materials { - inject_guest_tls_materials(overlay_disk, tls)?; - } - if let Some(token) = sandbox_token { - inject_guest_sandbox_token(overlay_disk, token)?; - } - Ok(()) + clone_or_copy_sparse_file(template_path, overlay_disk) } fn prepare_sandbox_overlay_image( template_path: &Path, overlay_disk: &Path, - tls_materials: Option<&GuestTlsMaterials>, - sandbox_token: Option<&str>, preparation: OverlayPreparation, expected_size_bytes: u64, ) -> Result<(), String> { if preparation == OverlayPreparation::PreserveExisting { match fs::metadata(overlay_disk) { Ok(metadata) if metadata.is_file() && metadata.len() == expected_size_bytes => { - if let Some(tls) = tls_materials { - inject_guest_tls_materials(overlay_disk, tls)?; - } - if let Some(token) = sandbox_token { - inject_guest_sandbox_token(overlay_disk, token)?; - } return Ok(()); } Ok(metadata) if metadata.is_file() => { @@ -5034,37 +5106,15 @@ fn prepare_sandbox_overlay_image( } } - create_sandbox_overlay_image_from_template( - template_path, - overlay_disk, - tls_materials, - sandbox_token, - ) + create_sandbox_overlay_image_from_template(template_path, overlay_disk) } -fn inject_guest_tls_materials( - overlay_disk: &Path, - materials: &GuestTlsMaterials, -) -> Result<(), String> { - write_rootfs_image_file( - overlay_disk, - &overlay_upper_path(GUEST_TLS_CA_PATH), - &materials.ca, - )?; - write_rootfs_image_file( - overlay_disk, - &overlay_upper_path(GUEST_TLS_CERT_PATH), - &materials.cert, - )?; - let key_path = overlay_upper_path(GUEST_TLS_KEY_PATH); - write_rootfs_image_file(overlay_disk, &key_path, &materials.key)?; - set_rootfs_image_file_mode(overlay_disk, &key_path, 0o600) -} - -fn inject_guest_sandbox_token(overlay_disk: &Path, token: &str) -> Result<(), String> { - let token_path = overlay_upper_path(GUEST_SANDBOX_TOKEN_PATH); - write_rootfs_image_file(overlay_disk, &token_path, format!("{token}\n").as_bytes())?; - set_rootfs_image_file_mode(overlay_disk, &token_path, 0o600) +fn inject_guest_boundary_config(overlay_disk: &Path, config: &GuestConfig) -> Result<(), String> { + let config = serde_json::to_vec(config) + .map_err(|error| format!("encode VM guest boundary configuration: {error}"))?; + let config_path = overlay_upper_path(GUEST_BOUNDARY_CONFIG_PATH); + write_rootfs_image_file(overlay_disk, &config_path, &config)?; + set_rootfs_image_file_mode(overlay_disk, &config_path, 0o600) } #[allow(clippy::result_large_err)] @@ -5311,47 +5361,6 @@ fn dir_size_bytes(path: &Path) -> Result { Ok(total) } -#[cfg(test)] -fn stage_guest_tls_materials( - staging_dir: &Path, - materials: &GuestTlsMaterials, -) -> Result<(), String> { - let tls_dir = staging_dir - .join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')) - .parent() - .ok_or_else(|| "guest TLS CA path has no parent".to_string())? - .to_path_buf(); - fs::create_dir_all(&tls_dir) - .map_err(|err| format!("create guest TLS dir {}: {err}", tls_dir.display()))?; - - let ca_path = staging_dir - .join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')); - let cert_path = staging_dir - .join("upper") - .join(GUEST_TLS_CERT_PATH.trim_start_matches('/')); - let key_path = staging_dir - .join("upper") - .join(GUEST_TLS_KEY_PATH.trim_start_matches('/')); - fs::write(&ca_path, &materials.ca) - .map_err(|err| format!("write guest TLS CA {}: {err}", ca_path.display()))?; - fs::write(&cert_path, &materials.cert) - .map_err(|err| format!("write guest TLS cert {}: {err}", cert_path.display()))?; - fs::write(&key_path, &materials.key) - .map_err(|err| format!("write guest TLS key {}: {err}", key_path.display()))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - - fs::set_permissions(&key_path, fs::Permissions::from_mode(0o600)) - .map_err(|err| format!("chmod guest TLS key {}: {err}", key_path.display()))?; - } - - Ok(()) -} - fn overlay_staging_dir(overlay_disk: &Path) -> PathBuf { let parent = overlay_disk.parent().unwrap_or_else(|| Path::new(".")); parent.join(format!( @@ -5391,6 +5400,7 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { vm.backend = ?backend, ) )] +#[allow(dead_code)] fn spawn_vm_launcher( command: &mut Command, sandbox_id: &str, @@ -6048,7 +6058,7 @@ mod tests { let parent = tracing::info_span!("vm.provision"); let result = driver - .prepare_runtime_overlay(Path::new("/unused"), None, None, OverlayPreparation::Fresh) + .prepare_runtime_overlay(Path::new("/unused"), OverlayPreparation::Fresh) .instrument(parent) .await; assert!(result.is_err(), "overflow should stop before disk I/O"); @@ -6780,8 +6790,6 @@ mod tests { prepare_sandbox_overlay_image( &template, &overlay, - None, - None, OverlayPreparation::PreserveExisting, "saved-overlay".len() as u64, ) @@ -6803,8 +6811,6 @@ mod tests { prepare_sandbox_overlay_image( &template, &overlay, - None, - None, OverlayPreparation::PreserveExisting, "fresh-overlay".len() as u64, ) @@ -6818,8 +6824,8 @@ mod tests { #[test] fn overlay_upper_path_targets_overlay_upperdir() { assert_eq!( - overlay_upper_path(GUEST_TLS_KEY_PATH), - "/upper/opt/openshell/tls/tls.key" + overlay_upper_path(GUEST_BOUNDARY_CONFIG_PATH), + "/upper/etc/openshell/vm-guest.json" ); } @@ -7033,7 +7039,7 @@ mod tests { } #[test] - fn build_guest_environment_sets_supervisor_defaults() { + fn build_guest_environment_sets_process_leaf_boot_metadata() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() @@ -7045,16 +7051,18 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!(env.contains(&"HOME=/root".to_string())); - assert!(env.contains(&format!( - "OPENSHELL_ENDPOINT=http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/" - ))); assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); assert!(env.contains(&"OPENSHELL_SANDBOX=breezy-rhinoceros".to_string())); - assert!(env.contains(&format!( - "OPENSHELL_SSH_SOCKET_PATH={GUEST_SSH_SOCKET_PATH}" - ))); + assert!( + !env.iter() + .any(|entry| entry.starts_with("OPENSHELL_ENDPOINT=")) + ); + assert!( + !env.iter() + .any(|entry| entry.starts_with("OPENSHELL_SSH_SOCKET_PATH=")) + ); } #[test] @@ -7068,78 +7076,45 @@ mod tests { } #[test] - fn persisted_legacy_sandbox_without_command_uses_scratch_main() { + fn build_guest_environment_keeps_user_values_in_child_channel() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() }; - // Requests persisted before the canonical-main contract have a - // present DriverSandboxSpec but no command or tty fields. - let sandbox = Sandbox { - id: "legacy-sandbox".to_string(), - name: "legacy-sandbox".to_string(), - spec: Some(SandboxSpec::default()), - ..Default::default() - }; - - let env = build_guest_environment(&sandbox, &config, None); - let encoded = env - .iter() - .find_map(|entry| { - entry.strip_prefix(&format!( - "{}=", - openshell_core::sandbox_env::MAIN_PROCESS_SPEC - )) - }) - .expect("main process environment"); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded) - .expect("legacy persisted request should produce a valid main config"); - - assert_eq!( - main, - openshell_core::sandbox_env::MainProcessConfig::scratch() - ); - } - - #[test] - fn build_guest_environment_preserves_main_command_spaces() { - let config = VmDriverConfig { - openshell_endpoint: "https://127.0.0.1:8080".to_string(), - ..Default::default() - }; - let command = vec![ - "sh".to_string(), - "-lc".to_string(), - "echo ready; while true; do sleep 1; done".to_string(), - ]; let sandbox = Sandbox { - id: "space-command".to_string(), - name: "space-command".to_string(), + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), spec: Some(SandboxSpec { - command: command.clone(), + environment: HashMap::from([ + ("LD_PRELOAD".to_string(), "/workload/evil.so".to_string()), + ("BAD;touch /root/pwned".to_string(), "value".to_string()), + ]), ..Default::default() }), ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); - let encoded = env - .iter() - .find_map(|entry| { - entry.strip_prefix(&format!( - "{}=", - openshell_core::sandbox_env::MAIN_PROCESS_SPEC - )) - }) - .expect("main process environment"); + let env = build_guest_environment(&sandbox, &config); - assert!(!encoded.contains(char::is_whitespace)); - let main = openshell_core::sandbox_env::MainProcessConfig::decode(encoded).unwrap(); - assert_eq!(main.command, command); + assert!(!env.iter().any(|entry| entry.starts_with("LD_PRELOAD="))); + assert!(!env.iter().any(|entry| entry.starts_with("BAD;"))); + assert!( + !env.iter() + .any(|entry| { entry.starts_with(openshell_core::sandbox_env::USER_ENVIRONMENT) }) + ); + let child_env = merged_environment(&sandbox); + assert_eq!( + child_env.get("LD_PRELOAD"), + Some(&"/workload/evil.so".to_string()) + ); + assert_eq!( + child_env.get("BAD;touch /root/pwned"), + Some(&"value".to_string()) + ); } #[test] - fn build_guest_environment_uses_token_file_without_raw_token_env() { + fn build_guest_environment_excludes_all_gateway_credentials() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), ..Default::default() @@ -7158,16 +7133,16 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!(!env.iter().any(|v| v.starts_with(&format!( "{}=", openshell_core::sandbox_env::SANDBOX_TOKEN )))); - assert!(env.contains(&format!( - "{}={GUEST_SANDBOX_TOKEN_PATH}", + assert!(!env.iter().any(|v| v.starts_with(&format!( + "{}=", openshell_core::sandbox_env::SANDBOX_TOKEN_FILE - ))); + )))); } #[test] @@ -7189,7 +7164,7 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); assert!( !env.iter().any(|v| v.starts_with(&format!( @@ -7226,7 +7201,7 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); + let env = build_guest_environment(&sandbox, &config); let telemetry_entries = env .iter() .filter(|entry| { @@ -7247,6 +7222,7 @@ mod tests { } #[test] + #[cfg(any())] fn build_guest_environment_clears_unsupported_network_capabilities() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), @@ -7277,6 +7253,7 @@ mod tests { } #[test] + #[cfg(any())] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { openshell_endpoint: "http://127.0.0.1:8080".to_string(), @@ -7305,6 +7282,7 @@ mod tests { } #[test] + #[cfg(any())] fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_host_alias() { assert_eq!( guest_visible_openshell_endpoint("http://127.0.0.1:8080"), @@ -7321,6 +7299,7 @@ mod tests { } #[test] + #[cfg(any())] fn guest_visible_openshell_endpoint_preserves_non_loopback_hosts() { assert_eq!( guest_visible_openshell_endpoint(&format!( @@ -7479,7 +7458,7 @@ mod tests { } #[test] - fn build_guest_environment_includes_tls_paths_for_https_endpoint() { + fn build_guest_environment_keeps_tls_paths_host_side() { let config = VmDriverConfig { openshell_endpoint: "https://127.0.0.1:8443".to_string(), guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), @@ -7494,10 +7473,8 @@ mod tests { ..Default::default() }; - let env = build_guest_environment(&sandbox, &config, None); - assert!(env.contains(&format!("OPENSHELL_TLS_CA={GUEST_TLS_CA_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_CERT={GUEST_TLS_CERT_PATH}"))); - assert!(env.contains(&format!("OPENSHELL_TLS_KEY={GUEST_TLS_KEY_PATH}"))); + let env = build_guest_environment(&sandbox, &config); + assert!(!env.iter().any(|entry| entry.starts_with("OPENSHELL_TLS_"))); } #[test] @@ -7562,6 +7539,7 @@ mod tests { record.state_dir = retry_state_dir; record.process = Some(Arc::new(Mutex::new(VmProcess { child: spawn_exited_child(), + supervisor: spawn_exited_child(), deleting: false, }))); } @@ -7769,14 +7747,14 @@ mod tests { } #[test] - fn bootstrap_image_cache_identity_includes_rootfs_layout_and_openshell_version() { - assert_eq!( - bootstrap_image_cache_identity("sha256:bootstrap-image"), - format!( - "sandbox-bootstrap-rootfs-ext4-v3:openshell-{}:sha256:bootstrap-image", - openshell_core::VERSION - ) - ); + fn bootstrap_image_cache_identity_includes_rootfs_layout_version_and_guest_runtime() { + let identity = bootstrap_image_cache_identity("sha256:bootstrap-image"); + assert!(identity.starts_with(&format!( + "sandbox-bootstrap-rootfs-ext4-v4:openshell-{}:guest-", + openshell_core::VERSION + ))); + assert!(identity.ends_with(":sha256:bootstrap-image")); + assert!(identity.contains(&sandbox_guest_runtime_identity())); } #[test] @@ -7876,66 +7854,6 @@ mod tests { ); } - #[tokio::test] - async fn read_guest_tls_materials_reports_missing_input() { - let base = unique_temp_dir(); - let source_dir = base.join("missing-source"); - - let err = read_guest_tls_materials(&VmDriverTlsPaths { - ca: source_dir.join("ca.crt"), - cert: source_dir.join("tls.crt"), - key: source_dir.join("tls.key"), - }) - .await - .expect_err("missing TLS materials should fail before image injection"); - - assert!(err.contains("ca.crt")); - - let _ = std::fs::remove_dir_all(base); - } - - #[cfg(unix)] - #[test] - fn stage_guest_tls_materials_places_files_in_overlay_upper_with_private_key_mode() { - use std::os::unix::fs::PermissionsExt as _; - - let base = unique_temp_dir(); - let materials = GuestTlsMaterials { - ca: b"ca".to_vec(), - cert: b"cert".to_vec(), - key: b"key".to_vec(), - }; - - stage_guest_tls_materials(&base, &materials).expect("stage TLS materials"); - - assert_eq!( - fs::read( - base.join("upper") - .join(GUEST_TLS_CA_PATH.trim_start_matches('/')) - ) - .unwrap(), - b"ca" - ); - assert_eq!( - fs::read( - base.join("upper") - .join(GUEST_TLS_CERT_PATH.trim_start_matches('/')) - ) - .unwrap(), - b"cert" - ); - let key_path = base - .join("upper") - .join(GUEST_TLS_KEY_PATH.trim_start_matches('/')); - assert_eq!(fs::read(&key_path).unwrap(), b"key"); - assert_eq!( - fs::metadata(&key_path).unwrap().permissions().mode() & 0o777, - 0o600 - ); - - let _ = std::fs::remove_dir_all(base); - } - #[test] fn subnet_allocator_assigns_and_releases() { let mut alloc = SubnetAllocator::new(Ipv4Addr::new(10, 0, 128, 0), 17); @@ -8010,6 +7928,7 @@ mod tests { }; let process = Arc::new(Mutex::new(VmProcess { child, + supervisor: spawn_exited_child(), deleting: false, })); diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs index 423ad6f05b..bdb1d3f509 100644 --- a/crates/openshell-driver-vm/src/ffi.rs +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -52,6 +52,8 @@ type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunAddVsock = unsafe extern "C" fn(ctx_id: u32, tsi_features: u32) -> i32; +type KrunAddVsockPort2 = + unsafe extern "C" fn(ctx_id: u32, port: u32, filepath: *const c_char, listen: bool) -> i32; #[cfg(target_os = "macos")] type KrunAddNetUnixgram = unsafe extern "C" fn( ctx_id: u32, @@ -86,6 +88,7 @@ pub struct LibKrun { pub krun_start_enter: KrunStartEnter, pub krun_disable_implicit_vsock: KrunDisableImplicitVsock, pub krun_add_vsock: KrunAddVsock, + pub krun_add_vsock_port2: KrunAddVsockPort2, #[cfg(target_os = "macos")] pub krun_add_net_unixgram: KrunAddNetUnixgram, #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode. @@ -151,6 +154,7 @@ impl LibKrun { &libkrun_path, )?, krun_add_vsock: load_symbol(library, b"krun_add_vsock\0", &libkrun_path)?, + krun_add_vsock_port2: load_symbol(library, b"krun_add_vsock_port2\0", &libkrun_path)?, #[cfg(target_os = "macos")] krun_add_net_unixgram: load_symbol(library, b"krun_add_net_unixgram\0", &libkrun_path)?, krun_add_net_unixstream: load_symbol( diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 98ba6b0c9a..7a8bedc4c3 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -19,6 +19,6 @@ pub use lifecycle::{ RestoreContext, }; pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, + VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, VsockPortMap, cleanup_stale_tap_interfaces, configured_runtime_dir, run_vm, }; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 949d4ce05c..366a208ddc 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -9,7 +9,9 @@ use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServ use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer; #[cfg(target_os = "macos")] use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; -use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; +use openshell_driver_vm::{ + VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, VsockPortMap, procguard, run_vm, +}; use std::io; use std::net::SocketAddr; use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; @@ -166,6 +168,12 @@ struct Args { #[arg(long, hide = true)] vm_gateway_port: Option, + + #[arg(long, hide = true)] + vm_vsock_control_port: Option, + + #[arg(long, hide = true)] + vm_vsock_control_socket: Option, } #[tokio::main] @@ -551,6 +559,23 @@ fn build_vm_launch_config(args: &Args) -> std::result::Result Some(VsockPortMap { + guest_port, + host_socket, + host_initiated: true, + }), + (None, None) => None, + _ => { + return Err( + "--vm-vsock-control-port and --vm-vsock-control-socket must be set together" + .to_string(), + ); + } + }, }) } diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs index 9046913c9d..000d50d697 100644 --- a/crates/openshell-driver-vm/src/rootfs.rs +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use sha2::{Digest, Sha256}; use std::fs; use std::fs::File; #[cfg(test)] @@ -11,6 +12,8 @@ use std::process::Command; use std::sync::atomic::{AtomicU64, Ordering}; const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst")); +const SUPERVISOR_RUNTIME: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/openshell-runtime.tar.zst")); const UMOCI: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/umoci.zst")); const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; @@ -18,6 +21,7 @@ const SANDBOX_SUPERVISOR_PATH: &str = openshell_core::driver_utils::SUPERVISOR_C const SANDBOX_UMOCI_PATH: &str = openshell_core::container_paths::VM_UMOCI_PATH; const SANDBOX_OWNER_NORMALIZED_MARKER: &str = openshell_core::container_paths::VM_SANDBOX_OWNER_NORMALIZED_MARKER; +const SANDBOX_SUPERVISOR_RUNTIME_PATH: &str = "/opt/openshell/bin/openshell-runtime"; const ROOTFS_IMAGE_MIN_SIZE_BYTES: u64 = 512 * 1024 * 1024; const ROOTFS_IMAGE_MIN_HEADROOM_BYTES: u64 = 256 * 1024 * 1024; const EXT4_IMAGE_MIN_HEADROOM_BYTES: u64 = 16 * 1024 * 1024; @@ -27,6 +31,44 @@ pub const fn sandbox_guest_init_path() -> &'static str { SANDBOX_GUEST_INIT_PATH } +/// Identity of every embedded artifact materialized into a bootstrap rootfs. +/// +/// Including this in the image-cache key makes local, uncommitted guest-leaf +/// changes invalidate the cache even when the `OpenShell` version is unchanged. +pub fn sandbox_guest_runtime_identity() -> String { + let mut hasher = Sha256::new(); + hasher.update(SUPERVISOR); + hasher.update(SUPERVISOR_RUNTIME); + hasher.update(UMOCI); + hasher.update(include_bytes!("../scripts/openshell-vm-sandbox-init.sh")); + format!("{:x}", hasher.finalize()) +} + +/// Materialize the supervisor embedded in the VM driver for host-side use. +pub fn extract_host_supervisor(path: &Path) -> Result<(), String> { + if SUPERVISOR.is_empty() { + return Err( + "host supervisor is not embedded; run `mise run vm:supervisor` and rebuild openshell-driver-vm" + .to_string(), + ); + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|error| format!("create {}: {error}", parent.display()))?; + } + let supervisor = zstd::decode_all(Cursor::new(SUPERVISOR)) + .map_err(|error| format!("decompress host supervisor: {error}"))?; + fs::write(path, supervisor).map_err(|error| format!("write {}: {error}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + fs::set_permissions(path, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("chmod {}: {error}", path.display()))?; + } + Ok(()) +} + #[allow(clippy::similar_names)] pub fn prepare_sandbox_rootfs_from_image_root( rootfs: &Path, @@ -376,6 +418,8 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> } ensure_supervisor_binary(rootfs)?; + ensure_supervisor_runtime(rootfs)?; + ensure_guest_init_ip(rootfs)?; ensure_umoci_binary(rootfs)?; let opt_dir = rootfs.join("opt/openshell"); @@ -392,9 +436,55 @@ fn prepare_sandbox_rootfs(rootfs: &Path, sandbox_uid: u32, sandbox_gid: u32) -> Ok(()) } +fn ensure_guest_init_ip(rootfs: &Path) -> Result<(), String> { + const IP_PATHS: [&str; 4] = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"]; + if IP_PATHS.iter().any(|path| rootfs.join(path).is_file()) { + return Ok(()); + } + + // Guest init runs before the process leaf can enter its trusted helper + // runtime. Images such as stock Ubuntu do not ship iproute2, so install a + // driver-owned launcher that executes the embedded musl helper explicitly. + // The helper and loader are both materialized from the trusted runtime, + // never from the workload image. + let path = rootfs.join("usr/sbin/ip"); + let parent = path + .parent() + .ok_or_else(|| format!("guest ip launcher path has no parent: {}", path.display()))?; + fs::create_dir_all(parent).map_err(|error| format!("create {}: {error}", parent.display()))?; + fs::write( + &path, + r#"#!/bin/sh +set -eu +runtime=/opt/openshell/bin/openshell-runtime +for loader in "$runtime"/lib/ld-musl-*.so.1; do + if [ -x "$loader" ]; then + for helper in "$runtime"/sbin/ip "$runtime"/usr/sbin/ip "$runtime"/bin/ip "$runtime"/usr/bin/ip; do + if [ -x "$helper" ]; then + exec "$loader" --library-path "$runtime/lib:$runtime/usr/lib" "$helper" "$@" + fi + done + fi +done +echo "trusted OpenShell ip helper is unavailable" >&2 +exit 127 +"#, + ) + .map_err(|error| format!("write {}: {error}", path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)) + .map_err(|error| format!("chmod {}: {error}", path.display()))?; + } + Ok(()) +} + pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> { require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?; require_rootfs_path(rootfs, SANDBOX_SUPERVISOR_PATH)?; + validate_supervisor_runtime(rootfs)?; require_rootfs_path(rootfs, SANDBOX_UMOCI_PATH)?; require_any_rootfs_path(rootfs, &["/bin/bash"])?; require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?; @@ -871,6 +961,78 @@ fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> { Ok(()) } +fn ensure_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + if validate_supervisor_runtime(rootfs).is_ok() { + return Ok(()); + } + if SUPERVISOR_RUNTIME.is_empty() { + return Err( + "trusted supervisor helper runtime not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:supervisor` first" + .to_string(), + ); + } + + install_supervisor_runtime_archive(rootfs, SUPERVISOR_RUNTIME) +} + +fn install_supervisor_runtime_archive(rootfs: &Path, archive_bytes: &[u8]) -> Result<(), String> { + let destination = rootfs.join("opt/openshell/bin"); + fs::create_dir_all(&destination) + .map_err(|e| format!("create {}: {e}", destination.display()))?; + let decoder = zstd::Decoder::new(Cursor::new(archive_bytes)) + .map_err(|e| format!("decompress supervisor runtime: {e}"))?; + let mut archive = tar::Archive::new(decoder); + for entry in archive + .entries() + .map_err(|e| format!("open supervisor runtime archive: {e}"))? + { + let mut entry = entry.map_err(|e| format!("read supervisor runtime archive: {e}"))?; + let kind = entry.header().entry_type(); + if !kind.is_file() && !kind.is_dir() { + return Err( + "supervisor runtime archive contains a non-materialized link or special file" + .to_string(), + ); + } + if !entry + .unpack_in(&destination) + .map_err(|e| format!("extract supervisor runtime archive: {e}"))? + { + return Err("supervisor runtime archive contains a path outside its root".to_string()); + } + } + validate_supervisor_runtime(rootfs) +} + +fn validate_supervisor_runtime(rootfs: &Path) -> Result<(), String> { + let runtime = rootfs.join(SANDBOX_SUPERVISOR_RUNTIME_PATH.trim_start_matches('/')); + let has_ip = ["sbin/ip", "usr/sbin/ip", "bin/ip", "usr/bin/ip"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_nft = ["sbin/nft", "usr/sbin/nft", "usr/bin/nft"] + .iter() + .any(|path| runtime.join(path).is_file()); + let has_loader = fs::read_dir(runtime.join("lib")) + .ok() + .into_iter() + .flatten() + .filter_map(Result::ok) + .any(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("ld-musl-") && name.ends_with(".so.1")) + }); + if has_ip && has_nft && has_loader { + Ok(()) + } else { + Err(format!( + "trusted supervisor helper runtime '{}' is incomplete", + runtime.display() + )) + } +} + fn ensure_umoci_binary(rootfs: &Path) -> Result<(), String> { let path = rootfs.join(SANDBOX_UMOCI_PATH.trim_start_matches('/')); if UMOCI.is_empty() { @@ -944,10 +1106,28 @@ fn remove_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + #[cfg(unix)] + use std::os::unix::fs::PermissionsExt as _; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn guest_init_gets_driver_owned_ip_launcher_when_image_omits_iproute2() { + let rootfs = tempfile::tempdir().expect("create rootfs"); + ensure_guest_init_ip(rootfs.path()).expect("install guest ip launcher"); + + let launcher = rootfs.path().join("usr/sbin/ip"); + let contents = fs::read_to_string(&launcher).expect("read guest ip launcher"); + assert!(contents.contains("/opt/openshell/bin/openshell-runtime")); + assert!(contents.contains("ld-musl-")); + #[cfg(unix)] + assert_eq!( + fs::metadata(launcher).unwrap().permissions().mode() & 0o777, + 0o755 + ); + } + #[test] fn prepare_sandbox_rootfs_rewrites_guest_layout() { let dir = unique_temp_dir(); @@ -979,6 +1159,16 @@ mod tests { assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); assert!(rootfs.join("opt/openshell/bin/umoci").is_file()); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + let init_script = fs::read_to_string(rootfs.join("srv/openshell-vm-sandbox-init.sh")) + .expect("read guest init"); + assert!(init_script.contains("vm-guest /etc/openshell/vm-guest.json")); + assert!(!init_script.contains("--topology-backend-name=in-pod")); + assert!(!init_script.contains("@ISOLATION_INTERFACE_VERSION@")); assert!(rootfs.join("sandbox").is_dir()); assert!(rootfs.join("image-cache").is_dir()); assert!(rootfs.join("lower").is_dir()); @@ -1010,6 +1200,44 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn supervisor_runtime_archive_materializes_below_the_trusted_path() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + let mut tar_bytes = Vec::new(); + { + let mut archive = tar::Builder::new(&mut tar_bytes); + for (path, bytes, mode) in [ + ("openshell-runtime/usr/sbin/ip", b"ip".as_slice(), 0o755), + ("openshell-runtime/usr/sbin/nft", b"nft".as_slice(), 0o755), + ( + "openshell-runtime/lib/ld-musl-test.so.1", + b"loader".as_slice(), + 0o755, + ), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(mode); + header.set_entry_type(tar::EntryType::Regular); + header.set_cksum(); + archive + .append_data(&mut header, path, bytes) + .expect("append runtime entry"); + } + archive.finish().expect("finish runtime archive"); + } + let compressed = zstd::encode_all(Cursor::new(tar_bytes), 1).expect("compress runtime"); + + install_supervisor_runtime_archive(&rootfs, &compressed).expect("install runtime"); + validate_supervisor_runtime(&rootfs).expect("validate runtime"); + assert!( + rootfs + .join("opt/openshell/bin/openshell-runtime/usr/sbin/nft") + .is_file() + ); + } + #[test] fn prepare_sandbox_rootfs_preserves_image_workdir_contents_in_rootfs() { let dir = unique_temp_dir(); @@ -1230,6 +1458,13 @@ mod tests { } fn write_fake_runtime_binaries(rootfs: &Path) { + let helper_runtime = rootfs.join("opt/openshell/bin/openshell-runtime"); + fs::create_dir_all(helper_runtime.join("usr/sbin")).expect("create helper bin directory"); + fs::create_dir_all(helper_runtime.join("lib")).expect("create helper lib directory"); + fs::write(helper_runtime.join("usr/sbin/ip"), b"ip").expect("write ip helper"); + fs::write(helper_runtime.join("usr/sbin/nft"), b"nft").expect("write nft helper"); + fs::write(helper_runtime.join("lib/ld-musl-test.so.1"), b"loader") + .expect("write helper loader"); fs::write( rootfs.join("opt/openshell/bin/openshell-sandbox"), b"sandbox", diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index f6020af829..93d5f96b65 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -31,6 +31,13 @@ pub enum VmBackend { Qemu, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VsockPortMap { + pub guest_port: u32, + pub host_socket: PathBuf, + pub host_initiated: bool, +} + // virtio-net feature bits (see Linux `include/uapi/linux/virtio_net.h`). const NET_FEATURE_CSUM: u32 = 1 << 0; const NET_FEATURE_GUEST_CSUM: u32 = 1 << 1; @@ -66,6 +73,7 @@ pub struct VmLaunchConfig { pub vsock_cid: Option, pub guest_mac: Option, pub gateway_port: Option, + pub vsock_port_map: Option, } pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { @@ -818,6 +826,10 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { vm.disable_implicit_vsock()?; vm.add_vsock(0)?; + if let Some(port_map) = &config.vsock_port_map { + let _ = std::fs::remove_file(&port_map.host_socket); + vm.add_vsock_port(port_map)?; + } let mac: [u8; 6] = [0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xee]; @@ -1118,6 +1130,21 @@ impl VmContext { ) } + fn add_vsock_port(&self, port_map: &VsockPortMap) -> Result<(), String> { + let socket_c = path_to_cstring(&port_map.host_socket)?; + check( + unsafe { + (self.krun.krun_add_vsock_port2)( + self.ctx_id, + port_map.guest_port, + socket_c.as_ptr(), + port_map.host_initiated, + ) + }, + "krun_add_vsock_port2", + ) + } + #[cfg(target_os = "macos")] fn add_net_unixgram( &self, @@ -1437,6 +1464,7 @@ mod tests { vsock_cid: Some(4), guest_mac: Some("02:00:00:00:00:01".to_string()), gateway_port: Some(8080), + vsock_port_map: None, } } diff --git a/crates/openshell-isolation-vm/Cargo.toml b/crates/openshell-isolation-vm/Cargo.toml new file mode 100644 index 0000000000..6e7ef1415f --- /dev/null +++ b/crates/openshell-isolation-vm/Cargo.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-isolation-vm" +description = "Shared authenticated VM boundary transport for OpenShell isolation backends" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } +openshell-supervisor-network = { path = "../openshell-supervisor-network" } +openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +async-trait = "0.1" +libc = "0.2" +nix = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/openshell-isolation-vm/src/backend.rs b/crates/openshell-isolation-vm/src/backend.rs new file mode 100644 index 0000000000..d424b11a81 --- /dev/null +++ b/crates/openshell-isolation-vm/src/backend.rs @@ -0,0 +1,884 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned VM. + +#![allow(unsafe_code)] + +use std::fmt; +use std::mem::size_of; +use std::os::fd::{FromRawFd as _, IntoRawFd as _}; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use openshell_isolation::AgentSpec; +use openshell_isolation::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryInput, BoundaryOutput, BoundaryPortForward, BoundaryProcess, BoundarySignal, + BoundaryTerminal, ExecSession, ExecSpec, INTERFACE_VERSION, IsolationBackend, LoopbackTarget, + MediatedConnection, NetworkMediationSource, ReadyBoundary, RunningBoundary, SandboxContext, + VerifiedTopologyDescriptor, +}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; +use tokio::sync::Notify; + +use crate::protocol::{ + AgentSpecWire, ExecSpecWire, ExitStatusWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, + Response, ResponseEnvelope, STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, + STREAM_STDOUT, SandboxPolicyWire, SignalWire, decode_frame, encode_frame, read_stream_frame, + write_stream_frame, +}; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; + +/// Hypervisor-specific host-to-guest control transport. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum VmTransport { + /// A raw Unix stream mapped to a guest vsock port by libkrun. + MappedUnix { socket_path: PathBuf }, + /// A Linux host `AF_VSOCK` connection to a QEMU guest. + HostVsock { guest_cid: u32, control_port: u32 }, +} + +/// Backend-private provisioned topology payload. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VmTopology { + pub boundary_id: String, + pub transport: VmTransport, + pub bootstrap_token: String, +} + +impl fmt::Debug for VmTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("VmTopology") + .field("boundary_id", &self.boundary_id) + .field("transport", &self.transport) + .field("bootstrap_token", &"") + .finish() + } +} + +impl VmTopology { + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}"))) + } +} + +/// Host-side VM implementation registered with the supervisor. +#[derive(Debug)] +pub struct VmHostBackend { + backend_name: String, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +impl VmHostBackend { + pub fn new( + backend_name: impl Into, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, + ) -> Self { + Self { + backend_name: backend_name.into(), + ca_file_paths, + provider_env, + } + } +} + +#[async_trait] +impl IsolationBackend for VmHostBackend { + fn backend_name(&self) -> &str { + &self.backend_name + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: VmTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox)?; + let client = Arc::new(GuestClient::new(topology)); + expect_response( + client + .call_idempotent(Request::Attach { + policy: Box::new(SandboxPolicyWire::from(sandbox.policy.clone())), + }) + .await?, + "attached", + )?; + Ok(Box::new(VmBound { + client: client.clone(), + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(VmNetworkMediation { client }), + ca_file_paths: self.ca_file_paths.clone(), + provider_env: self.provider_env.clone(), + })) + } +} + +fn validate_topology(topology: &VmTopology, sandbox: &SandboxContext) -> Result<(), BackendError> { + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "VM boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "VM bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + match &topology.transport { + VmTransport::MappedUnix { socket_path } => validate_socket_path(socket_path)?, + VmTransport::HostVsock { + guest_cid, + control_port, + } => { + if *guest_cid < 3 { + return Err(BackendError::Descriptor( + "VM guest CID must be at least 3".to_string(), + )); + } + validate_control_port(*control_port)?; + } + } + Ok(()) +} + +fn validate_socket_path(path: &std::path::Path) -> Result<(), BackendError> { + if path.is_absolute() { + Ok(()) + } else { + Err(BackendError::Descriptor( + "VM control Unix socket path must be absolute".to_string(), + )) + } +} + +fn validate_control_port(port: u32) -> Result<(), BackendError> { + if port == 0 { + Err(BackendError::Descriptor( + "VM guest control port must be nonzero".to_string(), + )) + } else { + Ok(()) + } +} + +struct VmBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +#[async_trait] +impl BoundBoundary for VmBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + async fn confirm(self: Box) -> Result, BackendError> { + expect_response( + self.client.call_idempotent(Request::Confirm).await?, + "confirmed", + )?; + Ok(Box::new(VmReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + ca_file_paths: self.ca_file_paths, + provider_env: self.provider_env, + })) + } +} + +struct VmReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + ca_file_paths: Arc>>, + provider_env: std::collections::HashMap, +} + +#[async_trait] +impl ReadyBoundary for VmReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let ca_paths = self + .ca_file_paths + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + let (ca_cert, ca_bundle) = if let Some((ca_cert, ca_bundle)) = ca_paths { + let ca_cert = tokio::fs::read(&ca_cert).await.map_err(|error| { + BackendError::Process(format!("read host proxy CA {}: {error}", ca_cert.display())) + })?; + let ca_bundle = tokio::fs::read(&ca_bundle).await.map_err(|error| { + BackendError::Process(format!( + "read host proxy CA bundle {}: {error}", + ca_bundle.display() + )) + })?; + (Some(ca_cert), Some(ca_bundle)) + } else { + (None, None) + }; + let response = self + .client + .call(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + ca_cert, + ca_bundle, + provider_env: self.provider_env, + }) + .await?; + let Response::Started { process_id } = response else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(VmProcess { + client: self.client.clone(), + process_id, + }); + Ok(Box::new(VmRunning { + process, + exec: Arc::new(VmExec { + client: self.client.clone(), + }), + port_forward: Arc::new(VmPortForward { + client: self.client, + }), + })) + } +} + +struct VmRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for VmRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct VmProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for VmProcess { + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +struct VmExec { + client: Arc, +} + +#[async_trait] +impl BoundaryExec for VmExec { + async fn exec(&self, spec: ExecSpec) -> Result { + open_exec_session(self.client.clone(), spec).await + } +} + +struct VmPortForward { + client: Arc, +} + +#[async_trait] +impl BoundaryPortForward for VmPortForward { + async fn connect(&self, target: LoopbackTarget) -> Result { + let (stream, response) = self + .client + .call_stream(Request::PortForward { + host: target.host(), + port: target.port(), + }) + .await?; + match response { + Response::PortConnected => Ok(stream), + response => Err(unexpected_response("port_connected", &response)), + } + } +} + +struct RemoteExecProcess { + client: Arc, + process_id: String, + exit: Arc, +} + +struct RemoteExit { + result: std::sync::Mutex>>, + changed: Notify, +} + +impl RemoteExit { + fn new() -> Self { + Self { + result: std::sync::Mutex::new(None), + changed: Notify::new(), + } + } + + fn set(&self, result: Result) { + let mut current = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if current.is_none() { + *current = Some(result); + self.changed.notify_waiters(); + } + } + + async fn wait(&self) -> Result { + loop { + let changed = self.changed.notified(); + let result = self + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + if let Some(result) = result { + return result.map_err(BackendError::Terminated); + } + changed.await; + } + } +} + +#[async_trait] +impl BoundaryProcess for RemoteExecProcess { + async fn wait(&self) -> Result { + self.exit.wait().await + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + expect_response( + self.client + .call(Request::ExecSignal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?, + "signaled", + ) + } + + async fn terminate(&self) -> Result<(), BackendError> { + self.signal(BoundarySignal::Kill).await + } +} + +struct RemoteTerminal { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryTerminal for RemoteTerminal { + async fn resize(&self, cols: u16, rows: u16) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Resize { + process_id: self.process_id.clone(), + cols, + rows, + }) + .await?; + if matches!(response, Response::Resized) { + Ok(()) + } else { + Err(unexpected_response("resized", &response)) + } + } +} + +async fn open_exec_session( + client: Arc, + spec: ExecSpec, +) -> Result { + let (stream, response) = client + .call_stream(Request::Exec { + spec: ExecSpecWire::from(spec), + }) + .await?; + let Response::ExecStarted { process_id, pty } = response else { + return Err(unexpected_response("exec_started", &response)); + }; + let (network_reader, network_writer) = tokio::io::split(stream); + let (stdin, stdin_pump) = tokio::io::duplex(64 * 1024); + let (stdout, stdout_pump) = tokio::io::duplex(64 * 1024); + let (stderr, stderr_pump) = tokio::io::duplex(64 * 1024); + let exit = Arc::new(RemoteExit::new()); + tokio::spawn(pump_exec_input(stdin_pump, network_writer)); + tokio::spawn(pump_exec_responses( + network_reader, + stdout_pump, + stderr_pump, + exit.clone(), + )); + + let process: Arc = Arc::new(RemoteExecProcess { + client: client.clone(), + process_id: process_id.clone(), + exit, + }); + let terminal: Option> = if pty { + Some(Arc::new(RemoteTerminal { client, process_id })) + } else { + None + }; + let stdin: BoundaryInput = Box::new(stdin); + let stdout: BoundaryOutput = Box::new(stdout); + let stderr: Option = if pty { None } else { Some(Box::new(stderr)) }; + Ok(ExecSession { + process, + stdin: Some(stdin), + stdout, + stderr, + terminal, + }) +} + +async fn pump_exec_input( + mut input: tokio::io::DuplexStream, + mut network: tokio::io::WriteHalf, +) { + let mut buffer = vec![0; 16 * 1024]; + loop { + match input.read(&mut buffer).await { + Ok(0) => { + let _ = write_stream_frame(&mut network, STREAM_STDIN_CLOSED, &[]).await; + return; + } + Ok(read) => { + if write_stream_frame(&mut network, STREAM_STDIN, &buffer[..read]) + .await + .is_err() + { + return; + } + } + Err(_) => return, + } + } +} + +async fn pump_exec_responses( + mut network: tokio::io::ReadHalf, + mut stdout: tokio::io::DuplexStream, + mut stderr: tokio::io::DuplexStream, + exit: Arc, +) { + loop { + match read_stream_frame(&mut network).await { + Ok(Some((STREAM_STDOUT, payload))) => { + if stdout.write_all(&payload).await.is_err() { + exit.set(Err("VM exec stdout consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_STDERR, payload))) => { + if stderr.write_all(&payload).await.is_err() { + exit.set(Err("VM exec stderr consumer closed".to_string())); + return; + } + } + Ok(Some((STREAM_EXIT, payload))) => { + let result = serde_json::from_slice::(&payload) + .map(BoundaryExitStatus::from) + .map_err(|error| format!("decode VM exec exit: {error}")); + exit.set(result); + return; + } + Ok(Some((channel, _))) => { + exit.set(Err(format!( + "VM exec returned unexpected stream channel {channel}" + ))); + return; + } + Ok(None) => { + exit.set(Err("VM exec stream closed before exit status".to_string())); + return; + } + Err(error) => { + exit.set(Err(format!("read VM exec stream: {error}"))); + return; + } + } + } +} + +/// Pulls guest proxy connections over one authenticated vsock stream each. +struct VmNetworkMediation { + client: Arc, +} + +#[async_trait] +impl NetworkMediationSource for VmNetworkMediation { + async fn accept(&self) -> Result { + let (stream, response) = self.client.open_exchange(Request::AcceptNetwork).await?; + let Response::NetworkConnected { identity } = response else { + return Err(unexpected_response("network_connected", &response)); + }; + Ok(MediatedConnection { + stream, + binary_identity: identity.into_result(), + }) + } +} + +struct GuestClient { + topology: VmTopology, + next_request_id: AtomicU64, +} + +impl GuestClient { + fn new(topology: VmTopology) -> Self { + Self { + topology, + next_request_id: AtomicU64::new(1), + } + } + + async fn call(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request)) + .await + .map_err(|_| BackendError::Unavailable("guest control request timed out".to_string()))? + } + + async fn call_idempotent(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, async { + loop { + match self.exchange(request.clone()).await { + Ok(response) => return Ok(response), + Err(BackendError::Unavailable(_)) => { + tokio::time::sleep(Duration::from_millis(25)).await; + } + Err(error) => return Err(error), + } + } + }) + .await + .map_err(|_| { + BackendError::Unavailable( + "guest idempotent control request timed out while waiting for VM boot".to_string(), + ) + })? + } + + async fn call_wait(&self, request: Request) -> Result { + self.exchange(request).await + } + + async fn call_stream( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + tokio::time::timeout(REQUEST_TIMEOUT, self.open_exchange(request)) + .await + .map_err(|_| BackendError::Unavailable("guest stream request timed out".to_string()))? + } + + async fn exchange(&self, request: Request) -> Result { + let (_, response) = self.open_exchange(request).await?; + Ok(response) + } + + async fn open_exchange( + &self, + request: Request, + ) -> Result<(BoundaryDuplexStream, Response), BackendError> { + let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed); + let envelope = RequestEnvelope { + request_id, + boundary_id: self.topology.boundary_id.clone(), + bootstrap_token: self.topology.bootstrap_token.clone(), + request, + }; + let mut stream = self.connect_vsock().await?; + let frame = encode_frame(&envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write guest control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read guest control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "guest control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read guest control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "guest response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + let response = match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + }?; + Ok((stream, response)) + } + + async fn connect_vsock(&self) -> Result { + loop { + match self.connect_vsock_once().await { + Ok(stream) => return Ok(stream), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } + + async fn connect_vsock_once(&self) -> Result { + match &self.topology.transport { + VmTransport::MappedUnix { socket_path } => { + let stream = UnixStream::connect(socket_path).await.map_err(|error| { + BackendError::Unavailable(format!( + "connect to mapped VM control socket {}: {error}", + socket_path.display() + )) + })?; + Ok(Box::new(stream)) + } + VmTransport::HostVsock { + guest_cid, + control_port, + } => connect_host_vsock(*guest_cid, *control_port), + } + } +} + +#[cfg(target_os = "linux")] +fn connect_host_vsock( + guest_cid: u32, + control_port: u32, +) -> Result { + let fd = unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(BackendError::Unavailable(format!( + "create host vsock: {}", + std::io::Error::last_os_error() + ))); + } + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(fd) }; + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address family: {error}")) + })?; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: control_port, + svm_cid: guest_cid, + svm_zero: [0; 4], + }; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|error| { + BackendError::Unavailable(format!("convert host vsock address length: {error}")) + })?; + let result = unsafe { + libc::connect( + std::os::fd::AsRawFd::as_raw_fd(&fd), + (&raw const address).cast::(), + address_length, + ) + }; + if result != 0 { + return Err(BackendError::Unavailable(format!( + "connect host vsock CID {guest_cid} port {control_port}: {}", + std::io::Error::last_os_error() + ))); + } + let stream = unsafe { std::os::unix::net::UnixStream::from_raw_fd(fd.into_raw_fd()) }; + stream.set_nonblocking(true).map_err(|error| { + BackendError::Unavailable(format!("set host vsock nonblocking: {error}")) + })?; + let stream = UnixStream::from_std(stream).map_err(|error| { + BackendError::Unavailable(format!("register host vsock with Tokio: {error}")) + })?; + Ok(Box::new(stream)) +} + +#[cfg(not(target_os = "linux"))] +fn connect_host_vsock( + _guest_cid: u32, + _control_port: u32, +) -> Result { + Err(BackendError::Unavailable( + "host AF_VSOCK transport is supported only on Linux".to_string(), + )) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached, "attached") + | (Response::Confirmed, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected guest response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("VM guest process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + + use super::*; + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = VmTopology { + boundary_id: "sandbox-1".to_string(), + transport: VmTransport::MappedUnix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = VmTopology { + boundary_id: "other".to_string(), + transport: VmTransport::MappedUnix { + socket_path: PathBuf::from("/tmp/vsock.sock"), + }, + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } +} diff --git a/crates/openshell-isolation-vm/src/guest.rs b/crates/openshell-isolation-vm/src/guest.rs new file mode 100644 index 0000000000..fc5dd61ca1 --- /dev/null +++ b/crates/openshell-isolation-vm/src/guest.rs @@ -0,0 +1,1112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Private guest mode shared by VM isolation drivers. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the host authorizes `start_agent`, it invokes the existing +//! `openshell-supervisor-process` implementation inside the VM. + +#![allow(unsafe_code)] + +use serde::{Deserialize, Serialize}; +use std::path::Path; + +const DEFAULT_CONTROL_PORT: u32 = 5500; +const DEFAULT_AGENT_UID: u32 = 10_001; +const DEFAULT_AGENT_GID: u32 = 10_001; + +/// Driver-private configuration injected into the guest image at provision time. +#[derive(Clone, Serialize, Deserialize)] +pub struct GuestConfig { + pub boundary_id: String, + pub bootstrap_token: String, + #[serde(default = "default_control_port")] + pub control_port: u32, + #[serde(default = "default_agent_uid")] + pub agent_uid: u32, + #[serde(default = "default_agent_gid")] + pub agent_gid: u32, + /// Absolute, driver-owned helper runtime used for namespace setup. + #[serde(default = "default_trusted_runtime_root")] + pub trusted_runtime_root: std::path::PathBuf, + /// Driver-resolved environment exposed only to workload processes. + #[serde(default)] + pub child_env: std::collections::HashMap, +} + +impl std::fmt::Debug for GuestConfig { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GuestConfig") + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("control_port", &self.control_port) + .field("agent_uid", &self.agent_uid) + .field("agent_gid", &self.agent_gid) + .field("trusted_runtime_root", &self.trusted_runtime_root) + .field("child_env_keys", &self.child_env.keys().collect::>()) + .finish() + } +} + +const fn default_control_port() -> u32 { + DEFAULT_CONTROL_PORT +} + +const fn default_agent_uid() -> u32 { + DEFAULT_AGENT_UID +} + +const fn default_agent_gid() -> u32 { + DEFAULT_AGENT_GID +} + +fn default_trusted_runtime_root() -> std::path::PathBuf { + std::path::PathBuf::from("/opt/openshell/bin/openshell-runtime") +} + +#[cfg(target_os = "linux")] +mod linux { + #[cfg(test)] + use super::{DEFAULT_AGENT_GID, DEFAULT_AGENT_UID, default_trusted_runtime_root}; + use super::{GuestConfig, Path}; + use std::ffi::CString; + use std::fs::File; + use std::io::{self, Read, Write}; + use std::mem::size_of; + use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd as _, OwnedFd}; + use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + use openshell_core::proposals::AgentProposals; + use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_isolation::contract::{ + BoundaryExec, BoundaryPortForward, BoundaryProcess, BoundaryTerminal, ExecSession, + LoopbackTarget, + }; + use openshell_supervisor_network::identity_source::ProcfsIdentityResolver; + use openshell_supervisor_process::boundary_io::BoundaryRuntimeState; + use openshell_supervisor_process::netns::{ + NetworkNamespace, create_conformant_netns_for_proxy, + }; + use openshell_supervisor_process::process::{ + ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity, + }; + use openshell_supervisor_process::run::{AgentSignaler, spawn_workload}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + use crate::protocol::{ + AgentSpecWire, BinaryIdentityWire, ExecSpecWire, ExitStatusWire, Request, RequestEnvelope, + Response, ResponseEnvelope, STREAM_EXIT, STREAM_STDERR, STREAM_STDIN, STREAM_STDIN_CLOSED, + STREAM_STDOUT, SandboxPolicyWire, SignalWire, read_frame, read_stream_frame, write_frame, + write_stream_frame, + }; + + const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + + pub fn run_guest(config_path: &Path) -> Result<(), String> { + if std::process::id() == 1 { + prepare_pid1_filesystems()?; + } + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read guest config {}: {error}", config_path.display()))?; + let config: GuestConfig = serde_json::from_slice(&bytes) + .map_err(|error| format!("decode guest config {}: {error}", config_path.display()))?; + validate_config(&config)?; + openshell_supervisor_process::netns::configure_trusted_runtime_root( + config.trusted_runtime_root.clone(), + ) + .map_err(|error| format!("configure trusted guest helper runtime: {error}"))?; + let child_env = serde_json::to_string(&config.child_env) + .map_err(|error| format!("encode guest workload environment: {error}"))?; + // This runs before the Tokio runtime or control threads exist. The process + // supervisor consumes the serialized map and applies values only to + // workload children. + unsafe { + std::env::set_var(openshell_core::sandbox_env::USER_ENVIRONMENT, child_env); + } + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create guest process runtime: {error}"))?; + let runtime = Arc::new(GuestRuntime::new( + config.clone(), + process_runtime.handle().clone(), + )); + serve(config.control_port, runtime) + } + + fn validate_config(config: &GuestConfig) -> Result<(), String> { + if config.boundary_id.is_empty() { + return Err("guest boundary ID must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("guest bootstrap token must contain at least 32 bytes".to_string()); + } + if config.control_port == 0 { + return Err("guest control port must be nonzero".to_string()); + } + if config.agent_uid == 0 || config.agent_gid == 0 { + return Err("guest agent UID and GID must be nonzero".to_string()); + } + if !config.trusted_runtime_root.is_absolute() { + return Err("guest trusted helper runtime root must be absolute".to_string()); + } + Ok(()) + } + + fn prepare_pid1_filesystems() -> Result<(), String> { + for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] { + std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?; + } + mount_if_needed("proc", "/proc", "proc")?; + mount_if_needed("sysfs", "/sys", "sysfs")?; + mount_if_needed("devtmpfs", "/dev", "devtmpfs")?; + std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?; + mount_if_needed("devpts", "/dev/pts", "devpts")?; + Ok(()) + } + + fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> { + let source = CString::new(source).map_err(|error| error.to_string())?; + let target_c = CString::new(target).map_err(|error| error.to_string())?; + let file_system = CString::new(file_system).map_err(|error| error.to_string())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + file_system.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EBUSY) { + Ok(()) + } else { + Err(format!("mount {file_system:?} on {target}: {error}")) + } + } + + fn serve(port: u32, runtime: Arc) -> Result<(), String> { + let listener = VsockListener::bind(port) + .map_err(|error| format!("bind guest control vsock port {port}: {error}"))?; + eprintln!("VM process supervisor leaf listening on vsock port {port}"); + loop { + match listener.accept() { + Ok(stream) => { + let runtime = runtime.clone(); + std::thread::spawn(move || { + if let Err(error) = serve_one(stream, &runtime) { + eprintln!("VM guest control request failed: {error}"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(format!("accept guest control connection: {error}")), + } + } + } + + fn serve_one(mut stream: VsockStream, runtime: &GuestRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + if !runtime.authenticate(&request) { + let response = ResponseEnvelope { + request_id: request.request_id, + response: guest_error("denied", "control authentication failed"), + }; + return write_frame(&mut stream, &response) + .map_err(|error| format!("write control frame: {error}")); + } + match request.request.clone() { + Request::Exec { spec } => { + let (process_id, session) = match runtime.start_exec(spec) { + Ok(started) => started, + Err(response) => { + return write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response, + }, + ) + .map_err(|error| format!("write exec error response: {error}")); + } + }; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::ExecStarted { + process_id: process_id.clone(), + pty: session.terminal.is_some(), + }, + }, + ) + .map_err(|error| format!("write exec start response: {error}"))?; + return runtime.stream_exec(stream, &process_id, session); + } + Request::PortForward { host, port } => { + let target = LoopbackTarget::new(host, port) + .map_err(|error| format!("validate port-forward target: {error}"))?; + let mut target = runtime + .connect_port(target) + .map_err(|error| format!("connect guest loopback port: {error}"))?; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::PortConnected, + }, + ) + .map_err(|error| format!("write port-forward response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge guest loopback stream: {error}")) + })?; + return Ok(()); + } + Request::AcceptNetwork => { + let (mut target, identity) = runtime.accept_network()?; + write_frame( + &mut stream, + &ResponseEnvelope { + request_id: request.request_id, + response: Response::NetworkConnected { identity }, + }, + ) + .map_err(|error| format!("write network mediation response: {error}"))?; + runtime.process_runtime.block_on(async move { + let mut stream = stream.into_tokio()?; + tokio::io::copy_bidirectional(&mut stream, &mut target) + .await + .map_err(|error| format!("bridge guest network mediation stream: {error}")) + })?; + return Ok(()); + } + _ => {} + } + let response = ResponseEnvelope { + request_id: request.request_id, + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response).map_err(|error| format!("write control frame: {error}")) + } + + struct GuestRuntime { + config: GuestConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, + next_exec_id: AtomicU64, + exec_handles: Mutex>, + } + + struct ExecHandle { + process: Arc, + terminal: Option>, + } + + enum RuntimeState { + AwaitingAttach, + Bound(PreparedBoundary), + Ready(PreparedBoundary), + Running(Arc), + } + + #[derive(Clone)] + struct PreparedBoundary { + netns: Option>, + network_listener: Option>, + proxy_port: u16, + } + + async fn bridge_exec_stream( + stream: tokio::net::UnixStream, + session: ExecSession, + ) -> Result<(), String> { + let ExecSession { + process, + stdin, + stdout, + stderr, + terminal: _, + } = session; + let (mut network_reader, network_writer) = tokio::io::split(stream); + let network_writer = Arc::new(tokio::sync::Mutex::new(network_writer)); + + let stdin_task = stdin.map(|mut stdin| { + tokio::spawn(async move { + while let Some((channel, payload)) = read_stream_frame(&mut network_reader).await? { + match channel { + STREAM_STDIN => stdin.write_all(&payload).await?, + STREAM_STDIN_CLOSED => { + stdin.shutdown().await?; + break; + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected host-to-guest stream channel", + )); + } + } + } + Ok::<(), io::Error>(()) + }) + }); + + let stdout_task = tokio::spawn(pump_exec_output( + stdout, + STREAM_STDOUT, + network_writer.clone(), + )); + let stderr_task = stderr.map(|stderr| { + tokio::spawn(pump_exec_output( + stderr, + STREAM_STDERR, + network_writer.clone(), + )) + }); + + let status = process + .wait() + .await + .map_err(|error| format!("wait for guest exec: {error}"))?; + stdout_task + .await + .map_err(|error| format!("join guest exec stdout: {error}"))? + .map_err(|error| format!("stream guest exec stdout: {error}"))?; + if let Some(stderr_task) = stderr_task { + stderr_task + .await + .map_err(|error| format!("join guest exec stderr: {error}"))? + .map_err(|error| format!("stream guest exec stderr: {error}"))?; + } + let exit = serde_json::to_vec(&ExitStatusWire::from(status)) + .map_err(|error| format!("encode guest exec exit: {error}"))?; + write_stream_frame(&mut *network_writer.lock().await, STREAM_EXIT, &exit) + .await + .map_err(|error| format!("write guest exec exit: {error}"))?; + if let Some(stdin_task) = stdin_task { + stdin_task.abort(); + } + Ok(()) + } + + async fn pump_exec_output( + mut output: openshell_isolation::contract::BoundaryOutput, + channel: u8, + writer: Arc>>, + ) -> io::Result<()> { + let mut buffer = vec![0; 16 * 1024]; + loop { + let read = output.read(&mut buffer).await?; + if read == 0 { + return Ok(()); + } + write_stream_frame(&mut *writer.lock().await, channel, &buffer[..read]).await?; + } + } + + impl GuestRuntime { + fn new(config: GuestConfig, process_runtime: tokio::runtime::Handle) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + next_exec_id: AtomicU64::new(1), + exec_handles: Mutex::new(std::collections::HashMap::new()), + } + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !self.authenticate(&envelope) { + return guest_error("denied", "control authentication failed"); + } + match envelope.request { + Request::Attach { policy } => self.attach((*policy).into()), + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + ca_cert, + ca_bundle, + provider_env, + } => self.start_agent(sandbox_id, spec, *policy, ca_cert, ca_bundle, provider_env), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + Request::ExecSignal { process_id, signal } => self.signal_exec(&process_id, signal), + Request::Resize { + process_id, + cols, + rows, + } => self.resize_exec(&process_id, cols, rows), + Request::Exec { .. } | Request::PortForward { .. } | Request::AcceptNetwork => { + guest_error("invalid", "streaming request used on control path") + } + } + } + + fn authenticate(&self, envelope: &RequestEnvelope) -> bool { + constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) && constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) + } + + fn start_exec(&self, spec: ExecSpecWire) -> Result<(String, ExecSession), Response> { + let executor = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + process.boundary_exec() + }; + let session = self + .process_runtime + .block_on(executor.exec(spec.into())) + .map_err(|error| guest_error("failed", error.to_string()))?; + let process_id = format!("exec-{}", self.next_exec_id.fetch_add(1, Ordering::Relaxed)); + lock(&self.exec_handles).insert( + process_id.clone(), + ExecHandle { + process: session.process.clone(), + terminal: session.terminal.clone(), + }, + ); + Ok((process_id, session)) + } + + fn signal_exec(&self, process_id: &str, signal: SignalWire) -> Response { + let process = lock(&self.exec_handles) + .get(process_id) + .map(|handle| handle.process.clone()); + let Some(process) = process else { + return guest_error("invalid", "unknown exec process ID"); + }; + match self.process_runtime.block_on(process.signal(signal.into())) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn resize_exec(&self, process_id: &str, cols: u16, rows: u16) -> Response { + let terminal = lock(&self.exec_handles) + .get(process_id) + .and_then(|handle| handle.terminal.clone()); + let Some(terminal) = terminal else { + return guest_error("invalid", "exec process has no terminal"); + }; + match self.process_runtime.block_on(terminal.resize(cols, rows)) { + Ok(()) => Response::Resized, + Err(error) => guest_error("failed", error.to_string()), + } + } + + fn connect_port( + &self, + target: LoopbackTarget, + ) -> Result { + let port_forward = { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err("agent process has not been started".to_string()); + }; + process.port_forward() + }; + self.process_runtime + .block_on(port_forward.connect(target)) + .map_err(|error| error.to_string()) + } + + fn accept_network(&self) -> Result<(tokio::net::TcpStream, BinaryIdentityWire), String> { + let process = loop { + let running = { + let state = lock(&self.state); + match &*state { + RuntimeState::Running(process) => Some(process.clone()), + _ => None, + } + }; + if let Some(process) = running { + break process; + } + std::thread::sleep(Duration::from_millis(10)); + }; + let listener = process + .network_listener() + .ok_or_else(|| "network mediation requested for a non-proxy policy".to_string())?; + let (stream, workload_addr) = self + .process_runtime + .block_on(listener.accept()) + .map_err(|error| format!("accept guest proxy connection: {error}"))?; + let proxy_addr = stream + .local_addr() + .map_err(|error| format!("read guest proxy address: {error}"))?; + let identity = process + .identity_resolver() + .resolve_connection(workload_addr, proxy_addr); + Ok((stream, BinaryIdentityWire::from(identity))) + } + + fn stream_exec( + &self, + stream: VsockStream, + process_id: &str, + session: ExecSession, + ) -> Result<(), String> { + let process_id = process_id.to_string(); + self.process_runtime.block_on(async move { + let stream = stream.into_tokio()?; + bridge_exec_stream(stream, session).await + })?; + lock(&self.exec_handles).remove(&process_id); + Ok(()) + } + + fn attach(&self, policy: openshell_core::policy::SandboxPolicy) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::AwaitingAttach => { + let prepared = match PreparedBoundary::establish(&self.process_runtime, &policy) + { + Ok(prepared) => prepared, + Err(error) => return guest_error("failed", error), + }; + *state = RuntimeState::Bound(prepared); + Response::Attached + } + RuntimeState::Bound(_) => Response::Attached, + _ => guest_error("invalid", "boundary has already advanced past attach"), + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match &*state { + RuntimeState::Bound(prepared) => { + if let Err(error) = prepared.confirm(&self.process_runtime) { + return guest_error("failed", error); + } + *state = RuntimeState::Ready(prepared.clone()); + Response::Confirmed + } + RuntimeState::Ready(_) => Response::Confirmed, + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + RuntimeState::Running(_) => { + guest_error("invalid", "boundary has already started its agent") + } + } + } + + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ca_cert: Option>, + ca_bundle: Option>, + provider_env: std::collections::HashMap, + ) -> Response { + let mut state = lock(&self.state); + let RuntimeState::Ready(prepared) = &*state else { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + }; + let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { + Ok(paths) => paths, + Err(error) => return guest_error("failed", error), + }; + let launch = ManagedProcessLaunch { + sandbox_id, + spec, + policy: policy.into(), + resolved_identity: ResolvedProcessIdentity::new( + Some(self.config.agent_uid), + Some(self.config.agent_gid), + ), + provider_env, + ca_file_paths, + }; + let process = + match ManagedProcess::spawn(&self.process_runtime, launch, prepared.clone()) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *state = RuntimeState::Running(process); + Response::Started { process_id } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } + } + + impl PreparedBoundary { + fn establish( + runtime: &tokio::runtime::Handle, + policy: &openshell_core::policy::SandboxPolicy, + ) -> Result { + let netns = create_conformant_netns_for_proxy(policy) + .map_err(|error| format!("establish guest workload network namespace: {error}"))? + .map(Arc::new); + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + let network_listener = if let Some(netns) = netns.as_ref() { + let address = std::net::SocketAddr::new(netns.host_ip(), proxy_port); + Some(Arc::new( + runtime + .block_on(tokio::net::TcpListener::bind(address)) + .map_err(|error| { + format!("bind guest mediation listener {address}: {error}") + })?, + )) + } else { + None + }; + Ok(Self { + netns, + network_listener, + proxy_port, + }) + } + + fn confirm(&self, runtime: &tokio::runtime::Handle) -> Result<(), String> { + if let Some(netns) = self.netns.as_ref() { + runtime + .block_on( + netns + .egress_ceiling_verifier() + .verify_bounded(self.proxy_port, Duration::from_secs(2)), + ) + .map_err(|error| format!("verify guest egress ceiling: {error}"))?; + if self.network_listener.is_none() { + return Err("guest proxy namespace has no mediation listener".to_string()); + } + } + Ok(()) + } + } + + fn install_ca_material( + ca_cert: Option>, + ca_bundle: Option>, + ) -> Result, String> { + let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), + (None, None) => return Ok(None), + _ => { + return Err( + "VM proxy CA certificate and bundle must be supplied together".to_string(), + ); + } + }; + let directory = std::path::PathBuf::from("/run/openshell/proxy-ca"); + std::fs::create_dir_all(&directory) + .map_err(|error| format!("create guest proxy CA directory: {error}"))?; + let ca_path = directory.join("ca.crt"); + let bundle_path = directory.join("ca-bundle.crt"); + std::fs::write(&ca_path, ca_cert) + .map_err(|error| format!("write guest proxy CA: {error}"))?; + std::fs::write(&bundle_path, ca_bundle) + .map_err(|error| format!("write guest proxy CA bundle: {error}"))?; + Ok(Some((ca_path, bundle_path))) + } + + type ProcessExit = Result; + type SharedProcessExit = Arc<(Mutex>, Condvar)>; + + struct ManagedProcess { + pid: i32, + signaler: AgentSignaler, + exit: SharedProcessExit, + boundary_exec: Arc, + port_forward: Arc, + network_listener: Option>, + identity_resolver: ProcfsIdentityResolver, + _netns: Option>, + } + + struct ManagedProcessLaunch { + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + } + + impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + launch: ManagedProcessLaunch, + prepared: PreparedBoundary, + ) -> Result { + let ManagedProcessLaunch { + sandbox_id, + spec, + policy, + resolved_identity, + provider_env, + ca_file_paths, + } = launch; + if spec.program.is_empty() { + return Err("agent program must not be empty".to_string()); + } + let boundary_runtime = BoundaryRuntimeState::new(); + let entrypoint_pid = Arc::new(AtomicU32::new(0)); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + resolved_identity, + ProcessEnforcementMode::Full, + entrypoint_pid.clone(), + None, + ProviderCredentialState::from_child_env_snapshot(0, provider_env.clone()), + provider_env, + ca_file_paths, + AgentProposals::default(), + prepared.netns.as_deref(), + None, + None, + Some(boundary_runtime), + )) + .map_err(|error| format!("start process supervisor leaf: {error}"))?; + let pid = i32::try_from(spawned.pid()) + .map_err(|_| "process supervisor PID does not fit i32".to_string())?; + let signaler = spawned.signaler(); + let boundary_exec = spawned.boundary_exec(); + let port_forward = spawned.port_forward(); + let network_listener = prepared.network_listener.clone(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + pid, + signaler, + exit, + boundary_exec, + port_forward, + network_listener, + identity_resolver: ProcfsIdentityResolver { entrypoint_pid }, + _netns: prepared.netns, + }) + } + + fn process_id(&self) -> String { + self.pid.to_string() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } + + fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + fn network_listener(&self) -> Option> { + self.network_listener.clone() + } + + fn identity_resolver(&self) -> ProcfsIdentityResolver { + self.identity_resolver.clone() + } + } + + fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) + } + + fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } + } + + fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 + } + + struct VsockListener { + fd: OwnedFd, + } + + impl VsockListener { + fn bind(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = libc::socklen_t::try_from(size_of::()) + .map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = + unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(Self { fd }) + } + + fn accept(&self) -> io::Result { + let raw_fd = unsafe { + libc::accept4( + self.fd.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(VsockStream { + file: unsafe { File::from_raw_fd(raw_fd) }, + }) + } + } + } + + struct VsockStream { + file: File, + } + + impl VsockStream { + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + let option_length = + libc::socklen_t::try_from(size_of::()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t") + })?; + let timeout = libc::timeval { + tv_sec: timeout.as_secs().try_into().unwrap_or(libc::time_t::MAX), + tv_usec: timeout.subsec_micros().into(), + }; + for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] { + let result = unsafe { + libc::setsockopt( + self.file.as_raw_fd(), + libc::SOL_SOCKET, + option, + (&raw const timeout).cast(), + option_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } + + fn into_tokio(self) -> Result { + let stream = + unsafe { std::os::unix::net::UnixStream::from_raw_fd(self.file.into_raw_fd()) }; + stream + .set_nonblocking(true) + .map_err(|error| format!("set guest vsock nonblocking: {error}"))?; + tokio::net::UnixStream::from_std(stream) + .map_err(|error| format!("register guest vsock with Tokio: {error}")) + } + } + + impl Read for VsockStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.file.read(buffer) + } + } + + impl Write for VsockStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.file.write(buffer) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn guest_config_debug_redacts_token() { + let config = GuestConfig { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + control_port: 5500, + agent_uid: DEFAULT_AGENT_UID, + agent_gid: DEFAULT_AGENT_GID, + trusted_runtime_root: default_trusted_runtime_root(), + child_env: std::collections::HashMap::new(), + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::run_guest; + +#[cfg(not(target_os = "linux"))] +pub fn run_guest(_config_path: &Path) -> Result<(), String> { + Err("the VM guest process leaf is supported only on Linux guests".to_string()) +} diff --git a/crates/openshell-isolation-vm/src/lib.rs b/crates/openshell-isolation-vm/src/lib.rs new file mode 100644 index 0000000000..3ffe3a4429 --- /dev/null +++ b/crates/openshell-isolation-vm/src/lib.rs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared authenticated VM boundary transport. +//! +//! Hypervisor drivers provision a VM and mint a [`VmTopology`]. The logical +//! supervisor remains on the host and drives the RFC 0012 lifecycle through +//! [`VmHostBackend`]. Inside the guest, [`run_guest`] invokes the existing +//! `openshell-supervisor-process` implementation; this crate does not define a +//! second supervisor or agent model. + +mod backend; +mod guest; +mod protocol; + +pub use backend::{VmHostBackend, VmTopology, VmTransport}; +pub use guest::{GuestConfig, run_guest}; diff --git a/crates/openshell-isolation-vm/src/protocol.rs b/crates/openshell-isolation-vm/src/protocol.rs new file mode 100644 index 0000000000..fc123e6772 --- /dev/null +++ b/crates/openshell-isolation-vm/src/protocol.rs @@ -0,0 +1,620 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-private, length-delimited JSON protocol carried over virtio-vsock. + +use std::fmt; +use std::io::{self, Read, Write}; + +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use openshell_isolation::AgentSpec; +use openshell_isolation::contract::{ + BinaryIdentity, BoundaryExitStatus, BoundarySignal, ExecSpec, ResolveError, Sha256Digest, +}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; +pub const STREAM_STDIN: u8 = 0; +pub const STREAM_STDOUT: u8 = 1; +pub const STREAM_STDERR: u8 = 2; +pub const STREAM_EXIT: u8 = 3; +pub const STREAM_STDIN_CLOSED: u8 = 4; +pub const MAX_STREAM_FRAME_BYTES: usize = 64 * 1024; + +pub async fn write_stream_frame( + writer: &mut (impl AsyncWrite + Unpin), + channel: u8, + payload: &[u8], +) -> io::Result<()> { + if payload.len() > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "VM stream frame exceeds limit", + )); + } + writer.write_u8(channel).await?; + writer + .write_u32(payload.len().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "VM stream frame length overflow", + ) + })?) + .await?; + writer.write_all(payload).await?; + writer.flush().await +} + +pub async fn read_stream_frame( + reader: &mut (impl AsyncRead + Unpin), +) -> io::Result)>> { + let channel = match reader.read_u8().await { + Ok(channel) => channel, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + }; + let declared = reader.read_u32().await? as usize; + if declared > MAX_STREAM_FRAME_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("VM stream frame is too large: {declared} bytes"), + )); + } + let mut payload = vec![0; declared]; + reader.read_exact(&mut payload).await?; + Ok(Some((channel, payload))) +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + pub request_id: u64, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach { + policy: Box, + }, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + ca_cert: Option>, + ca_bundle: Option>, + provider_env: std::collections::HashMap, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, + Exec { + spec: ExecSpecWire, + }, + ExecSignal { + process_id: String, + signal: SignalWire, + }, + Resize { + process_id: String, + cols: u16, + rows: u16, + }, + PortForward { + host: std::net::IpAddr, + port: u16, + }, + AcceptNetwork, +} + +impl fmt::Debug for Request { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Attach { policy: _ } => formatter + .debug_struct("Attach") + .field("policy", &"") + .finish(), + Self::Confirm => formatter.write_str("Confirm"), + Self::StartAgent { + sandbox_id, + spec, + policy: _, + ca_cert, + ca_bundle, + provider_env, + } => formatter + .debug_struct("StartAgent") + .field("sandbox_id", sandbox_id) + .field("spec", spec) + .field("policy", &"") + .field("ca_cert_present", &ca_cert.is_some()) + .field("ca_bundle_present", &ca_bundle.is_some()) + .field( + "provider_env_keys", + &provider_env.keys().collect::>(), + ) + .finish(), + Self::Wait { process_id } => formatter + .debug_struct("Wait") + .field("process_id", process_id) + .finish(), + Self::Signal { process_id, signal } => formatter + .debug_struct("Signal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Terminate { process_id } => formatter + .debug_struct("Terminate") + .field("process_id", process_id) + .finish(), + Self::Exec { spec } => formatter.debug_tuple("Exec").field(spec).finish(), + Self::ExecSignal { process_id, signal } => formatter + .debug_struct("ExecSignal") + .field("process_id", process_id) + .field("signal", signal) + .finish(), + Self::Resize { + process_id, + cols, + rows, + } => formatter + .debug_struct("Resize") + .field("process_id", process_id) + .field("cols", cols) + .field("rows", rows) + .finish(), + Self::PortForward { host, port } => formatter + .debug_struct("PortForward") + .field("host", host) + .field("port", port) + .finish(), + Self::AcceptNetwork => formatter.write_str("AcceptNetwork"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: u64, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached, + Confirmed, + Started { process_id: String }, + Exited { status: ExitStatusWire }, + Signaled, + Terminated, + ExecStarted { process_id: String, pty: bool }, + Resized, + PortConnected, + NetworkConnected { identity: BinaryIdentityWire }, + Error { kind: String, message: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinaryIdentityWire { + pub binary_path: Option, + pub binary_digest: Option, + pub ancestors: Vec, + pub cmdline_paths: Vec, + pub resolve_error: Option, +} + +impl From> for BinaryIdentityWire { + fn from(identity: Result) -> Self { + match identity { + Ok(identity) => Self { + binary_path: Some(identity.binary_path), + binary_digest: identity.binary_digest.map(|digest| digest.to_string()), + ancestors: identity.ancestors, + cmdline_paths: identity.cmdline_paths, + resolve_error: None, + }, + Err(error) => Self { + binary_path: None, + binary_digest: None, + ancestors: Vec::new(), + cmdline_paths: Vec::new(), + resolve_error: Some(error.to_string()), + }, + } + } +} + +impl BinaryIdentityWire { + pub fn into_result(self) -> Result { + if let Some(error) = self.resolve_error { + return Err(ResolveError::Failed(error)); + } + let binary_path = self + .binary_path + .ok_or_else(|| ResolveError::Failed("VM identity omitted binary path".to_string()))?; + let binary_digest = self + .binary_digest + .map(|digest| digest.parse::()) + .transpose()?; + Ok(BinaryIdentity { + binary_path, + binary_digest, + ancestors: self.ancestors, + cmdline_paths: self.cmdline_paths, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ExecSpecWire { + pub program: String, + pub args: Vec, + pub env: Vec<(String, String)>, + pub workdir: Option, + pub pty: bool, +} + +impl From for ExecSpecWire { + fn from(spec: ExecSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +impl From for ExecSpec { + fn from(spec: ExecSpecWire) -> Self { + Self { + program: spec.program, + args: spec.args, + env: spec.env, + workdir: spec.workdir, + pty: spec.pty, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + Self { + version: policy.version, + read_only: policy.filesystem.read_only, + read_write: policy.filesystem.read_write, + include_workdir: policy.filesystem.include_workdir, + network: NetworkModeWire::from(policy.network.mode), + proxy_addr: policy.network.proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(policy.landlock.compatibility), + run_as_user: policy.process.run_as_user, + run_as_group: policy.process.run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +impl From for BoundarySignal { + fn from(signal: SignalWire) -> Self { + match signal { + SignalWire::Term => Self::Term, + SignalWire::Kill => Self::Kill, + SignalWire::Int => Self::Int, + SignalWire::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +impl From for ExitStatusWire { + fn from(status: BoundaryExitStatus) -> Self { + match status { + BoundaryExitStatus::Exited(code) => Self::Exited(code), + BoundaryExitStatus::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: 7, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + ca_cert: Some(b"test certificate".to_vec()), + ca_bundle: Some(b"test bundle".to_vec()), + provider_env: std::collections::HashMap::from([( + "OPENAI_API_KEY".to_string(), + "test credential".to_string(), + )]), + }, + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + assert!(!debug.contains("test credential")); + assert!(!debug.contains("test certificate")); + assert!(!debug.contains("test bundle")); + assert!(debug.contains("OPENAI_API_KEY")); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index c653db84dd..869a8b4765 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -16,10 +16,12 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } -openshell-extension-core = { path = "../openshell-extension-core" } +openshell-isolation = { path = "../openshell-isolation" } +openshell-isolation-vm = { path = "../openshell-isolation-vm" } +base64 = { workspace = true } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } -openshell-supervisor-network = { path = "../openshell-supervisor-network", default-features = false } +openshell-supervisor-network = { path = "../openshell-supervisor-network" } openshell-supervisor-middleware = { path = "../openshell-supervisor-middleware" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } openshell-supervisor-process = { path = "../openshell-supervisor-process" } @@ -50,19 +52,15 @@ prost = { workspace = true } # Logging tracing = { workspace = true } -uuid = { workspace = true } tracing-subscriber = { workspace = true } tracing-appender = { workspace = true } [features] -default = ["telemetry", "bundled-ca-roots"] -## Convenience alias: all defaults except bundled CA roots. Use -## `--no-default-features --features system-ca-roots` to build a supervisor -## that uses the platform trust store with telemetry intact. -system-ca-roots = ["telemetry"] - +default = ["telemetry"] +## Compile in telemetry activity collection (forwards to openshell-core/telemetry). +## On by default; build with `--no-default-features` for a telemetry-free sandbox +## supervisor that never collects or forwards activity summaries. telemetry = ["openshell-core/telemetry"] -bundled-ca-roots = ["openshell-supervisor-network/bundled-ca-roots"] [dev-dependencies] tempfile = "3" diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b1c226cebd..2f82397536 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -16,7 +16,6 @@ mod sidecar_control; use miette::{IntoDiagnostic, Result, WrapErr}; use std::future::Future; -use std::pin::Pin; use std::sync::Arc; #[cfg(target_os = "linux")] use std::sync::atomic::Ordering; @@ -27,9 +26,9 @@ use tracing::{debug, info, warn}; use openshell_core::PolicyValidationFailureMode; use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfidenceId, ConfigStateChangeBuilder, - DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, - StateId, StatusId, ocsf_emit, + ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, + DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, + ocsf_emit, }; // --------------------------------------------------------------------------- @@ -63,7 +62,6 @@ use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPol use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_network::proxy::ProxyHandle; use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; @@ -72,18 +70,9 @@ use tokio::sync::mpsc::UnboundedSender; use tokio::time::timeout; const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; - -#[cfg(any(test, target_os = "linux"))] -fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> bool { - capabilities.is_some_and(|capabilities| { - capabilities - .split(',') - .any(|capability| capability.trim() == required) - }) -} const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; @@ -94,7 +83,6 @@ const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; /// Returns an error if the command fails to start or encounters a fatal error. #[allow( clippy::too_many_arguments, - clippy::implicit_hasher, clippy::similar_names, clippy::fn_params_excessive_bools )] @@ -116,6 +104,7 @@ pub async fn run_sandbox( network_enabled: bool, process_enabled: bool, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + topology_descriptor: Option, ) -> Result { let (program, args) = command .split_first() @@ -167,11 +156,6 @@ pub async fn run_sandbox( None }; - // Extension credentials are owned by this supervisor and shared by every - // gateway connection it opens, so the middleware registry's bearer slots - // and the policy poll loop that rotates them stay the same objects. - let extension_credentials = openshell_extension_core::ExtensionCredentialStore::new(); - // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); @@ -182,7 +166,6 @@ pub async fn run_sandbox( middleware_registry_status, loaded_policy_origin, initial_agent_proposals_enabled, - initial_extension_authentication_enabled, ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { let (policy, opa_engine, retained_proto, loaded_policy_origin) = load_policy_from_sidecar_bootstrap(bootstrap)?; @@ -193,7 +176,6 @@ pub async fn run_sandbox( MiddlewareRegistryStatus::Synchronized, loaded_policy_origin, bootstrap.agent_proposals_enabled, - false, ) } else { load_policy( @@ -202,159 +184,101 @@ pub async fn run_sandbox( openshell_endpoint.clone(), policy_rules, policy_data, - &extension_credentials, ) .await? }; // Normalize the active driver's identity contract once, while both the // policy and launched image filesystem are available. Kubernetes and - // OpenShift retain their authoritative numeric pair; Docker fills only - // omitted policy fields from OCI Config.User. + // OpenShift retain their authoritative numeric pair; Docker and Podman + // fill only omitted policy fields from OCI Config.User. #[cfg(unix)] - let (resolved_process_identity, workspace) = { + let resolved_process_identity = { let driver_identity = openshell_supervisor_process::identity::DriverIdentity::from_env()?; - let use_workdir_as_home = matches!( - &driver_identity, - openshell_supervisor_process::identity::DriverIdentity::OciUser { .. } - ); - let resolved = openshell_supervisor_process::identity::resolve_process_identity( + openshell_supervisor_process::identity::resolve_process_identity( &mut policy, &driver_identity, - )?; - ( - resolved, - openshell_supervisor_process::process::ResolvedWorkspace::new( - workdir.clone(), - use_workdir_as_home, - ), - ) + )? }; #[cfg(not(unix))] - let (resolved_process_identity, workspace) = ( - openshell_supervisor_process::process::ResolvedProcessIdentity::default(), - openshell_supervisor_process::process::ResolvedWorkspace::new(workdir.clone(), false), - ); + let resolved_process_identity = + openshell_supervisor_process::process::ResolvedProcessIdentity::default(); #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = if let Some(bootstrap) = - sidecar_bootstrap.as_ref() - { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - result.static_credential_bindings, - result.non_secret_environment_keys, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Failed to fetch provider environment; no provider credentials are active: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - } - } + let (provider_credentials, mut provider_env) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + bootstrap.provider_env_revision, + bootstrap.provider_child_env.clone(), + ); + (provider_credentials, bootstrap.provider_child_env.clone()) } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - Vec::new(), - ) - }; - - let dynamic_credentials_fallback = dynamic_credentials.clone(); - let provider_credentials = match ProviderCredentialState::from_bound_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - static_credential_bindings, - non_secret_environment_keys, - ) { - Ok(credentials) => credentials, - Err(error) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment bindings; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - ProviderCredentialState::from_environment( - provider_env_revision, + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { + ( + 0, + std::collections::HashMap::new(), std::collections::HashMap::new(), std::collections::HashMap::new(), - dynamic_credentials_fallback, ) - } - }; - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - if credential_gating_unavailable( - &loaded_policy_origin, - provider_credentials.resolver().is_some(), - network_enabled, - ) { - report_credential_gating_unavailable(); - } + }; - // Canonical-process overrides are deliberately applied only to the main - // child. Keep the provider snapshot pristine because Kubernetes forwards - // it to the process sidecar for later exec/editor/SFTP children. + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; // Shared agent-proposals feature flag. Seed from the same initial settings // snapshot that produced the policy so networking and process setup agree @@ -364,10 +288,6 @@ pub async fn run_sandbox( let process_control_writer = process_control_connection .as_ref() .map(|connection| connection.writer.clone()); - let process_exit_ack = Arc::new(tokio::sync::Mutex::new(None)); - let initial_provider_env_generation = sidecar_bootstrap - .as_ref() - .map_or(0, |bootstrap| bootstrap.provider_env_generation); let mut process_control_closed = None; if let Some(connection) = process_control_connection { process_control_closed = Some(connection.closed); @@ -375,8 +295,6 @@ pub async fn run_sandbox( connection.updates, provider_credentials.clone(), agent_proposals.clone(), - Arc::clone(&process_exit_ack), - initial_provider_env_generation, ); } @@ -384,99 +302,6 @@ pub async fn run_sandbox( // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - #[cfg(target_os = "linux")] - let transparent_tcp_requested = opa_engine - .as_ref() - .map(|engine| engine.policy_dns_eligibility_snapshot()) - .transpose()? - .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); - #[cfg(target_os = "linux")] - let runtime_capabilities = - std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); - #[cfg(target_os = "linux")] - let transparent_tcp_capable = has_network_runtime_capability( - runtime_capabilities.as_deref(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, - ); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_capable = false; - #[cfg(target_os = "linux")] - let transparent_runtime = if transparent_tcp_requested { - if !transparent_tcp_capable { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_runtime") - .message( - "Policy DNS and transparent TCP unavailable: runtime capability is missing" - ) - .build() - ); - return Err(miette::miette!( - "policy contains protocol: tcp endpoints, but the selected runtime does not advertise policy DNS and transparent TCP support" - )); - } - if sidecar_network_enforcement { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Disabled, "unsupported_topology") - .message("Policy DNS and transparent TCP unavailable: sidecar topology is unsupported") - .build() - ); - return Err(miette::miette!( - "policy DNS and transparent TCP are not yet supported by the sidecar topology" - )); - } - let namespace = netns.as_ref().ok_or_else(|| { - miette::miette!("policy DNS and transparent TCP require a workload network namespace") - })?; - let listeners = namespace - .bind_transparent_tcp_listeners() - .await - .into_diagnostic() - .wrap_err("failed to bind transparent TCP listeners")?; - let (dns_udp, dns_tcp) = namespace - .bind_policy_dns_sockets() - .await - .into_diagnostic() - .wrap_err("failed to bind policy DNS listeners")?; - let proxy_port = policy - .network - .proxy - .as_ref() - .and_then(|proxy| proxy.http_addr) - .map_or(3128, |address| address.port()); - let runtime = openshell_supervisor_network::run::TransparentRuntimeSetup::new( - listeners, - dns_udp, - dns_tcp, - sandbox_id.as_deref(), - )?; - let (ipv4_cidr, ipv6_cidr) = runtime.synthetic_cidrs(); - namespace.install_transparent_tcp_rules(proxy_port, &ipv4_cidr, &ipv6_cidr)?; - Some(runtime) - } else { - None - }; - #[cfg(target_os = "linux")] - let transparent_tcp_substrate_ready = transparent_runtime.is_some(); - #[cfg(not(target_os = "linux"))] - let transparent_tcp_substrate_ready = false; // The denial channel is owned by the orchestrator: the proxy (in the // networking leaf) and the bypass monitor (in the process leaf) both // produce DenialEvents that the denial aggregator (orchestrator-side) @@ -516,7 +341,243 @@ pub async fn run_sandbox( // API read the current value so proposals target the correct workspace. let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - let mut networking = if network_enabled { + if let Some(descriptor) = topology_descriptor { + if sidecar_network_enforcement || !network_enabled || !process_enabled { + return Err(miette::miette!( + "the VM isolation backend requires combined network,process mode" + )); + } + if descriptor.backend_name != "vm" { + return Err(miette::miette!( + "unsupported prototype isolation backend {:?}; expected \"vm\"", + descriptor.backend_name + )); + } + + let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); + let proxy_bind_ip = Arc::new(std::sync::Mutex::new(None)); + let admitted_backend_name = descriptor.backend_name.clone(); + let backend: Arc = + Arc::new(openshell_isolation_vm::VmHostBackend::new( + "vm", + ca_file_paths.clone(), + provider_env.clone(), + )); + let mut registry = openshell_isolation::contract::BackendRegistry::new(); + registry + .register(backend) + .map_err(|error| miette::miette!(error.to_string()))?; + let (backend, verified) = registry + .resolve(descriptor, &admitted_backend_name) + .map_err(|error| miette::miette!(error.to_string()))?; + let context = openshell_isolation::contract::SandboxContext { + sandbox_id: sandbox_id.clone().unwrap_or_default(), + policy: policy.clone(), + agent: openshell_isolation::AgentSpec { + program: program.clone(), + args: args.to_vec(), + workdir: workdir.clone(), + timeout_secs, + interactive, + }, + }; + let bound = backend + .attach(verified, context) + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary attached"); + let network_mediation_source = bound.network_mediation_source(); + let mediation_bind_ip = *proxy_bind_ip.lock().expect("proxy bind IP lock"); + let networking = openshell_supervisor_network::run::run_networking( + &policy, + mediation_bind_ip, + opa_engine.as_ref(), + retained_proto.as_ref(), + entrypoint_pid.clone(), + process_enabled, + &provider_credentials, + sandbox_id.as_deref(), + sandbox_name_for_agg.as_deref(), + openshell_endpoint_for_proxy.as_deref(), + inference_routes.as_deref(), + denial_tx, + activity_tx, + agent_proposals.clone(), + workspace_rx.clone(), + &upstream_proxy_args, + Some(network_mediation_source), + ) + .await?; + info!( + backend = %admitted_backend_name, + "Host network supervisor connected to isolation boundary" + ); + ca_file_paths + .lock() + .expect("ca paths lock") + .clone_from(&networking.ca_file_paths); + let ready = bound + .confirm() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary enforcement confirmed"); + + if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let flush_interval_secs = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(10); + let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); + let workspace_gate = workspace_rx.clone(); + let workspace = workspace_rx.clone(); + tokio::spawn(async move { + aggregator + .run( + |summaries| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = workspace.borrow().clone(); + async move { + if let Err(error) = flush_proposals_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summaries, + ) + .await + { + warn!(%error, "Failed to flush denial summaries to gateway"); + } + } + }, + move || !workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { + let agg_name = sandbox_name_for_agg + .clone() + .or_else(|| sandbox_id.clone()) + .unwrap_or_default(); + let agg_endpoint = endpoint.to_string(); + let interval = activity_aggregator::activity_flush_interval_secs_from_env( + std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") + .ok() + .as_deref(), + ); + let aggregator = activity_aggregator::ActivityAggregator::new(rx, interval); + let workspace_gate = workspace_rx.clone(); + let workspace = workspace_rx.clone(); + tokio::spawn(async move { + aggregator + .run( + move |summary| { + let endpoint = agg_endpoint.clone(); + let sandbox_name = agg_name.clone(); + let workspace = workspace.borrow().clone(); + async move { + if let Err(error) = flush_activity_to_gateway( + &endpoint, + &sandbox_name, + &workspace, + summary, + ) + .await + { + warn!(%error, "Failed to flush activity summary to gateway"); + } + } + }, + move || !workspace_gate.borrow().is_empty(), + ) + .await; + }); + } + + if let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) { + let poll_ctx = PolicyPollLoopContext { + endpoint: endpoint.to_string(), + sandbox_id: id.to_string(), + opa_engine: engine.clone(), + loaded_policy_origin, + entrypoint_pid: entrypoint_pid.clone(), + interval_secs: std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(10), + ocsf_enabled: ocsf_enabled.clone(), + provider_credentials: provider_credentials.clone(), + policy_local_ctx: Some(networking.policy_local_ctx.clone()), + agent_proposals: agent_proposals.clone(), + middleware_registry_status, + sidecar_control_publisher: None, + workspace_tx, + }; + tokio::spawn(async move { + if let Err(error) = run_policy_poll_loop(poll_ctx).await { + warn!(%error, "Policy poll loop exited with error"); + } + }); + } + + let running = ready + .start_agent() + .await + .map_err(|error| miette::miette!(error.to_string()))?; + info!(backend = %admitted_backend_name, "Isolation boundary agent started"); + let boundary_access = openshell_supervisor_process::run::start_boundary_access( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + ssh_socket_path.as_deref(), + false, + networking.ca_file_paths.clone(), + process_enforcement_mode, + running.exec(), + running.port_forward(), + ) + .await?; + info!(backend = %admitted_backend_name, "Host supervisor access plane started"); + let agent = running.agent(); + // Mediation-source failure is already audited by the proxy. The + // default-deny ceiling remains standing, so new egress fails static + // while the workload continues under its last confirmed enforcement. + let result = agent + .wait() + .await + .map(|status| match status { + openshell_isolation::contract::BoundaryExitStatus::Exited(code) => code, + openshell_isolation::contract::BoundaryExitStatus::Signaled(signal) => { + 128_i32.saturating_add(signal) + } + }) + .map_err(|error| miette::miette!(error.to_string())); + drop(running); + drop(boundary_access); + drop(networking); + + return result; + } + + // Existing container topologies retain their legacy lifecycle. The VM + // backend above is the prototype RFC 0012 implementation. + #[cfg(target_os = "linux")] + let netns = if network_enabled && !sidecar_network_enforcement { + openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? + } else { + None + }; + + let networking = if network_enabled { #[cfg(target_os = "linux")] let proxy_bind_ip = netns .as_ref() @@ -542,8 +603,7 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, - #[cfg(target_os = "linux")] - transparent_runtime, + None, ) .await?, ) @@ -611,15 +671,12 @@ pub async fn run_sandbox( sidecar_control_task = Some(connection_task); spawn_sidecar_entrypoint_handler( entrypoint_rx, - SidecarEntrypointHandler { - entrypoint_pid: entrypoint_pid.clone(), - opa_engine: opa_engine.clone(), - retained_proto: retained_proto.clone(), - openshell_endpoint: openshell_endpoint.clone(), - sandbox_id: sandbox_id.clone(), - trusted_ssh_socket_path: std::path::PathBuf::from(trusted_ssh_socket_path), - control_publisher: sidecar_control_publisher.clone(), - }, + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + std::path::PathBuf::from(trusted_ssh_socket_path), ); } @@ -754,13 +811,6 @@ pub async fn run_sandbox( middleware_registry_status, sidecar_control_publisher: sidecar_control_publisher.clone(), workspace_tx, - extension_credentials: extension_credentials.clone(), - extension_authentication_enabled: initial_extension_authentication_enabled, - middleware_connector: default_middleware_connector(), - transparent_tcp: TransparentTcpReloadState { - capable: transparent_tcp_capable, - substrate_ready: transparent_tcp_substrate_ready, - }, }; tokio::spawn(async move { @@ -817,7 +867,6 @@ pub async fn run_sandbox( } let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let main_env = provider_env.clone(); let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { bootstrap .proxy_ca_cert_path @@ -825,19 +874,6 @@ pub async fn run_sandbox( .zip(bootstrap.proxy_ca_bundle_path.clone()) }); - let proxy_exited: Pin + Send>> = if let Some(rx) = networking - .as_mut() - .and_then(|n| n.proxy.as_mut()) - .and_then(ProxyHandle::take_exit_receiver) - { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(proxy_exited); - let exit_code = if process_enabled { let ca_file_paths = networking .as_ref() @@ -852,92 +888,47 @@ pub async fn run_sandbox( } }); - let (ssh_exit_tx, ssh_exit_rx) = if ssh_socket_path.is_some() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - (Some(tx), Some(rx)) - } else { - (None, None) - }; - let ssh_exited: Pin + Send>> = if let Some(rx) = ssh_exit_rx { - Box::pin(async { - let _ = rx.await; - }) - } else { - Box::pin(std::future::pending()) - }; - tokio::pin!(ssh_exited); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok((pid, instance_id)) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid, instance_id) - .await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); + let entrypoint_started_tx = if process_uses_sidecar_control + && let Some(writer) = process_control_writer.clone() + { + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = + sidecar_control::send_entrypoint_started(&writer, pid, String::new()) + .await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); } } - }); - Some(tx) - } else { - None - }; - let sidecar_exit_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let exit_ack = Arc::clone(&process_exit_ack); - let (tx, mut rx) = tokio::sync::mpsc::channel::< - openshell_supervisor_process::run::SidecarExitReport, - >(1); - tokio::spawn(async move { - while let Some((instance_id, exit_code, ack)) = rx.recv().await { - let (durable_tx, durable_rx) = tokio::sync::oneshot::channel(); - *exit_ack.lock().await = Some((instance_id.clone(), durable_tx)); - let result = match sidecar_control::send_main_process_exited( - &writer, - instance_id, - exit_code, - ) - .await - { - Ok(()) => durable_rx.await.map_err(|_| { - "sidecar durable exit acknowledgement closed".to_string() - }), - Err(error) => Err(error.to_string()), - }; - let _ = ack.send(result); + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); } - }); - Some(tx) - } else { - None - }; + } + }); + Some(tx) + } else { + None + }; let process = openshell_supervisor_process::run::run_process( program, args, - workspace, + workdir.as_deref(), timeout_secs, interactive, sandbox_id.as_deref(), openshell_endpoint.as_deref(), ssh_socket_path, sidecar_network_enforcement, - ssh_exit_tx, &process_policy, resolved_process_identity, process_enforcement_mode, entrypoint_pid, entrypoint_started_tx, - sidecar_exit_tx, provider_credentials, - main_env, + provider_env, ca_file_paths, agent_proposals.clone(), #[cfg(target_os = "linux")] @@ -966,71 +957,9 @@ pub async fn run_sandbox( "authoritative network-sidecar control channel closed" )); } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } } } else { - tokio::select! { - result = process => result?, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - () = &mut ssh_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "SSH accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "SSH accept loop exited unexpectedly" - )); - } - } + process.await? } } else { // Network-only sidecar mode: keep the proxy and its background @@ -1046,62 +975,15 @@ pub async fn run_sandbox( warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); 1 } - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } } } else { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } + wait_for_shutdown_signal().await; + 0 } #[cfg(not(target_os = "linux"))] { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - () = &mut proxy_exited => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Proxy accept loop exited unexpectedly; terminating sandbox" - ) - .build() - ); - return Err(miette::miette!( - "proxy accept loop exited unexpectedly" - )); - } - } + wait_for_shutdown_signal().await; + 0 } }; @@ -1191,9 +1073,6 @@ type LoadedPolicyBundle = ( LoadedPolicyOrigin, ); -type MainProcessExitAckWaiter = - Arc)>>>; - fn load_policy_from_sidecar_bootstrap( bootstrap: &sidecar_control::BootstrapData, ) -> Result { @@ -1216,30 +1095,26 @@ fn spawn_sidecar_control_update_watcher( mut updates: tokio::sync::mpsc::UnboundedReceiver, provider_credentials: ProviderCredentialState, agent_proposals: AgentProposals, - exit_ack: MainProcessExitAckWaiter, - mut provider_env_generation: u64, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { while let Some(update) = updates.recv().await { match update { sidecar_control::ControlUpdate::ProviderEnv { revision, - generation, provider_child_env, + .. } => { - if generation <= provider_env_generation { + if revision <= provider_credentials.snapshot().revision { continue; } let env_count = provider_credentials .install_child_env_snapshot(revision, provider_child_env); - provider_env_generation = generation; ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) .status(StatusId::Success) .state(StateId::Enabled, "loaded") .unmapped("provider_env_revision", serde_json::json!(revision)) - .unmapped("provider_env_generation", serde_json::json!(generation)) .message(format!( "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" )) @@ -1271,80 +1146,27 @@ fn spawn_sidecar_control_update_watcher( skills::install_static_skills, ); } - sidecar_control::ControlUpdate::MainProcessExitAck { instance_id } => { - let mut waiter = exit_ack.lock().await; - if waiter - .as_ref() - .is_some_and(|(expected, _)| expected == &instance_id) - && let Some((_, ack)) = waiter.take() - { - let _ = ack.send(()); - } - } + sidecar_control::ControlUpdate::MainProcessExitAck { .. } => {} } } }) } #[cfg(target_os = "linux")] -struct SidecarEntrypointHandler { +fn spawn_sidecar_entrypoint_handler( + mut entrypoint_rx: tokio::sync::mpsc::Receiver, entrypoint_pid: Arc, opa_engine: Option>, retained_proto: Option, openshell_endpoint: Option, sandbox_id: Option, trusted_ssh_socket_path: std::path::PathBuf, - control_publisher: Option, -} - -#[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - handler: SidecarEntrypointHandler, ) { tokio::spawn(async move { - let SidecarEntrypointHandler { - entrypoint_pid, - opa_engine, - retained_proto, - openshell_endpoint, - sandbox_id, - trusted_ssh_socket_path, - control_publisher, - } = handler; let mut session_started = false; let mut trusted_supervisor_pid = None; let terminating = Arc::new(AtomicBool::new(false)); while let Some(started) = entrypoint_rx.recv().await { - if let Some(exit_code) = started.exit_code { - terminating.store(true, Ordering::Release); - if let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let mut delay = Duration::from_millis(250); - loop { - match openshell_supervisor_process::supervisor_session::report_main_process_exit( - endpoint, - id, - &started.instance_id, - exit_code, - ) - .await - { - Ok(()) => break, - Err(error) => { - warn!(%error, "sidecar main-process exit report failed; retrying"); - tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); - } - } - } - if let Some(publisher) = control_publisher.as_ref() { - publisher.publish_main_process_exit_ack(started.instance_id.clone()); - } - } - break; - } entrypoint_pid.store(started.pid, Ordering::Release); if started.start_session { info!( @@ -1390,10 +1212,13 @@ fn spawn_sidecar_entrypoint_handler( endpoint.clone(), id.clone(), trusted_ssh_socket_path.clone(), - None, + Arc::new( + openshell_supervisor_process::boundary_io::NetnsPortForward::new( + None, None, + ), + ), Some(supervisor_pid), Arc::clone(&terminating), - started.instance_id.clone(), ); session_started = true; info!("sidecar supervisor session task spawned"); @@ -1567,9 +1392,9 @@ const PROXY_BASELINE_READ_ONLY: &[&str] = &[ "/dev/urandom", ]; -/// Minimum read-write paths required for a proxy-mode sandbox child process. -/// The active workspace is granted separately through `include_workdir`. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/tmp"]; +/// Minimum read-write paths required for a proxy-mode sandbox child process: +/// user working directory and temporary files. +const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; /// GPU read-only paths. /// @@ -1883,7 +1708,6 @@ fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { mod baseline_tests { use super::*; use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - use std::path::PathBuf; #[test] fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { @@ -1919,10 +1743,10 @@ mod baseline_tests { } #[test] - fn baseline_read_write_does_not_hardcode_sandbox() { + fn baseline_read_write_always_includes_sandbox_and_tmp() { let (_ro, rw) = baseline_enrichment_paths(); + assert!(rw.contains(&"/sandbox".to_string())); assert!(rw.contains(&"/tmp".to_string())); - assert!(!rw.contains(&"/sandbox".to_string())); } #[test] @@ -2115,7 +1939,7 @@ mod baseline_tests { let mut policy = SandboxPolicy { version: 1, filesystem: FilesystemPolicy { - read_only: vec![PathBuf::from("/tmp")], + read_only: vec![std::path::PathBuf::from("/tmp")], read_write: vec![], include_workdir: false, }, @@ -2130,14 +1954,17 @@ mod baseline_tests { enrich_sandbox_baseline_paths(&mut policy); assert!( - policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), + policy + .filesystem + .read_only + .contains(&std::path::PathBuf::from("/tmp")), "explicit read_only baseline path should be preserved" ); assert!( !policy .filesystem .read_write - .contains(&PathBuf::from("/tmp")), + .contains(&std::path::PathBuf::from("/tmp")), "baseline enrichment must not promote explicit read_only /tmp to read_write" ); } @@ -2245,7 +2072,6 @@ async fn load_policy( openshell_endpoint: Option, policy_rules: Option, policy_data: Option, - extension_credentials: &openshell_extension_core::ExtensionCredentialStore, ) -> Result<( SandboxPolicy, Option>, @@ -2253,7 +2079,6 @@ async fn load_policy( MiddlewareRegistryStatus, LoadedPolicyOrigin, bool, - bool, )> { // File mode: load OPA engine from rego rules + YAML data (dev override) if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { @@ -2303,7 +2128,6 @@ async fn load_policy( MiddlewareRegistryStatus::Synchronized, LoadedPolicyOrigin::LocalOverride, false, - false, )); } @@ -2466,32 +2290,10 @@ async fn load_policy( let middleware_registry_status = if middleware_services.is_empty() { MiddlewareRegistryStatus::Synchronized } else if let Err(error) = grpc_retry("Middleware connect", || { - let middleware_services = middleware_services.clone(); - let extension_credentials = extension_credentials.clone(); - let extension_authentication_enabled = snapshot.extension_authentication_enabled; - async move { - let credentials = if extension_authentication_enabled { - // Share the supervisor's store so the slots installed here - // are the ones the policy poll loop later rotates in place. - openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - endpoint, - extension_credentials, - ) - .await? - .refresh_extension_credentials(&middleware_services) - .await? - } else { - std::collections::HashMap::new() - }; - connect_middleware_registry( - &middleware_services, - &MiddlewareAuthentication { - credentials, - enabled: extension_authentication_enabled, - }, - ) - .await - } + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + middleware_services.clone(), + ) }) .await .and_then(|registry| engine.replace_middleware_registry(registry)) @@ -2534,7 +2336,6 @@ async fn load_policy( has_last_valid_policy, }, agent_proposals_enabled_from_settings(&snapshot.settings), - snapshot.extension_authentication_enabled, )); } @@ -2663,14 +2464,12 @@ enum MiddlewareRegistryStatus { #[derive(Debug)] enum GatewayRuntimeReloadError { PolicyValidation(miette::Report), - TransparentTcpPrerequisite(miette::Report), MiddlewareRegistry(miette::Report), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GatewayRuntimeFailureClass { PolicyValidation, - TransparentTcpPrerequisite, MiddlewareRegistry, } @@ -2678,9 +2477,6 @@ impl GatewayRuntimeReloadError { fn class(&self) -> GatewayRuntimeFailureClass { match self { Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, - Self::TransparentTcpPrerequisite(_) => { - GatewayRuntimeFailureClass::TransparentTcpPrerequisite - } Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, } } @@ -2703,46 +2499,18 @@ impl FailedRuntimeRevision { } } -struct MiddlewareReloadContext<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: &'a MiddlewareAuthentication, - registry_changed: bool, - connector: &'a MiddlewareConnector, -} - async fn reload_gateway_policy_runtime( engine: &OpaEngine, policy: Option<&openshell_core::proto::SandboxPolicy>, entrypoint_pid: u32, - middleware: MiddlewareReloadContext<'_>, - transparent_tcp: TransparentTcpReloadState, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + middleware_registry_changed: bool, ) -> std::result::Result<(), GatewayRuntimeReloadError> { - if let Some(policy) = policy - && policy_contains_explicit_tcp(policy) - { - if !transparent_tcp.capable { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" - ), - )); - } - if !transparent_tcp.substrate_ready { - return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( - miette::miette!( - "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" - ), - )); - } - } match policy { - Some(policy) if middleware.registry_changed => { - let registry = (middleware.connector)( - middleware.desired_services.to_vec(), - middleware.authentication.clone(), - ) - .await - .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; + Some(policy) if middleware_registry_changed => { + let registry = connect_middleware_registry(desired_services) + .await + .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; engine .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) .map_err(GatewayRuntimeReloadError::PolicyValidation) @@ -2759,20 +2527,6 @@ async fn reload_gateway_policy_runtime( } } -fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { - policy.network_policies.values().any(|rule| { - rule.endpoints - .iter() - .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) - }) -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct TransparentTcpReloadState { - capable: bool, - substrate_ready: bool, -} - /// True when the installed middleware registry no longer matches the desired /// service set and must be rebuilt (reconnecting every delivered service). /// @@ -2873,13 +2627,7 @@ struct PolicyStatusUpdate { version: u32, loaded: bool, error: String, - success_event: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum PolicyStatusSuccessEvent { - InitialAcknowledgement { policy_hash: String }, - UnchangedAcknowledgement { policy_hash: String }, + initial_policy_hash: Option, } impl PolicyStatusUpdate { @@ -2888,9 +2636,7 @@ impl PolicyStatusUpdate { version: ack.version, loaded: true, error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::InitialAcknowledgement { - policy_hash: ack.policy_hash.clone(), - }), + initial_policy_hash: Some(ack.policy_hash.clone()), } } @@ -2899,16 +2645,7 @@ impl PolicyStatusUpdate { version, loaded: true, error: String::new(), - success_event: None, - } - } - - fn unchanged_loaded(version: u32, policy_hash: String) -> Self { - Self { - version, - loaded: true, - error: String::new(), - success_event: Some(PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash }), + initial_policy_hash: None, } } @@ -2917,7 +2654,7 @@ impl PolicyStatusUpdate { version, loaded: false, error, - success_event: None, + initial_policy_hash: None, } } } @@ -2978,167 +2715,18 @@ fn initial_poll_disposition( } } -fn unchanged_policy_revision_candidate( - reloads_gateway_policy: bool, - recovering_rejected_policy: bool, - current_policy_version: u32, - current_policy_hash: &str, - result: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - (reloads_gateway_policy - && !recovering_rejected_policy - && !current_policy_hash.is_empty() - && result.policy_source == openshell_core::proto::PolicySource::Sandbox - && result.version > current_policy_version - && result.policy_hash == current_policy_hash) - .then_some(result.version) -} - -fn unchanged_policy_revision_ready_to_ack( - candidate: Option, - policy_runtime_changed: bool, - policy_runtime_reconciled: bool, -) -> Option { - candidate.filter(|_| !policy_runtime_changed || policy_runtime_reconciled) -} - -/// Whether the credential-provenance gates cannot apply to the loaded policy. -/// -/// The gateway derives `provider_credentialed` and deliberately keeps it out of -/// the policy YAML schema, so a local-file policy never carries it and never -/// will: gateway revisions are observed for settings and providers but must not -/// replace the local OPA policy. Provider credentials still arrive from the -/// gateway on that path, so the raw-tunnel and WebSocket binary-frame refusals -/// have nothing to match on. The request-body backstop is unaffected because it -/// keys off the secret resolver rather than endpoint provenance. -fn credential_gating_unavailable( - origin: &LoadedPolicyOrigin, - has_resolver: bool, - network_enabled: bool, -) -> bool { - network_enabled && has_resolver && matches!(origin, LoadedPolicyOrigin::LocalOverride) -} - -/// Report that credential provenance is unavailable for the loaded policy. -/// -/// Carries no credential name, host, or value: the finding states which -/// controls are inactive, nothing about what they would have protected. -fn report_credential_gating_unavailable() { - ocsf_emit!( - DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::High) - .confidence(ConfidenceId::High) - .is_alert(true) - .finding_info( - FindingInfo::new( - "credential-gating-unavailable", - "Credential Provenance Unavailable", - ) - .with_desc( - "Provider credentials are injected, but the loaded policy comes from local \ - files and carries no gateway-derived credential provenance. Uninspected \ - credentialed tunnels and WebSocket binary frames are not refused. Load \ - policy from the gateway to enable these controls." - ), - ) - .evidence_pairs(&[ - ("policy_source", "local-override"), - ("uninspected_connect_gate", "inactive"), - ("websocket_binary_gate", "inactive"), - ("request_body_backstop", "active"), - ]) - .remediation( - "Remove the local policy override so the gateway-delivered effective policy \ - applies, or detach provider credentials from this sandbox." - ) - .message( - "Credential provenance unavailable for local-file policy; uninspected credential gates inactive" - ) - .build() - ); -} - /// Deliver policy status updates independently from policy reconciliation. /// /// The channel is FIFO, so a delayed older status can never arrive after a /// newer status and move the gateway's active version backward. Delivery uses /// the existing bounded retry, but failures never delay policy enforcement. -#[tonic::async_trait] -trait PolicyGatewayClient: Clone + Send + Sync + 'static { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result; - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()>; - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - Ok(()) - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - Ok(std::collections::HashMap::new()) - } - - fn workspace(&self) -> String; -} - -#[tonic::async_trait] -impl PolicyGatewayClient for openshell_core::grpc_client::CachedOpenShellClient { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn refresh_installed_extension_credentials(&self) -> Result<()> { - self.refresh_installed_extension_credentials().await - } - - async fn extension_credentials_for( - &self, - services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> { - self.extension_credentials_for(services).await - } - - fn workspace(&self) -> String { - self.workspace() - } -} - -async fn run_policy_status_reporter( - client: C, +async fn run_policy_status_reporter( + client: openshell_core::grpc_client::CachedOpenShellClient, sandbox_id: String, mut updates: tokio::sync::mpsc::UnboundedReceiver, ) { 'updates: while let Some(update) = updates.recv().await { - let operation = if matches!( - update.success_event, - Some(PolicyStatusSuccessEvent::InitialAcknowledgement { .. }) - ) { + let operation = if update.initial_policy_hash.is_some() { "Initial policy acknowledgement" } else { "Policy status report" @@ -3178,23 +2766,7 @@ async fn run_policy_status_reporter( } } - if let Some(event) = update.success_event { - let (policy_hash, message) = match event { - PolicyStatusSuccessEvent::InitialAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - ), - ), - PolicyStatusSuccessEvent::UnchangedAcknowledgement { policy_hash } => ( - policy_hash, - format!( - "Acknowledged unchanged policy revision as loaded [version:{}]", - update.version - ), - ), - }; + if let Some(policy_hash) = update.initial_policy_hash { ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -3202,7 +2774,10 @@ async fn run_policy_status_reporter( .state(StateId::Enabled, "loaded") .unmapped("version", serde_json::json!(update.version)) .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(message) + .message(format!( + "Acknowledged initial policy revision as loaded [version:{}]", + update.version + )) .build() ); } @@ -3286,57 +2861,16 @@ struct PolicyPollLoopContext { middleware_registry_status: MiddlewareRegistryStatus, sidecar_control_publisher: Option, workspace_tx: tokio::sync::watch::Sender, - extension_credentials: openshell_extension_core::ExtensionCredentialStore, - extension_authentication_enabled: bool, - middleware_connector: MiddlewareConnector, - /// Immutable driver capability and startup substrate state. - transparent_tcp: TransparentTcpReloadState, -} - -type MiddlewareConnector = Arc< - dyn Fn( - Vec, - MiddlewareAuthentication, - ) -> Pin< - Box< - dyn std::future::Future< - Output = Result, - > + Send, - >, - > + Send - + Sync, ->; - -#[derive(Clone, Default)] -struct MiddlewareAuthentication { - credentials: std::collections::HashMap, - enabled: bool, -} - -fn default_middleware_connector() -> MiddlewareConnector { - Arc::new(|services, authentication| { - Box::pin(async move { connect_middleware_registry(&services, &authentication).await }) - }) } async fn connect_middleware_registry( services: &[openshell_core::proto::SupervisorMiddlewareService], - authentication: &MiddlewareAuthentication, ) -> Result { - if authentication.enabled { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services_authenticated( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - &authentication.credentials, - ) - .await - } else { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await - } + openshell_supervisor_middleware::MiddlewareRegistry::connect_services( + openshell_supervisor_middleware_builtins::services(), + services.to_vec(), + ) + .await } async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { @@ -3348,76 +2882,26 @@ async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<( opa_engine.replace_middleware_registry(registry) } -/// Wait the configured poll interval, but never past the point at which an -/// installed extension credential must be rotated. -fn next_poll_delay( - store: &openshell_extension_core::ExtensionCredentialStore, - interval: Duration, -) -> Duration { - let now_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| { - i64::try_from(elapsed.as_millis()).unwrap_or(i64::MAX) - }); - store.next_refresh_delay(interval, now_ms) -} - -/// Drop credentials for services no longer in the installed registry. -/// -/// Call only after a registry swap succeeds, so a failed candidate cannot -/// invalidate the last-known-good clients. -fn retain_extension_credentials( - store: &openshell_extension_core::ExtensionCredentialStore, - installed: &[openshell_core::proto::SupervisorMiddlewareService], - extension_authentication_enabled: bool, -) { - let retained = if extension_authentication_enabled { - installed - .iter() - .map(|service| service.name.as_str()) - .collect() - } else { - std::collections::HashSet::default() - }; - store.retain(&retained); -} - -struct MiddlewareRegistryReconciliation<'a> { - desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], - authentication: MiddlewareAuthentication, - registry_changed: bool, - extension_credentials: &'a openshell_extension_core::ExtensionCredentialStore, - current_services: &'a mut Vec, - status: &'a mut MiddlewareRegistryStatus, -} - async fn reconcile_middleware_registry( opa_engine: &OpaEngine, - middleware_connector: &MiddlewareConnector, - reconciliation: MiddlewareRegistryReconciliation<'_>, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + current_services: &mut Vec, + status: &mut MiddlewareRegistryStatus, ) { - if !reconciliation.registry_changed { + if *status == MiddlewareRegistryStatus::Synchronized + && desired_services == current_services.as_slice() + { return; } - match middleware_connector( - reconciliation.desired_services.to_vec(), - reconciliation.authentication.clone(), - ) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) + match connect_middleware_registry(desired_services) + .await + .and_then(|registry| opa_engine.replace_middleware_registry(registry)) { Ok(()) => { - retain_extension_credentials( - reconciliation.extension_credentials, - reconciliation.desired_services, - reconciliation.authentication.enabled, - ); - reconciliation.current_services.clear(); - reconciliation - .current_services - .extend_from_slice(reconciliation.desired_services); - *reconciliation.status = MiddlewareRegistryStatus::Synchronized; + current_services.clear(); + current_services.extend_from_slice(desired_services); + *status = MiddlewareRegistryStatus::Synchronized; ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -3425,11 +2909,11 @@ async fn reconcile_middleware_registry( .state(StateId::Enabled, "loaded") .unmapped( "supervisor_middleware_service_count", - serde_json::json!(reconciliation.current_services.len()) + serde_json::json!(current_services.len()) ) .message(format!( "Supervisor middleware registry reloaded [service_count:{}]", - reconciliation.current_services.len() + current_services.len() )) .build() ); @@ -3437,7 +2921,7 @@ async fn reconcile_middleware_registry( Err(error) => { // Emit only on the transition into the failed state to avoid // repeating the same finding on every poll during an outage. - if *reconciliation.status == MiddlewareRegistryStatus::Synchronized { + if *status == MiddlewareRegistryStatus::Synchronized { ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Medium) @@ -3449,7 +2933,7 @@ async fn reconcile_middleware_registry( .build() ); } - *reconciliation.status = MiddlewareRegistryStatus::NeedsReconciliation; + *status = MiddlewareRegistryStatus::NeedsReconciliation; } } } @@ -3477,10 +2961,6 @@ enum GatewayRuntimeFailureDisposition { MiddlewareUnavailable { error: String, }, - TransparentTcpExpansionRejected { - error: String, - active_generation: u64, - }, } fn apply_gateway_runtime_reload_failure( @@ -3502,12 +2982,6 @@ fn apply_gateway_runtime_reload_failure( )?; Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) } - GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error: error.to_string(), - active_generation: engine.current_generation(), - }, - ), GatewayRuntimeReloadError::MiddlewareRegistry(error) => { Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error: error.to_string(), @@ -3516,30 +2990,6 @@ fn apply_gateway_runtime_reload_failure( } } -fn emit_transparent_tcp_expansion_rejection( - version: u32, - policy_hash: &str, - active_generation: u64, - error: &str, -) { - let message = format!( - "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Enabled, "retained_previous_policy") - .unmapped("candidate_version", serde_json::json!(version)) - .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) - .unmapped("previous_policy_active", serde_json::json!(true)) - .unmapped("active_generation", serde_json::json!(active_generation)) - .unmapped("validation_error", serde_json::json!(error)) - .message(message) - .build() - ); -} - fn apply_policy_validation_failure( engine: &OpaEngine, configured_mode: PolicyValidationFailureMode, @@ -3670,21 +3120,11 @@ fn emit_policy_validation_failure( } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - let client = openshell_core::grpc_client::CachedOpenShellClient::connect_with_credentials( - &ctx.endpoint, - ctx.extension_credentials.clone(), - ) - .await?; - run_policy_poll_loop_with_client(ctx, client).await -} - -async fn run_policy_poll_loop_with_client( - ctx: PolicyPollLoopContext, - client: C, -) -> Result<()> { + use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::PolicySource; use std::sync::atomic::Ordering; + let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(run_policy_status_reporter( client.clone(), @@ -3694,10 +3134,8 @@ async fn run_policy_poll_loop_with_client( let mut current_config_revision: u64 = 0; let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_version: u32 = 0; let mut current_policy_hash = String::new(); let mut current_middleware_services = Vec::new(); - let mut current_extension_authentication_enabled = ctx.extension_authentication_enabled; let mut middleware_registry_status = ctx.middleware_registry_status; let mut current_settings: std::collections::HashMap< String, @@ -3731,11 +3169,8 @@ async fn run_policy_poll_loop_with_client( skills::install_static_skills, ); current_config_revision = candidate.config_revision; - current_policy_version = candidate.version; current_policy_hash.clone_from(&candidate.policy_hash); current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; current_settings = result.settings; enqueue_policy_status( &status_sender, @@ -3760,8 +3195,6 @@ async fn run_policy_poll_loop_with_client( current_config_revision = result.config_revision; current_policy_hash = result.policy_hash.clone(); current_middleware_services = result.supervisor_middleware_services; - current_extension_authentication_enabled = - result.extension_authentication_enabled; current_settings = result.settings; debug!( config_revision = current_config_revision, @@ -3780,7 +3213,7 @@ async fn run_policy_poll_loop_with_client( let result = if let Some(result) = pending_result.take() { result } else { - tokio::time::sleep(next_poll_delay(&ctx.extension_credentials, interval)).await; + tokio::time::sleep(interval).await; match client.poll_settings(&ctx.sandbox_id).await { Ok(result) => { let _ = ctx.workspace_tx.send(client.workspace()); @@ -3788,50 +3221,19 @@ async fn run_policy_poll_loop_with_client( } Err(e) => { debug!(error = %e, "Settings poll: server unreachable, will retry"); - if current_extension_authentication_enabled - && let Err(refresh_error) = - client.refresh_installed_extension_credentials().await - { - warn!( - error = %refresh_error, - "Settings poll: extension credential refresh failed while configuration was unavailable" - ); - } continue; } } }; - // Reuse installed per-service credentials, rotating only when one is - // missing or due. Rotation happens on the existing gateway channel and - // updates slots in place, so it is independent of config revision and - // registry equality. - let middleware_credentials = if result.extension_authentication_enabled { - match client - .extension_credentials_for(&result.supervisor_middleware_services) - .await - { - Ok(credentials) => credentials, - Err(error) => { - warn!(error = %error, "Settings poll: extension credential refresh failed"); - std::collections::HashMap::new() - } - } - } else { - std::collections::HashMap::new() - }; - let config_changed = result.config_revision != current_config_revision; let provider_env_changed = result.provider_env_revision != current_provider_env_revision; let policy_changed = result.policy_hash != current_policy_hash; - let extension_authentication_changed = - current_extension_authentication_enabled != result.extension_authentication_enabled; - let middleware_registry_changed = extension_authentication_changed - || middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); + let middleware_registry_changed = middleware_registry_needs_rebuild( + middleware_registry_status, + ¤t_middleware_services, + &result.supervisor_middleware_services, + ); // A valid candidate may intentionally restore byte-for-byte policy // content that was active before a rejected update. Its hash then // equals `current_policy_hash`, but the runtime is still quarantined @@ -3841,7 +3243,6 @@ async fn run_policy_poll_loop_with_client( .as_ref() .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); let policy_runtime_changed = recovering_rejected_policy - || extension_authentication_changed || gateway_policy_runtime_needs_reconciliation( reloads_gateway_policy, ¤t_policy_hash, @@ -3850,17 +3251,6 @@ async fn run_policy_poll_loop_with_client( &result.supervisor_middleware_services, middleware_registry_status, ); - // Recovery already has its own acknowledgement path below. Giving it - // precedence here prevents a restored last-known-good policy from - // also being acknowledged as an ordinary same-hash revision. - let unchanged_policy_revision = unchanged_policy_revision_candidate( - reloads_gateway_policy, - recovering_rejected_policy, - current_policy_version, - ¤t_policy_hash, - &result, - ); - let mut policy_runtime_reconciled = false; // A local policy override is not coupled to the gateway policy // snapshot, so its service registry can still be reconciled alone. @@ -3869,30 +3259,14 @@ async fn run_policy_poll_loop_with_client( if !reloads_gateway_policy { reconcile_middleware_registry( &ctx.opa_engine, - &ctx.middleware_connector, - MiddlewareRegistryReconciliation { - desired_services: &result.supervisor_middleware_services, - authentication: MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - extension_credentials: &ctx.extension_credentials, - current_services: &mut current_middleware_services, - status: &mut middleware_registry_status, - }, + &result.supervisor_middleware_services, + &mut current_middleware_services, + &mut middleware_registry_status, ) .await; - if middleware_registry_status == MiddlewareRegistryStatus::Synchronized { - current_extension_authentication_enabled = result.extension_authentication_enabled; - } } - if !config_changed - && !provider_env_changed - && !policy_runtime_changed - && unchanged_policy_revision.is_none() - { + if !config_changed && !provider_env_changed && !policy_runtime_changed { continue; } @@ -3947,67 +3321,42 @@ async fn run_policy_poll_loop_with_client( .await { Ok(env_result) => { - let provider_env_revision = env_result.provider_env_revision; - let install_result = ctx.provider_credentials.install_bound_environment( - provider_env_revision, + ctx.provider_credentials.install_environment( + env_result.provider_env_revision, env_result.environment, env_result.credential_expires_at_ms, env_result.dynamic_credentials, - env_result.static_credential_bindings, - env_result.non_secret_environment_keys, ); - if let Err(error) = install_result { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message(format!( - "Rejected provider environment refresh; static provider credentials were revoked; fetched dynamic token grants remain active: {error}" - )) - .build() - ); - } else { - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher - .publish_provider_env(provider_env_revision, child_env.clone()); - } - current_provider_env_revision = provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{provider_env_revision} env_count:{env_count}]" - )) - .build() + let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); + let env_count = child_env.len(); + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_provider_env( + env_result.provider_env_revision, + child_env.clone(), ); } + current_provider_env_revision = env_result.provider_env_revision; + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped( + "provider_env_revision", + serde_json::json!(env_result.provider_env_revision) + ) + .message(format!( + "Provider environment refreshed [revision:{} env_count:{env_count}]", + env_result.provider_env_revision + )) + .build() + ); } Err(e) => { - ctx.provider_credentials - .revoke_static_provider_environment(result.provider_env_revision); warn!( error = %e, provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment; static provider credentials were revoked; previous dynamic token grants remain active" - ); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::High) - .status(StatusId::Failure) - .state(StateId::Disabled, "fail_closed") - .message( - "Provider environment refresh failed; static provider credentials were revoked; previous dynamic token grants remain active" - ) - .build() + "Settings poll: failed to refresh provider environment" ); } } @@ -4019,22 +3368,13 @@ async fn run_policy_poll_loop_with_client( &ctx.opa_engine, result.policy.as_ref(), pid, - MiddlewareReloadContext { - desired_services: &result.supervisor_middleware_services, - authentication: &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, - }, - registry_changed: middleware_registry_changed, - connector: &ctx.middleware_connector, - }, - ctx.transparent_tcp, + &result.supervisor_middleware_services, + middleware_registry_changed, ) .await; match runtime_result { Ok(()) => { - policy_runtime_reconciled = true; let policy = result .policy .as_ref() @@ -4084,7 +3424,6 @@ async fn run_policy_poll_loop_with_client( &status_sender, PolicyStatusUpdate::loaded(result.version), ); - current_policy_version = result.version; } } else if recovering_rejected_policy && result.version > 0 @@ -4106,7 +3445,6 @@ async fn run_policy_poll_loop_with_client( &status_sender, PolicyStatusUpdate::loaded(result.version), ); - current_policy_version = result.version; } if middleware_registry_changed { @@ -4127,13 +3465,6 @@ async fn run_policy_poll_loop_with_client( current_policy_hash.clone_from(&result.policy_hash); current_middleware_services.clone_from(&result.supervisor_middleware_services); - current_extension_authentication_enabled = - result.extension_authentication_enabled; - retain_extension_credentials( - &ctx.extension_credentials, - &result.supervisor_middleware_services, - result.extension_authentication_enabled, - ); middleware_registry_status = MiddlewareRegistryStatus::Synchronized; last_failed_runtime_revision = None; } @@ -4192,26 +3523,6 @@ async fn run_policy_poll_loop_with_client( )) .build()); } - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - error, - active_generation, - } => { - emit_transparent_tcp_expansion_rejection( - result.version, - &result.policy_hash, - active_generation, - &error, - ); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, error), - ); - } - } } } last_failed_runtime_revision = Some(failed_revision); @@ -4223,18 +3534,6 @@ async fn run_policy_poll_loop_with_client( } } - if let Some(version) = unchanged_policy_revision_ready_to_ack( - unchanged_policy_revision, - policy_runtime_changed, - policy_runtime_reconciled, - ) { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::unchanged_loaded(version, result.policy_hash.clone()), - ); - current_policy_version = version; - } - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); @@ -4415,21 +3714,6 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String )] mod tests { use super::*; - - #[test] - fn transparent_tcp_capability_requires_exact_driver_marker() { - let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; - assert!(!has_network_runtime_capability(None, required)); - assert!(!has_network_runtime_capability(Some(""), required)); - assert!(!has_network_runtime_capability( - Some("policy-dns-transparent-tcp-extra"), - required - )); - assert!(has_network_runtime_capability( - Some("other, policy-dns-transparent-tcp"), - required - )); - } use openshell_core::policy::{ FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, }; @@ -4499,24 +3783,21 @@ mod tests { } #[tokio::test] - async fn sidecar_control_provider_env_update_orders_by_generation() { + async fn sidecar_control_provider_env_update_installs_newer_revision() { let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - u64::MAX, + 1, std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), ); - let agent_proposals = AgentProposals::new(true); let handle = spawn_sidecar_control_update_watcher( rx, provider_credentials.clone(), - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 10, + AgentProposals::default(), ); tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - generation: 11, + revision: 2, + generation: 1, provider_child_env: std::collections::HashMap::from([( "TOKEN".to_string(), "new".to_string(), @@ -4526,7 +3807,7 @@ mod tests { timeout(Duration::from_secs(1), async { loop { - if provider_credentials.snapshot().revision == 1 { + if provider_credentials.snapshot().revision == 2 { break; } tokio::time::sleep(Duration::from_millis(10)).await; @@ -4535,33 +3816,22 @@ mod tests { .await .unwrap(); let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 1); + assert_eq!(snapshot.revision, 2); assert_eq!( snapshot.child_env.get("TOKEN").map(String::as_str), Some("new") ); tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 11, + revision: 1, + generation: 2, provider_child_env: std::collections::HashMap::from([( "TOKEN".to_string(), - "duplicate-generation".to_string(), + "stale".to_string(), )]), }) .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 1, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; assert_eq!( provider_credentials .snapshot() @@ -4570,54 +3840,6 @@ mod tests { .map(String::as_str), Some("new") ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - generation: 12, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "newest".to_string(), - )]), - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: u64::MAX, - generation: 11, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: true, - config_revision: 2, - }) - .unwrap(); - timeout(Duration::from_secs(1), async { - while !agent_proposals.enabled() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("newest") - ); handle.abort(); } @@ -4627,13 +3849,8 @@ mod tests { let provider_credentials = ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); let agent_proposals = AgentProposals::new(true); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials, - agent_proposals.clone(), - Arc::new(tokio::sync::Mutex::new(None)), - 0, - ); + let handle = + spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); tx.send(sidecar_control::ControlUpdate::AgentProposals { enabled: false, @@ -4830,24 +4047,6 @@ filesystem_policy: openshell_policy::restrictive_default_policy() } - fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::parse_sandbox_policy( - r#" -version: 1 -network_policies: - redis: - name: redis - endpoints: - - host: redis.example.com - port: 6379 - protocol: tcp - binaries: - - path: /usr/bin/redis-cli -"#, - ) - .expect("parse TCP policy") - } - fn settings_poll_result( policy: Option, version: u32, @@ -4862,603 +4061,11 @@ network_policies: settings: std::collections::HashMap::new(), global_policy_version: 0, provider_env_revision: 0, + extension_authentication_enabled: false, supervisor_middleware_services: Vec::new(), workspace: String::new(), policy_validation_failure_mode: PolicyValidationFailureMode::default(), - extension_authentication_enabled: false, - } - } - - #[derive(Clone)] - struct ScriptedPolicyGateway { - polls: Arc< - tokio::sync::Mutex< - tokio::sync::mpsc::UnboundedReceiver< - openshell_core::grpc_client::SettingsPollResult, - >, - >, - >, - reports: UnboundedSender<(u32, bool, String)>, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for ScriptedPolicyGateway { - async fn poll_settings( - &self, - _sandbox_id: &str, - ) -> Result { - self.polls - .lock() - .await - .recv() - .await - .ok_or_else(|| miette::miette!("scripted policy poll channel closed")) - } - - async fn report_policy_status( - &self, - _sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.reports - .send((version, loaded, error.to_string())) - .map_err(|_| miette::miette!("scripted policy report channel closed")) - } - - fn workspace(&self) -> String { - "test-workspace".to_string() - } - } - - #[derive(Clone)] - struct CredentialRejectingPolicyGateway { - inner: ScriptedPolicyGateway, - credential_requests: Arc, - } - - #[tonic::async_trait] - impl PolicyGatewayClient for CredentialRejectingPolicyGateway { - async fn poll_settings( - &self, - sandbox_id: &str, - ) -> Result { - self.inner.poll_settings(sandbox_id).await - } - - async fn report_policy_status( - &self, - sandbox_id: &str, - version: u32, - loaded: bool, - error: &str, - ) -> Result<()> { - self.inner - .report_policy_status(sandbox_id, version, loaded, error) - .await - } - - async fn extension_credentials_for( - &self, - _services: &[openshell_core::proto::SupervisorMiddlewareService], - ) -> Result> - { - self.credential_requests.fetch_add(1, Ordering::SeqCst); - Err(miette::miette!( - "gateway extension authentication is unavailable" - )) - } - - fn workspace(&self) -> String { - self.inner.workspace() - } - } - - fn scripted_policy_gateway() -> ( - ScriptedPolicyGateway, - UnboundedSender, - tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - let (poll_tx, poll_rx) = tokio::sync::mpsc::unbounded_channel(); - let (report_tx, report_rx) = tokio::sync::mpsc::unbounded_channel(); - ( - ScriptedPolicyGateway { - polls: Arc::new(tokio::sync::Mutex::new(poll_rx)), - reports: report_tx, - }, - poll_tx, - report_rx, - ) - } - - fn policy_poll_test_context( - opa_engine: Arc, - loaded_policy_origin: LoadedPolicyOrigin, - middleware_connector: MiddlewareConnector, - ) -> PolicyPollLoopContext { - let (workspace_tx, _workspace_rx) = tokio::sync::watch::channel(String::new()); - PolicyPollLoopContext { - endpoint: String::new(), - sandbox_id: "sandbox-test".to_string(), - opa_engine, - loaded_policy_origin, - entrypoint_pid: Arc::new(AtomicU32::new(0)), - interval_secs: 0, - ocsf_enabled: Arc::new(AtomicBool::new(false)), - provider_credentials: ProviderCredentialState::from_child_env_snapshot( - 0, - std::collections::HashMap::new(), - ), - policy_local_ctx: None, - agent_proposals: AgentProposals::default(), - middleware_registry_status: MiddlewareRegistryStatus::Synchronized, - sidecar_control_publisher: None, - workspace_tx, - extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), - extension_authentication_enabled: false, - middleware_connector, - transparent_tcp: TransparentTcpReloadState::default(), - } - } - - async fn expect_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - version: u32, - ) { - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("policy report timed out") - .expect("policy reporter stopped"); - assert_eq!(report, (version, true, String::new())); - } - - async fn expect_no_policy_report( - reports: &mut tokio::sync::mpsc::UnboundedReceiver<(u32, bool, String)>, - ) { - assert!( - timeout(Duration::from_millis(50), reports.recv()) - .await - .is_err(), - "unexpected policy status report" - ); - } - - #[tokio::test] - async fn same_hash_poll_revision_is_acknowledged_once_without_opa_reload() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - expect_policy_report(&mut reports, 2).await; - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - - assert_eq!( - engine.current_generation(), - 0, - "same-hash acknowledgement must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_tcp_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let active_generation = engine.current_generation(); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let mut ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - ctx.transparent_tcp = TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }; - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - let report = timeout(Duration::from_secs(1), reports.recv()) - .await - .expect("TCP rejection report timed out") - .expect("policy reporter stopped"); - - assert_eq!(report.0, 2); - assert!(!report.1); - assert!(report.2.contains("recreate the sandbox"), "{}", report.2); - assert!(report.2.contains("previous policy remains active")); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "scripted-guard".to_string(), - grpc_endpoint: "http://scripted.invalid".to_string(), - ..Default::default() - }]; - - let connector_attempts = Arc::new(AtomicUsize::new(0)); - let (attempt_tx, mut attempt_rx) = tokio::sync::mpsc::unbounded_channel(); - let middleware_connector: MiddlewareConnector = { - let connector_attempts = connector_attempts.clone(); - Arc::new(move |_services, _authentication| { - let attempt = connector_attempts.fetch_add(1, Ordering::SeqCst) + 1; - attempt_tx.send(attempt).unwrap(); - Box::pin(async move { - if attempt == 1 { - Err(miette::miette!("scripted middleware connection failure")) - } else { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - } - }) - }) - }; - - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - middleware_connector, - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(1) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(engine.current_generation(), 0); - - polls.send(v2.clone()).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), attempt_rx.recv()) - .await - .unwrap(), - Some(2) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(engine.current_generation(), 1); - - polls.send(v2).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!(connector_attempts.load(Ordering::SeqCst), 2); - handle.abort(); - } - - #[tokio::test] - async fn no_signer_capability_uses_legacy_middleware_connector_without_credentials() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "legacy-guard".to_string(), - grpc_endpoint: "http://legacy.invalid".to_string(), - ..Default::default() - }]; - assert!(!v2.extension_authentication_enabled); - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - connect_middleware_registry(&[], &MiddlewareAuthentication::default()).await - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, false)) - ); - expect_policy_report(&mut reports, 2).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 0); - handle.abort(); - } - - #[tokio::test] - async fn enabled_extension_authentication_keeps_credential_failure_fail_closed() { - let mut v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - v1.policy_hash = "same-policy".to_string(); - let mut v2 = v1.clone(); - v2.version = 2; - v2.config_revision = 200; - v2.extension_authentication_enabled = true; - v2.supervisor_middleware_services = - vec![openshell_core::proto::SupervisorMiddlewareService { - name: "authenticated-guard".to_string(), - grpc_endpoint: "https://guard.invalid".to_string(), - ..Default::default() - }]; - - let (inner, polls, mut reports) = scripted_policy_gateway(); - let credential_requests = Arc::new(AtomicUsize::new(0)); - let client = CredentialRejectingPolicyGateway { - inner, - credential_requests: credential_requests.clone(), - }; - let (connector_tx, mut connector_rx) = tokio::sync::mpsc::unbounded_channel(); - let connector: MiddlewareConnector = Arc::new(move |_services, authentication| { - connector_tx - .send((authentication.credentials.len(), authentication.enabled)) - .unwrap(); - Box::pin(async move { - if authentication.enabled && authentication.credentials.is_empty() { - Err(miette::miette!( - "missing authenticated middleware credential" - )) - } else { - connect_middleware_registry(&[], &authentication).await - } - }) - }); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let ctx = policy_poll_test_context( - engine, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - connector, - ); - - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - assert_eq!( - timeout(Duration::from_secs(1), connector_rx.recv()) - .await - .unwrap(), - Some((0, true)) - ); - expect_no_policy_report(&mut reports).await; - assert_eq!(credential_requests.load(Ordering::SeqCst), 1); - handle.abort(); - } - - async fn assert_poll_does_not_use_same_hash_acknowledgement( - initial: openshell_core::grpc_client::SettingsPollResult, - next: openshell_core::grpc_client::SettingsPollResult, - origin: LoadedPolicyOrigin, - initial_report: Option, - ) { - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context(engine.clone(), origin, default_middleware_connector()); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(initial).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - if let Some(version) = initial_report { - expect_policy_report(&mut reports, version).await; - } else { - expect_no_policy_report(&mut reports).await; } - - polls.send(next).unwrap(); - expect_no_policy_report(&mut reports).await; - assert_eq!( - engine.current_generation(), - 0, - "negative same-hash scope must not reload OPA" - ); - handle.abort(); - } - - #[tokio::test] - async fn same_hash_ack_poll_loop_rejects_local_global_empty_equal_and_older_scopes() { - let mut sandbox_v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - sandbox_v1.policy_hash = "same-policy".to_string(); - let loaded_v1 = LoadedPolicyRevision::from_snapshot(&sandbox_v1); - let mut sandbox_v2 = sandbox_v1.clone(); - sandbox_v2.version = 2; - sandbox_v2.config_revision = 200; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v2.clone(), - LoadedPolicyOrigin::LocalOverride, - None, - ) - .await; - - let mut global_v2 = sandbox_v2.clone(); - global_v2.policy_source = openshell_core::proto::PolicySource::Global; - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - global_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let mut empty_v1 = sandbox_v1.clone(); - empty_v1.policy_hash.clear(); - let empty_loaded = LoadedPolicyRevision::from_snapshot(&empty_v1); - let mut empty_v2 = sandbox_v2.clone(); - empty_v2.policy_hash.clear(); - assert_poll_does_not_use_same_hash_acknowledgement( - empty_v1, - empty_v2, - LoadedPolicyOrigin::Gateway { - revision: Some(empty_loaded), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v1.clone(), - sandbox_v1.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v1.clone()), - has_last_valid_policy: true, - }, - Some(1), - ) - .await; - - let loaded_v2 = LoadedPolicyRevision::from_snapshot(&sandbox_v2); - assert_poll_does_not_use_same_hash_acknowledgement( - sandbox_v2, - sandbox_v1, - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_v2), - has_last_valid_policy: true, - }, - Some(2), - ) - .await; - } - - #[tokio::test] - async fn changed_hash_poll_uses_normal_opa_reload_and_status_path() { - let v1 = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let v2 = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); - let engine = - Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); - let ctx = policy_poll_test_context( - engine.clone(), - LoadedPolicyOrigin::Gateway { - revision: Some(loaded_revision), - has_last_valid_policy: true, - }, - default_middleware_connector(), - ); - let (client, polls, mut reports) = scripted_policy_gateway(); - polls.send(v1).unwrap(); - let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); - - expect_policy_report(&mut reports, 1).await; - polls.send(v2).unwrap(); - expect_policy_report(&mut reports, 2).await; - assert_eq!( - engine.current_generation(), - 1, - "changed policy content must still reload OPA" - ); - handle.abort(); } #[tokio::test] @@ -5476,7 +4083,7 @@ network_policies: max_payload_bytes: 1024, ..Default::default() }; - connect_middleware_registry(&[invalid_external], &MiddlewareAuthentication::default()) + connect_middleware_registry(&[invalid_external]) .await .expect_err("unavailable external service must not replace built-ins"); @@ -5501,13 +4108,8 @@ network_policies: &engine, Some(&proto_policy_fixture()), 0, - MiddlewareReloadContext { - desired_services: &[unavailable_service], - authentication: &MiddlewareAuthentication::default(), - registry_changed: true, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), + &[unavailable_service], + true, ) .await .expect_err("unavailable middleware must fail candidate preparation"); @@ -5528,74 +4130,6 @@ network_policies: assert!(engine.fail_closed_reason().is_none()); } - #[tokio::test] - async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - let active_generation = engine.current_generation(); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState { - capable: true, - substrate_ready: false, - }, - ) - .await - .expect_err("TCP expansion must require startup substrate"); - let disposition = apply_gateway_runtime_reload_failure( - &engine, - failure, - PolicyValidationFailureMode::FailClosed, - true, - 2, - ) - .expect("runtime prerequisite failure handling must succeed"); - - assert!(matches!( - disposition, - GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { - active_generation: generation, - .. - } if generation == active_generation - )); - assert_eq!(engine.current_generation(), active_generation); - assert!(engine.fail_closed_reason().is_none()); - } - - #[tokio::test] - async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - - let failure = reload_gateway_policy_runtime( - &engine, - Some(&proto_tcp_policy_fixture()), - 0, - MiddlewareReloadContext { - desired_services: &[], - authentication: &MiddlewareAuthentication::default(), - registry_changed: false, - connector: &default_middleware_connector(), - }, - TransparentTcpReloadState::default(), - ) - .await - .expect_err("unsupported runtime must reject TCP expansion"); - - assert!(matches!( - failure, - GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) - )); - assert_eq!(engine.current_generation(), 0); - } - #[test] fn policy_rejection_after_middleware_outage_is_not_deduplicated() { let engine = OpaEngine::from_strings( @@ -5889,120 +4423,6 @@ network_policies: assert!(origin.allows_gateway_policy_reload()); } - #[test] - fn unchanged_sandbox_policy_revision_candidate_is_strictly_scoped() { - let sandbox_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ) - }; - - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &sandbox_result), - Some(2) - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 2, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate( - true, - false, - 1, - "different-policy", - &sandbox_result, - ), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(false, false, 1, "same-policy", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "", &sandbox_result), - None - ); - assert_eq!( - unchanged_policy_revision_candidate(true, true, 1, "same-policy", &sandbox_result), - None - ); - - let global_result = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "same-policy".to_string(), - ..settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Global, - ) - }; - assert_eq!( - unchanged_policy_revision_candidate(true, false, 1, "same-policy", &global_result), - None - ); - } - - #[test] - fn unchanged_policy_revision_waits_for_required_runtime_reconciliation() { - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), false, false), - Some(2), - "a same-hash revision needs no OPA reload" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, false), - None, - "failed runtime reconciliation must keep the revision pending" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(Some(2), true, true), - Some(2), - "successful runtime reconciliation permits acknowledgement" - ); - assert_eq!( - unchanged_policy_revision_ready_to_ack(None, false, true), - None, - "runtime success cannot manufacture a revision candidate" - ); - } - - #[test] - fn credential_gating_unavailable_for_local_override_with_credentials() { - assert!(credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - true - )); - } - - #[test] - fn credential_gating_available_without_local_override_or_credentials() { - // A gateway policy is stamped with provenance, so the gates apply. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::Gateway { - revision: None, - has_last_valid_policy: true, - }, - true, - true - )); - // No provider credentials means there is nothing to leak. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - false, - true - )); - // Without networking the proxy never evaluates endpoint provenance. - assert!(!credential_gating_unavailable( - &LoadedPolicyOrigin::LocalOverride, - true, - false - )); - } - #[test] fn policy_status_outbox_preserves_all_revision_order() { let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); @@ -6164,22 +4584,12 @@ network_policies: "fail_closed" ); assert_eq!(config["unmapped"]["previous_policy_active"], false); - assert_eq!( - config["unmapped"]["validation_error"], - "conflicting tls metadata" - ); assert!( config["message"] .as_str() .unwrap() .contains("previous policy IS NOT active") ); - assert!( - config["message"] - .as_str() - .unwrap() - .contains("error:conflicting tls metadata") - ); let finding = finding.to_json().unwrap(); assert_eq!(finding["class_uid"], 2004); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 6d244fb6bc..bc38433165 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -23,6 +23,7 @@ use openshell_sandbox::run_sandbox; /// to copy the binary out. Invoking the binary itself with this argument /// performs the copy in pure Rust. const COPY_SELF_SUBCOMMAND: &str = "copy-self"; +const VM_GUEST_SUBCOMMAND: &str = "vm-guest"; /// Subcommand for one-shot debug RPCs from inside a sandbox container. /// @@ -32,14 +33,13 @@ const COPY_SELF_SUBCOMMAND: &str = "copy-self"; /// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` /// to confirm the cross-sandbox IDOR guard fires. const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; -const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; #[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR; +const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; #[cfg(target_os = "linux")] const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; #[cfg(target_os = "linux")] @@ -111,7 +111,8 @@ impl std::str::FromStr for Mode { #[command(about = "Process sandbox and monitor", long_about = None)] struct Args { /// Command to execute in the sandbox. - /// Defaults to `/bin/bash -l` if neither this nor the driver specification is provided. + /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. + /// Defaults to `/bin/bash` if neither is provided. #[arg(trailing_var_arg = true)] command: Vec, @@ -230,55 +231,17 @@ struct Args { #[arg(long)] upstream_proxy_connect_by_hostname: bool, - /// Path to a PEM CA bundle trusted for the corporate proxy: the TLS - /// handshake with an `https://` proxy and, for TLS-intercepting proxies, - /// re-signed upstream certificates and the sandbox trust bundle. + /// Backend named by the compute driver's topology descriptor. #[arg(long)] - upstream_proxy_ca_bundle: Option, -} + topology_backend_name: Option, -/// Internal one-shot command used by the privileged supervisor to validate an -/// image-provided workdir as the final sandbox identity. -#[derive(Parser, Debug)] -#[command(name = "validate-workspace", hide = true)] -struct ValidateWorkspaceArgs { - #[arg(long)] - workdir: String, + /// Isolation Backend interface version named by the topology descriptor. #[arg(long)] - expected_uid: u32, - #[arg(long)] - expected_gid: u32, -} + topology_version: Option, -#[cfg(target_os = "linux")] -fn validate_workspace(args: &[String]) -> Result<()> { - let args = ValidateWorkspaceArgs::try_parse_from( - std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), - ) - .into_diagnostic()?; - let actual = ( - nix::unistd::geteuid().as_raw(), - nix::unistd::getegid().as_raw(), - ); - if actual != (args.expected_uid, args.expected_gid) { - return Err(miette::miette!( - "workspace validator privilege drop failed: expected {}:{}, got {}:{}", - args.expected_uid, - args.expected_gid, - actual.0, - actual.1 - )); - } - openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new( - &args.workdir, - )) -} - -#[cfg(not(target_os = "linux"))] -fn validate_workspace(_args: &[String]) -> Result<()> { - Err(miette::miette!( - "workspace validation is only supported on Unix" - )) + /// Base64-encoded opaque topology-descriptor payload. + #[arg(long)] + topology_payload_base64: Option, } /// Copy the running executable to `dest`, creating parent directories as @@ -477,23 +440,16 @@ fn run_network_init( #[cfg(target_os = "linux")] fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 - && !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_user_id) - { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( - "--proxy-uid must be 0 or in range [{}, {}]", - openshell_policy::MIN_SANDBOX_PROXY_UID, - openshell_policy::MAX_SANDBOX_UID, + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID )); } - if !(openshell_policy::MIN_SANDBOX_UID..=openshell_policy::MAX_SANDBOX_UID) - .contains(&proxy_primary_group_id) - { + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { return Err(miette::miette!( - "--proxy-gid must be in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID )); } Ok(()) @@ -522,6 +478,15 @@ fn main() -> Result<()> { })?; return copy_self(dest); } + if raw_args.get(1).map(String::as_str) == Some(VM_GUEST_SUBCOMMAND) { + let [_, _, config] = raw_args.as_slice() else { + return Err(miette::miette!( + "usage: openshell-sandbox {VM_GUEST_SUBCOMMAND} " + )); + }; + return openshell_isolation_vm::run_guest(Path::new(config)) + .map_err(|error| miette::miette!(error)); + } // Handle `debug-rpc [args]` before clap. Uses a small // dedicated runtime so we don't pay the supervisor's full startup cost. @@ -536,9 +501,6 @@ fn main() -> Result<()> { std::process::exit(exit); }); } - if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { - return validate_workspace(&raw_args[2..]); - } let args = Args::parse(); @@ -655,19 +617,11 @@ fn main() -> Result<()> { (None, None) }; - // Resolve an exact canonical process. Explicit offline/test argv wins; - // drivers otherwise provide a versioned JSON transport so argument - // boundaries are never reconstructed with shell parsing. - let workdir = args.workdir.clone(); - let (command, interactive) = if !args.command.is_empty() { - (args.command, args.interactive) - } else if let Ok(json) = std::env::var(openshell_core::sandbox_env::MAIN_PROCESS_SPEC) { - let config = openshell_core::sandbox_env::MainProcessConfig::decode(&json) - .map_err(|error| miette::miette!("{error}"))?; - (config.command, config.tty) + // Get command - either from CLI args, environment variable, or default to /bin/bash + let command = if args.command.is_empty() { + vec!["/bin/bash".to_string()] } else { - let config = openshell_core::sandbox_env::MainProcessConfig::scratch(); - (config.command, config.tty) + args.command }; info!(command = ?command, "Starting sandbox"); @@ -681,14 +635,37 @@ fn main() -> Result<()> { proxy_auth_file: args.upstream_proxy_auth_file, proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - proxy_ca_bundle: args.upstream_proxy_ca_bundle, + proxy_ca_bundle: None, + }; + + let topology_descriptor = match ( + args.topology_backend_name, + args.topology_version, + args.topology_payload_base64, + ) { + (None, None, None) => None, + (Some(backend_name), Some(version), Some(payload)) => { + use base64::Engine as _; + Some(openshell_isolation::contract::TopologyDescriptor { + backend_name, + version, + payload: base64::engine::general_purpose::STANDARD + .decode(payload) + .into_diagnostic()?, + }) + } + _ => { + return Err(miette::miette!( + "topology descriptor requires backend name, version, and payload" + )); + } }; run_sandbox( command, - workdir, + args.workdir, args.timeout, - interactive, + args.interactive, args.sandbox_id, args.sandbox, args.openshell_endpoint, @@ -702,6 +679,7 @@ fn main() -> Result<()> { args.mode.network, args.mode.process, upstream_proxy_args, + topology_descriptor, ) .await })?; @@ -714,29 +692,19 @@ mod tests { use super::*; use std::os::unix::fs::PermissionsExt; - #[cfg(target_os = "linux")] #[test] - fn workspace_validation_subcommand_uses_final_policy_identity() { - let uid = nix::unistd::geteuid().as_raw(); - let gid = nix::unistd::getegid().as_raw(); - if uid < 1000 || gid < 1000 { - return; - } - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - let args = vec![ - "--workdir".to_string(), - root.display().to_string(), - "--expected-uid".to_string(), - uid.to_string(), - "--expected-gid".to_string(), - gid.to_string(), - ]; - - validate_workspace(&args).expect("current identity should retain workspace authority"); + fn topology_descriptor_selects_vm_backend() { + let args = Args::try_parse_from([ + "openshell-sandbox", + "--topology-backend-name=vm", + "--topology-version=1", + "--topology-payload-base64=", + "/bin/true", + ]) + .expect("parse VM backend flags"); + assert_eq!(args.topology_backend_name.as_deref(), Some("vm")); + assert_eq!(args.topology_version, Some(1)); + assert_eq!(args.topology_payload_base64.as_deref(), Some("")); } /// Drives `copy_self`'s file-copy logic against an arbitrary source path @@ -824,23 +792,17 @@ mod tests { #[cfg(target_os = "linux")] #[test] fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, 30).unwrap(); + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); } #[cfg(target_os = "linux")] #[test] - fn network_init_still_rejects_low_non_root_proxy_uid_and_root_gid() { + fn network_init_still_rejects_low_non_root_proxy_ids() { let uid_err = validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); assert!(uid_err.to_string().contains("--proxy-uid")); - let gid_err = validate_network_init_ids(0, 0).unwrap_err(); + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); assert!(gid_err.to_string().contains("--proxy-gid")); } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_non_root_system_proxy_group() { - validate_network_init_ids(openshell_policy::MIN_SANDBOX_PROXY_UID, 30).unwrap(); - } } diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs index 658a20132c..91d7c3c9d3 100644 --- a/crates/openshell-sandbox/src/sidecar_control.rs +++ b/crates/openshell-sandbox/src/sidecar_control.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Local control channel for Kubernetes sidecar topology. //! //! The network sidecar owns gateway credentials. The process supervisor in the diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index d3def44743..a81d4a9572 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -8,7 +8,7 @@ //! store, terminates TLS from the client (presenting dynamic certs per hostname), //! inspects the plaintext HTTP, then re-encrypts to upstream using real root CAs. -use miette::{IntoDiagnostic, Result, miette}; +use miette::{IntoDiagnostic, Result}; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ServerConfig}; @@ -182,7 +182,7 @@ where Ok(tls_stream) } -/// Connect TLS to an upstream server, verifying against the configured CA roots. +/// Connect TLS to an upstream server, verifying against webpki-roots. /// /// Returns a TLS stream for re-encrypted upstream communication. pub async fn tls_connect_upstream( @@ -199,75 +199,34 @@ pub async fn tls_connect_upstream( Ok(tls_stream) } -/// Build a rustls `ClientConfig` using the configured CA root source. +/// Build a rustls `ClientConfig` with Mozilla + system root CAs for upstream connections. /// -/// In `bundled-ca-roots` mode this uses Mozilla roots from `webpki-roots` overlaid -/// with any locally-installed CAs from `system_ca_bundle` (e.g. corporate or private -/// CAs added to `/etc/pki/ca-trust`). Duplicates with the Mozilla bundle are harmless. -/// -/// Without `bundled-ca-roots` this uses the platform/native trust store exclusively; -/// `system_ca_bundle` is ignored because the native store already reflects all -/// operator-installed trust anchors. -pub fn build_upstream_client_config(system_ca_bundle: &str) -> Result> { - let mut config = ClientConfig::builder() - .with_root_certificates(build_upstream_root_store(system_ca_bundle)?) - .with_no_client_auth(); - config.alpn_protocols = vec![b"http/1.1".to_vec()]; - - Ok(Arc::new(config)) -} - -fn build_upstream_root_store(system_ca_bundle: &str) -> Result { +/// `system_ca_bundle` is the pre-read PEM contents of the system CA bundle +/// (from [`read_system_ca_bundle`]). Pass the same string to [`write_ca_files`] +/// to avoid reading the bundle from disk twice. +pub fn build_upstream_client_config(system_ca_bundle: &str) -> Arc { let mut root_store = rustls::RootCertStore::empty(); + root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - #[cfg(feature = "bundled-ca-roots")] - { - root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); - // Overlay system/corporate CAs so custom trust anchors are honoured in - // default upstream builds. Duplicates with webpki-roots are harmless. - let (added, ignored) = load_pem_certs_into_store(&mut root_store, system_ca_bundle); - if added > 0 { - tracing::debug!(added, "loaded system CA certificates for upstream TLS"); - } - if ignored > 0 { - tracing::warn!( - ignored, - "some system CA certificates could not be parsed and were ignored" - ); - } - } - - #[cfg(not(feature = "bundled-ca-roots"))] - { - let _ = system_ca_bundle; // native store already includes operator-installed CAs - add_native_roots(&mut root_store)?; + // System bundles typically overlap with webpki-roots (Mozilla roots); + // duplicates are harmless and ensure we also pick up any custom/corporate CAs. + let (added, ignored) = load_pem_certs_into_store(&mut root_store, system_ca_bundle); + if added > 0 { + tracing::debug!(added, "Loaded system CA certificates for upstream TLS"); } - - if root_store.is_empty() { - return Err(miette!("no TLS root certificates available")); - } - - Ok(root_store) -} - -#[cfg(not(feature = "bundled-ca-roots"))] -fn add_native_roots(root_store: &mut rustls::RootCertStore) -> Result<()> { - let native_certs = rustls_native_certs::load_native_certs(); - let cert_count = native_certs.certs.len(); - let (added, ignored) = root_store.add_parsable_certificates(native_certs.certs); - let ignored = ignored + native_certs.errors.len(); - if ignored > 0 { - tracing::debug!(ignored, "ignored unparsable native root certificates"); + tracing::warn!( + ignored, + "Some system CA certificates could not be parsed and were ignored" + ); } - if added == 0 { - return Err(miette!( - "no usable native TLS root certificates found ({cert_count} loaded, {ignored} ignored)" - )); - } + let mut config = ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; - Ok(()) + Arc::new(config) } /// Write CA certificate files for the sandbox trust store. @@ -277,7 +236,8 @@ fn add_native_roots(root_store: &mut rustls::RootCertStore) -> Result<()> { /// 2. Combined bundle: system CAs + sandbox CA (for `SSL_CERT_FILE` which replaces default) /// /// `system_ca_bundle` is the pre-read PEM contents of the system CA bundle -/// (from [`read_system_ca_bundle`]). +/// (from [`read_system_ca_bundle`]). Pass the same string to +/// [`build_upstream_client_config`] to avoid reading the bundle from disk twice. /// /// Returns `(ca_cert_path, combined_bundle_path)`. pub fn write_ca_files( @@ -308,7 +268,6 @@ pub fn write_ca_files( /// Returns `(added, ignored)` counts. Invalid or unparseable certificates /// are silently ignored, matching the behavior of /// `RootCertStore::add_parsable_certificates`. -#[cfg_attr(not(feature = "bundled-ca-roots"), allow(dead_code))] fn load_pem_certs_into_store( root_store: &mut rustls::RootCertStore, pem_data: &str, @@ -332,7 +291,7 @@ fn load_pem_certs_into_store( /// /// Returns the PEM contents of the first non-empty bundle found, or an empty /// string if none of the well-known paths exist. Call once and pass the result -/// to [`write_ca_files`]. +/// to both [`write_ca_files`] and [`build_upstream_client_config`]. pub fn read_system_ca_bundle() -> String { for path in SYSTEM_CA_PATHS { if let Ok(contents) = std::fs::read_to_string(path) @@ -342,6 +301,7 @@ pub fn read_system_ca_bundle() -> String { } } // No system bundle found — combined file will contain only the sandbox CA. + // This is acceptable since the proxy uses webpki-roots independently. String::new() } @@ -468,7 +428,7 @@ mod tests { #[test] fn upstream_config_alpn() { let _ = rustls::crypto::ring::default_provider().install_default(); - let config = build_upstream_client_config("").unwrap(); + let config = build_upstream_client_config(""); assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]); } diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index a828f75fba..e81ed02d50 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -13,12 +13,12 @@ pub mod identity_source; pub mod inference_routes; pub mod l7; pub mod opa; -pub(crate) mod policy_dns; pub mod policy_local; pub mod procfs; pub mod proxy; pub mod run; pub mod sigv4; +mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 35cef601d8..aff19cbc04 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -810,6 +810,7 @@ impl OpaEngine { /// generation comparison and callback linearizes state derived from an OPA /// snapshot with every policy reload and fail-closed transition. Callers /// must not perform I/O or other long-running work in `operation`. + #[allow(dead_code)] pub(crate) fn with_current_generation( &self, expected_generation: u64, diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index dc2736a4ea..3025e1b0d2 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,35 +10,39 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; -#[cfg(target_os = "linux")] -use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; use openshell_core::activity::{ActivitySender, try_record_activity}; use openshell_core::denial::DenialEvent; -use openshell_core::net::{ - connect_tcp_nodelay_best_effort, is_always_blocked_ip, is_internal_ip, is_link_local_ip, - set_tcp_nodelay_best_effort, -}; +use openshell_core::net::{is_always_blocked_ip, is_internal_ip, is_link_local_ip}; use openshell_core::policy::ProxyPolicy; -use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; +use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; +use openshell_isolation::contract::{ + BinaryIdentity as ContractBinaryIdentity, BoundaryDuplexStream, NetworkMediationSource, + ResolveError, +}; use openshell_ocsf::{ - ActionId, ActivityId, AiModel, ApiActivityBuilder, DispositionId, Endpoint, - HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, - Url as OcsfUrl, ocsf_emit, + ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, + NetworkActivityBuilder, Process, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; -#[cfg(target_os = "linux")] -use std::mem::size_of; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{ - AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, AsyncWriteExt, + AsyncBufReadExt, AsyncRead as TokioAsyncRead, AsyncReadExt, AsyncWrite as TokioAsyncWrite, + AsyncWriteExt, }; use tokio::net::{TcpListener, TcpStream}; + +type ProxyClient = tokio::io::BufReader; + +enum ProxyAcceptError { + Listener(std::io::Error), + Source(openshell_isolation::contract::BackendError), +} use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; @@ -64,35 +68,9 @@ const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(1); const INFERENCE_LOCAL_HOST: &str = "inference.local"; const INFERENCE_LOCAL_PORT: u16 = 443; -const FORWARD_ENCODED_SLASH_REJECTION_DETAIL: &str = - "request-target contains an encoded '/' (%2F) which is not allowed on this endpoint"; #[cfg(target_os = "linux")] const SIDECAR_SUPERVISOR_TOPOLOGY: &str = "sidecar"; -fn emit_credential_endpoint_mismatch(host: &str, port: u16, policy_name: &str) { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(host, port)) - .firewall_rule(policy_name, "credential-binding") - .message(format!( - "Credential use denied: credential is not authorized for {host}:{port}" - )) - .status_detail("credential_endpoint_mismatch") - .build(); - ocsf_emit!(event); - let finding = crate::l7::build_credential_endpoint_mismatch_finding( - policy_name, - host, - None, - "Provider credential endpoint binding mismatch; request denied", - ); - ocsf_emit!(finding); -} - /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host /// machine. Traffic to these names is eligible for the trusted-gateway SSRF /// exemption when the resolved IP matches the driver-injected value read from @@ -103,27 +81,6 @@ const HOST_GATEWAY_ALIASES: &[&str] = &[ "host.docker.internal", ]; -fn revision_scoped_dynamic_credentials( - snapshot: &ProviderCredentialSnapshot, -) -> std::collections::HashMap { - snapshot - .dynamic_credentials - .iter() - .map(|(key, credential)| { - let scoped_key = key.rsplit_once('\t').map_or_else( - || format!("rev:{}\t{key}", snapshot.revision), - |(endpoint_selector, provider_credential)| { - format!( - "{endpoint_selector}\trev:{}\t{provider_credential}", - snapshot.revision - ) - }, - ); - (scoped_key, credential.clone()) - }) - .collect() -} - /// Cloud instance metadata IPs that are NEVER exempted from SSRF blocking, /// even when they coincidentally match a host-gateway alias resolution. /// This list covers the well-known IMDS endpoints across major cloud providers. @@ -228,11 +185,12 @@ impl InferenceContext { } } +#[derive(Debug)] pub struct ProxyHandle { #[allow(dead_code)] http_addr: Option, join: JoinHandle<()>, - exited_rx: Option>, + source_failure: tokio::sync::watch::Receiver>, } impl ProxyHandle { @@ -255,6 +213,7 @@ impl ProxyHandle { activity_tx: Option, engine_ready: tokio::sync::watch::Receiver, upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, + network_mediation_source: Option>, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -270,8 +229,15 @@ impl ProxyHandle { )); } - let listener = TcpListener::bind(http_addr).await.into_diagnostic()?; - let local_addr = listener.local_addr().into_diagnostic()?; + let listener = if network_mediation_source.is_none() { + Some(TcpListener::bind(http_addr).await.into_diagnostic()?) + } else { + None + }; + let local_addr = match listener.as_ref() { + Some(listener) => listener.local_addr().into_diagnostic()?, + None => http_addr, + }; { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Listen) @@ -337,13 +303,8 @@ impl ProxyHandle { ocsf_emit!(event); } - let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); + let (source_failure_tx, source_failure) = tokio::sync::watch::channel(None); let join = tokio::spawn(async move { - // Hold the sender for the lifetime of this task — when the task - // exits (panic, abort, or loop break), the sender drops and the - // receiver fires, notifying the sandbox that the proxy is gone. - let _proxy_exit_guard = exited_tx; - // Wait for the OPA engine's symlink resolution reload to complete // before accepting connections. This prevents requests from // observing a generation transition mid-flight, which would cause @@ -371,11 +332,33 @@ impl ProxyHandle { let mut consecutive_resource_errors: u32 = 0; let mut consecutive_unknown_errors: u32 = 0; loop { - match listener.accept().await { - Ok((stream, _addr)) => { + let accepted = if let Some(source) = network_mediation_source.as_ref() { + source + .accept() + .await + .map(|connection| { + (connection.stream, Some(connection.binary_identity), None) + }) + .map_err(ProxyAcceptError::Source) + } else { + let listener = listener + .as_ref() + .expect("listener exists without a mediation source"); + listener + .accept() + .await + .map(|(stream, _)| { + let workload_addr = stream.peer_addr().ok(); + let proxy_addr = stream.local_addr().ok(); + let stream: BoundaryDuplexStream = Box::new(stream); + (stream, None, workload_addr.zip(proxy_addr)) + }) + .map_err(ProxyAcceptError::Listener) + }; + match accepted { + Ok((stream, supplied_identity, socket_addrs)) => { consecutive_resource_errors = 0; consecutive_unknown_errors = 0; - set_tcp_nodelay_best_effort(&stream); let opa = opa_engine.clone(); let cache = identity_cache.clone(); let spid = entrypoint_pid.clone(); @@ -385,21 +368,22 @@ impl ProxyHandle { let proposals = agent_proposals.clone(); let gw = trusted_host_gateway.clone(); let up_proxy = upstream_proxy.clone(); - let credentials = provider_credentials.clone(); let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); let dynamic_credentials = provider_credentials.as_ref().map(|state| { - Arc::new(std::sync::RwLock::new(revision_scoped_dynamic_credentials( - &state.snapshot(), - ))) + Arc::new(std::sync::RwLock::new( + state.snapshot().dynamic_credentials.clone(), + )) }); let dtx = denial_tx.clone(); let atx = activity_tx.clone(); tokio::spawn(async move { #[allow(clippy::large_futures)] - if let Err(err) = handle_tcp_connection( - stream, + if let Err(err) = handle_mediated_connection( + tokio::io::BufReader::new(stream), + supplied_identity, + socket_addrs, opa, cache, spid, @@ -409,7 +393,6 @@ impl ProxyHandle { proposals, gw, up_proxy, - credentials, resolver, dynamic_credentials, dtx, @@ -427,37 +410,35 @@ impl ProxyHandle { } }); } - Err(err) => { - match classify_accept_error( - &err, + Err(ProxyAcceptError::Source(err)) => { + let _ = source_failure_tx.send(Some(err.to_string())); + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message(format!("Network-mediation source failed: {err}")) + .build(); + ocsf_emit!(event); + break; + } + Err(ProxyAcceptError::Listener(error)) => { + let outcome = handle_accept_error( + &error, &mut consecutive_resource_errors, &mut consecutive_unknown_errors, - ) { - AcceptAction::Terminal => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "Proxy accept loop exiting on terminal error: {err}", - )) - .build(); - ocsf_emit!(event); - break; - } - AcceptAction::Retry { backoff, severity } => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "Proxy accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build(); - ocsf_emit!(event); - tokio::time::sleep(backoff).await; - } + ); + + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(outcome.severity) + .status(StatusId::Failure) + .message(outcome.message) + .build(); + ocsf_emit!(event); + + match outcome.backoff { + Some(backoff) => tokio::time::sleep(backoff).await, + None => break, } } } @@ -467,7 +448,7 @@ impl ProxyHandle { Ok(Self { http_addr: Some(local_addr), join, - exited_rx: Some(exited_rx), + source_failure, }) } @@ -476,8 +457,18 @@ impl ProxyHandle { self.http_addr } - pub fn take_exit_receiver(&mut self) -> Option> { - self.exited_rx.take() + /// Wait until a backend-provided mediation source fails terminally. + pub async fn wait_for_source_failure(&self) -> String { + let mut receiver = self.source_failure.clone(); + loop { + let current = receiver.borrow().clone(); + if let Some(error) = current { + return error; + } + if receiver.changed().await.is_err() { + return "network mediation stopped".to_string(); + } + } } } @@ -487,641 +478,168 @@ impl Drop for ProxyHandle { } } -/// RAII handle for transparent TCP accept loops. -#[cfg(target_os = "linux")] -pub(crate) struct TransparentTcpHandle { - joins: Vec>, +fn emit_activity(tx: &Option, denied: bool, deny_group: &'static str) { + if let Some(tx) = tx { + let _ = try_record_activity(tx, denied, deny_group); + } } -#[cfg(target_os = "linux")] -impl TransparentTcpHandle { - #[allow(clippy::too_many_arguments)] - pub(crate) fn start( - listeners: Vec, - store: Arc, - opa_engine: Arc, - identity_cache: Arc, - entrypoint_pid: Arc, - agent_proposals: openshell_core::proposals::AgentProposals, - denial_tx: Option>, - activity_tx: Option, - upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, - engine_ready: tokio::sync::watch::Receiver, - ) -> Result { - let upstream_proxy = Arc::new( - UpstreamProxyConfig::from_args(upstream_proxy_args) - .map_err(|error| miette::miette!(error))?, - ); - let mut joins = Vec::with_capacity(listeners.len()); - for listener in listeners { - let store = store.clone(); - let engine = opa_engine.clone(); - let cache = identity_cache.clone(); - let pid = entrypoint_pid.clone(); - let proposals = agent_proposals.clone(); - let denial_tx = denial_tx.clone(); - let activity_tx = activity_tx.clone(); - let upstream_proxy = upstream_proxy.clone(); - let mut engine_ready = engine_ready.clone(); - joins.push(tokio::spawn(async move { - if tokio::time::timeout( - std::time::Duration::from_secs(15), - engine_ready.wait_for(|ready| *ready), - ) - .await - .is_err() - { - warn!( - "Engine readiness signal not received within 15s; proceeding with transparent TCP accept loop" - ); - } - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - set_tcp_nodelay_best_effort(&stream); - let store = store.clone(); - let engine = engine.clone(); - let cache = cache.clone(); - let pid = pid.clone(); - let proposals = proposals.clone(); - let denial_tx = denial_tx.clone(); - let activity_tx = activity_tx.clone(); - let upstream_proxy = upstream_proxy.clone(); - tokio::spawn(async move { - if let Err(error) = handle_transparent_tcp_connection( - stream, - store, - engine, - cache, - pid, - proposals, - denial_tx, - activity_tx, - upstream_proxy, - ) - .await - { - ocsf_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .message(format!("Transparent TCP connection error: {error}")) - .build() - ); - } - }); - } - })); - } - Ok(Self { joins }) - } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AcceptErrorClass { + Transient, + Terminal, + Unknown, } -#[cfg(target_os = "linux")] -impl Drop for TransparentTcpHandle { - fn drop(&mut self) { - for join in &self.joins { - join.abort(); - } - } +#[cfg(unix)] +fn classify_accept_error(err: &std::io::Error) -> AcceptErrorClass { + match err.raw_os_error() { + Some( + libc::EMFILE + | libc::ENFILE + | libc::ENOBUFS + | libc::ENOMEM + | libc::ECONNABORTED + | libc::ECONNRESET + | libc::EINTR + | libc::ENETDOWN + | libc::EPROTO + | libc::ENOPROTOOPT + | libc::EHOSTDOWN + | libc::EHOSTUNREACH + | libc::EOPNOTSUPP + | libc::ENETUNREACH + | libc::ENOSR + | libc::ESOCKTNOSUPPORT + | libc::EPROTONOSUPPORT + | libc::ETIMEDOUT, + ) => AcceptErrorClass::Transient, + #[cfg(target_os = "linux")] + Some(libc::ENONET) => AcceptErrorClass::Transient, + Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) => AcceptErrorClass::Terminal, + _ => AcceptErrorClass::Unknown, + } +} + +#[cfg(not(unix))] +fn classify_accept_error(_err: &std::io::Error) -> AcceptErrorClass { + AcceptErrorClass::Unknown +} + +#[cfg(unix)] +fn is_resource_pressure_error(err: &std::io::Error) -> bool { + matches!( + err.raw_os_error(), + Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) + ) } -#[cfg(target_os = "linux")] -#[allow(clippy::too_many_arguments)] -async fn handle_transparent_tcp_connection( - mut client: TcpStream, - store: Arc, - opa_engine: Arc, - identity_cache: Arc, - entrypoint_pid: Arc, - agent_proposals: openshell_core::proposals::AgentProposals, - denial_tx: Option>, - activity_tx: Option, - upstream_proxy: Arc>, -) -> Result<()> { - let workload_addr = client.peer_addr().into_diagnostic()?; - let original = original_destination(&client).into_diagnostic()?; - let current_generation = opa_engine.current_generation(); - let mapping = match store.lookup( - original.ip(), - original.port(), - current_generation, - std::time::Instant::now(), - ) { - Ok(mapping) => mapping, - Err(error) => { - emit_transparent_mapping_denial(workload_addr, original, error); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - } - }; - let host = mapping.record.normalized_name.as_str().to_string(); - let port = original.port(); - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, original); - let intent = EgressIntent::transparent_tcp(host.clone(), port); - let engine = opa_engine.clone(); - let cache = identity_cache.clone(); - let pid = entrypoint_pid.clone(); - let decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &engine, &cache, &pid, intent) - }) - .await - .map_err(|error| miette::miette!("identity resolution task panicked: {error}"))?; +#[cfg(not(unix))] +fn is_resource_pressure_error(_err: &std::io::Error) -> bool { + false +} - if let NetworkAction::Deny { reason } = &decision.action { - emit_transparent_policy_denial(&decision, workload_addr, &host, port); - emit_denial( - &denial_tx, - &host, - port, - decision - .binary - .as_ref() - .map_or("-", |path| path.to_str().unwrap_or("-")), - &decision, - reason, - "transparent-tcp", - ); - emit_activity(&activity_tx, true, "transparent_tcp_policy"); - return Ok(()); - } +const ACCEPT_BACKOFF_BASE_MS: u64 = 100; +const ACCEPT_BACKOFF_MAX_MS: u64 = 5_000; +const MAX_CONSECUTIVE_UNKNOWN_ERRORS: u32 = 5; - // Authorization may race a policy reload. Re-pin the exact generation - // that produced the decision, then reacquire the DNS mapping against that - // generation before correlating endpoint identity or constructing a - // connector. This prevents combining an old DNS answer with a newer - // policy decision (or vice versa). - let Ok(generation_guard) = - relay::pin_policy_generation(&opa_engine, decision.policy_generation) - else { - emit_transparent_mapping_denial(workload_addr, original, MappingLookupError::StalePolicy); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - }; - let mapping = match store.lookup( - original.ip(), - original.port(), - decision.policy_generation, - std::time::Instant::now(), - ) { - Ok(mapping) => mapping, - Err(error) => { - emit_transparent_mapping_denial(workload_addr, original, error); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - } - }; +fn accept_backoff(consecutive_errors: u32) -> std::time::Duration { + let exponent = consecutive_errors.saturating_sub(1).min(7); + let ms = ACCEPT_BACKOFF_BASE_MS + .saturating_mul(1u64 << exponent) + .min(ACCEPT_BACKOFF_MAX_MS); + std::time::Duration::from_millis(ms) +} - let endpoint_id = decision - .endpoint - .matched_endpoints - .iter() - .map(|endpoint| PolicyEndpointId { - policy_name: endpoint.policy_name.clone(), - endpoint_index: endpoint.endpoint_index, - }) - .find(|candidate| mapping.endpoint_ids().any(|mapped| mapped == candidate)); - let Some(endpoint_id) = endpoint_id else { - let reason = "authorized endpoint did not match DNS correlation"; - emit_transparent_policy_denial(&decision, workload_addr, &host, port); - emit_denial( - &denial_tx, - &host, - port, - decision - .binary - .as_ref() - .map_or("-", |path| path.to_str().unwrap_or("-")), - &decision, - reason, - "transparent-tcp", - ); - emit_activity(&activity_tx, true, "transparent_tcp_policy"); - return Ok(()); - }; +struct AcceptErrorOutcome { + severity: SeverityId, + message: String, + backoff: Option, +} - let connector = mapping.connector_for(&endpoint_id).await.map_err(|error| { - miette::miette!("transparent TCP pinned destination is invalid: {error}") - })?; - let mut ctx = relay::http_context( - &decision, - None, - None, - activity_tx.clone(), - None, - agent_proposals, - // The transparent TCP path carries no PolicyLocalContext, so no - // workspace is available here; matches the CONNECT path default when - // policy-local context is absent. - String::new(), - ); - let middleware_gate = middleware_uninspectable_gate(&opa_engine, &ctx)?; - if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::Deny { - crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", true); - return Ok(()); - } - if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding { - crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", false); - } - let approved_real_ip_candidates = connector.addrs().to_vec(); - generation_guard.ensure_current()?; - let mut upstream = - dial_transparent_upstream(&upstream_proxy, &host, port, &approved_real_ip_candidates) - .await - .into_diagnostic()?; - let upstream_socket_peer = upstream.peer_addr().into_diagnostic()?; - let (connected_real_destination, dial_mode) = match upstream.connect_target() { - Some(upstream_proxy::ConnectTarget::Ip(ip)) => ( - Some(SocketAddr::new(ip, port)), - "upstream_proxy_validated_ip", - ), - Some(upstream_proxy::ConnectTarget::Hostname) => { - // Transparent TCP authorization is correlated to the resolver's - // validated address set. A hostname-mode CONNECT would make the - // corporate proxy resolve again and break that binding. Treat a - // future invariant regression as an audited denial, not a panic. - emit_transparent_policy_denial(&decision, workload_addr, &host, port); - emit_denial( - &denial_tx, - &host, - port, - decision - .binary - .as_ref() - .map_or("-", |path| path.to_str().unwrap_or("-")), - &decision, - "upstream proxy did not preserve the validated IP target", - "transparent-tcp", - ); - emit_activity(&activity_tx, true, "transparent_tcp_destination"); - return Ok(()); +fn handle_accept_error( + err: &std::io::Error, + consecutive_resource_errors: &mut u32, + consecutive_unknown_errors: &mut u32, +) -> AcceptErrorOutcome { + let class = classify_accept_error(err); + + match class { + AcceptErrorClass::Terminal => AcceptErrorOutcome { + severity: SeverityId::High, + message: format!("Proxy accept error (terminal, exiting): {err}"), + backoff: None, + }, + AcceptErrorClass::Unknown => { + *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); + if *consecutive_unknown_errors > MAX_CONSECUTIVE_UNKNOWN_ERRORS { + AcceptErrorOutcome { + severity: SeverityId::High, + message: format!( + "Proxy accept error (exceeded {MAX_CONSECUTIVE_UNKNOWN_ERRORS} retries, exiting): {err}" + ), + backoff: None, + } + } else { + let backoff = accept_backoff(*consecutive_unknown_errors); + AcceptErrorOutcome { + severity: SeverityId::Medium, + message: format!( + "Proxy accept error (retry {}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS} in {}ms): {err}", + *consecutive_unknown_errors, + backoff.as_millis(), + ), + backoff: Some(backoff), + } + } } - None => (Some(upstream_socket_peer), "direct"), - }; - generation_guard.ensure_current()?; - ctx.request_default_port = None; - let policy_name = match &decision.action { - NetworkAction::Allow { matched_policy } => matched_policy.as_deref().unwrap_or("-"), - NetworkAction::Deny { .. } => "-", - }; - let binary = decision - .binary - .as_ref() - .map_or_else(|| "-".to_string(), |path| path.display().to_string()); - let pid = decision - .binary_pid - .map_or_else(|| "-".to_string(), |pid| pid.to_string()); - ocsf_emit!(build_transparent_tcp_allow_ocsf_event( - TransparentTcpAllowAudit { - workload: workload_addr, - synthetic_destination: original, - normalized_domain: &host, - approved_real_ip_candidates: &approved_real_ip_candidates, - connected_real_destination, - upstream_socket_peer, - dial_mode, - mapping_id: mapping.record.mapping_id, - mapping_generation: mapping.record.mapping_generation, - mapping_policy_generation: mapping.record.policy_generation, - authorization_policy_generation: decision.policy_generation, - binary: &binary, - pid: &pid, - policy_name, + AcceptErrorClass::Transient => { + *consecutive_unknown_errors = 0; + if is_resource_pressure_error(err) { + *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); + let backoff = accept_backoff(*consecutive_resource_errors); + AcceptErrorOutcome { + severity: SeverityId::Medium, + message: format!( + "Proxy accept error (retrying in {}ms): {err}", + backoff.as_millis(), + ), + backoff: Some(backoff), + } + } else { + AcceptErrorOutcome { + severity: SeverityId::Low, + message: format!("Proxy accept error (retrying in 100ms): {err}"), + backoff: Some(std::time::Duration::from_millis(100)), + } + } } - )); - emit_activity(&activity_tx, false, "transparent_tcp"); - relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await + } } -#[cfg(any(target_os = "linux", test))] -struct TransparentTcpAllowAudit<'a> { - workload: SocketAddr, - synthetic_destination: SocketAddr, - normalized_domain: &'a str, - approved_real_ip_candidates: &'a [SocketAddr], - connected_real_destination: Option, - upstream_socket_peer: SocketAddr, - dial_mode: &'a str, - mapping_id: uuid::Uuid, - mapping_generation: u64, - mapping_policy_generation: u64, - authorization_policy_generation: u64, - binary: &'a str, - pid: &'a str, - policy_name: &'a str, +fn l7_inspection_active(l7_route: Option<&L7RouteSnapshot>) -> bool { + l7_route.is_some_and(|route| !route.configs.is_empty()) } -#[cfg(any(target_os = "linux", test))] -fn build_transparent_tcp_allow_ocsf_event( - audit: TransparentTcpAllowAudit<'_>, -) -> openshell_ocsf::OcsfEvent { - let logical_destination = format!( - "{}:{}", - audit.normalized_domain, - audit.synthetic_destination.port() - ); - let mapping_id = audit.mapping_id.to_string(); - let actual_target = audit - .connected_real_destination - .map_or_else(|| "proxy-resolved".to_string(), |target| target.to_string()); - let message = format!( - "Transparent TCP mapping_id={mapping_id} synthetic={} real={actual_target}", - audit.synthetic_destination, - ); - let approved_real_ip_candidates = audit - .approved_real_ip_candidates - .iter() - .map(ToString::to_string) - .collect::>(); - let mut builder = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .dst_endpoint(Endpoint::from_domain( - audit.normalized_domain, - audit.synthetic_destination.port(), - )) - .src_endpoint_addr(audit.workload.ip(), audit.workload.port()) - .actor_process(Process::from_bypass(audit.binary, audit.pid, "")) - .firewall_rule(audit.policy_name, "opa") - .unmapped("matched_policy", audit.policy_name) - .unmapped("normalized_domain", audit.normalized_domain) - .unmapped("logical_destination", logical_destination) - .unmapped( - "synthetic_destination", - audit.synthetic_destination.to_string(), - ) - .unmapped( - "approved_real_ip_candidates", - serde_json::json!(approved_real_ip_candidates), - ) - .unmapped( - "upstream_socket_peer", - audit.upstream_socket_peer.to_string(), - ) - .unmapped("dial_mode", audit.dial_mode) - .unmapped("mapping_id", mapping_id) - .unmapped("mapping_generation", audit.mapping_generation) - .unmapped("policy_generation", audit.mapping_policy_generation) - .unmapped("mapping_policy_generation", audit.mapping_policy_generation) - .unmapped( - "authorization_policy_generation", - audit.authorization_policy_generation, - ) - .message(message) - .status_detail("transparent_tcp_allowed"); - if let Some(destination) = audit.connected_real_destination { - builder = builder.unmapped("connected_real_destination", destination.to_string()); - } - builder.build() +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TunnelProtocol { + Tls, + Http1, + H2cPriorKnowledge, + Unsupported, } -#[cfg(target_os = "linux")] -fn original_destination(stream: &TcpStream) -> std::io::Result { - use std::os::fd::AsRawFd; - let fd = stream.as_raw_fd(); - if stream.local_addr()?.is_ipv4() { - #[allow(unsafe_code)] - unsafe { - let mut address: libc::sockaddr_in = std::mem::zeroed(); - let mut length = libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in size fits socklen_t"); - if libc::getsockopt( - fd, - libc::SOL_IP, - 80, // SO_ORIGINAL_DST - std::ptr::addr_of_mut!(address).cast(), - std::ptr::addr_of_mut!(length), - ) != 0 - { - return Err(std::io::Error::last_os_error()); - } - return Ok(SocketAddr::new( - IpAddr::V4(std::net::Ipv4Addr::from( - address.sin_addr.s_addr.to_ne_bytes(), - )), - u16::from_be(address.sin_port), - )); - } +fn classify_tunnel_protocol(peek: &[u8]) -> TunnelProtocol { + if crate::l7::tls::looks_like_tls(peek) { + return TunnelProtocol::Tls; } - #[allow(unsafe_code)] - unsafe { - let mut address: libc::sockaddr_in6 = std::mem::zeroed(); - let mut length = libc::socklen_t::try_from(size_of::()) - .expect("sockaddr_in6 size fits socklen_t"); - if libc::getsockopt( - fd, - libc::SOL_IPV6, - 80, // IP6T_SO_ORIGINAL_DST - std::ptr::addr_of_mut!(address).cast(), - std::ptr::addr_of_mut!(length), - ) != 0 - { - return Err(std::io::Error::last_os_error()); - } - Ok(SocketAddr::new( - IpAddr::V6(std::net::Ipv6Addr::from(address.sin6_addr.s6_addr)), - u16::from_be(address.sin6_port), - )) + if crate::l7::rest::looks_like_http(peek) { + return TunnelProtocol::Http1; + } + if crate::l7::rest::looks_like_http2_prior_knowledge(peek) { + return TunnelProtocol::H2cPriorKnowledge; } -} - -#[cfg(target_os = "linux")] -fn emit_transparent_mapping_denial( - workload: SocketAddr, - original: SocketAddr, - error: MappingLookupError, -) { - let detail = match error { - MappingLookupError::Missing => "transparent_tcp_mapping_missing", - MappingLookupError::Expired => "transparent_tcp_mapping_expired", - MappingLookupError::StalePolicy => "transparent_tcp_mapping_stale_policy", - MappingLookupError::PortMismatch => "transparent_tcp_port_mismatch", - MappingLookupError::EndpointMismatch - | MappingLookupError::InvalidMapping - | MappingLookupError::LockPoisoned => "transparent_tcp_destination_denied", - }; - ocsf_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_ip(original.ip(), original.port())) - .src_endpoint_addr(workload.ip(), workload.port()) - .message(format!("Transparent TCP denied: {error}")) - .status_detail(detail) - .build() - ); -} - -#[cfg(target_os = "linux")] -fn emit_transparent_policy_denial( - decision: &EgressDecision, - workload: SocketAddr, - host: &str, - port: u16, -) { - let status_detail = if matches!(decision.action, NetworkAction::Deny { .. }) { - "transparent_tcp_identity_denied" - } else { - "transparent_tcp_destination_denied" - }; - let binary = decision - .binary - .as_ref() - .map_or_else(|| "-".to_string(), |path| path.display().to_string()); - let pid = decision - .binary_pid - .map_or_else(|| "-".to_string(), |pid| pid.to_string()); - ocsf_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(host, port)) - .src_endpoint_addr(workload.ip(), workload.port()) - .actor_process(Process::from_bypass(&binary, &pid, "-")) - .firewall_rule("-", "opa") - .message(format!("Transparent TCP denied {host}:{port}")) - .status_detail(status_detail) - .build() - ); -} - -const MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS: u32 = 10; - -#[derive(Debug, PartialEq)] -enum AcceptAction { - Terminal, - Retry { - backoff: std::time::Duration, - severity: SeverityId, - }, -} - -fn classify_accept_error( - err: &std::io::Error, - consecutive_resource_errors: &mut u32, - consecutive_unknown_errors: &mut u32, -) -> AcceptAction { - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) - ) { - return AcceptAction::Terminal; - } - - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some( - libc::EMFILE - | libc::ENFILE - | libc::ENOBUFS - | libc::ENOMEM - | libc::ECONNABORTED - | libc::ECONNRESET - | libc::EINTR - | libc::ENETDOWN - | libc::EPROTO - | libc::ENOPROTOOPT - | libc::EHOSTDOWN - | libc::EHOSTUNREACH - | libc::EOPNOTSUPP - | libc::ENETUNREACH - | libc::ENOSR - | libc::ESOCKTNOSUPPORT - | libc::EPROTONOSUPPORT - | libc::ETIMEDOUT - ) - ) { - *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), - Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) - ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { - *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) - .min(5_000); - return AcceptAction::Retry { - backoff: std::time::Duration::from_millis(backoff_ms), - severity: SeverityId::Medium, - }; - } - - *consecutive_resource_errors = 0; - return AcceptAction::Retry { - backoff: std::time::Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - #[cfg(unix)] - #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; - *consecutive_resource_errors = 0; - return AcceptAction::Retry { - backoff: std::time::Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); - if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS { - return AcceptAction::Terminal; - } - AcceptAction::Retry { - backoff: std::time::Duration::from_millis(100), - severity: SeverityId::Low, - } -} - -fn emit_activity(tx: &Option, denied: bool, deny_group: &'static str) { - if let Some(tx) = tx { - let _ = try_record_activity(tx, denied, deny_group); - } -} - -fn l7_inspection_active(l7_route: Option<&L7RouteSnapshot>) -> bool { - l7_route.is_some_and(|route| !route.configs.is_empty()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TunnelProtocol { - Tls, - Http1, - H2cPriorKnowledge, - Unsupported, -} - -fn classify_tunnel_protocol(peek: &[u8]) -> TunnelProtocol { - if crate::l7::tls::looks_like_tls(peek) { - return TunnelProtocol::Tls; - } - if crate::l7::rest::looks_like_http(peek) { - return TunnelProtocol::Http1; - } - if crate::l7::rest::looks_like_http2_prior_knowledge(peek) { - return TunnelProtocol::H2cPriorKnowledge; - } - TunnelProtocol::Unsupported + TunnelProtocol::Unsupported } fn could_be_tls_prefix(peek: &[u8]) -> bool { @@ -1192,21 +710,24 @@ fn middleware_uninspectable_gate( Ok(crate::l7::middleware::uninspectable_traffic_gate(&chain)) } -async fn peek_tunnel_protocol(client: &TcpStream) -> Result> { - let mut peek_buf = [0u8; TUNNEL_PROTOCOL_PEEK_BYTES]; +async fn peek_tunnel_protocol(client: &mut C) -> Result> +where + C: tokio::io::AsyncBufRead + Unpin, +{ let deadline = tokio::time::Instant::now() + TUNNEL_PROTOCOL_PEEK_TIMEOUT; loop { - let n = client.peek(&mut peek_buf).await.into_diagnostic()?; - if n == 0 { + let available = client.fill_buf().await.into_diagnostic()?; + if available.is_empty() { return Ok(None); } - let peek = &peek_buf[..n]; + let n = available.len().min(TUNNEL_PROTOCOL_PEEK_BYTES); + let peek = &available[..n]; let protocol = classify_tunnel_protocol(peek); if protocol != TunnelProtocol::Unsupported || !could_be_supported_tunnel_protocol_prefix(peek) - || n == peek_buf.len() + || n == TUNNEL_PROTOCOL_PEEK_BYTES || tokio::time::Instant::now() >= deadline { return Ok(Some(protocol)); @@ -1424,50 +945,6 @@ fn build_forward_allow_ocsf_event( .build() } -fn build_forward_parse_error_ocsf_event(path: &str) -> openshell_ocsf::OcsfEvent { - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .message(format!("FORWARD parse error for {path}")) - .build() -} - -#[allow(clippy::too_many_arguments)] -fn build_forward_l7_parse_rejection_ocsf_event( - peer_addr: SocketAddr, - method: &str, - host: &str, - port: u16, - path: &str, - binary: &str, - pid: &str, - ancestors: &str, - cmdline: &str, - policy: &str, - detail: &str, -) -> openshell_ocsf::OcsfEvent { - HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", host, path, port), - )) - .dst_endpoint(Endpoint::from_domain(host, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) - .firewall_rule(policy, "l7") - .message(format!( - "FORWARD_L7 denied non-canonical request-target for {method} {host}:{port}{path}" - )) - .status_detail(detail) - .build() -} - #[allow(clippy::too_many_arguments)] fn build_forward_policy_deny_ocsf_event( peer_addr: SocketAddr, @@ -1585,7 +1062,7 @@ fn build_forward_destination_deny_ocsf_event( #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -1631,7 +1108,7 @@ async fn deny_connect_destination( #[allow(clippy::too_many_arguments)] async fn deny_forward_destination( - client: &mut TcpStream, + client: &mut (impl TokioAsyncWrite + Unpin), denial: &DestinationDenial, peer_addr: SocketAddr, method: &str, @@ -1681,9 +1158,58 @@ async fn deny_forward_destination( // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. +#[cfg(test)] #[allow(clippy::too_many_arguments)] async fn handle_tcp_connection( - mut client: TcpStream, + client: TcpStream, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + tls_state: Option>, + inference_ctx: Option>, + policy_local_ctx: Option>, + agent_proposals: openshell_core::proposals::AgentProposals, + trusted_host_gateway: Arc>, + upstream_proxy: Arc>, + secret_resolver: Option>, + dynamic_credentials: Option< + Arc< + std::sync::RwLock< + std::collections::HashMap, + >, + >, + >, + denial_tx: Option>, + activity_tx: Option, +) -> Result<()> { + let socket_addrs = client.peer_addr().ok().zip(client.local_addr().ok()); + let stream: BoundaryDuplexStream = Box::new(client); + Box::pin(handle_mediated_connection( + tokio::io::BufReader::new(stream), + None, + socket_addrs, + opa_engine, + identity_cache, + entrypoint_pid, + tls_state, + inference_ctx, + policy_local_ctx, + agent_proposals, + trusted_host_gateway, + upstream_proxy, + secret_resolver, + dynamic_credentials, + denial_tx, + activity_tx, + )) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn handle_mediated_connection( + mut client: ProxyClient, + supplied_identity: Option>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, @@ -1693,7 +1219,6 @@ async fn handle_tcp_connection( agent_proposals: openshell_core::proposals::AgentProposals, trusted_host_gateway: Arc>, upstream_proxy: Arc>, - provider_credentials: Option, secret_resolver: Option>, dynamic_credentials: Option< Arc< @@ -1757,13 +1282,14 @@ async fn handle_tcp_connection( &buf[..], used, &mut client, + supplied_identity.as_ref(), + socket_addrs, opa_engine, identity_cache, entrypoint_pid, policy_local_ctx, agent_proposals, trusted_host_gateway, - provider_credentials, secret_resolver, dynamic_credentials, denial_tx.as_ref(), @@ -1772,9 +1298,8 @@ async fn handle_tcp_connection( .await; } - let (raw_host, port) = parse_target(target)?; - let host = normalize_host(&raw_host); - let (host_lc, raw_host_lc) = (host.to_ascii_lowercase(), raw_host.to_ascii_lowercase()); + let (host, port) = parse_target(target)?; + let host_lc = host.to_ascii_lowercase(); if host_lc == INFERENCE_LOCAL_HOST && port == INFERENCE_LOCAL_PORT { respond(&mut client, b"HTTP/1.1 200 Connection Established\r\n\r\n").await?; @@ -1803,22 +1328,27 @@ async fn handle_tcp_connection( return Ok(()); } - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - // Evaluate OPA policy with process-identity binding. - // Wrapped in spawn_blocking because identity resolution does heavy sync I/O: - // /proc scanning + SHA256 hashing of binaries (e.g. node at 124MB). - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::connect(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity.as_ref() { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -1929,13 +1459,12 @@ async fn handle_tcp_connection( // allowed_ips validation below — so an internal-address CONNECT still gets // the SSRF 403 and telemetry in degraded state — but before the upstream // connect and before `200 Connection Established`. - hydrate_tls_mode(&mut decision); + hydrate_tls_mode(&opa_engine, &mut decision); let effective_tls_skip = decision.endpoint.tls_mode == crate::l7::TlsMode::Skip; - let credential_guard = query_endpoint_credential_guard(&opa_engine, &decision, &host_lc, port)?; let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - match hydrate_destination_plan(&mut decision, *trusted_host_gateway) { + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { Ok(()) => {} Err(denial) => { deny_connect_destination( @@ -1965,7 +1494,7 @@ async fn handle_tcp_connection( // Defense-in-depth: resolve DNS and reject connections to internal IPs. let dns_connect_start = std::time::Instant::now(); let connector = match validate_destination(DestinationRequest { - host: &raw_host, + host: &host, port, sandbox_entrypoint_pid, plan: destination_plan, @@ -2035,54 +1564,10 @@ async fn handle_tcp_connection( return Ok(()); } - if credential_guard.blocks_connect() { - const DETAIL: &str = - "credentialed endpoint requires L7 inspection; raw tunnel is not explicitly allowed"; - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "credentials") - .message(format!( - "CONNECT refused for {host_lc}:{port}: uninspected credential traffic" - )) - .status_detail(DETAIL) - .build(); - ocsf_emit!(event); - crate::l7::emit_uninspected_credential_finding( - &host_lc, - policy_str, - if effective_tls_skip { "tls-skip" } else { "l4" }, - ); - emit_activity_simple(activity_tx.as_ref(), true, "uninspected_credentials"); - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - DETAIL, - "connect-uninspected-credentials", - ); - respond( - &mut client, - &build_json_error_response(403, "Forbidden", "uninspected_credentials", DETAIL), - ) - .await?; - return Ok(()); - } - // CONNECT must use one policy generation from authorization through route - // materialization and relay startup. - hydrate_l7_route(&mut decision); + // hydration and relay startup. A later L7 lookup must never make a stale + // L4 allow appear current. + hydrate_l7_route(&opa_engine, &mut decision); let l7_route = decision.endpoint.l7_route.as_ref(); if let Err(error) = relay::validate_route_generation(l7_route, connect_generation_guard.captured_generation()) @@ -2093,7 +1578,7 @@ async fn handle_tcp_connection( } let upstream_result = tokio::select! { - result = dial_upstream(&upstream_proxy, &host_lc, &raw_host_lc, port, connector.addrs()) => Some(result), + result = dial_upstream(&upstream_proxy, &host_lc, port, connector.addrs()) => Some(result), () = connect_generation_guard.wait_until_stale() => None, }; let Some(upstream_result) = upstream_result else { @@ -2151,9 +1636,9 @@ async fn handle_tcp_connection( .as_ref() .map(|ctx| ctx.workspace()) .unwrap_or_default(); - let mut ctx = relay::http_context( + let ctx = relay::http_context( &decision, - provider_credentials, + None, secret_resolver.clone(), activity_tx.clone(), dynamic_credentials.clone(), @@ -2205,14 +1690,13 @@ async fn handle_tcp_connection( // Auto-detect the tunnel payload. L7-configured endpoints must only // enter relays that can enforce their configured protocol; unsupported // bytes fail closed below instead of falling through to raw relay. - let Some(tunnel_protocol) = peek_tunnel_protocol(&client).await? else { + let Some(tunnel_protocol) = peek_tunnel_protocol(&mut client).await? else { return Ok(()); }; if tunnel_protocol == TunnelProtocol::Tls { // TLS detected — terminate unconditionally. if let Some(ref tls) = tls_state { - ctx.request_default_port = Some(443); let tls_result = async { let mut tls_client = crate::l7::tls::tls_terminate_client(client, tls, &host_lc).await?; @@ -2292,7 +1776,6 @@ async fn handle_tcp_connection( } } else if tunnel_protocol == TunnelProtocol::Http1 { // Plaintext HTTP detected. - ctx.request_default_port = Some(80); let is_l7_relay = l7_route.is_some_and(|route| !route.configs.is_empty()); let Some(relay_context) = relay::prepare_http_relay(l7_route, &opa_engine, &decision, &ctx) else { @@ -2693,13 +2176,13 @@ fn authorize_egress_intent( cmdline_paths: cmdline_paths.clone(), }; - let result = match engine.authorize_egress(&input) { - Ok(authorization) => EgressDecision { + let result = match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { intent: intent.clone(), - action: authorization.action.clone(), - policy_generation: authorization.generation, + action, + policy_generation: generation, identity: ProcessIdentityEvidence::Available, - endpoint: EndpointDecision::from_authorization(&authorization), + endpoint: EndpointDecision::default(), binary: Some(bin_path), binary_pid: Some(binary_pid), ancestors, @@ -2748,15 +2231,15 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres cmdline_paths: vec![], }; - match engine.authorize_egress(&input) { - Ok(authorization) => EgressDecision { + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { intent, - action: authorization.action.clone(), - policy_generation: authorization.generation, + action, + policy_generation: generation, identity: ProcessIdentityEvidence::Unavailable( IdentityUnavailableReason::EndpointOnlyMode, ), - endpoint: EndpointDecision::from_authorization(&authorization), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], @@ -2780,6 +2263,77 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres } } +/// Evaluate an egress intent using identity already bound to the accepted +/// connection by an isolation backend. This is the RFC 0012 path; legacy +/// listeners continue to resolve through procfs in `authorize_egress_intent`. +fn authorize_supplied_identity( + engine: &OpaEngine, + intent: EgressIntent, + identity: &Result, +) -> EgressDecision { + let deny = |reason: String, + binary: Option, + ancestors: Vec, + cmdline_paths: Vec| EgressDecision { + intent: intent.clone(), + action: NetworkAction::Deny { reason }, + policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), + endpoint: EndpointDecision::default(), + binary, + binary_pid: None, + ancestors, + cmdline_paths, + }; + + let identity = match identity { + Ok(identity) => identity, + Err(error) => { + return deny( + format!("backend identity resolution failed: {error}"), + None, + vec![], + vec![], + ); + } + }; + let Some(digest) = identity.binary_digest else { + return deny( + "backend identity did not include the required binary digest".to_string(), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ); + }; + let input = crate::opa::NetworkInput { + host: intent.destination.host.clone(), + port: intent.destination.port, + binary_path: identity.binary_path.clone(), + binary_sha256: digest.to_string(), + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }; + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => EgressDecision { + intent, + action, + policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(identity.binary_path.clone()), + binary_pid: None, + ancestors: identity.ancestors.clone(), + cmdline_paths: identity.cmdline_paths.clone(), + }, + Err(error) => deny( + format!("policy evaluation error: {error}"), + Some(identity.binary_path.clone()), + identity.ancestors.clone(), + identity.cmdline_paths.clone(), + ), + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn authorize_egress_intent( @@ -2825,7 +2379,7 @@ const INITIAL_INFERENCE_BUF: usize = 65536; /// Returns [`InferenceOutcome::Routed`] if at least one request was successfully /// routed, or [`InferenceOutcome::Denied`] with a reason for all denial cases. async fn handle_inference_interception( - client: TcpStream, + client: ProxyClient, host: &str, port: u16, tls_state: Option<&Arc>, @@ -2954,90 +2508,6 @@ async fn process_inference_keepalive Option { - #[derive(serde::Deserialize)] - struct Req { - model: Option, - } - serde_json::from_slice::(body) - .ok() - .and_then(|r| r.model) - .filter(|m| !m.is_empty()) -} - -/// Extract token usage from an inference response body. -fn extract_usage_from_response(body: &[u8]) -> (Option, Option) { - #[derive(serde::Deserialize)] - struct Resp { - usage: Option, - } - #[derive(serde::Deserialize)] - struct Usage { - prompt_tokens: Option, - completion_tokens: Option, - } - match serde_json::from_slice::(body) { - Ok(r) => { - let u = r.usage.unwrap_or(Usage { - prompt_tokens: None, - completion_tokens: None, - }); - (u.prompt_tokens, u.completion_tokens) - } - Err(_) => (None, None), - } -} - -/// Emit an OCSF API Activity [6003] event with the `ai_operation` profile after an inference call. -#[allow(clippy::too_many_arguments)] -fn emit_ai_inference( - method: &str, - path: &str, - route_model: Option<&str>, - route_provider: Option<&str>, - status: StatusId, - input_tokens: Option, - output_tokens: Option, - latency: std::time::Duration, -) { - let model_name = route_model.unwrap_or("unknown"); - let provider_name = route_provider.unwrap_or("unknown"); - let latency_ms = u64::try_from(latency.as_millis()).unwrap_or(u64::MAX); - - let operation = format!("{method} {path}"); - - let mut builder = ApiActivityBuilder::new(openshell_ocsf::ctx::ctx(), &operation) - .severity(SeverityId::Informational) - .status(status) - .ai_model(AiModel::new(model_name, provider_name)) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("https", INFERENCE_LOCAL_HOST, path, 443), - )) - .dst_endpoint(Endpoint::from_domain( - INFERENCE_LOCAL_HOST, - INFERENCE_LOCAL_PORT, - )) - .unmapped("latency_ms", latency_ms); - - if let Some(t) = input_tokens { - builder = builder.unmapped("input_tokens", t); - } - if let Some(t) = output_tokens { - builder = builder.unmapped("output_tokens", t); - } - - let msg = format!( - "Model call: {model_name} via {provider_name} ({}in, {}out)", - input_tokens.map_or_else(|| "?".to_string(), |t| t.to_string()), - output_tokens.map_or_else(|| "?".to_string(), |t| t.to_string()), - ); - builder = builder.message(msg); - - ocsf_emit!(builder.build()); -} - /// Route a parsed inference request locally via the sandbox router, or deny it. /// /// Returns `Ok(true)` if the request was routed to an inference backend, @@ -3093,16 +2563,6 @@ async fn route_inference_request( // the body on a size-cap or idle-timeout truncation, corrupting a // payload the client parses as one JSON object. Framing is declared per // protocol on the matched pattern. - let _req_model = extract_model_from_request(&request.body); - let normalized_protocol = pattern.protocol.to_ascii_lowercase(); - let selected_route = routes - .iter() - .find(|r| r.protocols.iter().any(|p| p == &normalized_protocol)) - .or_else(|| routes.first()); - let route_model = selected_route.map(|r| r.model.clone()); - let route_endpoint = selected_route.map(|r| r.endpoint.clone()); - let infer_start = std::time::Instant::now(); - if pattern.is_buffered() { match ctx .router @@ -3117,40 +2577,11 @@ async fn route_inference_request( .await { Ok(resp) => { - let (input_tokens, output_tokens) = extract_usage_from_response(&resp.body); - let resp_status = if (200..300).contains(&resp.status) { - StatusId::Success - } else { - StatusId::Failure - }; - emit_ai_inference( - &request.method, - &normalized_path, - resp.route_model.as_deref(), - resp.route_endpoint.as_deref(), - resp_status, - input_tokens, - output_tokens, - infer_start.elapsed(), - ); - let resp_headers = sanitize_inference_response_headers(resp.headers); let response = format_http_response(resp.status, &resp_headers, &resp.body); write_all(tls_client, &response).await?; } - Err(e) => { - emit_ai_inference( - &request.method, - &normalized_path, - route_model.as_deref(), - route_endpoint.as_deref(), - StatusId::Failure, - None, - None, - infer_start.elapsed(), - ); - write_inference_router_error(tls_client, &e).await?; - } + Err(e) => write_inference_router_error(tls_client, &e).await?, } return Ok(true); } @@ -3173,9 +2604,6 @@ async fn route_inference_request( format_sse_error, }; - let stream_route_model = resp.route_model.take(); - let stream_route_endpoint = resp.route_endpoint.take(); - let resp_headers = sanitize_inference_response_headers( std::mem::take(&mut resp.headers).into_iter().collect(), ); @@ -3192,9 +2620,7 @@ async fn route_inference_request( // coalesce the framing header + data + trailer into a single // write_all call, reducing the number of TLS records per chunk // from 3 to 1 while preserving incremental delivery. - let resp_status_ok = (200..300).contains(&resp.status); let mut total_bytes: usize = 0; - let mut stream_failed = false; loop { match tokio::time::timeout(CHUNK_IDLE_TIMEOUT, resp.next_chunk()).await { Ok(Ok(Some(chunk))) => { @@ -3209,7 +2635,6 @@ async fn route_inference_request( "response truncated: exceeded maximum streaming body size", ); let _ = write_all(tls_client, &format_chunk(&err)).await; - stream_failed = true; break; } let encoded = format_chunk(&chunk); @@ -3230,7 +2655,6 @@ async fn route_inference_request( ocsf_emit!(event); let err = format_sse_error("response truncated: upstream read error"); let _ = write_all(tls_client, &format_chunk(&err)).await; - stream_failed = true; break; } Err(_) => { @@ -3248,7 +2672,6 @@ async fn route_inference_request( let err = format_sse_error("response truncated: chunk idle timeout exceeded"); let _ = write_all(tls_client, &format_chunk(&err)).await; - stream_failed = true; break; } } @@ -3256,37 +2679,8 @@ async fn route_inference_request( // Terminate the chunked stream. write_all(tls_client, format_chunk_terminator()).await?; - - // Emit API Activity for the completed streaming call. - let stream_status = if stream_failed || !resp_status_ok { - StatusId::Failure - } else { - StatusId::Success - }; - emit_ai_inference( - &request.method, - &normalized_path, - stream_route_model.as_deref(), - stream_route_endpoint.as_deref(), - stream_status, - None, - None, - infer_start.elapsed(), - ); - } - Err(e) => { - emit_ai_inference( - &request.method, - &normalized_path, - route_model.as_deref(), - route_endpoint.as_deref(), - StatusId::Failure, - None, - None, - infer_start.elapsed(), - ); - write_inference_router_error(tls_client, &e).await?; } + Err(e) => write_inference_router_error(tls_client, &e).await?, } Ok(true) } else { @@ -3420,7 +2814,7 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette } async fn reject_stale_connect_policy( - client: &mut TcpStream, + client: &mut ProxyClient, host: &str, port: u16, activity_tx: Option<&ActivitySender>, @@ -3450,25 +2844,27 @@ async fn reject_stale_connect_policy( /// /// Returns `Some(L7EndpointConfig)` if the matched endpoint has L7 config (protocol field), /// `None` for L4-only endpoints. -fn hydrate_l7_route(decision: &mut EgressDecision) { +fn hydrate_l7_route(engine: &OpaEngine, decision: &mut EgressDecision) { let host = decision.intent.destination.host.clone(); let port = decision.intent.destination.port; - decision.endpoint.l7_route = query_l7_route_snapshot(decision, &host, port); + decision.endpoint.l7_route = query_l7_route_snapshot(engine, decision, &host, port); } -fn hydrate_tls_mode(decision: &mut EgressDecision) { +fn hydrate_tls_mode(engine: &OpaEngine, decision: &mut EgressDecision) { let host = decision.intent.destination.host.clone(); let port = decision.intent.destination.port; - decision.endpoint.tls_mode = query_tls_mode(decision, &host, port); + decision.endpoint.tls_mode = query_tls_mode(engine, decision, &host, port); } fn hydrate_destination_plan( + engine: &OpaEngine, decision: &mut EgressDecision, trusted_host_gateway: Option, ) -> std::result::Result<(), DestinationDenial> { let host = decision.intent.destination.host.clone(); - let raw_allowed_ips = query_allowed_ips(decision); - let exact_declared_host = decision.endpoint.exact_declared_host; + let port = decision.intent.destination.port; + let raw_allowed_ips = query_allowed_ips(engine, decision, &host, port); + let exact_declared_host = query_exact_declared_endpoint_host(engine, decision, &host, port); let plan = build_validation_plan( &host, &host.to_ascii_lowercase(), @@ -3481,6 +2877,7 @@ fn hydrate_destination_plan( } fn query_l7_route_snapshot( + engine: &OpaEngine, decision: &EgressDecision, host: &str, port: u16, @@ -3494,27 +2891,46 @@ fn query_l7_route_snapshot( return None; } - let configs: Vec<_> = decision - .endpoint - .policy_configs - .iter() - .filter_map(crate::l7::parse_l7_config) - .map(|config| L7ConfigSnapshot { config }) - .collect(); - if configs.is_empty() { - return None; - } - debug!( - host, + let input = crate::opa::NetworkInput { + host: host.to_string(), port, - generation = decision.policy_generation, - config_count = configs.len(), - "Egress L7 route materialized from authorization snapshot" - ); - Some(L7RouteSnapshot { - configs, - l7_policy_generation: decision.policy_generation, - }) + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), + }; + + match engine.query_endpoint_configs_with_generation(&input) { + Ok((vals, generation)) => { + let configs: Vec<_> = vals + .into_iter() + .filter_map(|val| crate::l7::parse_l7_config(&val)) + .map(|config| L7ConfigSnapshot { config }) + .collect(); + debug!( + host, + port, + generation, + config_count = configs.len(), + "Forward proxy L7 route lookup complete" + ); + Some(L7RouteSnapshot { + configs, + l7_policy_generation: generation, + }) + } + Err(e) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!("Failed to query L7 endpoint config: {e}")) + .build(); + ocsf_emit!(event); + None + } + } } fn select_l7_config_for_path<'a>( @@ -3530,34 +2946,18 @@ fn select_l7_config_for_path<'a>( /// Query the TLS mode for an endpoint, independent of L7 config. /// /// This extracts `tls: skip` from the endpoint even when no `protocol` is set. -fn query_tls_mode(decision: &EgressDecision, _host: &str, _port: u16) -> crate::l7::TlsMode { - let has_policy = match &decision.action { - NetworkAction::Allow { matched_policy } => matched_policy.is_some(), - NetworkAction::Deny { .. } => false, - }; - if !has_policy { - return crate::l7::TlsMode::Auto; - } - - decision - .endpoint - .policy_configs - .first() - .map_or(crate::l7::TlsMode::Auto, crate::l7::parse_tls_mode) -} - -fn query_endpoint_credential_guard( +fn query_tls_mode( engine: &OpaEngine, decision: &EgressDecision, host: &str, port: u16, -) -> Result { +) -> crate::l7::TlsMode { let has_policy = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.is_some(), NetworkAction::Deny { .. } => false, }; if !has_policy { - return Ok(crate::l7::EndpointCredentialGuard::default()); + return crate::l7::TlsMode::Auto; } let input = crate::opa::NetworkInput { @@ -3568,31 +2968,11 @@ fn query_endpoint_credential_guard( ancestors: decision.ancestors.clone(), cmdline_paths: decision.cmdline_paths.clone(), }; - let values = engine.query_endpoint_credential_guards(&input)?; - let credentialed: Vec<_> = values - .iter() - .map(crate::l7::parse_endpoint_credential_guard) - .filter(|guard| guard.provider_credentialed) - .collect(); - if credentialed.is_empty() { - return Ok(crate::l7::EndpointCredentialGuard::default()); - } - Ok(crate::l7::EndpointCredentialGuard { - provider_credentialed: true, - allow_uninspected_credentials: credentialed - .iter() - .all(|guard| guard.allow_uninspected_credentials), - has_l7_protocol: credentialed.iter().all(|guard| guard.has_l7_protocol), - tls: if credentialed - .iter() - .any(|guard| guard.tls == crate::l7::TlsMode::Skip) - { - crate::l7::TlsMode::Skip - } else { - crate::l7::TlsMode::Auto - }, - }) + match engine.query_endpoint_config(&input) { + Ok(Some(val)) => crate::l7::parse_tls_mode(&val), + _ => crate::l7::TlsMode::Auto, + } } /// When the policy endpoint host is a literal IP address, the user has @@ -3622,11 +3002,9 @@ fn implicit_allowed_ips_for_ip_host(host: &str) -> Vec { } fn normalize_host_lookup_key(host: &str) -> &str { - let h = host - .strip_prefix('[') + host.strip_prefix('[') .and_then(|trimmed| trimmed.strip_suffix(']')) - .unwrap_or(host); - h.strip_suffix('.').unwrap_or(h) + .unwrap_or(host) } /// Returns `true` if `host` is one of the well-known driver-injected aliases @@ -3661,7 +3039,7 @@ fn is_cloud_metadata_ip(ip: IpAddr) -> bool { /// entry exists, the entry cannot be parsed, or the mapped IP is a cloud /// metadata address. #[cfg(any(target_os = "linux", test))] -pub(crate) fn detect_trusted_host_gateway() -> Option { +fn detect_trusted_host_gateway() -> Option { let contents = std::fs::read_to_string("/etc/hosts").ok()?; let ips = parse_hosts_file_for_host(&contents, "host.openshell.internal"); @@ -3709,7 +3087,7 @@ pub(crate) fn detect_trusted_host_gateway() -> Option { } #[cfg(not(any(target_os = "linux", test)))] -pub(crate) fn detect_trusted_host_gateway() -> Option { +fn detect_trusted_host_gateway() -> Option { None } @@ -3867,18 +3245,15 @@ async fn resolve_socket_addrs( return Ok(addrs); } - let dns_host = host - .strip_prefix('[') - .and_then(|h| h.strip_suffix(']')) - .unwrap_or(host); - let addrs: Vec = tokio::net::lookup_host((dns_host, port)) + let lookup_host = normalize_host_lookup_key(host); + let addrs: Vec = tokio::net::lookup_host((lookup_host, port)) .await - .map_err(|e| format!("DNS resolution failed for {dns_host}:{port}: {e}"))? + .map_err(|e| format!("DNS resolution failed for {lookup_host}:{port}: {e}"))? .collect(); if addrs.is_empty() { return Err(format!( - "DNS resolution returned no addresses for {dns_host}:{port}" + "DNS resolution returned no addresses for {lookup_host}:{port}" )); } @@ -4009,7 +3384,6 @@ fn validate_declared_endpoint_resolved_addrs( async fn dial_upstream( upstream_proxy: &Option, host_lc: &str, - raw_host_lc: &str, port: u16, addrs: &[SocketAddr], ) -> std::io::Result { @@ -4019,7 +3393,7 @@ async fn dial_upstream( if cfg.connect_by_hostname() { upstream_proxy::connect_via( endpoint, - raw_host_lc, + host_lc, port, upstream_proxy::ConnectTarget::Hostname, ) @@ -4032,43 +3406,13 @@ async fn dial_upstream( } upstream_proxy::ProxyDecision::Direct(direct_addrs) => { Ok(upstream_proxy::PrefixedStream::without_prefix( - connect_tcp_nodelay_best_effort(&direct_addrs[..]).await?, - )) - } - }; - } - Ok(upstream_proxy::PrefixedStream::without_prefix( - connect_tcp_nodelay_best_effort(addrs).await?, - )) -} - -/// Dial a policy-DNS-correlated transparent TCP destination. -/// -/// Unlike explicit proxy traffic, transparent TCP must never honor the -/// operator hostname-CONNECT compatibility mode: the corporate proxy must -/// receive one of the resolver-approved addresses so it cannot perform a -/// second, policy-bypassing DNS resolution. -#[cfg(target_os = "linux")] -async fn dial_transparent_upstream( - upstream_proxy: &Option, - host_lc: &str, - port: u16, - addrs: &[SocketAddr], -) -> std::io::Result { - if let Some(cfg) = upstream_proxy.as_ref() { - return match cfg.decision(host_lc, port, addrs) { - upstream_proxy::ProxyDecision::Proxy(endpoint) => { - upstream_proxy::connect_via_validated(endpoint, host_lc, port, addrs).await - } - upstream_proxy::ProxyDecision::Direct(direct_addrs) => { - Ok(upstream_proxy::PrefixedStream::without_prefix( - connect_tcp_nodelay_best_effort(&direct_addrs[..]).await?, + TcpStream::connect(&direct_addrs[..]).await?, )) } }; } Ok(upstream_proxy::PrefixedStream::without_prefix( - connect_tcp_nodelay_best_effort(addrs).await?, + TcpStream::connect(addrs).await?, )) } @@ -4202,8 +3546,13 @@ fn parse_allowed_ips(raw: &[String]) -> std::result::Result, S } } -/// Read `allowed_ips` from the endpoint configs captured during authorization. -fn query_allowed_ips(decision: &EgressDecision) -> Vec { +/// Query `allowed_ips` from the matched endpoint config for a CONNECT decision. +fn query_allowed_ips( + engine: &OpaEngine, + decision: &EgressDecision, + host: &str, + port: u16, +) -> Vec { // Only query if action is Allow with a matched policy let has_policy = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.is_some(), @@ -4213,29 +3562,71 @@ fn query_allowed_ips(decision: &EgressDecision) -> Vec { return vec![]; } - decision - .endpoint - .policy_configs - .first() - .map(|config| endpoint_config_string_array(config, "allowed_ips")) - .unwrap_or_default() + let input = crate::opa::NetworkInput { + host: host.to_string(), + port, + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), + }; + + match engine.query_allowed_ips(&input) { + Ok(ips) => ips, + Err(e) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!( + "Failed to query allowed_ips from endpoint config: {e}" + )) + .build(); + ocsf_emit!(event); + vec![] + } + } } -fn endpoint_config_string_array(config: ®orus::Value, key: &str) -> Vec { - let regorus::Value::Object(fields) = config else { - return Vec::new(); +/// Query whether the matched endpoint was declared as this exact hostname. +fn query_exact_declared_endpoint_host( + engine: &OpaEngine, + decision: &EgressDecision, + host: &str, + port: u16, +) -> bool { + let has_policy = match &decision.action { + NetworkAction::Allow { matched_policy } => matched_policy.is_some(), + NetworkAction::Deny { .. } => false, }; - let key = regorus::Value::String(key.into()); - let Some(regorus::Value::Array(values)) = fields.get(&key) else { - return Vec::new(); + if !has_policy { + return false; + } + + let input = crate::opa::NetworkInput { + host: host.to_string(), + port, + binary_path: decision.binary.clone().unwrap_or_default(), + binary_sha256: String::new(), + ancestors: decision.ancestors.clone(), + cmdline_paths: decision.cmdline_paths.clone(), }; - values - .iter() - .filter_map(|value| match value { - regorus::Value::String(value) => Some(value.to_string()), - _ => None, - }) - .collect() + + match engine.query_exact_declared_endpoint_host(&input) { + Ok(is_exact_declared) => is_exact_declared, + Err(e) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .message(format!("Failed to query exact declared endpoint host: {e}")) + .build(); + ocsf_emit!(event); + false + } + } } /// Canonicalize the request-target for inference pattern detection. @@ -4297,32 +3688,23 @@ fn parse_proxy_uri(uri: &str) -> Result<(String, String, u16, String)> { .ok_or_else(|| miette::miette!("Missing scheme in proxy URI: {uri}"))?; let scheme = scheme.to_ascii_lowercase(); - // Split authority from the request target. A query may immediately follow - // the authority when the absolute URI has no explicit path, so `/` alone - // is not a sufficient delimiter. - let target_start = if rest.starts_with('[') { - // IPv6: [::1]:port/path or [::1]?query + // Split authority from path + let (authority, path) = if rest.starts_with('[') { + // IPv6: [::1]:port/path let bracket_end = rest .find(']') .ok_or_else(|| miette::miette!("Unclosed IPv6 bracket in URI: {uri}"))?; - rest[bracket_end + 1..] - .find(['/', '?', '#']) - .map(|position| bracket_end + 1 + position) + let after_bracket = &rest[bracket_end + 1..]; + after_bracket.find('/').map_or((rest, "/"), |slash_pos| { + ( + &rest[..=bracket_end + slash_pos], + &after_bracket[slash_pos..], + ) + }) + } else if let Some(slash_pos) = rest.find('/') { + (&rest[..slash_pos], &rest[slash_pos..]) } else { - rest.find(['/', '?', '#']) - }; - let (authority, target) = target_start.map_or((rest, ""), |position| rest.split_at(position)); - if target.contains('#') { - return Err(miette::miette!( - "Fragments are not allowed in proxy URI: {uri}" - )); - } - let path = match target.chars().next() { - None => "/".to_string(), - Some('/') => target.to_string(), - Some('?') => format!("/{target}"), - Some('#') => unreachable!("fragments were rejected above"), - Some(_) => unreachable!("target begins at a recognized delimiter"), + (rest, "/") }; // Parse host and port from authority @@ -4361,89 +3743,9 @@ fn parse_proxy_uri(uri: &str) -> Result<(String, String, u16, String)> { return Err(miette::miette!("Empty host in URI: {uri}")); } - Ok((scheme, host, port, path)) -} - -/// Return a query-free, credential-redacted path suitable for forward-proxy -/// telemetry. Malformed targets are represented by a fixed sentinel so parse -/// errors cannot expose query strings or credential environment-key names. -fn forward_telemetry_path(target_uri: &str) -> String { - let Ok((_, _, _, target)) = parse_proxy_uri(target_uri) else { - return "/[INVALID_REQUEST_TARGET]".to_string(); - }; - let path = target - .split_once('?') - .map_or(target.as_str(), |(path, _)| path); - secrets::redact_target_for_policy(path) - .unwrap_or_else(|_| "/[INVALID_REQUEST_TARGET]".to_string()) -} - -#[cfg(test)] -fn endpoint_secret_resolver( - provider_credentials: Option<&ProviderCredentialState>, - fallback: Option>, - host: &str, - port: u16, - canonical_path: &str, -) -> Option> { - endpoint_credentials_for_request(provider_credentials, fallback, host, port, canonical_path) - .resolver -} - -struct ForwardEndpointCredentials { - resolver: Option>, - revision: Option, -} - -fn endpoint_credentials_for_request( - provider_credentials: Option<&ProviderCredentialState>, - fallback: Option>, - host: &str, - port: u16, - canonical_path: &str, -) -> ForwardEndpointCredentials { - let Some(credentials) = provider_credentials else { - return ForwardEndpointCredentials { - resolver: fallback, - revision: None, - }; - }; - let (resolver, revision) = - credentials.resolver_for_endpoint_with_revision(host, port, canonical_path); - ForwardEndpointCredentials { - resolver, - revision: Some(revision), - } -} - -struct PreparedForwardTarget { - canonical_path: String, - raw_query: Option, - upstream_target: String, - telemetry_path: String, -} + let path = if path.is_empty() { "/" } else { path }; -fn prepare_forward_target( - target: &str, - canonicalize_options: crate::l7::path::CanonicalizeOptions, -) -> Result { - let (canonical, raw_query) = - crate::l7::path::canonicalize_request_target(target, &canonicalize_options)?; - let telemetry_path = secrets::redact_target_for_policy(&canonical.path) - .unwrap_or_else(|_| "/[INVALID_REQUEST_TARGET]".to_string()); - let upstream_target = raw_query - .as_deref() - .filter(|query| !query.is_empty()) - .map_or_else( - || canonical.path.clone(), - |query| format!("{}?{query}", canonical.path), - ); - Ok(PreparedForwardTarget { - canonical_path: canonical.path, - raw_query, - upstream_target, - telemetry_path, - }) + Ok((scheme, host, port, path.to_string())) } /// Build the HTTP/1.1 `Host` value for a plain-HTTP absolute-form target. @@ -4640,16 +3942,18 @@ fn rewrite_forward_request( } // Fail-closed: scan for any remaining unresolved placeholders - let scan_end = if request_body_credential_rewrite { - rewritten_header_end - } else { - output.len() - }; - let output_str = String::from_utf8_lossy(&output[..scan_end]); - if output_str.contains(secrets::PLACEHOLDER_PREFIX_PUBLIC) - || output_str.contains(secrets::PROVIDER_ALIAS_MARKER_PUBLIC) - { - return Err(secrets::UnresolvedPlaceholderError::unavailable("header")); + if secret_resolver.is_some() { + let scan_end = if request_body_credential_rewrite { + rewritten_header_end + } else { + output.len() + }; + let output_str = String::from_utf8_lossy(&output[..scan_end]); + if output_str.contains(secrets::PLACEHOLDER_PREFIX_PUBLIC) + || output_str.contains(secrets::PROVIDER_ALIAS_MARKER_PUBLIC) + { + return Err(secrets::UnresolvedPlaceholderError::unavailable("header")); + } } Ok(output) @@ -4717,16 +4021,9 @@ fn complete_chunked_body_prefix_len(bytes: &[u8]) -> Option { struct ForwardRelayOptions<'a> { generation_guard: &'a PolicyGenerationGuard, - credential_generation: Option>, websocket_extensions: crate::l7::rest::WebSocketExtensionMode, secret_resolver: Option<&'a SecretResolver>, request_body_credential_rewrite: bool, - deny_uninspected_credentials: bool, - credential_signing: crate::l7::CredentialSigning, - signing_service: &'a str, - signing_region: &'a str, - host: &'a str, - port: u16, } async fn relay_rewritten_forward_request( @@ -4762,16 +4059,16 @@ where upstream, crate::l7::rest::RelayRequestOptions { resolver: options.secret_resolver, - credential_generation: options.credential_generation, + credential_generation: None, generation_guard: Some(options.generation_guard), websocket_extensions: options.websocket_extensions, request_body_credential_rewrite: options.request_body_credential_rewrite, - deny_uninspected_credentials: options.deny_uninspected_credentials, - credential_signing: options.credential_signing, - signing_service: options.signing_service, - signing_region: options.signing_region, - host: options.host, - port: options.port, + deny_uninspected_credentials: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: "", + signing_region: "", + host: "", + port: 0, }, ) .await @@ -4818,14 +4115,15 @@ async fn handle_forward_proxy( target_uri: &str, buf: &[u8], used: usize, - client: &mut TcpStream, + client: &mut ProxyClient, + supplied_identity: Option<&Result>, + socket_addrs: Option<(SocketAddr, SocketAddr)>, opa_engine: Arc, identity_cache: Arc, entrypoint_pid: Arc, policy_local_ctx: Option>, agent_proposals: openshell_core::proposals::AgentProposals, trusted_host_gateway: Arc>, - provider_credentials: Option, secret_resolver: Option>, dynamic_credentials: Option< Arc< @@ -4837,18 +4135,24 @@ async fn handle_forward_proxy( denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, ) -> Result<()> { - let mut telemetry_path = forward_telemetry_path(target_uri); - // 1. Parse the absolute-form URI. Every external forward target is - // canonicalized below before credential binding, policy-path evaluation, - // upstream bytes, or telemetry consume it. - let Ok((scheme, host, port, mut path)) = parse_proxy_uri(target_uri) else { - ocsf_emit!(build_forward_parse_error_ocsf_event(&telemetry_path)); - respond(client, b"HTTP/1.1 400 Bad Request\r\n\r\n").await?; - return Ok(()); + // 1. Parse the absolute-form URI. `path` is marked `mut` so that, when an + // L7 config applies, the canonicalized form produced below replaces it + // in-place — keeping OPA evaluation and the bytes written onto the wire + // in sync. See the L7 block below. + let (scheme, host, port, mut path) = match parse_proxy_uri(target_uri) { + Ok(parsed) => parsed, + Err(e) => { + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!("FORWARD parse error for {target_uri}: {e}")) + .build(); + ocsf_emit!(event); + respond(client, b"HTTP/1.1 400 Bad Request\r\n\r\n").await?; + return Ok(()); + } }; - - let raw_host = host; - let host = normalize_host(&raw_host); let host_lc = host.to_ascii_lowercase(); if host_lc == POLICY_LOCAL_HOST { @@ -4922,19 +4226,27 @@ async fn handle_forward_proxy( canonicalize_forward_host_header(&buf[..used], &canonical_authority)?; // 2. Evaluate OPA policy (same identity binding as CONNECT) - let workload_addr = client.peer_addr().into_diagnostic()?; - let proxy_addr = client.local_addr().into_diagnostic()?; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); - - let opa_clone = opa_engine.clone(); - let cache_clone = identity_cache.clone(); - let pid_clone = entrypoint_pid.clone(); + let workload_addr = socket_addrs.map_or_else( + || SocketAddr::from(([0, 0, 0, 0], 0)), + |(workload, _)| workload, + ); let intent = EgressIntent::forward_http(host_lc.clone(), port); - let mut decision = tokio::task::spawn_blocking(move || { - authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) - }) - .await - .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + let mut decision = if let Some(identity) = supplied_identity { + authorize_supplied_identity(&opa_engine, intent, identity) + } else { + let (workload_addr, proxy_addr) = socket_addrs.ok_or_else(|| { + miette::miette!("legacy proxy connection is missing socket addresses") + })?; + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, proxy_addr); + let opa_clone = opa_engine.clone(); + let cache_clone = identity_cache.clone(); + let pid_clone = entrypoint_pid.clone(); + tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) + }) + .await + .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))? + }; debug!( transport = ?decision.intent.transport, @@ -4980,7 +4292,7 @@ async fn handle_forward_proxy( method, &host_lc, port, - &telemetry_path, + &path, &binary_str, &pid_str, &ancestors_str, @@ -5003,7 +4315,7 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; @@ -5017,7 +4329,7 @@ async fn handle_forward_proxy( binary = %binary_str, binary_pid = %pid_str, matched_policy = %policy_str, - policy_generation = decision.policy_generation, + l4_policy_generation = decision.policy_generation, current_generation = opa_engine.current_generation(), action = ?decision.action, "Forward proxy L4 policy decision" @@ -5032,7 +4344,7 @@ async fn handle_forward_proxy( warn!( host = %host_lc, port, - policy_generation = decision.policy_generation, + l4_policy_generation = decision.policy_generation, current_generation = opa_engine.current_generation(), error = %e, "Forward proxy rejected request because policy generation changed after L4 decision" @@ -5045,13 +4357,14 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; return Ok(()); } }; + let mut upstream_target = path.clone(); let mut websocket_extensions = crate::l7::rest::WebSocketExtensionMode::Preserve; let mut forward_tunnel_engine: Option = None; // L7 endpoint config and evaluated request info, carried past the L7 @@ -5065,7 +4378,19 @@ async fn handle_forward_proxy( let mut forward_websocket_request = crate::l7::rest::request_is_websocket_upgrade(&forward_request_bytes); let mut request_body_credential_rewrite = false; - let mut deny_uninspected_credentials = false; + let workspace = policy_local_ctx + .as_ref() + .map(|ctx| ctx.workspace()) + .unwrap_or_default(); + let l7_ctx = relay::http_context( + &decision, + None, + secret_resolver.clone(), + activity_tx.cloned(), + dynamic_credentials.clone(), + agent_proposals, + workspace, + ); let mut l7_activity_pending = false; // 4b. If the endpoint has L7 config, evaluate the request against @@ -5073,70 +4398,7 @@ async fn handle_forward_proxy( // connection, so a single evaluation suffices. The shared HTTP relay // strips hop-by-hop `Connection` headers and drops the upstream after // the response instead of asking the upstream to close it. - hydrate_l7_route(&mut decision); - let canonicalize_options = crate::l7::path::CanonicalizeOptions { - allow_encoded_slash: decision.endpoint.l7_route.as_ref().is_some_and(|route| { - route - .configs - .iter() - .any(|snapshot| snapshot.config.allow_encoded_slash) - }), - ..Default::default() - }; - let prepared_target = match prepare_forward_target(&path, canonicalize_options) { - Ok(prepared) => prepared, - Err(error) => { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!( - "FORWARD rejecting non-canonical request-target: {error}" - )) - .build(); - ocsf_emit!(event); - emit_activity_simple(activity_tx, true, "forward_parse_rejection"); - respond( - client, - &build_json_error_response( - 400, - "Bad Request", - "invalid_request_target", - "request-target must be canonical", - ), - ) - .await?; - return Ok(()); - } - }; - path = prepared_target.canonical_path; - telemetry_path = prepared_target.telemetry_path; - let upstream_target = prepared_target.upstream_target; - let query_params = prepared_target - .raw_query - .as_deref() - .map_or_else(std::collections::HashMap::new, |query| { - crate::l7::rest::parse_query_params(query).unwrap_or_default() - }); - let workspace = policy_local_ctx - .as_ref() - .map(|ctx| ctx.workspace()) - .unwrap_or_default(); - let mut l7_ctx = relay::http_context( - &decision, - provider_credentials, - secret_resolver.clone(), - activity_tx.cloned(), - dynamic_credentials.clone(), - agent_proposals, - workspace, - ); - l7_ctx.request_default_port = match scheme.as_str() { - "http" => Some(80), - "https" => Some(443), - _ => None, - }; + hydrate_l7_route(&opa_engine, &mut decision); if let Some(route) = decision .endpoint .l7_route @@ -5147,7 +4409,7 @@ async fn handle_forward_proxy( warn!( host = %host_lc, port, - policy_generation = decision.policy_generation, + l4_policy_generation = decision.policy_generation, l4_guard_generation = forward_generation_guard.captured_generation(), l7_policy_generation = route.l7_policy_generation, current_generation = opa_engine.current_generation(), @@ -5169,7 +4431,7 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; @@ -5194,9 +4456,7 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!( - "{method} {host_lc}:{port}{telemetry_path} not permitted by policy" - ), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; @@ -5204,20 +4464,63 @@ async fn handle_forward_proxy( } }; - let Ok(redacted_path) = secrets::redact_target_for_policy(&path) else { - respond( - client, - &build_json_error_response( - 400, - "Bad Request", - "invalid_credential_placeholder", - "request-target contains an invalid credential placeholder", - ), - ) - .await?; - return Ok(()); + // Canonicalize the request-target. The canonical form is fed to OPA + // AND reassigned to the outer `path` variable so the later call to + // `rewrite_forward_request` writes canonical bytes to the upstream. + // This closes the policy/upstream parser-differential at this site; + // without this reassignment, OPA would evaluate the canonical form + // while the upstream re-normalizes the raw input and dispatches on a + // potentially different path. + let canonicalize_options = crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: route + .configs + .iter() + .any(|snapshot| snapshot.config.allow_encoded_slash), + ..Default::default() }; - let Some(l7_config) = select_l7_config_for_path(&route.configs, &redacted_path) else { + let query_params = + match crate::l7::path::canonicalize_request_target(&path, &canonicalize_options) { + Ok((canon, query)) => { + upstream_target = match query.as_deref() { + Some(raw_query) if !raw_query.is_empty() => { + format!("{}?{raw_query}", canon.path) + } + _ => canon.path.clone(), + }; + let params = query + .as_deref() + .map_or_else(std::collections::HashMap::new, |q| { + crate::l7::rest::parse_query_params(q).unwrap_or_default() + }); + path = canon.path; + params + } + Err(e) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(format!( + "FORWARD_L7 rejecting non-canonical request-target: {e}" + )) + .build(); + ocsf_emit!(event); + emit_activity_simple(activity_tx, true, "l7_parse_rejection"); + respond( + client, + &build_json_error_response( + 400, + "Bad Request", + "invalid_request_target", + "request-target must be canonical", + ), + ) + .await?; + return Ok(()); + } + }; + let Some(l7_config) = select_l7_config_for_path(&route.configs, &path) else { emit_activity_simple(activity_tx, true, "l7_policy"); respond( client, @@ -5225,50 +4528,12 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!( - "{method} {host_lc}:{port}{telemetry_path} did not match an L7 endpoint path" - ), + &format!("{method} {host_lc}:{port}{path} did not match an L7 endpoint path"), ), ) .await?; return Ok(()); }; - // `canonicalize_options` was built before the matching config was - // known, so `allow_encoded_slash` was taken permissively across every - // config on this route. Re-check it against the config that actually - // matched: the opt-in is per-endpoint, and one endpoint enabling it - // must not loosen parsing for the others. Rejecting here yields the - // same response the parser would have produced had the option been - // scoped correctly from the start. - if !l7_config.config.allow_encoded_slash - && crate::l7::path::canonical_path_has_encoded_slash(&path) - { - ocsf_emit!(build_forward_l7_parse_rejection_ocsf_event( - workload_addr, - method, - &host_lc, - port, - &telemetry_path, - &binary_str, - &pid_str, - &ancestors_str, - &cmdline_str, - policy_str, - FORWARD_ENCODED_SLASH_REJECTION_DETAIL, - )); - emit_activity_simple(activity_tx, true, "forward_parse_rejection"); - respond( - client, - &build_json_error_response( - 400, - "Bad Request", - "invalid_request_target", - "request-target must be canonical", - ), - ) - .await?; - return Ok(()); - } if crate::l7::rest::request_is_h2c_upgrade(&forward_request_bytes) { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) @@ -5278,7 +4543,7 @@ async fn handle_forward_proxy( .status(StatusId::Failure) .http_request(HttpRequest::new( method, - OcsfUrl::new("http", &host_lc, &telemetry_path, port), + OcsfUrl::new("http", &host_lc, &path, port), )) .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) @@ -5288,7 +4553,7 @@ async fn handle_forward_proxy( ) .firewall_rule(policy_str, "l7") .message(format!( - "FORWARD_L7 denied unsupported h2c upgrade for {method} {host_lc}:{port}{telemetry_path}" + "FORWARD_L7 denied unsupported h2c upgrade for {method} {host_lc}:{port}{path}" )) .status_detail(crate::l7::rest::UNSUPPORTED_H2C_UPGRADE_DETAIL) .build(); @@ -5320,9 +4585,6 @@ async fn handle_forward_proxy( websocket_extensions = crate::l7::relay::websocket_extension_mode(&l7_config.config, false); request_body_credential_rewrite = l7_config.config.protocol == crate::l7::L7Protocol::Rest && l7_config.config.request_body_credential_rewrite; - deny_uninspected_credentials = l7_config - .config - .deny_uninspected_body_credentials(secret_resolver.is_some()); forward_upgrade_config = Some(l7_config.config.clone()); forward_upgrade_target = path.clone(); forward_upgrade_query_params = query_params.clone(); @@ -5438,7 +4700,7 @@ async fn handle_forward_proxy( }; let request_info = crate::l7::L7RequestInfo { action: method.to_string(), - target: redacted_path, + target: path.clone(), query_params, graphql, jsonrpc, @@ -5501,11 +4763,11 @@ async fn handle_forward_proxy( "FORWARD_L7" }; format!( - "{message_prefix} {decision_str} {method} {host_lc}:{port}{telemetry_path} reason={reason}" + "{message_prefix} {decision_str} {method} {host_lc}:{port}{path} reason={reason}" ) }, |jsonrpc_info| { - let endpoint = format!("{host_lc}:{port}{telemetry_path}"); + let endpoint = format!("{host_lc}:{port}{path}"); crate::l7::relay::jsonrpc_log_message( decision_str, method, @@ -5523,7 +4785,7 @@ async fn handle_forward_proxy( .severity(severity) .http_request(HttpRequest::new( method, - OcsfUrl::new("http", &host_lc, &telemetry_path, port), + OcsfUrl::new("http", &host_lc, &path, port), )) .dst_endpoint(Endpoint::from_domain(&host_lc, port)) .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) @@ -5557,9 +4819,7 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!( - "{method} {host_lc}:{port}{telemetry_path} denied by L7 policy: {reason}" - ), + &format!("{method} {host_lc}:{port}{path} denied by L7 policy: {reason}"), ), ) .await?; @@ -5580,7 +4840,7 @@ async fn handle_forward_proxy( // - Otherwise: reject internal IPs, allow public IPs through. // When the policy host is already a literal IP address, treat it as // implicitly allowed — the user explicitly declared the destination. - match hydrate_destination_plan(&mut decision, *trusted_host_gateway) { + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { Ok(()) => {} Err(denial) => { deny_forward_destination( @@ -5590,7 +4850,7 @@ async fn handle_forward_proxy( method, &host_lc, port, - &telemetry_path, + &path, &binary_str, &pid_str, &ancestors_str, @@ -5611,7 +4871,7 @@ async fn handle_forward_proxy( .expect("destination plan hydrated"); let connector = match validate_destination(DestinationRequest { - host: &raw_host, + host: &host, port, sandbox_entrypoint_pid, plan: destination_plan, @@ -5627,7 +4887,7 @@ async fn handle_forward_proxy( method, &host_lc, port, - &telemetry_path, + &path, &binary_str, &pid_str, &ancestors_str, @@ -5659,13 +4919,53 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; return Ok(()); } + // 6. Connect upstream. Plain-HTTP requests always dial the destination + // directly: only TLS (CONNECT) tunnels chain through the corporate + // proxy, since plain-HTTP forwarding would need absolute-form requests + // rather than a CONNECT tunnel. + let mut upstream = match connector.connect().await { + Ok(s) => s, + Err(e) => { + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", &host_lc, &path, port), + )) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .message(format!( + "FORWARD upstream connect failed for {host_lc}:{port}: {e}" + )) + .build(); + ocsf_emit!(event); + respond( + client, + &build_json_error_response( + 502, + "Bad Gateway", + "upstream_unreachable", + &format!("connection to {host_lc}:{port} failed"), + ), + ) + .await?; + return Ok(()); + } + }; + let middleware_path = path.split_once('?').map_or(path.as_str(), |(path, _)| path); let middleware_input = crate::opa::NetworkInput { host: host_lc.clone(), @@ -5693,13 +4993,12 @@ async fn handle_forward_proxy( 403, "Forbidden", "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), + &format!("{method} {host_lc}:{port}{path} not permitted by policy"), ), ) .await?; return Ok(()); } - let websocket_chain = forward_websocket_request.then(|| chain.clone()); if !chain.is_empty() { let middleware_runner = opa_engine.middleware_runner()?; let request = crate::l7::rest::request_from_buffered_http( @@ -5736,61 +5035,12 @@ async fn handle_forward_proxy( } crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { emit_activity_simple(activity_tx, true, "middleware"); - let response = build_middleware_unavailable_response(&l7_ctx.policy_name); + let response = build_middleware_failure_response(&l7_ctx.policy_name); respond(client, &response).await?; return Ok(()); } }; } - let mut middleware_session = if let Some(chain) = websocket_chain.as_deref() { - let request = crate::l7::rest::request_from_buffered_http( - method, - middleware_path, - &upstream_target, - forward_request_bytes.clone(), - )?; - let middleware_runner = opa_engine.middleware_runner()?; - let preflight = crate::l7::relay::websocket_middleware_preflight( - &request, - chain, - &middleware_runner, - &l7_ctx, - "ws", - ) - .await; - let preflight = match preflight { - Ok(preflight) => preflight, - Err(error) => { - warn!(error = %error, "Plaintext WebSocket middleware preflight failed"); - respond( - client, - &build_json_error_response( - 502, - "Bad Gateway", - "middleware_failed", - "WebSocket middleware preflight failed", - ), - ) - .await?; - return Ok(()); - } - }; - crate::l7::middleware::emit_websocket_preflight_events(&l7_ctx, &preflight); - if preflight.terminal_reason.is_some() { - let response = preflight.denial.as_ref().map_or_else( - || build_middleware_failure_response(&l7_ctx.policy_name), - |denial| build_middleware_deny_response(&l7_ctx.policy_name, denial), - ); - respond(client, &response).await?; - return Ok(()); - } - preflight.session - } else { - None - }; - if middleware_session.is_some() { - websocket_extensions = crate::l7::rest::WebSocketExtensionMode::PermessageDeflate; - } forward_request_bytes = match inject_token_grant_for_forward_request( method, &upstream_target, @@ -5807,11 +5057,6 @@ async fn handle_forward_proxy( error = %e, "token grant failed in forward proxy" ); - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::Cancellation) - .await; - } respond( client, &build_json_error_response( @@ -5825,30 +5070,6 @@ async fn handle_forward_proxy( return Ok(()); } }; - // Static credentials are intentionally acquired only after every - // asynchronous admission step. Holding an endpoint-scoped resolver across - // middleware or token-grant awaits would let a revoked generation reach - // the upstream. - let endpoint_credentials = endpoint_credentials_for_request( - l7_ctx.provider_credentials.as_ref(), - l7_ctx.secret_resolver.clone(), - &host_lc, - port, - &path, - ); - let secret_resolver = endpoint_credentials.resolver; - let credential_generation = match ( - l7_ctx.provider_credentials.as_ref(), - endpoint_credentials.revision, - ) { - (Some(state), Some(revision)) => Some(crate::l7::rest::CredentialGenerationGuard::new( - state, revision, - )), - _ => None, - }; - if let Some(guard) = credential_generation { - guard.ensure_current()?; - } // 9. Rewrite request and forward to upstream let rewritten = match rewrite_forward_request( @@ -5867,106 +5088,13 @@ async fn handle_forward_proxy( error = %e, "credential injection failed in forward proxy" ); - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::Cancellation) - .await; - } - if e.is_endpoint_mismatch() { - emit_credential_endpoint_mismatch(&host_lc, port, policy_str); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "credential_endpoint_mismatch", - "credential is not authorized for this request endpoint", - ), - ) - .await?; - } else { - respond( - client, - &build_json_error_response( - 500, - "Internal Server Error", - "credential_injection_failed", - "unresolved credential placeholder in request", - ), - ) - .await?; - } - return Ok(()); - } - }; - - if let Err(e) = forward_generation_guard.ensure_current() { - warn!( - host = %host_lc, - port, - captured_generation = forward_generation_guard.captured_generation(), - current_generation = forward_generation_guard.current_generation(), - error = %e, - "Forward proxy rejected request because policy changed before relay" - ); - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::PolicyReload) - .await; - } - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "policy_denied", - &format!("{method} {host_lc}:{port}{telemetry_path} not permitted by policy"), - ), - ) - .await?; - return Ok(()); - } - // Plain-HTTP requests dial the destination directly: only TLS (CONNECT) - // tunnels chain through the corporate proxy, since plain-HTTP forwarding - // would need absolute-form requests rather than a CONNECT tunnel. Dial - // only after every local authorization and transformation step so a - // rejected WebSocket preflight cannot contact the destination. - let dial_result = connector.connect().await; - let mut upstream = match dial_result { - Ok(s) => s, - Err(e) => { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .message(format!( - "FORWARD upstream connect failed for {host_lc}:{port}: {e}" - )) - .build(); - ocsf_emit!(event); - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::UpstreamRejected) - .await; - } respond( client, &build_json_error_response( - 502, - "Bad Gateway", - "upstream_unreachable", - &format!("connection to {host_lc}:{port} failed"), + 500, + "Internal Server Error", + "credential_injection_failed", + "unresolved credential placeholder in request", ), ) .await?; @@ -5981,14 +5109,9 @@ async fn handle_forward_proxy( captured_generation = forward_generation_guard.captured_generation(), current_generation = forward_generation_guard.current_generation(), error = %e, - "Forward proxy rejected request because policy changed during upstream connect" + "Forward proxy rejected request because policy changed before relay" ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::PolicyReload) - .await; - } respond( client, &build_json_error_response( @@ -6001,62 +5124,19 @@ async fn handle_forward_proxy( .await?; return Ok(()); } - - let credential_signing = forward_upgrade_config - .as_ref() - .map_or(crate::l7::CredentialSigning::None, |config| { - config.credential_signing - }); - let signing_service = forward_upgrade_config - .as_ref() - .map_or("", |config| config.signing_service.as_str()); - let signing_region = forward_upgrade_config - .as_ref() - .map_or("", |config| config.signing_region.as_str()); - let outcome_result = relay_rewritten_forward_request( + let outcome = relay_rewritten_forward_request( method, - &upstream_target, + &path, rewritten, client, &mut upstream, ForwardRelayOptions { generation_guard: &forward_generation_guard, - credential_generation, websocket_extensions, secret_resolver: secret_resolver.as_deref(), request_body_credential_rewrite, - deny_uninspected_credentials, - credential_signing, - signing_service, - signing_region, - host: &host_lc, - port, }, ) - .await; - let outcome_result = match outcome_result { - Err(report) => { - if let Some(error) = report.downcast_ref::() { - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::Cancellation) - .await; - } - crate::l7::relay::reject_credential_resolution(client, &l7_ctx, error).await?; - return Ok(()); - } - Err(report) - } - outcome => outcome, - }; - let outcome = crate::l7::relay::finalize_websocket_pre_upgrade( - &mut middleware_session, - &forward_generation_guard, - &host_lc, - port, - policy_str, - outcome_result, - ) .await?; // The request has now survived middleware, token grant, credential @@ -6067,7 +5147,7 @@ async fn handle_forward_proxy( method, &host_lc, port, - &telemetry_path, + &path, &binary_str, &pid_str, &ancestors_str, @@ -6076,55 +5156,40 @@ async fn handle_forward_proxy( )); emit_forward_success_activity(activity_tx, l7_activity_pending); - match outcome { - crate::l7::provider::RelayOutcome::Reusable - | crate::l7::provider::RelayOutcome::Consumed => { - if let Some(session) = middleware_session.take() { - session - .end(openshell_core::proto::WebSocketSessionEndReason::UpstreamRejected) - .await; + if let crate::l7::provider::RelayOutcome::Upgraded { + overflow, + websocket_permessage_deflate, + .. + } = outcome + { + let mut upgrade_options = if let (Some(config), Some(engine)) = ( + forward_upgrade_config.as_ref(), + forward_tunnel_engine.as_ref(), + ) { + crate::l7::relay::upgrade_options( + config, + &l7_ctx, + forward_websocket_request, + &forward_upgrade_target, + &forward_upgrade_query_params, + Some(engine), + ) + } else { + crate::l7::relay::UpgradeRelayOptions { + websocket_request: forward_websocket_request, + ..Default::default() } - } - crate::l7::provider::RelayOutcome::Upgraded { + }; + upgrade_options.websocket.permessage_deflate = websocket_permessage_deflate; + crate::l7::relay::handle_upgrade( + client, + &mut upstream, overflow, - websocket_permessage_deflate, - websocket_subprotocol, - } => { - let mut upgrade_options = if let (Some(config), Some(engine)) = ( - forward_upgrade_config.as_ref(), - forward_tunnel_engine.as_ref(), - ) { - crate::l7::relay::upgrade_options( - config, - &l7_ctx, - forward_websocket_request, - &forward_upgrade_target, - &forward_upgrade_query_params, - Some(engine), - ) - } else { - crate::l7::relay::UpgradeRelayOptions { - websocket_request: forward_websocket_request, - ctx: Some(&l7_ctx), - policy_name: l7_ctx.policy_name.clone(), - ..Default::default() - } - }; - upgrade_options.generation_guard = Some(&forward_generation_guard); - upgrade_options.assembly_budget = Some(opa_engine.websocket_assembly_budget()); - upgrade_options.websocket.permessage_deflate = websocket_permessage_deflate; - upgrade_options.middleware_session = middleware_session.take(); - upgrade_options.selected_subprotocol = websocket_subprotocol; - crate::l7::relay::handle_upgrade( - client, - &mut upstream, - overflow, - &host_lc, - port, - upgrade_options, - ) - .await?; - } + &host_lc, + port, + upgrade_options, + ) + .await?; } Ok(()) @@ -6156,11 +5221,7 @@ fn parse_target(target: &str) -> Result<(String, u16)> { Ok((host.to_string(), port)) } -fn normalize_host(raw_host: &str) -> &str { - raw_host.strip_suffix('.').unwrap_or(raw_host) -} - -async fn respond(client: &mut TcpStream, bytes: &[u8]) -> Result<()> { +async fn respond(client: &mut (impl TokioAsyncWrite + Unpin), bytes: &[u8]) -> Result<()> { client.write_all(bytes).await.into_diagnostic()?; Ok(()) } @@ -6250,14 +5311,6 @@ fn build_middleware_deny_response( } fn build_middleware_failure_response(policy_name: &str) -> Vec { - build_middleware_platform_response(policy_name, "403 Forbidden") -} - -fn build_middleware_unavailable_response(policy_name: &str) -> Vec { - build_middleware_platform_response(policy_name, "503 Service Unavailable") -} - -fn build_middleware_platform_response(policy_name: &str, status: &str) -> Vec { let body = serde_json::json!({ "error": "middleware_failed", "detail": "Request could not be processed by configured middleware", @@ -6265,7 +5318,7 @@ fn build_middleware_platform_response(policy_name: &str, status: &str) -> Vec, ) -> std::result::Result< - tonic::Response, - tonic::Status, + openshell_isolation::contract::MediatedConnection, + openshell_isolation::contract::BackendError, > { - Ok(tonic::Response::new( - openshell_core::proto::ValidateConfigResponse { - valid: true, - reason: String::new(), - }, + Err(openshell_isolation::contract::BackendError::Unavailable( + "test source unavailable".to_string(), )) } - - async fn evaluate_http_request( - &self, - _request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { - Err(tonic::Status::unimplemented("WebSocket-only test service")) - } - - async fn open_websocket_session( - &self, - mut receiver: mpsc::Receiver, - ) -> std::result::Result - { - let (responses, response_stream) = mpsc::channel(1); - tokio::spawn(async move { - while let Some(event) = receiver.recv().await { - if matches!( - event.event, - Some(openshell_core::proto::web_socket_session_event::Event::Preflight(_)) - ) { - let _ = responses - .send(Ok(openshell_core::proto::WebSocketSessionEventResult { - result: Some( - openshell_core::proto::web_socket_session_event_result::Result::PreflightDecision( - openshell_core::proto::WebSocketPreflightDecision { - action: openshell_core::proto::WebSocketPreflightAction::Deny as i32, - reason: "test denial".into(), - reason_code: "test_denial".into(), - ..Default::default() - }, - ), - ), - })) - .await; - break; - } - } - }); - Ok(Box::pin(tokio_stream::wrappers::ReceiverStream::new( - response_stream, - ))) - } - } - - struct BlockingForwardMiddleware { - entered: Arc, - release: Arc, - } - - #[tonic::async_trait] - impl openshell_core::middleware::InProcessMiddleware for BlockingForwardMiddleware { - async fn describe(&self) -> openshell_core::proto::MiddlewareManifest { - openshell_core::proto::MiddlewareManifest { - name: "test/blocking-forward".into(), - service_version: "test".into(), - bindings: vec![openshell_core::proto::MiddlewareBinding { - operation: openshell_core::proto::SupervisorMiddlewareOperation::HttpRequest - as i32, - phase: openshell_core::proto::SupervisorMiddlewarePhase::PreCredentials as i32, - max_payload_bytes: 8192, - timeout: String::new(), - }], - expected_audience: String::new(), - } - } - - async fn validate_config( - &self, - _middleware_name: &str, - _config: &prost_types::Struct, - ) -> Result<()> { - Ok(()) - } - - async fn evaluate_http_request( - &self, - _request: openshell_core::middleware::HttpRequestView<'_>, - ) -> Result { - self.entered.notify_one(); - self.release.notified().await; - Ok(openshell_core::proto::HttpRequestResult { - decision: openshell_core::proto::Decision::Allow as i32, - ..Default::default() - }) - } - } - - fn non_loopback_test_ipv4() -> Option { - let socket = std::net::UdpSocket::bind("0.0.0.0:0").ok()?; - socket.connect("192.0.2.1:9").ok()?; - match socket.local_addr().ok()?.ip() { - IpAddr::V4(ip) if !ip.is_loopback() && !ip.is_unspecified() => Some(ip), - _ => None, - } } async fn drive_raw_request_through_handler(raw: Vec) -> Vec { @@ -6534,7 +5460,6 @@ network_policies: {} None, None, None, - None, )) .await .expect("malformed request should be handled"); @@ -6542,266 +5467,62 @@ network_policies: {} } #[tokio::test] - async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { - for host in ["api.example.com", "unmatched.example.com"] { - let raw = format!( - "GET http://{host}/ HTTP/1.1\r\nHost: {host}\r\nX-Guard: before\0after\r\n\r\n" + async fn terminal_mediation_source_failure_stops_accepting_fail_static() { + let policy = include_str!("../data/sandbox-policy.rego"); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + policy, + "network_policies: {}", + true, ) - .into_bytes(); - let response = Box::pin(drive_raw_request_through_handler(raw)).await; - assert!( - response.starts_with(b"HTTP/1.1 400 Bad Request"), - "malformed request for {host} must fail at ingress" - ); - } - } - - #[tokio::test] - async fn plaintext_websocket_preflight_denial_does_not_connect_upstream() { - if !cfg!(target_os = "linux") { - eprintln!("skipping: handler identity binding requires /proc (Linux)"); - return; - } - let Some(upstream_ip) = non_loopback_test_ipv4() else { - eprintln!("skipping: no routable non-loopback IPv4 test address"); - return; - }; - - let upstream_listener = TcpListener::bind((upstream_ip, 0)) - .await - .expect("bind upstream listener"); - let upstream_port = upstream_listener.local_addr().unwrap().port(); - let executable = std::env::current_exe().expect("current executable"); - let data = format!( - r#" -network_middlewares: - deny-upgrade: - middleware: test/deny-websocket - on_error: fail_closed - endpoints: - include: ["{upstream_ip}"] -network_policies: - allow-upstream: - name: allow-upstream - endpoints: - - host: "{upstream_ip}" - port: {upstream_port} - binaries: - - {{ path: "{executable}" }} -"#, - executable = executable.display(), + .expect("engine"), ); - let engine = Arc::new( - OpaEngine::from_strings(include_str!("../data/sandbox-policy.rego"), &data) - .expect("load policy"), - ); - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - vec![openshell_supervisor_middleware::in_process_endpoint( - Arc::new(DenyWebSocketPreflight), - )], - Vec::new(), + let (_ready_tx, ready_rx) = tokio::sync::watch::channel(true); + let handle = ProxyHandle::start_with_bind_addr( + &ProxyPolicy { http_addr: None }, + Some(([127, 0, 0, 1], 3128).into()), + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(1)), + None, + None, + None, + None, + None, + None, + ready_rx, + &upstream_proxy::UpstreamProxyArgs::default(), + Some(Arc::new(FailedMediationSource)), ) .await - .expect("connect test middleware"); - engine - .replace_middleware_registry(registry) - .expect("install test middleware"); + .expect("proxy starts before source accept"); - let proxy_listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind proxy listener"); - let proxy_address = proxy_listener.local_addr().unwrap(); - let target = format!("http://{upstream_ip}:{upstream_port}/ws"); - let request = format!( - "GET {target} HTTP/1.1\r\nHost: {upstream_ip}:{upstream_port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n" - ); - let client = tokio::spawn(async move { - let mut socket = TcpStream::connect(proxy_address) - .await - .expect("connect proxy"); - let mut response = Vec::new(); - socket - .read_to_end(&mut response) - .await - .expect("read proxy response"); - response - }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); - - tokio::time::timeout( - std::time::Duration::from_secs(30), - handle_forward_proxy( - "GET", - &target, - request.as_bytes(), - request.len(), - &mut proxy_connection, - engine, - Arc::new(BinaryIdentityCache::new()), - Arc::new(AtomicU32::new(std::process::id())), - None, - AgentProposals::default(), - Arc::new(None), - None, - None, - None, - None, - None, - ), + let failure = tokio::time::timeout( + std::time::Duration::from_secs(1), + handle.wait_for_source_failure(), ) .await - .expect("denied preflight must complete without an upstream response") - .expect("handle denied plaintext WebSocket upgrade"); - drop(proxy_connection); - - let response = String::from_utf8(client.await.unwrap()).expect("UTF-8 response"); - assert!( - response.contains("\"error\":\"middleware_denied\""), - "preflight must deny the upgrade: {response}" - ); + .expect("source failure must be observed"); + assert!(failure.contains("test source unavailable")); assert!( - tokio::time::timeout( - std::time::Duration::from_millis(100), - upstream_listener.accept() - ) - .await - .is_err(), - "denied preflight must not establish an upstream connection" + handle.join.is_finished(), + "accept loop must stop after source loss" ); } #[tokio::test] - async fn plaintext_websocket_middleware_inspects_compressed_ws_messages() { - if !cfg!(target_os = "linux") { - eprintln!("skipping: handler identity binding requires /proc (Linux)"); - return; - } - let Some(upstream_ip) = non_loopback_test_ipv4() else { - eprintln!("skipping: no routable non-loopback IPv4 test address"); - return; - }; - - let upstream_listener = TcpListener::bind((upstream_ip, 0)) - .await - .expect("bind upstream listener"); - let upstream_port = upstream_listener.local_addr().unwrap().port(); - let executable = std::env::current_exe().expect("current executable"); - let data = format!( - r#" -network_middlewares: - redact: - middleware: openshell/regex - on_error: fail_closed - endpoints: - include: ["{upstream_ip}"] -network_policies: - allow-upstream: - name: allow-upstream - endpoints: - - host: "{upstream_ip}" - port: {upstream_port} - binaries: - - {{ path: "{executable}" }} -"#, - executable = executable.display(), - ); - let engine = Arc::new( - OpaEngine::from_strings(include_str!("../data/sandbox-policy.rego"), &data) - .expect("load policy"), - ); - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await - .expect("connect built-in middleware"); - engine - .replace_middleware_registry(registry) - .expect("install built-in middleware"); - - let upstream = tokio::spawn(async move { - let (mut socket, _) = upstream_listener.accept().await.unwrap(); - let request = read_http_headers_unbounded(&mut socket).await; - let request = String::from_utf8_lossy(&request); - assert!(request.starts_with("GET /ws HTTP/1.1\r\n")); - assert!(request.contains( - "Sec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover\r\n" - )); - socket - .write_all( - b"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\nSec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover\r\n\r\n", - ) - .await - .unwrap(); - let frame = crate::l7::websocket::read_frame_for_test(&mut socket).await; - crate::l7::websocket::decode_compressed_masked_text_frame_for_test(&frame) - }); - - let proxy_listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("bind proxy listener"); - let proxy_address = proxy_listener.local_addr().unwrap(); - let target = format!("http://{upstream_ip}:{upstream_port}/ws"); - let request = format!( - "GET {target} HTTP/1.1\r\nHost: {upstream_ip}:{upstream_port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Extensions: permessage-deflate; client_no_context_takeover\r\n\r\n" - ); - let client = tokio::spawn(async move { - let mut socket = TcpStream::connect(proxy_address) - .await - .expect("connect proxy"); - let response = read_http_headers_unbounded(&mut socket).await; - assert!(String::from_utf8_lossy(&response).contains("101 Switching Protocols")); - socket - .write_all( - &crate::l7::websocket::compressed_masked_text_frame_for_test( - br#"{"token":"sk-1234567890abcdef"}"#, - ), - ) - .await - .unwrap(); - }); - let (mut proxy_connection, _) = proxy_listener.accept().await.unwrap(); - - let handler = tokio::spawn(async move { - handle_forward_proxy( - "GET", - &target, - request.as_bytes(), - request.len(), - &mut proxy_connection, - engine, - Arc::new(BinaryIdentityCache::new()), - Arc::new(AtomicU32::new(std::process::id())), - None, - AgentProposals::default(), - Arc::new(None), - None, - None, - None, - None, - None, + async fn malformed_forward_headers_are_rejected_before_route_or_middleware_dispatch() { + for host in ["api.example.com", "unmatched.example.com"] { + let raw = format!( + "GET http://{host}/ HTTP/1.1\r\nHost: {host}\r\nX-Guard: before\0after\r\n\r\n" ) - .await - }); - let scenario = tokio::time::timeout(std::time::Duration::from_secs(60), async { - let (client, upstream) = tokio::join!(client, upstream); - client.expect("join plaintext WebSocket client"); - assert_eq!( - upstream.expect("join plaintext WebSocket upstream"), - r#"{"token":"[REDACTED]"}"# + .into_bytes(); + let response = Box::pin(drive_raw_request_through_handler(raw)).await; + assert!( + response.starts_with(b"HTTP/1.1 400 Bad Request"), + "malformed request for {host} must fail at ingress" ); - }) - .await; - if handler.is_finished() { - handler - .await - .expect("join plaintext WebSocket handler") - .expect("handle compressed plaintext WebSocket upgrade"); - } else { - handler.abort(); - let _ = handler.await; } - scenario.expect("compressed plaintext WebSocket scenario should complete"); } #[tokio::test] @@ -6851,49 +5572,6 @@ network_policies: } } - #[tokio::test] - async fn dial_upstream_preserves_trailing_dot_in_hostname_connect() { - // Fake upstream proxy: capture the request line, then 200. - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy_addr = listener.local_addr().unwrap(); - let handle = tokio::spawn(async move { - let (mut sock, _) = listener.accept().await.unwrap(); - let mut buf = [0_u8; 1024_usize]; - let n = sock.read(&mut buf).await.unwrap(); - sock.write_all(b"HTTP/1.1 200 Connection established\r\n\r\n") - .await - .unwrap(); - String::from_utf8_lossy(&buf[..n]).into_owned() - }); - - // Operator config: proxy set + connect-by-hostname opt-in. - let cfg = UpstreamProxyConfig::from_args(&upstream_proxy::UpstreamProxyArgs { - https_proxy: Some(format!("http://{proxy_addr}")), - proxy_connect_by_hostname: true, - ..Default::default() - }) - .unwrap(); - - // host_lc = normalized (undotted), raw_host_lc = absolute (dotted). - let stream = dial_upstream( - &cfg, - "api.example.com", - "api.example.com.", - 443, - &[], // addrs unused in the hostname branch - ) - .await - .unwrap(); - - drop(stream); - - let request = handle.await.unwrap(); - assert!( - request.starts_with("CONNECT api.example.com.:443 HTTP/1.1\r\n"), - "proxy must receive the absolute FQDN: {request}" - ); - } - #[test] fn middleware_failure_response_uses_platform_text_without_policy_guidance() { let response = build_middleware_failure_response("api-policy"); @@ -6912,28 +5590,6 @@ network_policies: assert!(body.get("agent_guidance").is_none()); } - #[test] - fn middleware_unavailable_response_is_complete_and_has_no_retry_hint() { - let response = build_middleware_unavailable_response("api-policy"); - let response = String::from_utf8(response).expect("UTF-8 error response"); - assert!(response.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); - assert!(!response.to_ascii_lowercase().contains("retry-after")); - let (headers, body) = response.split_once("\r\n\r\n").expect("HTTP response"); - let content_length = headers - .lines() - .find_map(|line| { - line.strip_prefix("Content-Length: ") - .and_then(|value| value.parse::().ok()) - }) - .expect("Content-Length"); - assert_eq!(content_length, body.len()); - let body: serde_json::Value = serde_json::from_str(body).expect("JSON response"); - assert_eq!(body["error"], "middleware_failed"); - assert_eq!(body["policy"], "api-policy"); - assert!(body.get("middleware").is_none()); - assert!(body.get("reason_code").is_none()); - } - #[test] fn policy_deny_response_includes_reason() { let response = build_json_error_response_with_reason( @@ -6980,272 +5636,22 @@ network_policies: fn forward_policy_denial_ocsf_includes_validation_rationale() { let reason = "policy validation failed; fail-closed quarantine is active; candidate version 7 rejected: conflicting tls metadata"; let event = build_forward_policy_deny_ocsf_event( - "127.0.0.1:45123".parse().unwrap(), - "GET", - "api.example.com", - 80, - "/v1/models", - "/usr/bin/curl", - "42", - "/usr/bin/bash", - "curl http://api.example.com/v1/models", - reason, - ); - let json = event.to_json().unwrap(); - - assert_eq!(json["status_detail"], reason); - assert_eq!(json["action"], "Denied"); - assert_eq!(json["disposition"], "Blocked"); - } - - #[test] - fn forward_l7_parse_rejection_ocsf_includes_denial_context() { - let event = build_forward_l7_parse_rejection_ocsf_event( - "127.0.0.1:45123".parse().unwrap(), - "GET", - "api.example.com", - 80, - "/admin/x%2Fy", - "/usr/bin/curl", - "42", - "/usr/bin/bash", - "curl http://api.example.com/admin/x%2Fy", - "allow_api", - FORWARD_ENCODED_SLASH_REJECTION_DETAIL, - ); - let json = event.to_json().unwrap(); - - assert_eq!(json["class_name"], "HTTP Activity"); - assert_eq!(json["activity_name"], "Other"); - assert_eq!(json["action"], "Denied"); - assert_eq!(json["disposition"], "Blocked"); - assert_eq!(json["severity"], "Medium"); - assert_eq!(json["status"], "Failure"); - assert_eq!(json["http_request"]["http_method"], "GET"); - assert_eq!(json["http_request"]["url"]["path"], "/admin/x%2Fy"); - assert_eq!(json["dst_endpoint"]["domain"], "api.example.com"); - assert_eq!(json["dst_endpoint"]["port"], 80); - assert_eq!(json["actor"]["process"]["name"], "/usr/bin/curl"); - assert_eq!(json["firewall_rule"]["name"], "allow_api"); - assert_eq!(json["firewall_rule"]["type"], "l7"); - assert_eq!( - json["status_detail"], - FORWARD_ENCODED_SLASH_REJECTION_DETAIL - ); - } - - #[test] - fn transparent_tcp_allow_ocsf_exposes_correlated_dns_and_dial_chain() { - let mapping_id = uuid::Uuid::new_v4(); - let event = build_transparent_tcp_allow_ocsf_event(TransparentTcpAllowAudit { - workload: "127.0.0.1:45123".parse().unwrap(), - synthetic_destination: "198.18.0.7:6379".parse().unwrap(), - normalized_domain: "redis.openshell.demo", - approved_real_ip_candidates: &[ - "172.18.0.4:6379".parse().unwrap(), - "172.18.0.5:6379".parse().unwrap(), - ], - connected_real_destination: Some("172.18.0.5:6379".parse().unwrap()), - upstream_socket_peer: "172.18.0.5:6379".parse().unwrap(), - dial_mode: "direct", - mapping_id, - mapping_generation: 4, - mapping_policy_generation: 7, - authorization_policy_generation: 7, - binary: "/sandbox/.venv/bin/python3", - pid: "42", - policy_name: "redis", - }); - let json = event.to_json().unwrap(); - - assert_eq!(json["actor"]["process"]["pid"], 42); - assert_eq!( - json["actor"]["process"]["name"], - "/sandbox/.venv/bin/python3" - ); - assert!(json["actor"]["process"].get("parent_process").is_none()); - assert_eq!(json["dst_endpoint"]["domain"], "redis.openshell.demo"); - assert_eq!(json["firewall_rule"]["name"], "redis"); - assert_eq!(json["unmapped"]["synthetic_destination"], "198.18.0.7:6379"); - assert_eq!( - json["unmapped"]["connected_real_destination"], - "172.18.0.5:6379" - ); - assert_eq!( - json["unmapped"]["approved_real_ip_candidates"], - serde_json::json!(["172.18.0.4:6379", "172.18.0.5:6379"]) - ); - assert_eq!(json["unmapped"]["mapping_id"], mapping_id.to_string()); - assert_eq!(json["unmapped"]["mapping_generation"], 4); - assert_eq!(json["unmapped"]["policy_generation"], 7); - assert_eq!(json["unmapped"]["mapping_policy_generation"], 7); - assert_eq!(json["unmapped"]["matched_policy"], "redis"); - assert_eq!(json["unmapped"]["dial_mode"], "direct"); - - let shorthand = event.format_shorthand(); - assert!(shorthand.contains("/sandbox/.venv/bin/python3(42)")); - assert!(shorthand.contains("redis.openshell.demo:6379")); - assert!(shorthand.contains("synthetic=198.18.0.7:6379")); - assert!(shorthand.contains("real=172.18.0.5:6379")); - assert!(shorthand.contains(&format!("mapping_id={mapping_id}"))); - } - - #[test] - fn transparent_tcp_proxy_audit_reports_validated_connect_target() { - let event = build_transparent_tcp_allow_ocsf_event(TransparentTcpAllowAudit { - workload: "127.0.0.1:45123".parse().unwrap(), - synthetic_destination: "198.18.0.7:6379".parse().unwrap(), - normalized_domain: "redis.openshell.demo", - approved_real_ip_candidates: &["172.18.0.4:6379".parse().unwrap()], - connected_real_destination: Some("172.18.0.4:6379".parse().unwrap()), - upstream_socket_peer: "192.0.2.20:3128".parse().unwrap(), - dial_mode: "upstream_proxy_validated_ip", - mapping_id: uuid::Uuid::new_v4(), - mapping_generation: 4, - mapping_policy_generation: 7, - authorization_policy_generation: 7, - binary: "/usr/bin/redis-cli", - pid: "43", - policy_name: "redis", - }); - let json = event.to_json().unwrap(); - - assert_eq!( - json["unmapped"]["connected_real_destination"], - "172.18.0.4:6379" - ); - assert_eq!(json["unmapped"]["upstream_socket_peer"], "192.0.2.20:3128"); - assert_eq!(json["unmapped"]["dial_mode"], "upstream_proxy_validated_ip"); - assert!(event.format_shorthand().contains("real=172.18.0.4:6379")); - } - - #[cfg(target_os = "linux")] - #[tokio::test] - async fn transparent_tcp_ignores_proxy_hostname_mode_and_connects_to_validated_ip() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let proxy_addr = listener.local_addr().unwrap(); - let (request_tx, request_rx) = tokio::sync::oneshot::channel(); - let proxy = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.unwrap(); - let mut request = Vec::new(); - loop { - let mut byte = [0_u8; 1]; - stream.read_exact(&mut byte).await.unwrap(); - request.push(byte[0]); - if request.ends_with(b"\r\n\r\n") { - break; - } - } - request_tx.send(request).unwrap(); - stream - .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n") - .await - .unwrap(); - }); - let config = UpstreamProxyConfig::from_args(&upstream_proxy::UpstreamProxyArgs { - https_proxy: Some(format!("http://{proxy_addr}")), - proxy_connect_by_hostname: true, - ..Default::default() - }) - .unwrap() - .unwrap(); - let approved = "203.0.113.27:6379".parse().unwrap(); - - let stream = - dial_transparent_upstream(&Some(config), "redis.openshell.demo", 6379, &[approved]) - .await - .unwrap(); - let request = String::from_utf8(request_rx.await.unwrap()).unwrap(); - - assert!(request.starts_with("CONNECT 203.0.113.27:6379 HTTP/1.1\r\n")); - assert!(!request.contains("CONNECT redis.openshell.demo:6379")); - assert!(matches!( - stream.connect_target(), - Some(upstream_proxy::ConnectTarget::Ip(ip)) if ip == approved.ip() - )); - proxy.await.unwrap(); - } - - #[test] - fn forward_ocsf_events_omit_queries_and_credential_key_names() { - let peer = "127.0.0.1:45123".parse().unwrap(); - let path = forward_telemetry_path( - "http://api.example.com/v1/openshell:resolve:env:API_TOKEN?token=real-secret", - ); - assert_eq!(path, "/v1/[CREDENTIAL]"); - - let allowed = build_forward_allow_ocsf_event( - peer, - "GET", - "api.example.com", - 80, - &path, - "/usr/bin/curl", - "42", - "/usr/bin/bash", - "curl", - "allow_api", - ) - .to_json() - .unwrap(); - let denied = build_forward_policy_deny_ocsf_event( - peer, + "127.0.0.1:45123".parse().unwrap(), "GET", "api.example.com", 80, - &path, - "/usr/bin/curl", - "42", - "/usr/bin/bash", - "curl", - "policy denied", - ) - .to_json() - .unwrap(); - for event in [&allowed, &denied] { - assert_eq!(event["http_request"]["url"]["path"], "/v1/[CREDENTIAL]"); - let serialized = event.to_string(); - assert!(!serialized.contains("API_TOKEN"), "{serialized}"); - assert!(!serialized.contains("real-secret"), "{serialized}"); - assert!(!serialized.contains("?token="), "{serialized}"); - } - - let target = "http://api.example.com?token=real-secret"; - let (_, host, port, path) = parse_proxy_uri(target).expect("absolute URI without a path"); - assert_eq!(host, "api.example.com"); - assert_eq!(path, "/?token=real-secret"); - let no_path_query = build_forward_allow_ocsf_event( - peer, - "GET", - &host, - port, - &forward_telemetry_path(target), + "/v1/models", "/usr/bin/curl", "42", "/usr/bin/bash", - "curl", - "allow_api", - ) - .to_json() - .unwrap(); - assert_eq!(no_path_query["dst_endpoint"]["domain"], "api.example.com"); - assert_eq!(no_path_query["http_request"]["url"]["path"], "/"); - let serialized = no_path_query.to_string(); - assert!(!serialized.contains("real-secret"), "{serialized}"); - assert!(!serialized.contains("?token="), "{serialized}"); - - let malformed = build_forward_parse_error_ocsf_event(&forward_telemetry_path( - "not-a-uri?token=real-secret&key=openshell:resolve:env:API_TOKEN", - )) - .to_json() - .unwrap(); - assert_eq!( - malformed["message"], - "FORWARD parse error for /[INVALID_REQUEST_TARGET]" + "curl http://api.example.com/v1/models", + reason, ); - let serialized = malformed.to_string(); - assert!(!serialized.contains("API_TOKEN"), "{serialized}"); - assert!(!serialized.contains("real-secret"), "{serialized}"); + let json = event.to_json().unwrap(); + + assert_eq!(json["status_detail"], reason); + assert_eq!(json["action"], "Denied"); + assert_eq!(json["disposition"], "Blocked"); } #[test] @@ -7294,6 +5700,59 @@ network_policies: ); } + #[test] + fn backend_supplied_unresolved_identity_denies_an_allowed_endpoint() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, true) + .expect("identity-aware engine"); + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert!(matches!( + decision.identity, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed) + )); + } + + #[test] + fn backend_supplied_unresolved_identity_denies_in_endpoint_only_mode() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +network_policies: + test_allow: + name: test_allow + endpoints: + - { host: api.example.com, port: 443 } +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) + .expect("endpoint-only engine"); + temp_env::with_var( + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + Some("false"), + || { + let decision = authorize_supplied_identity( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + &Err(ResolveError::NotFound), + ); + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + }, + ); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, @@ -7354,13 +5813,14 @@ network_policies: let addr = listener.local_addr().unwrap(); let mut client = TcpStream::connect(addr).await.unwrap(); let (server, _) = listener.accept().await.unwrap(); + let mut server = tokio::io::BufReader::new(server); client .write_all(crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE) .await .unwrap(); - let protocol = peek_tunnel_protocol(&server) + let protocol = peek_tunnel_protocol(&mut server) .await .expect("peek should succeed") .expect("client sent bytes"); @@ -7471,29 +5931,6 @@ network_policies: ); } - #[test] - fn revision_scoped_dynamic_credentials_preserves_endpoint_selector_and_adds_revision() { - let mut dynamic_credentials = std::collections::HashMap::new(); - dynamic_credentials.insert( - "api.example.test\t443\t/v1/**\tprovider:access_token".to_string(), - openshell_core::proto::ProviderProfileCredential { - name: "access_token".to_string(), - ..Default::default() - }, - ); - let snapshot = ProviderCredentialSnapshot { - revision: 42, - child_env: std::collections::HashMap::new(), - dynamic_credentials, - }; - - let scoped = revision_scoped_dynamic_credentials(&snapshot); - - assert!( - scoped.contains_key("api.example.test\t443\t/v1/**\trev:42\tprovider:access_token") - ); - } - #[test] fn connect_activity_is_skipped_when_l7_will_count_the_request() { let (tx, mut rx) = mpsc::channel(4); @@ -7649,7 +6086,6 @@ network_policies: let ctx = crate::l7::relay::L7EvalContext { host: "api.example.test".into(), port: 80, - request_default_port: Some(80), policy_name: "jsonrpc_api".into(), binary_path: "/usr/bin/node".into(), ancestors: vec![], @@ -7701,131 +6137,6 @@ network_policies: } } - #[tokio::test] - async fn forward_reacquires_static_credentials_after_blocked_middleware() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - let policy = include_str!("../data/sandbox-policy.rego"); - let engine = OpaEngine::from_strings(policy, "network_policies: {}\n").unwrap(); - let guard = engine - .generation_guard(engine.current_generation()) - .expect("generation guard"); - let entered = Arc::new(tokio::sync::Notify::new()); - let release = Arc::new(tokio::sync::Notify::new()); - let runner = openshell_supervisor_middleware::ChainRunner::new(Arc::new( - BlockingForwardMiddleware { - entered: Arc::clone(&entered), - release: Arc::clone(&release), - }, - )); - let state = ProviderCredentialState::from_bound_environment( - 1, - TestHashMap::from([("API_TOKEN".to_string(), "real-secret".to_string())]), - TestHashMap::new(), - TestHashMap::new(), - TestHashMap::from([( - "API_TOKEN".to_string(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "api.example.test".to_string(), - port: 80, - path: "/allowed/**".to_string(), - }], - credential_identity: "provider-a:API_TOKEN".to_string(), - workload_credential_handle: String::new(), - }, - )]), - Vec::new(), - ) - .expect("bound provider state"); - let ctx = crate::l7::relay::L7EvalContext { - host: "api.example.test".into(), - port: 80, - request_default_port: Some(80), - policy_name: "forward".into(), - binary_path: "/usr/bin/node".into(), - provider_credentials: Some(state.clone()), - secret_resolver: state.resolver(), - ..Default::default() - }; - let raw = b"GET http://api.example.test/allowed/../outside HTTP/1.1\r\nHost: api.example.test\r\nAuthorization: Bearer openshell:resolve:env:v1_API_TOKEN\r\n\r\n"; - let prepared = prepare_forward_target( - "/allowed/../outside", - crate::l7::path::CanonicalizeOptions::default(), - ) - .expect("canonical target"); - assert_eq!(prepared.canonical_path, "/outside"); - let request = crate::l7::rest::request_from_buffered_http( - "GET", - &prepared.canonical_path, - &prepared.upstream_target, - canonicalize_forward_host_header(raw, "api.example.test").unwrap(), - ) - .unwrap(); - let pipeline = ForwardMiddlewarePipeline { - ctx: &ctx, - scheme: "http", - runner: &runner, - generation_guard: &guard, - l7_reevaluation: None, - }; - let chain = vec![openshell_supervisor_middleware::ChainEntry { - name: "blocker".into(), - implementation: "test/blocking-forward".into(), - order: 0, - config: prost_types::Struct::default(), - on_error: openshell_supervisor_middleware::OnError::FailClosed, - }]; - let (_app, mut client) = tokio::io::duplex(8192); - let revoke = async { - entered.notified().await; - state.revoke_static_provider_environment(2); - release.notify_one(); - }; - let (outcome, ()) = tokio::join!(pipeline.apply(request, &mut client, chain), revoke); - let request = match outcome.expect("middleware pipeline") { - crate::l7::middleware::MiddlewareApplyResult::Allowed(request) => request, - crate::l7::middleware::MiddlewareApplyResult::Denied { .. } => { - panic!("blocking middleware should allow after release") - } - crate::l7::middleware::MiddlewareApplyResult::AdmissionExhausted => { - panic!("blocking middleware should already hold admission") - } - }; - - let credentials = endpoint_credentials_for_request( - ctx.provider_credentials.as_ref(), - ctx.secret_resolver.clone(), - &ctx.host, - ctx.port, - &prepared.canonical_path, - ); - assert!( - credentials.resolver.is_none(), - "revoked live state must supersede the connection-open resolver" - ); - let rewrite = rewrite_forward_request( - &request.raw_header, - request.raw_header.len(), - &prepared.upstream_target, - "api.example.test", - credentials.resolver.as_deref(), - false, - ); - assert!( - rewrite.is_err(), - "revoked credential placeholder must fail before upstream relay" - ); - - let (proxy_upstream, mut upstream) = tokio::io::duplex(8192); - drop(proxy_upstream); - let mut forwarded = Vec::new(); - upstream.read_to_end(&mut forwarded).await.unwrap(); - assert!( - forwarded.is_empty(), - "revoked forward credential request must not reach upstream" - ); - } - #[test] fn forward_l7_allowed_activity_is_deferred_until_after_ssrf() { let (tx, mut rx) = mpsc::channel(4); @@ -7969,16 +6280,9 @@ network_policies: &mut proxy_to_upstream, ForwardRelayOptions { generation_guard: &guard, - credential_generation: None, websocket_extensions: crate::l7::rest::WebSocketExtensionMode::Preserve, secret_resolver: resolver, request_body_credential_rewrite, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::None, - signing_service: "", - signing_region: "", - host: "", - port: 0, }, ) .await?; @@ -7995,11 +6299,23 @@ network_policies: crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, ) { let provider_key = "api.example.test\t8080\t/v1/**\tprovider:access_token"; - let fixture = forward_token_grant_fixture(provider_key, resolver_response, false); + let fixture = match resolver_response { + Ok(token) => { + crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::success( + provider_key, + token, + ) + } + Err(error) => { + crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::failure( + provider_key, + error, + ) + } + }; let ctx = crate::l7::relay::L7EvalContext { host: "api.example.test".into(), port: 8080, - request_default_port: Some(8080), policy_name: "rest_api".into(), binary_path: "/usr/bin/curl".into(), ancestors: vec![], @@ -8013,54 +6329,6 @@ network_policies: (ctx, fixture) } - fn forward_token_exchange_context( - resolver_response: std::result::Result<&str, &str>, - ) -> ( - crate::l7::relay::L7EvalContext, - crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, - ) { - let (mut ctx, _) = forward_token_grant_context(Ok("unused-token")); - let provider_key = "api.example.test\t8080\t/v1/**\tprovider:access_token"; - let fixture = forward_token_grant_fixture(provider_key, resolver_response, true); - ctx.dynamic_credentials = Some(fixture.dynamic_credentials()); - ctx.token_grant_resolver = Some(fixture.resolver()); - - (ctx, fixture) - } - - fn forward_token_grant_fixture( - provider_key: &str, - resolver_response: std::result::Result<&str, &str>, - token_exchange: bool, - ) -> crate::l7::token_grant_injection::test_support::TokenGrantTestFixture { - match (resolver_response, token_exchange) { - (Ok(token), false) => { - crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::success( - provider_key, - token, - ) - } - (Ok(token), true) => { - crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::success_token_exchange( - provider_key, - token, - ) - } - (Err(error), false) => { - crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::failure( - provider_key, - error, - ) - } - (Err(error), true) => { - crate::l7::token_grant_injection::test_support::TokenGrantTestFixture::failure_token_exchange( - provider_key, - error, - ) - } - } - } - fn authorization_header_count(headers: &str) -> usize { headers .lines() @@ -8084,28 +6352,21 @@ network_policies: ) { let policy = include_str!("../data/sandbox-policy.rego"); let engine = OpaEngine::from_strings(policy, data).unwrap(); - let authorization = engine - .authorize_egress(&crate::opa::NetworkInput { - host: host.to_string(), - port, - binary_path: PathBuf::from("/usr/bin/node"), - binary_sha256: String::new(), - ancestors: vec![], - cmdline_paths: vec![], - }) - .expect("authorize egress"); let decision = EgressDecision { intent: EgressIntent::forward_http(host.to_string(), port), - action: authorization.action.clone(), - policy_generation: authorization.generation, + action: NetworkAction::Allow { + matched_policy: Some(policy_name.to_string()), + }, + policy_generation: engine.current_generation(), identity: ProcessIdentityEvidence::Available, - endpoint: EndpointDecision::from_authorization(&authorization), + endpoint: EndpointDecision::default(), binary: Some(PathBuf::from("/usr/bin/node")), binary_pid: None, ancestors: vec![], cmdline_paths: vec![], }; - let route = query_l7_route_snapshot(&decision, host, port).expect("L7 route should match"); + let route = + query_l7_route_snapshot(&engine, &decision, host, port).expect("L7 route should match"); let config = select_l7_config_for_path(&route.configs, path) .expect("path-specific L7 config should match") .config @@ -8116,7 +6377,6 @@ network_policies: let ctx = crate::l7::relay::L7EvalContext { host: host.to_string(), port, - request_default_port: Some(port), policy_name: policy_name.to_string(), binary_path: "/usr/bin/node".to_string(), ancestors: vec![], @@ -8128,36 +6388,14 @@ network_policies: } async fn read_http_headers(reader: &mut R) -> Vec { - read_http_headers_with_timeout(reader, std::time::Duration::from_secs(1)).await - } - - async fn read_http_headers_with_timeout( - reader: &mut R, - timeout: std::time::Duration, - ) -> Vec { - let mut bytes = Vec::new(); - let mut chunk = [0u8; 256]; - loop { - let n = tokio::time::timeout(timeout, reader.read(&mut chunk)) - .await - .expect("HTTP headers should arrive") - .expect("header read should succeed"); - assert!(n > 0, "stream closed before HTTP headers"); - bytes.extend_from_slice(&chunk[..n]); - if bytes.windows(4).any(|w| w == b"\r\n\r\n") { - return bytes; - } - } - } - - async fn read_http_headers_unbounded(reader: &mut R) -> Vec { let mut bytes = Vec::new(); let mut chunk = [0u8; 256]; loop { - let n = reader - .read(&mut chunk) - .await - .expect("header read should succeed"); + let n = + tokio::time::timeout(std::time::Duration::from_secs(1), reader.read(&mut chunk)) + .await + .expect("HTTP headers should arrive") + .expect("header read should succeed"); assert!(n > 0, "stream closed before HTTP headers"); bytes.extend_from_slice(&chunk[..n]); if bytes.windows(4).any(|w| w == b"\r\n\r\n") { @@ -8184,14 +6422,6 @@ network_policies: frame } - fn masked_close_code(frame: &[u8]) -> u16 { - assert_eq!(frame[0] & 0x0f, 0x08, "expected a close frame"); - assert_eq!(frame[1] & 0x7f, 2, "expected a two-byte close code"); - assert_ne!(frame[1] & 0x80, 0, "client-to-upstream close is masked"); - let decoded = [frame[6] ^ frame[2], frame[7] ^ frame[3]]; - u16::from_be_bytes(decoded) - } - async fn forward_websocket_denied_after_upgrade( config: crate::l7::L7EndpointConfig, tunnel_engine: crate::opa::TunnelPolicyEngine, @@ -8234,16 +6464,9 @@ network_policies: &mut proxy_to_upstream, ForwardRelayOptions { generation_guard: guard, - credential_generation: None, websocket_extensions, secret_resolver: None, request_body_credential_rewrite: false, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::None, - signing_service: "", - signing_region: "", - host: "", - port: 0, }, ) .await?; @@ -8328,7 +6551,6 @@ network_policies: let ctx = crate::l7::relay::L7EvalContext { host: "gateway.example.test".to_string(), port: 80, - request_default_port: Some(80), policy_name: "ws_api".to_string(), binary_path: "/usr/bin/node".to_string(), ancestors: vec![], @@ -8357,7 +6579,6 @@ network_policies: ); assert!(options.websocket.credential_rewrite); assert!(options.secret_resolver.is_some()); - assert!(options.generation_guard.is_some()); assert!(options.engine.is_some()); assert!(options.ctx.is_some()); assert!(matches!( @@ -8371,7 +6592,6 @@ network_policies: let ctx = crate::l7::relay::L7EvalContext { host: "gateway.example.test".to_string(), port: 80, - request_default_port: Some(80), policy_name: "rest_api".to_string(), binary_path: "/usr/bin/node".to_string(), ancestors: vec![], @@ -8399,41 +6619,6 @@ network_policies: )); } - #[test] - fn rest_websocket_upgrade_carries_guard_without_message_inspector() { - let engine = OpaEngine::from_strings( - include_str!("../data/sandbox-policy.rego"), - "network_policies: {}\n", - ) - .expect("test policy"); - let tunnel_engine = engine - .clone_engine_for_tunnel(engine.current_generation()) - .expect("tunnel engine"); - let ctx = crate::l7::relay::L7EvalContext { - host: "gateway.example.test".into(), - port: 443, - policy_name: "rest_api".into(), - binary_path: "/usr/bin/node".into(), - ..Default::default() - }; - let config = websocket_l7_config(crate::l7::L7Protocol::Rest, false); - let options = crate::l7::relay::upgrade_options( - &config, - &ctx, - true, - "/ws", - &std::collections::HashMap::new(), - Some(&tunnel_engine), - ); - - assert!(matches!( - options.websocket.message_policy, - crate::l7::relay::WebSocketMessagePolicy::None - )); - assert!(options.generation_guard.is_some()); - assert!(options.engine.is_some()); - } - #[tokio::test] async fn forward_websocket_upgrade_blocks_text_frame_by_policy() { let data = r#" @@ -8472,10 +6657,9 @@ network_policies: .await; assert!(err.to_string().contains("websocket text message denied")); - assert_eq!( - masked_close_code(&leaked), - 1008, - "only a policy close, not the denied text frame, may reach upstream" + assert!( + leaked.is_empty(), + "denied forward-proxy WebSocket text frames must not reach upstream" ); } @@ -8526,10 +6710,9 @@ network_policies: .await; assert!(err.to_string().contains("websocket GraphQL message denied")); - assert_eq!( - masked_close_code(&leaked), - 1008, - "only a policy close, not the denied GraphQL operation, may reach upstream" + assert!( + leaked.is_empty(), + "denied forward-proxy GraphQL WebSocket operations must not reach upstream" ); } @@ -10243,9 +8426,10 @@ network_policies: #[tokio::test] async fn test_resolve_check_allowed_ips_rejects_outside_allowlist() { - // 8.8.8.8 resolves to a public IP which is NOT in 10.0.0.0/8 + // A public IP outside 10.0.0.0/8 must be rejected. Use the literal so + // this security check does not depend on external DNS availability. let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("dns.google", 443, &nets, 0).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 443, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -10457,167 +8641,15 @@ network_policies: #[test] fn test_parse_proxy_uri_missing_path() { let (_, host, port, path) = parse_proxy_uri("http://10.0.0.1:9090").unwrap(); - assert_eq!(host, "10.0.0.1"); - assert_eq!(port, 9090); - assert_eq!(path, "/"); - } - - #[test] - fn test_parse_proxy_uri_with_query() { - let (_, _, _, path) = parse_proxy_uri("http://host:80/api?key=val&foo=bar").unwrap(); - assert_eq!(path, "/api?key=val&foo=bar"); - } - - #[test] - fn test_parse_proxy_uri_with_query_and_no_path() { - let (_, host, port, path) = parse_proxy_uri("http://host:8080?key=val&foo=bar").unwrap(); - assert_eq!(host, "host"); - assert_eq!(port, 8080); - assert_eq!(path, "/?key=val&foo=bar"); - } - - #[test] - fn forward_telemetry_path_omits_queries_and_redacts_credential_syntax() { - let target = "http://host:80/v1/openshell:resolve:env:API_TOKEN?token=real-secret"; - let redacted = forward_telemetry_path(target); - assert_eq!(redacted, "/v1/[CREDENTIAL]"); - assert!(!redacted.contains("API_TOKEN")); - assert!(!redacted.contains("real-secret")); - - let malformed = forward_telemetry_path( - "not-a-uri?token=real-secret&key=openshell:resolve:env:API_TOKEN", - ); - assert_eq!(malformed, "/[INVALID_REQUEST_TARGET]"); - assert!(!malformed.contains("API_TOKEN")); - assert!(!malformed.contains("real-secret")); - } - - #[test] - fn forward_credentials_capture_endpoint_resolver_and_revision_together() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - - let state = ProviderCredentialState::from_bound_environment( - 42, - TestHashMap::from([("API_TOKEN".to_string(), "secret".to_string())]), - TestHashMap::new(), - TestHashMap::new(), - TestHashMap::from([( - "API_TOKEN".to_string(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "api.example.com".to_string(), - port: 80, - path: "/allowed/**".to_string(), - }], - credential_identity: "provider-a:API_TOKEN".to_string(), - workload_credential_handle: String::new(), - }, - )]), - Vec::new(), - ) - .expect("bound provider state"); - - let credentials = endpoint_credentials_for_request( - Some(&state), - None, - "api.example.com", - 80, - "/allowed/v1", - ); - assert_eq!(credentials.revision, Some(42)); - assert_eq!( - credentials - .resolver - .expect("endpoint resolver") - .resolve_placeholder("openshell:resolve:env:v42_API_TOKEN"), - Some("secret") - ); - } - - #[test] - fn forward_binding_uses_canonical_path_for_dot_segment_traversal() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - - let state = ProviderCredentialState::from_bound_environment( - 1, - TestHashMap::from([("API_TOKEN".to_string(), "secret".to_string())]), - TestHashMap::new(), - TestHashMap::new(), - TestHashMap::from([( - "API_TOKEN".to_string(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "api.example.com".to_string(), - port: 80, - path: "/allowed/**".to_string(), - }], - credential_identity: "provider-a:API_TOKEN".to_string(), - workload_credential_handle: String::new(), - }, - )]), - Vec::new(), - ) - .expect("bound provider state"); - let placeholder = "openshell:resolve:env:v1_API_TOKEN"; - - for raw_path in ["/allowed/../outside", "/allowed/%2e%2e/outside"] { - let prepared = - prepare_forward_target(raw_path, crate::l7::path::CanonicalizeOptions::default()) - .expect("prepared target"); - assert_eq!(prepared.canonical_path, "/outside"); - let resolver = endpoint_secret_resolver( - Some(&state), - state.resolver(), - "api.example.com", - 80, - &prepared.canonical_path, - ) - .expect("scoped resolver"); - let error = resolver - .rewrite_header_value(placeholder) - .expect_err("canonical endpoint must deny traversal"); - assert!(error.is_endpoint_mismatch()); - } + assert_eq!(host, "10.0.0.1"); + assert_eq!(port, 9090); + assert_eq!(path, "/"); } #[test] - fn live_forward_state_is_authoritative_after_revocation() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - - let state = ProviderCredentialState::from_bound_environment( - 1, - TestHashMap::from([("API_TOKEN".to_string(), "secret".to_string())]), - TestHashMap::new(), - TestHashMap::new(), - TestHashMap::from([( - "API_TOKEN".to_string(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "api.example.com".to_string(), - port: 80, - path: "/**".to_string(), - }], - credential_identity: "provider-a:API_TOKEN".to_string(), - workload_credential_handle: String::new(), - }, - )]), - Vec::new(), - ) - .expect("bound provider state"); - let connection_open_resolver = state.resolver(); - state.revoke_static_provider_environment(2); - - assert!( - endpoint_secret_resolver( - Some(&state), - connection_open_resolver, - "api.example.com", - 80, - "/v1", - ) - .is_none(), - "live revocation must not fall back to the connection-open resolver" - ); + fn test_parse_proxy_uri_with_query() { + let (_, _, _, path) = parse_proxy_uri("http://host:80/api?key=val&foo=bar").unwrap(); + assert_eq!(path, "/api?key=val&foo=bar"); } #[test] @@ -10636,20 +8668,6 @@ network_policies: assert_eq!(path, "/path"); } - #[test] - fn test_parse_proxy_uri_ipv6_with_query_and_no_path() { - let (_, host, port, path) = parse_proxy_uri("http://[fe80::1]:8080?key=val").unwrap(); - assert_eq!(host, "fe80::1"); - assert_eq!(port, 8080); - assert_eq!(path, "/?key=val"); - } - - #[test] - fn test_parse_proxy_uri_rejects_fragment() { - assert!(parse_proxy_uri("http://example.com#secret").is_err()); - assert!(parse_proxy_uri("http://[fe80::1]#secret").is_err()); - } - #[test] fn test_parse_proxy_uri_missing_scheme() { let result = parse_proxy_uri("example.com/path"); @@ -10671,16 +8689,6 @@ network_policies: assert_eq!(port, 443); } - #[test] - fn test_normalize_host_strips_single_trailing_dot() { - assert_eq!(normalize_host("api.example.com."), "api.example.com"); - } - - #[test] - fn test_normalize_host_remains_the_same() { - assert_eq!(normalize_host("api.example.com"), "api.example.com"); - } - #[test] fn test_parse_target_preserves_case() { let (host, port) = parse_target("EXAMPLE.COM:443").unwrap(); @@ -10840,14 +8848,6 @@ network_policies: // -- parse_proxy_uri: hostname parser regression tests -- - #[test] - fn test_parse_proxy_uri_trailing_dot_host() { - let (_, host, port, _) = parse_proxy_uri("http://api.example.com.:80/path").unwrap(); - let host = normalize_host(&host); - assert_eq!(host, "api.example.com"); - assert_eq!(port, 80_u16); - } - #[test] fn test_parse_proxy_uri_nul_byte_in_host() { let (_, host, port, _) = parse_proxy_uri("http://evil.com\0.safe.com:80/path").unwrap(); @@ -10916,34 +8916,6 @@ network_policies: fixture.assert_one_request("api.example.test\t8080\t/v1/**\tprovider:access_token"); } - #[tokio::test] - async fn forward_proxy_injects_token_exchange_before_rewriting_request() { - let (ctx, fixture) = forward_token_exchange_context(Ok("grant-token")); - let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer stale-token\r\nConnection: close\r\n\r\n".to_vec(); - - let with_token = inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &ctx) - .await - .expect("forward token exchange should inject"); - let rewritten = rewrite_forward_request( - &with_token, - with_token.len(), - "/v1/projects", - "api.example.test:8080", - None, - false, - ) - .expect("forward request should rewrite"); - let rewritten = String::from_utf8_lossy(&rewritten); - - assert!(rewritten.starts_with("GET /v1/projects HTTP/1.1\r\n")); - assert!(rewritten.contains("Authorization: Bearer grant-token\r\n")); - assert!(!rewritten.contains("stale-token")); - assert_eq!(authorization_header_count(&rewritten), 1); - fixture.assert_one_token_exchange_request( - "api.example.test\t8080\t/v1/**\tprovider:access_token", - ); - } - #[tokio::test] async fn forward_proxy_token_grant_failure_returns_error_before_rewrite() { let (ctx, fixture) = forward_token_grant_context(Err("oauth unavailable")); @@ -10958,22 +8930,6 @@ network_policies: fixture.assert_one_request("api.example.test\t8080\t/v1/**\tprovider:access_token"); } - #[tokio::test] - async fn forward_proxy_token_exchange_failure_returns_error_before_rewrite() { - let (ctx, fixture) = forward_token_exchange_context(Err("oauth unavailable")); - let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - - let err = inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &ctx) - .await - .expect_err("forward token exchange failure should stop request rewriting"); - - assert!(err.to_string().contains("Token grant failed")); - assert!(err.to_string().contains("oauth unavailable")); - fixture.assert_one_token_exchange_request( - "api.example.test\t8080\t/v1/**\tprovider:access_token", - ); - } - #[test] fn test_rewrite_get_request() { let raw = @@ -11066,7 +9022,6 @@ network_policies: let ctx = crate::l7::relay::L7EvalContext { host: authority.into(), port: 80, - request_default_port: Some(80), policy_name: "test".into(), binary_path: "/usr/bin/node".into(), ancestors: vec![], @@ -11335,35 +9290,19 @@ network_policies: } #[tokio::test] - async fn forward_relay_body_endpoint_mismatch_is_typed_before_upstream_write() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - let state = ProviderCredentialState::from_bound_environment( - 1, - TestHashMap::from([("API_TOKEN".to_string(), "provider-real-token".to_string())]), - TestHashMap::new(), - TestHashMap::new(), - TestHashMap::from([( - "API_TOKEN".to_string(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "allowed.example.com".to_string(), - port: 80, - path: "/allowed/**".to_string(), - }], - credential_identity: "provider-a:API_TOKEN".to_string(), - workload_credential_handle: String::new(), - }, - )]), - Vec::new(), - ) - .expect("bound provider state"); - let resolver = state - .resolver_for_endpoint("api.example.com", 80, "/api/messages") - .expect("endpoint-scoped resolver"); - let body = "token=openshell:resolve:env:v1_API_TOKEN"; + async fn forward_relay_unresolved_body_placeholder_fails_before_upstream_write() { + let (_, resolver) = SecretResolver::from_provider_env( + [("API_TOKEN".to_string(), "provider-real-token".to_string())] + .into_iter() + .collect(), + ); + let resolver = resolver.expect("resolver"); + let alias = "provider-OPENSHELL-RESOLVE-ENV-API_TOKEN"; + let body = "token=provider-OPENSHELL-RESOLVE-ENV-MISSING_TOKEN"; let raw = format!( "POST http://api.example.com/api/messages HTTP/1.1\r\n\ Host: api.example.com\r\n\ + Authorization: Bearer {alias}\r\n\ Content-Type: application/x-www-form-urlencoded\r\n\ Content-Length: {}\r\n\r\n{}", body.len(), @@ -11390,27 +9329,16 @@ network_policies: &mut proxy_to_upstream, ForwardRelayOptions { generation_guard: &guard, - credential_generation: None, websocket_extensions: crate::l7::rest::WebSocketExtensionMode::Preserve, secret_resolver: Some(&resolver), request_body_credential_rewrite: true, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::None, - signing_service: "", - signing_region: "", - host: "", - port: 0, }, ) .await .expect_err("unresolved body placeholder should fail closed"); - let credential_error = err - .downcast_ref::() - .expect("body mismatch must retain its typed error"); - assert!(credential_error.is_endpoint_mismatch()); assert!(!err.to_string().contains("provider-real-token")); - assert!(!err.to_string().contains("API_TOKEN")); + assert!(!err.to_string().contains("MISSING_TOKEN")); drop(proxy_to_upstream); let mut forwarded = Vec::new(); upstream_side.read_to_end(&mut forwarded).await.unwrap(); @@ -11420,85 +9348,6 @@ network_policies: ); } - #[tokio::test] - async fn forward_relay_sigv4_endpoint_mismatch_is_typed_before_upstream_write() { - use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; - let values = TestHashMap::from([ - ("AWS_ACCESS_KEY_ID".to_string(), "access".to_string()), - ("AWS_SECRET_ACCESS_KEY".to_string(), "secret".to_string()), - ("AWS_SESSION_TOKEN".to_string(), "session".to_string()), - ]); - let bindings = values - .keys() - .map(|key| { - ( - key.clone(), - StaticCredentialBinding { - endpoints: vec![StaticCredentialEndpointBinding { - host: "allowed.example.com".to_string(), - port: 80, - path: "/allowed/**".to_string(), - }], - credential_identity: format!("provider-a:{key}"), - workload_credential_handle: String::new(), - }, - ) - }) - .collect(); - let state = ProviderCredentialState::from_bound_environment( - 1, - values, - TestHashMap::new(), - TestHashMap::new(), - bindings, - Vec::new(), - ) - .expect("bound provider state"); - let resolver = state - .resolver_for_endpoint("api.example.com", 80, "/api") - .expect("endpoint-scoped resolver"); - let guard = forward_test_guard(); - let rewritten = - b"GET /api HTTP/1.1\r\nHost: api.example.com\r\nContent-Length: 0\r\n\r\n".to_vec(); - let (mut proxy_to_upstream, mut upstream_side) = tokio::io::duplex(8192); - let (mut _app_side, mut proxy_to_client) = tokio::io::duplex(8192); - - let err = relay_rewritten_forward_request( - "GET", - "/api", - rewritten, - &mut proxy_to_client, - &mut proxy_to_upstream, - ForwardRelayOptions { - generation_guard: &guard, - credential_generation: None, - websocket_extensions: crate::l7::rest::WebSocketExtensionMode::Preserve, - secret_resolver: Some(&resolver), - request_body_credential_rewrite: false, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::SigV4NoBody, - signing_service: "execute-api", - signing_region: "us-west-2", - host: "api.example.com", - port: 80, - }, - ) - .await - .expect_err("SigV4 endpoint mismatch should fail closed"); - - let credential_error = err - .downcast_ref::() - .expect("SigV4 mismatch must retain its typed error"); - assert!(credential_error.is_endpoint_mismatch()); - drop(proxy_to_upstream); - let mut forwarded = Vec::new(); - upstream_side.read_to_end(&mut forwarded).await.unwrap(); - assert!( - forwarded.is_empty(), - "failed SigV4 credential lookup must not reach upstream" - ); - } - #[test] fn test_forward_rewrite_preserves_websocket_upgrade_connection_header() { let raw = "GET http://gateway.example.test/ws HTTP/1.1\r\n\ @@ -11560,16 +9409,9 @@ network_policies: &mut proxy_to_upstream, ForwardRelayOptions { generation_guard: &guard, - credential_generation: None, websocket_extensions: crate::l7::rest::WebSocketExtensionMode::Preserve, secret_resolver: None, request_body_credential_rewrite: false, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::None, - signing_service: "", - signing_region: "", - host: "", - port: 0, }, ) .await; @@ -11610,16 +9452,9 @@ network_policies: &mut proxy_to_upstream, ForwardRelayOptions { generation_guard: &guard, - credential_generation: None, websocket_extensions: crate::l7::rest::WebSocketExtensionMode::Preserve, secret_resolver: None, request_body_credential_rewrite: false, - deny_uninspected_credentials: false, - credential_signing: crate::l7::CredentialSigning::None, - signing_service: "", - signing_region: "", - host: "", - port: 0, }, ) .await; @@ -11644,9 +9479,9 @@ network_policies: #[tokio::test] async fn test_forward_public_ip_allowed_without_allowed_ips() { - // Public IPs (e.g. dns.google -> 8.8.8.8) should pass through - // resolve_and_reject_internal without needing allowed_ips. - let result = resolve_and_reject_internal("dns.google", 80, 0).await; + // Public IPs should pass through resolve_and_reject_internal without + // needing allowed_ips. Use a literal to keep the test hermetic. + let result = resolve_and_reject_internal("8.8.8.8", 80, 0).await; assert!( result.is_ok(), "Public IP should be allowed without allowed_ips: {result:?}" @@ -11657,7 +9492,7 @@ network_policies: for addr in &addrs { assert!( !is_internal_ip(addr.ip()), - "dns.google should resolve to public IPs, got {}", + "expected a public IP, got {}", addr.ip() ); } @@ -12100,7 +9935,6 @@ network_policies: AgentProposals::default(), // agent_proposals Arc::new(None), // trusted_host_gateway Arc::new(None), // upstream_proxy - None, // provider_credentials None, // secret_resolver None, // dynamic_credentials Some(denial_tx), // denial_tx — positive allow/deny signal @@ -12169,7 +10003,6 @@ network_policies: Arc::new(None), None, None, - None, Some(denial_tx), None, )) @@ -12348,8 +10181,10 @@ network_policies: ancestors: vec![], cmdline_paths: vec![], }; - let authorization = engine.authorize_egress(&input).expect("evaluate"); - match &authorization.action { + let (action, generation) = engine + .evaluate_network_action_with_generation(&input) + .expect("evaluate"); + match &action { NetworkAction::Allow { matched_policy } => { assert!(matched_policy.is_some(), "allow must carry the policy name"); } @@ -12359,16 +10194,16 @@ network_policies: } let decision = EgressDecision { intent: EgressIntent::connect("203.0.113.10".to_string(), 443), - action: authorization.action.clone(), - policy_generation: authorization.generation, + action, + policy_generation: generation, identity: ProcessIdentityEvidence::Available, - endpoint: EndpointDecision::from_authorization(&authorization), + endpoint: EndpointDecision::default(), binary: Some(input.binary_path), binary_pid: Some(1), ancestors: vec![], cmdline_paths: vec![], }; - query_tls_mode(&decision, "203.0.113.10", 443) + query_tls_mode(&engine, &decision, "203.0.113.10", 443) }; assert_eq!( @@ -12699,318 +10534,335 @@ network_policies: } } - #[tokio::test] - async fn test_exit_receiver_fires_when_task_exits() { - let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); - let handle = tokio::spawn(async move { - let _guard = exited_tx; - }); - handle.await.unwrap(); - // The sender was dropped when the task completed, so the receiver - // should resolve immediately with an Err (sender dropped). - assert!(exited_rx.await.is_err()); + #[test] + fn accept_backoff_exponential_progression() { + let ms = |n| accept_backoff(n).as_millis(); + assert_eq!(ms(1), 100); + assert_eq!(ms(2), 200); + assert_eq!(ms(3), 400); + assert_eq!(ms(4), 800); + assert_eq!(ms(5), 1_600); + assert_eq!(ms(6), 3_200); + assert_eq!(ms(7), 5_000); // 6400 capped to 5000 + assert_eq!(ms(8), 5_000); // stays at cap } - #[tokio::test] - async fn test_exit_receiver_fires_when_task_is_aborted() { - let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); - let handle = tokio::spawn(async move { - let _guard = exited_tx; - std::future::pending::<()>().await; - }); - handle.abort(); - // Abort drops the task's locals, including the sender guard. - assert!(exited_rx.await.is_err()); + #[test] + fn accept_backoff_zero_consecutive_errors() { + assert_eq!(accept_backoff(0).as_millis(), 100); } - #[tokio::test] - async fn test_take_exit_receiver_returns_real_receiver() { - let (tx, rx) = tokio::sync::oneshot::channel::<()>(); - let join = tokio::spawn(std::future::pending::<()>()); - let mut handle = ProxyHandle { - http_addr: None, - join, - exited_rx: Some(rx), - }; - let mut taken = handle - .take_exit_receiver() - .expect("first take should return Some"); - assert!(taken.try_recv().is_err()); - drop(tx); - assert!(taken.await.is_err()); + #[test] + fn accept_backoff_saturates_at_cap() { + assert_eq!(accept_backoff(100).as_millis(), 5_000); + assert_eq!(accept_backoff(u32::MAX).as_millis(), 5_000); } - #[tokio::test] - async fn test_take_exit_receiver_second_call_returns_none() { - let (_tx, rx) = tokio::sync::oneshot::channel::<()>(); - let join = tokio::spawn(std::future::pending::<()>()); - let mut handle = ProxyHandle { - http_addr: None, - join, - exited_rx: Some(rx), - }; - let _first = handle.take_exit_receiver(); - assert!(handle.take_exit_receiver().is_none()); + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_emfile() { + let err = std::io::Error::from_raw_os_error(libc::EMFILE); + assert!(is_resource_pressure_error(&err)); } - #[tokio::test] - async fn test_proxy_handle_drop_fires_exit_receiver() { - let (exited_tx, exited_rx) = tokio::sync::oneshot::channel::<()>(); - let join = tokio::spawn(async move { - let _guard = exited_tx; - std::future::pending::<()>().await; - }); - let mut handle = ProxyHandle { - http_addr: None, - join, - exited_rx: Some(exited_rx), - }; - let rx = handle.take_exit_receiver().expect("should return Some"); - drop(handle); - assert!(rx.await.is_err()); + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_enfile() { + let err = std::io::Error::from_raw_os_error(libc::ENFILE); + assert!(is_resource_pressure_error(&err)); } - // --- classify_accept_error tests --- + #[cfg(unix)] + #[test] + fn is_resource_pressure_detects_memory_pressure() { + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOBUFS) + )); + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOMEM) + )); + assert!(is_resource_pressure_error( + &std::io::Error::from_raw_os_error(libc::ENOSR) + )); + } #[cfg(unix)] #[test] - fn test_classify_terminal_error_ebadf() { - let err = std::io::Error::from_raw_os_error(libc::EBADF); - let mut fd = 0; - let mut unk = 0; + fn is_resource_pressure_rejects_other_errors() { + let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + assert!(!is_resource_pressure_error(&err)); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_fd_exhaustion_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EMFILE)), + AcceptErrorClass::Transient, + ); assert_eq!( - classify_accept_error(&err, &mut fd, &mut unk), - AcceptAction::Terminal + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENFILE)), + AcceptErrorClass::Transient, ); } #[cfg(unix)] #[test] - fn test_classify_terminal_error_einval() { - let err = std::io::Error::from_raw_os_error(libc::EINVAL); - let mut fd = 0; - let mut unk = 0; + fn classify_accept_error_connection_errors_are_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ECONNABORTED)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ECONNRESET)), + AcceptErrorClass::Transient, + ); assert_eq!( - classify_accept_error(&err, &mut fd, &mut unk), - AcceptAction::Terminal + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EINTR)), + AcceptErrorClass::Transient, ); } #[cfg(unix)] #[test] - fn test_classify_terminal_error_enotsock() { - let err = std::io::Error::from_raw_os_error(libc::ENOTSOCK); - let mut fd = 0; - let mut unk = 0; + fn classify_accept_error_broken_listener_is_terminal() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EBADF)), + AcceptErrorClass::Terminal, + ); assert_eq!( - classify_accept_error(&err, &mut fd, &mut unk), - AcceptAction::Terminal + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EINVAL)), + AcceptErrorClass::Terminal, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOTSOCK)), + AcceptErrorClass::Terminal, ); } #[cfg(unix)] #[test] - fn test_classify_fd_exhaustion_returns_retry_medium() { - let err = std::io::Error::from_raw_os_error(libc::EMFILE); - let mut fd = 0; - let mut unk = 0; - let action = classify_accept_error(&err, &mut fd, &mut unk); - assert!( - matches!( - action, - AcceptAction::Retry { - severity: SeverityId::Medium, - .. - } - ), - "expected Retry/Medium for EMFILE, got {action:?}", + fn classify_accept_error_unrecognized_errno_is_unknown() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::EPERM)), + AcceptErrorClass::Unknown, ); } #[cfg(unix)] #[test] - fn test_classify_unknown_error_returns_retry_low() { - let err = std::io::Error::from_raw_os_error(libc::ECONNREFUSED); - let mut fd = 0; + fn handle_accept_error_terminal_exits_immediately() { + let mut res = 0; let mut unk = 0; - let action = classify_accept_error(&err, &mut fd, &mut unk); - assert!( - matches!( - action, - AcceptAction::Retry { - severity: SeverityId::Low, - .. - } - ), - "expected Retry/Low for unknown error, got {action:?}", - ); + let err = std::io::Error::from_raw_os_error(libc::EBADF); + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_none()); + assert_eq!(outcome.severity, SeverityId::High); } #[cfg(unix)] #[test] - fn test_classify_fd_exhaustion_backoff_increases_and_caps() { - let mut fd = 0; + fn handle_accept_error_transient_retries_indefinitely() { + let mut res = 0; let mut unk = 0; let err = std::io::Error::from_raw_os_error(libc::EMFILE); - - let mut prev_backoff = std::time::Duration::ZERO; - for _ in 0..6 { - match classify_accept_error(&err, &mut fd, &mut unk) { - AcceptAction::Retry { backoff, .. } => { - assert!( - backoff > prev_backoff, - "backoff should increase: {backoff:?} <= {prev_backoff:?}", - ); - prev_backoff = backoff; - } - AcceptAction::Terminal => panic!("expected Retry, got Terminal"), - } - } - - // After enough consecutive errors the backoff should hit the 5s cap. - for _ in 6..12 { - classify_accept_error(&err, &mut fd, &mut unk); - } - match classify_accept_error(&err, &mut fd, &mut unk) { - AcceptAction::Retry { backoff, .. } => { - assert_eq!( - backoff, - std::time::Duration::from_secs(5), - "backoff should cap at 5000ms", - ); - } - AcceptAction::Terminal => panic!("expected Retry, got Terminal"), + for i in 1..=20 { + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_some(), "should retry on attempt {i}"); + assert_eq!(outcome.severity, SeverityId::Medium); } + assert_eq!(res, 20); } #[cfg(unix)] #[test] - fn test_classify_unknown_errors_exit_after_threshold() { - let mut fd = 0; + fn handle_accept_error_unknown_exits_after_limit() { + let mut res = 0; let mut unk = 0; - let err = std::io::Error::from_raw_os_error(libc::ECONNREFUSED); - - for i in 1..MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS { - let action = classify_accept_error(&err, &mut fd, &mut unk); + let err = std::io::Error::from_raw_os_error(libc::EPERM); + for i in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS { + let outcome = handle_accept_error(&err, &mut res, &mut unk); assert!( - matches!(action, AcceptAction::Retry { .. }), - "call {i} should be Retry, got {action:?}", + outcome.backoff.is_some(), + "should retry on attempt {i}/{MAX_CONSECUTIVE_UNKNOWN_ERRORS}", ); + assert_eq!(outcome.severity, SeverityId::Medium); } - let final_action = classify_accept_error(&err, &mut fd, &mut unk); - assert_eq!( - final_action, - AcceptAction::Terminal, - "call {MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS} should be Terminal", + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!( + outcome.backoff.is_none(), + "should exit after limit exceeded" ); + assert_eq!(outcome.severity, SeverityId::High); } #[cfg(unix)] #[test] - fn test_classify_success_resets_counters() { - let mut fd = 0; + fn handle_accept_error_transient_resets_unknown_counter() { + let mut res = 0; let mut unk = 0; - let err = std::io::Error::from_raw_os_error(libc::ECONNREFUSED); + let unknown_err = std::io::Error::from_raw_os_error(libc::EPERM); + let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); - for _ in 1..MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS { - classify_accept_error(&err, &mut fd, &mut unk); + // Accumulate unknowns up to the limit. + for _ in 1..=MAX_CONSECUTIVE_UNKNOWN_ERRORS { + handle_accept_error(&unknown_err, &mut res, &mut unk); } + assert_eq!(unk, MAX_CONSECUTIVE_UNKNOWN_ERRORS); - fd = 0; - unk = 0; + // A transient error resets the unknown counter. + let outcome = handle_accept_error(&transient_err, &mut res, &mut unk); + assert!(outcome.backoff.is_some()); + assert_eq!(unk, 0); - for i in 1..MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS { - let action = classify_accept_error(&err, &mut fd, &mut unk); - assert!( - matches!(action, AcceptAction::Retry { .. }), - "after reset, call {i} should be Retry, got {action:?}", - ); - } + // Unknown errors can retry again from zero. + let outcome = handle_accept_error(&unknown_err, &mut res, &mut unk); + assert!(outcome.backoff.is_some()); + assert_eq!(unk, 1); } #[cfg(unix)] #[test] - fn test_classify_fd_error_resets_unknown_counter() { - let mut fd = 0; + fn handle_accept_error_fd_exhaustion_uses_exponential_backoff() { + let mut res = 0; let mut unk = 0; - let unknown_err = std::io::Error::from_raw_os_error(libc::ECONNREFUSED); - let fd_err = std::io::Error::from_raw_os_error(libc::EMFILE); - - for _ in 0..5 { - classify_accept_error(&unknown_err, &mut fd, &mut unk); - } + let err = std::io::Error::from_raw_os_error(libc::EMFILE); - classify_accept_error(&fd_err, &mut fd, &mut unk); + let b1 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b2 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b3 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); - for i in 1..MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS { - let action = classify_accept_error(&unknown_err, &mut fd, &mut unk); - assert!( - matches!(action, AcceptAction::Retry { .. }), - "after FD reset, call {i} should be Retry, got {action:?}", - ); - } + assert_eq!(b1.as_millis(), 100); + assert_eq!(b2.as_millis(), 200); + assert_eq!(b3.as_millis(), 400); } #[cfg(unix)] #[test] - fn test_classify_transient_and_terminal_are_disjoint() { - let mut res = 0; - let mut unk = 0; - + fn classify_accept_error_network_errors_are_transient() { for errno in [ - libc::EMFILE, - libc::ENFILE, - libc::ENOBUFS, - libc::ENOMEM, - libc::ECONNABORTED, - libc::ECONNRESET, - libc::EINTR, libc::ENETDOWN, + libc::EPROTO, + libc::ENOPROTOOPT, libc::EHOSTDOWN, libc::EHOSTUNREACH, libc::EOPNOTSUPP, libc::ENETUNREACH, - libc::ENOSR, + libc::ESOCKTNOSUPPORT, + libc::EPROTONOSUPPORT, libc::ETIMEDOUT, ] { - let err = std::io::Error::from_raw_os_error(errno); - assert!( - matches!( - classify_accept_error(&err, &mut res, &mut unk), - AcceptAction::Retry { .. } - ), - "errno {errno} should be Retry", - ); - res = 0; - unk = 0; - } - - for errno in [libc::EBADF, libc::EINVAL, libc::ENOTSOCK] { - let err = std::io::Error::from_raw_os_error(errno); assert_eq!( - classify_accept_error(&err, &mut res, &mut unk), - AcceptAction::Terminal, - "errno {errno} should be Terminal", + classify_accept_error(&std::io::Error::from_raw_os_error(errno)), + AcceptErrorClass::Transient, + "errno {errno} should be transient", ); } } + #[cfg(target_os = "linux")] + #[test] + fn classify_accept_error_enonet_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENONET)), + AcceptErrorClass::Transient, + ); + } + + #[cfg(unix)] + #[test] + fn classify_accept_error_resource_pressure_is_transient() { + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOBUFS)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOMEM)), + AcceptErrorClass::Transient, + ); + assert_eq!( + classify_accept_error(&std::io::Error::from_raw_os_error(libc::ENOSR)), + AcceptErrorClass::Transient, + ); + } + #[cfg(unix)] #[test] - fn test_transient_errors_never_hit_unknown_budget() { + fn handle_accept_error_non_resource_transient_uses_fixed_backoff() { let mut res = 0; let mut unk = 0; let err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); - for i in 0..(MAX_CONSECUTIVE_UNKNOWN_ACCEPT_ERRORS + 5) { - let action = classify_accept_error(&err, &mut res, &mut unk); - assert!( - matches!(action, AcceptAction::Retry { .. }), - "transient error on call {i} should always Retry, got {action:?}", - ); - } + let o1 = handle_accept_error(&err, &mut res, &mut unk); + let o2 = handle_accept_error(&err, &mut res, &mut unk); + + assert_eq!(o1.severity, SeverityId::Low); + assert_eq!(o1.backoff.unwrap().as_millis(), 100); + assert_eq!(o2.backoff.unwrap().as_millis(), 100); + assert_eq!(res, 0); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_unknown_uses_exponential_backoff() { + let mut res = 0; + let mut unk = 0; + let err = std::io::Error::from_raw_os_error(libc::EPERM); + + let b1 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b2 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + let b3 = handle_accept_error(&err, &mut res, &mut unk) + .backoff + .unwrap(); + + assert_eq!(b1.as_millis(), 100); + assert_eq!(b2.as_millis(), 200); + assert_eq!(b3.as_millis(), 400); + } + + #[cfg(unix)] + #[test] + fn handle_accept_error_resource_counter_persists_across_mixed_transient() { + let mut res = 0; + let mut unk = 0; + let resource_err = std::io::Error::from_raw_os_error(libc::EMFILE); + let transient_err = std::io::Error::from_raw_os_error(libc::ECONNABORTED); + + let o1 = handle_accept_error(&resource_err, &mut res, &mut unk); + assert_eq!(res, 1); + assert_eq!(o1.backoff.unwrap().as_millis(), 100); + + let o2 = handle_accept_error(&transient_err, &mut res, &mut unk); + assert_eq!(res, 1); + assert_eq!(o2.backoff.unwrap().as_millis(), 100); + + let o3 = handle_accept_error(&resource_err, &mut res, &mut unk); + assert_eq!(res, 2); + assert_eq!(o3.backoff.unwrap().as_millis(), 200); } + #[cfg(unix)] + #[test] + fn handle_accept_error_terminal_leaves_counters_unchanged() { + let mut res = 3; + let mut unk = 2; + let err = std::io::Error::from_raw_os_error(libc::EBADF); + + let outcome = handle_accept_error(&err, &mut res, &mut unk); + assert!(outcome.backoff.is_none()); + assert_eq!(res, 3); + assert_eq!(unk, 2); + } #[path = "compatibility.rs"] mod compatibility; } diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 314596b048..55c4dd9099 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![allow(dead_code)] + //! Transport-neutral egress inputs and authorization results. //! //! Explicit proxy adapters normalize their protocol-specific request into an diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 186d156086..2546357c15 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -23,6 +23,23 @@ fn allowed_decision(intent: EgressIntent) -> EgressDecision { } } +fn compatibility_engine() -> OpaEngine { + OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + r#" +network_policies: + proxy_compatibility: + name: proxy_compatibility + endpoints: + - host: "target.example" + port: 443 + binaries: + - path: /** +"#, + ) + .unwrap() +} + async fn tcp_pair() -> (TcpStream, TcpStream) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let client = TcpStream::connect(listener.local_addr().unwrap()) @@ -281,23 +298,26 @@ fn representative_adapter_allows_preserve_ocsf_fields() { #[test] fn missing_authorized_l7_metadata_preserves_l4_only_fallback() { + let engine = compatibility_engine(); let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); - assert!(query_l7_route_snapshot(&decision, "target.example", 443).is_none()); + assert!(query_l7_route_snapshot(&engine, &decision, "target.example", 443).is_none()); } #[test] fn missing_authorized_tls_metadata_preserves_auto_fallback() { + let engine = compatibility_engine(); let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); assert_eq!( - query_tls_mode(&decision, "target.example", 443), + query_tls_mode(&engine, &decision, "target.example", 443), crate::l7::TlsMode::Auto ); } #[test] fn missing_authorized_allowed_ips_preserves_empty_fallback() { + let engine = compatibility_engine(); let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); - assert!(query_allowed_ips(&decision).is_empty()); + assert!(query_allowed_ips(&engine, &decision, "target.example", 443).is_empty()); } #[test] @@ -553,7 +573,6 @@ network_policies: None, None, None, - None, )) .await .unwrap(); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 2a71702b4b..106c6b1b1c 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -37,108 +37,7 @@ use crate::l7::tls::{ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; - -#[cfg(target_os = "linux")] -pub struct TransparentRuntimeSetup { - pub listeners: Vec, - pub dns_udp: tokio::net::UdpSocket, - pub dns_tcp: tokio::net::TcpListener, - config: crate::policy_dns::PolicyDnsRuntimeConfig, -} - -#[cfg(target_os = "linux")] -impl TransparentRuntimeSetup { - /// Build one boot-scoped synthetic allocation epoch. The epoch advances - /// before workload execution, so addresses cached across a supervisor - /// restart fall outside the newly installed capture ranges. - /// - /// # Errors - /// - /// Returns an error when the epoch cannot be read or atomically persisted, - /// or when the derived synthetic pools are invalid. - pub fn new( - listeners: Vec, - dns_udp: tokio::net::UdpSocket, - dns_tcp: tokio::net::TcpListener, - sandbox_id: Option<&str>, - ) -> Result { - let epoch = advance_allocation_epoch( - std::path::Path::new("/run/openshell/policy-dns-epoch"), - sandbox_id, - )?; - Ok(Self { - listeners, - dns_udp, - dns_tcp, - config: crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(epoch)?, - }) - } - - #[must_use] - pub fn synthetic_cidrs(&self) -> (String, String) { - ( - self.config.ipv4_cidr.to_string(), - self.config.ipv6_cidr.to_string(), - ) - } -} - -#[cfg(target_os = "linux")] -fn advance_allocation_epoch(path: &std::path::Path, sandbox_id: Option<&str>) -> Result { - use miette::{IntoDiagnostic, WrapErr}; - use std::io::Write as _; - - let seed = sandbox_id.map_or(0, |value| { - value - .as_bytes() - .iter() - .fold(0xcbf2_9ce4_8422_2325, |hash, byte| { - (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) - }) - }); - let previous = match std::fs::read_to_string(path) { - Ok(value) => value - .trim() - .parse::() - .into_diagnostic() - .wrap_err("policy DNS allocation epoch is invalid")?, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => seed, - Err(error) => { - return Err(error) - .into_diagnostic() - .wrap_err("failed to read policy DNS allocation epoch"); - } - }; - let epoch = previous.wrapping_add(1); - let parent = path - .parent() - .ok_or_else(|| miette::miette!("policy DNS allocation epoch has no parent directory"))?; - std::fs::create_dir_all(parent) - .into_diagnostic() - .wrap_err("failed to create policy DNS runtime directory")?; - let temporary = parent.join(format!( - ".policy-dns-epoch-{}-{}", - std::process::id(), - uuid::Uuid::new_v4() - )); - let result = (|| -> std::io::Result<()> { - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&temporary)?; - writeln!(file, "{epoch}")?; - file.sync_all()?; - std::fs::rename(&temporary, path)?; - std::fs::File::open(parent)?.sync_all() - })(); - if result.is_err() { - let _ = std::fs::remove_file(&temporary); - } - result - .into_diagnostic() - .wrap_err("failed to atomically persist policy DNS allocation epoch")?; - Ok(epoch) -} +use openshell_isolation::contract::NetworkMediationSource; /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. @@ -155,10 +54,6 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, - #[cfg(target_os = "linux")] - _policy_dns: Option, - #[cfg(target_os = "linux")] - _transparent_tcp: Option, } /// Set up the networking stack: ephemeral CA + TLS state, proxy server, @@ -196,7 +91,7 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, - #[cfg(target_os = "linux")] transparent_runtime: Option, + network_mediation_source: Option>, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -207,7 +102,7 @@ pub async fn run_networking( sandbox_name .map(str::to_string) .or_else(|| sandbox_id.map(str::to_string)), - agent_proposals.clone(), + agent_proposals, workspace_rx, )); @@ -217,10 +112,6 @@ pub async fn run_networking( // the race where an in-flight request observes a generation transition // during the OPA engine reload. let (engine_ready_tx, engine_ready_rx) = tokio::sync::watch::channel(false); - #[cfg(target_os = "linux")] - let transparent_engine_ready_rx = engine_ready_rx.clone(); - #[cfg(target_os = "linux")] - let policy_dns_engine_ready_rx = engine_ready_rx.clone(); // Spawn a task to resolve policy binary symlinks once the workload's mount // namespace becomes accessible via /proc//root/. The task starts @@ -319,36 +210,16 @@ pub async fn run_networking( match SandboxCa::generate() { Ok(ca) => { let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| openshell_core::container_paths::TLS_ROOT.to_string()); + .unwrap_or_else(|_| "/etc/openshell-tls".to_string()); let tls_dir = std::path::Path::new(&tls_dir); - let mut system_ca_bundle = read_system_ca_bundle(); - // A TLS-intercepting corporate proxy (issue #1792) re-signs - // tunneled server certificates with the corporate CA, so the - // operator-provided bundle must be trusted for upstream - // re-encryption (build_upstream_client_config below) and by - // sandbox processes (the combined bundle written by - // write_ca_files) — not only for the TLS handshake with an - // https:// proxy listener. Fail closed on an unreadable or - // certificate-free bundle, matching the rest of the - // operator-owned proxy configuration. - if let Some(path) = upstream_proxy_args.proxy_ca_bundle.as_deref() { - let pem = crate::upstream_proxy::read_proxy_ca_bundle( - path, - crate::upstream_proxy::ARG_PROXY_CA_BUNDLE, - ) - .map_err(|err| miette::miette!("{err}"))?; - if !system_ca_bundle.is_empty() && !system_ca_bundle.ends_with('\n') { - system_ca_bundle.push('\n'); - } - system_ca_bundle.push_str(&pem); - } + let system_ca_bundle = read_system_ca_bundle(); match write_ca_files(&ca, tls_dir, &system_ca_bundle) { Ok(paths) => { // /etc/openshell-tls is subsumed by the /etc baseline // path injected by enrich_*_baseline_paths(), so no // explicit Landlock entry is needed here. - let upstream_config = build_upstream_client_config(&system_ca_bundle)?; + let upstream_config = build_upstream_client_config(&system_ca_bundle); let cert_cache = CertCache::new(ca); let state = Arc::new(ProxyTlsState::new(cert_cache, upstream_config)); ocsf_emit!( @@ -443,10 +314,11 @@ pub async fn run_networking( inference_ctx, Some(provider_credentials.clone()), Some(policy_local_ctx.clone()), - denial_tx.clone(), - activity_tx.clone(), + denial_tx, + activity_tx, engine_ready_rx, upstream_proxy_args, + network_mediation_source, ) .await?; Some(proxy_handle) @@ -454,70 +326,9 @@ pub async fn run_networking( None }; - #[cfg(target_os = "linux")] - let (policy_dns, transparent_tcp) = if let Some(runtime) = transparent_runtime { - let engine = opa_engine - .cloned() - .ok_or_else(|| miette::miette!("transparent TCP requires an OPA policy engine"))?; - let cache = identity_cache - .clone() - .ok_or_else(|| miette::miette!("transparent TCP requires a process identity cache"))?; - let trusted_gateway = crate::proxy::detect_trusted_host_gateway(); - let dns = crate::policy_dns::PolicyDnsRuntime::start( - engine.clone(), - runtime.dns_udp, - runtime.dns_tcp, - trusted_gateway, - runtime.config, - policy_dns_engine_ready_rx, - )?; - let transparent = crate::proxy::TransparentTcpHandle::start( - runtime.listeners, - dns.store.clone(), - engine, - cache, - entrypoint_pid, - agent_proposals, - denial_tx, - activity_tx, - upstream_proxy_args, - transparent_engine_ready_rx, - )?; - (Some(dns), Some(transparent)) - } else { - (None, None) - }; - Ok(Networking { proxy: proxy_handle, ca_file_paths, policy_local_ctx, - #[cfg(target_os = "linux")] - _policy_dns: policy_dns, - #[cfg(target_os = "linux")] - _transparent_tcp: transparent_tcp, }) } - -#[cfg(all(test, target_os = "linux"))] -mod transparent_runtime_tests { - use super::*; - - #[test] - fn allocation_epoch_advances_across_restart() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("epoch"); - let first = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap(); - let second = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap(); - assert_eq!(second, first + 1); - } - - #[test] - fn invalid_allocation_epoch_fails_closed() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("epoch"); - std::fs::write(&path, "corrupt\n").unwrap(); - let error = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap_err(); - assert!(error.to_string().contains("allocation epoch is invalid")); - } -} diff --git a/crates/openshell-supervisor-network/src/spiffe_endpoint.rs b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs new file mode 100644 index 0000000000..b3b6816f1e --- /dev/null +++ b/crates/openshell-supervisor-network/src/spiffe_endpoint.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +/// Convert a path to a SPIFFE Workload API endpoint URL. +/// +/// If the path already has a scheme (`unix:` or `tcp:`), use it as-is. +/// Otherwise, assume it is a Unix socket path and prepend `unix:`. +#[allow(dead_code)] +pub fn workload_api_endpoint(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.starts_with("unix:") || path.starts_with("tcp:") { + path.into_owned() + } else { + format!("unix:{path}") + } +} diff --git a/crates/openshell-supervisor-network/src/upstream_proxy.rs b/crates/openshell-supervisor-network/src/upstream_proxy.rs index 253fa52263..7be7bbb6a6 100644 --- a/crates/openshell-supervisor-network/src/upstream_proxy.rs +++ b/crates/openshell-supervisor-network/src/upstream_proxy.rs @@ -655,7 +655,6 @@ fn build_proxy_tls_config(corporate_ca_pem: Option<&str>) -> Arc { bundle.push_str(pem); } crate::l7::tls::build_upstream_client_config(&bundle) - .expect("corporate proxy TLS config must be valid") } /// Build a `Proxy-Authorization: Basic ` header value from a raw @@ -2037,10 +2036,10 @@ mod tests { // Trusted CA; the client config trusts it, and the fake upstream // server presents a leaf for SERVER_HOSTNAME signed by it. let ca = tls::SandboxCa::generate().unwrap(); - let client_config = tls::build_upstream_client_config(ca.cert_pem()).unwrap(); + let client_config = tls::build_upstream_client_config(ca.cert_pem()); let tls_state = Arc::new(tls::ProxyTlsState::new( tls::CertCache::new(ca), - tls::build_upstream_client_config("").unwrap(), + tls::build_upstream_client_config(""), )); // Fake upstream TLS server: accepts tunneled connections and completes diff --git a/crates/openshell-supervisor-process/src/boundary_exec.rs b/crates/openshell-supervisor-process/src/boundary_exec.rs index 00d5239f29..7db1ed9d61 100644 --- a/crates/openshell-supervisor-process/src/boundary_exec.rs +++ b/crates/openshell-supervisor-process/src/boundary_exec.rs @@ -74,9 +74,7 @@ impl LocalBoundaryExec { } let mut command = Command::new(&spec.program); command.args(&spec.args); - let effective_workdir = spec.workdir.as_deref().or(self.base_workdir.as_deref()); - let (session_user, session_home) = - crate::process::session_user_and_home(&self.policy, effective_workdir); + let (session_user, session_home) = crate::ssh::session_user_and_home(&self.policy); crate::ssh::apply_child_env( &mut command, &session_home, @@ -136,7 +134,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command @@ -238,7 +237,8 @@ impl LocalBoundaryExec { self.enforcement_mode, #[cfg(target_os = "linux")] prepared, - ); + ) + .map_err(|error| BackendError::Process(error.to_string()))?; #[cfg(target_os = "linux")] let mut child_registry = crate::managed_children::lock(); let mut child = command diff --git a/crates/openshell-supervisor-process/src/boundary_io.rs b/crates/openshell-supervisor-process/src/boundary_io.rs index c09974d2df..aab272efc3 100644 --- a/crates/openshell-supervisor-process/src/boundary_io.rs +++ b/crates/openshell-supervisor-process/src/boundary_io.rs @@ -21,7 +21,7 @@ use openshell_isolation::contract::{ BackendError, BoundaryDuplexStream, BoundaryPortForward, LoopbackTarget, }; use std::collections::HashMap; -use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::fd::OwnedFd; use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex}; @@ -204,13 +204,9 @@ impl BoundaryPortForward for NetnsPortForward { runtime.ensure_active()?; } let addr = std::net::SocketAddr::new(target.host(), target.port()); - let addr_string = addr.to_string(); - let stream = crate::ssh::connect_in_netns( - &addr_string, - self.netns_fd.as_deref().map(AsRawFd::as_raw_fd), - ) - .await - .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; + let stream = crate::ssh::connect_in_netns(addr, self.netns_fd.clone()) + .await + .map_err(|e| BackendError::Process(format!("port-forward connect to {addr}: {e}")))?; if let Some(runtime) = &self.runtime { runtime.ensure_active()?; } diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index ee6bedeb22..e7f81524fd 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -15,7 +15,6 @@ pub mod debug_rpc; #[cfg(unix)] pub mod identity; pub mod log_push; -pub mod main_session; pub mod managed_children; pub mod process; pub mod run; diff --git a/crates/openshell-supervisor-process/src/managed_children.rs b/crates/openshell-supervisor-process/src/managed_children.rs index 04f4114a04..0f9817b232 100644 --- a/crates/openshell-supervisor-process/src/managed_children.rs +++ b/crates/openshell-supervisor-process/src/managed_children.rs @@ -26,29 +26,6 @@ pub struct ManagedChild { generation: u64, } -/// A managed-child registration accepted by [`unregister`]. -/// -/// New boundary-owned processes retain a generation-bearing token. Legacy -/// supervisor paths still identify their child by PID; supporting both keeps -/// the registry race-safe for new code without forcing an unrelated rewrite -/// of the canonical main-process and SSH paths. -pub enum ManagedChildRegistration { - Token(ManagedChild), - Pid(u32), -} - -impl From for ManagedChildRegistration { - fn from(value: ManagedChild) -> Self { - Self::Token(value) - } -} - -impl From for ManagedChildRegistration { - fn from(value: u32) -> Self { - Self::Pid(value) - } -} - /// Exclusive access to the managed-child registry. /// /// A process spawner holds this guard from immediately before `spawn` or @@ -89,28 +66,13 @@ pub fn lock() -> RegistryGuard { ) } -/// Register a child for a legacy caller that cannot retain a generation token. -pub fn register(pid: u32) { - let _ = lock().register(pid); -} - /// Remove exactly this supervised-child registration. A newer registration /// for a reused PID is preserved. -pub fn unregister(child: impl Into) { - if let Ok(mut children) = MANAGED_CHILDREN.lock() { - match child.into() { - ManagedChildRegistration::Token(child) - if children.get(&child.pid) == Some(&child.generation) => - { - children.remove(&child.pid); - } - ManagedChildRegistration::Pid(pid) => { - if let Ok(pid) = i32::try_from(pid) { - children.remove(&pid); - } - } - ManagedChildRegistration::Token(_) => {} - } +pub fn unregister(child: ManagedChild) { + if let Ok(mut children) = MANAGED_CHILDREN.lock() + && children.get(&child.pid) == Some(&child.generation) + { + children.remove(&child.pid); } } diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 2b4ea554ed..5f60ade475 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -11,9 +11,12 @@ mod nft_ruleset; use miette::{IntoDiagnostic, Result}; use std::net::IpAddr; +use std::os::fd::AsRawFd as _; +use std::os::fd::{BorrowedFd, OwnedFd}; use std::os::unix::io::RawFd; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; +use std::sync::OnceLock; use tracing::{debug, warn}; use uuid::Uuid; @@ -21,18 +24,80 @@ use uuid::Uuid; const SUBNET_PREFIX: &str = "10.200.0"; const HOST_IP_SUFFIX: u8 = 1; const SANDBOX_IP_SUFFIX: u8 = 2; -/// Unprivileged port owned by the supervisor's policy DNS service. Workload -/// queries still target the standard DNS port and nftables redirects them to -/// this listener before the bypass fence runs. -pub const POLICY_DNS_PORT: u16 = 15_053; -pub const TRANSPARENT_TCP_PORT: u16 = 15_001; -const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; -const NSENTER_SEARCH_PATHS: &[&str] = &[ - "/usr/bin/nsenter", - "/bin/nsenter", - "/usr/sbin/nsenter", - "/sbin/nsenter", -]; +const IP_SEARCH_PATHS: &[&str] = &["usr/sbin/ip", "sbin/ip", "usr/bin/ip", "bin/ip"]; +static TRUSTED_RUNTIME_ROOT: OnceLock = OnceLock::new(); + +/// Pin the driver-owned helper runtime used for conformant namespace setup. +/// +/// VM guest leaves call this before starting any control or workload threads +/// because their executable may be launched through a dynamic loader. In that +/// case `/proc/self/exe` identifies the loader rather than the supervisor +/// binary, so the default executable-relative lookup is not authoritative. +/// +/// # Errors +/// +/// Returns an error for a relative path or if another root was already pinned. +pub fn configure_trusted_runtime_root(root: PathBuf) -> Result<()> { + if !root.is_absolute() { + return Err(miette::miette!( + "trusted supervisor helper runtime root must be absolute" + )); + } + TRUSTED_RUNTIME_ROOT.set(root).map_err(|configured| { + miette::miette!( + "trusted supervisor helper runtime root is already configured as {}", + configured.display() + ) + }) +} + +#[derive(Clone, Debug)] +struct TrustedHelper { + executable: PathBuf, + loader: Option, + library_path: String, + xtables_path: PathBuf, +} + +impl TrustedHelper { + fn command(&self) -> Command { + self.loader.as_ref().map_or_else( + || Command::new(&self.executable), + |loader| { + let mut command = Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } + + fn tokio_command(&self) -> tokio::process::Command { + self.loader.as_ref().map_or_else( + || tokio::process::Command::new(&self.executable), + |loader| { + let mut command = tokio::process::Command::new(loader); + command + .env_clear() + .env("XTABLES_LIBDIR", &self.xtables_path) + .arg("--library-path") + .arg(&self.library_path) + .arg(&self.executable); + command + }, + ) + } +} + +#[derive(Clone, Copy, Debug)] +enum HelperSource { + LegacyWorkloadImage, + TrustedSupervisorRuntime, +} /// Handle to a network namespace with veth pair. /// @@ -44,13 +109,24 @@ pub struct NetworkNamespace { /// Host-side veth interface name veth_host: String, /// Sandbox-side veth interface name (inside namespace, used only during setup) - _veth_sandbox: String, + #[allow(dead_code)] + veth_sandbox: String, /// Host-side IP address (proxy binds here) host_ip: IpAddr, /// Sandbox-side IP address sandbox_ip: IpAddr, /// File descriptor for the namespace (for setns) ns_fd: Option, + helper_source: HelperSource, +} + +/// Cloneable coordinates for checking a live ceiling without retaining the +/// namespace fd or delaying namespace cleanup. +#[derive(Clone, Debug)] +pub struct EgressCeilingVerifier { + namespace: String, + host_ip: IpAddr, + helper_source: HelperSource, } impl NetworkNamespace { @@ -66,6 +142,14 @@ impl NetworkNamespace { /// /// Returns an error if namespace creation or network setup fails. pub fn create() -> Result { + Self::create_with_helper_source(HelperSource::LegacyWorkloadImage) + } + + fn create_conformant() -> Result { + Self::create_with_helper_source(HelperSource::TrustedSupervisorRuntime) + } + + fn create_with_helper_source(helper_source: HelperSource) -> Result { let id = Uuid::new_v4(); let short_id = &id.to_string()[..8]; let name = format!("sandbox-{short_id}"); @@ -89,84 +173,101 @@ impl NetworkNamespace { ); // Create the namespace - run_ip(&["netns", "add", &name])?; + run_ip(helper_source, &["netns", "add", &name])?; // Create veth pair - if let Err(e) = run_ip(&[ - "link", - "add", - &veth_host, - "type", - "veth", - "peer", - "name", - &veth_sandbox, - ]) { + if let Err(e) = run_ip( + helper_source, + &[ + "link", + "add", + &veth_host, + "type", + "veth", + "peer", + "name", + &veth_sandbox, + ], + ) { // Cleanup namespace on failure - let _ = run_ip(&["netns", "delete", &name]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Move sandbox veth into namespace - if let Err(e) = run_ip(&["link", "set", &veth_sandbox, "netns", &name]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["link", "set", &veth_sandbox, "netns", &name], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure host side let host_cidr = format!("{host_ip}/24"); - if let Err(e) = run_ip(&["addr", "add", &host_cidr, "dev", &veth_host]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip( + helper_source, + &["addr", "add", &host_cidr, "dev", &veth_host], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip(&["link", "set", &veth_host, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip(helper_source, &["link", "set", &veth_host, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Configure sandbox side (inside namespace) let sandbox_cidr = format!("{sandbox_ip}/24"); - if let Err(e) = run_ip_netns(&name, &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["addr", "add", &sandbox_cidr, "dev", &veth_sandbox], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } - if let Err(e) = run_ip_netns(&name, &["link", "set", &veth_sandbox, "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", &veth_sandbox, "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Bring up loopback in namespace - if let Err(e) = run_ip_netns(&name, &["link", "set", "lo", "up"]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns(helper_source, &name, &["link", "set", "lo", "up"]) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Add default route via host let host_ip_str = host_ip.to_string(); - if let Err(e) = run_ip_netns(&name, &["route", "add", "default", "via", &host_ip_str]) { - let _ = run_ip(&["link", "delete", &veth_host]); - let _ = run_ip(&["netns", "delete", &name]); + if let Err(e) = run_ip_netns( + helper_source, + &name, + &["route", "add", "default", "via", &host_ip_str], + ) { + let _ = run_ip(helper_source, &["link", "delete", &veth_host]); + let _ = run_ip(helper_source, &["netns", "delete", &name]); return Err(e); } // Open the namespace file descriptor for later use with setns - let ns_path = openshell_core::container_paths::netns_path(&name); + let ns_path = format!("/var/run/netns/{name}"); let ns_fd = match nix::fcntl::open( - ns_path.as_path(), + ns_path.as_str(), nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty(), ) { Ok(fd) => Some(fd), Err(e) => { - warn!(error = %e, "Failed to open namespace fd, will use nsenter fallback"); + warn!(error = %e, "Failed to retain network namespace fd"); None } }; @@ -185,10 +286,11 @@ impl NetworkNamespace { Ok(Self { name, veth_host, - _veth_sandbox: veth_sandbox, + veth_sandbox, host_ip, sandbox_ip, ns_fd, + helper_source, }) } @@ -249,25 +351,21 @@ impl NetworkNamespace { self.ns_fd } - /// Install nftables rules for bypass detection inside the namespace. - /// - /// Sets up OUTPUT chain rules that: - /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) - /// 2. ACCEPT loopback traffic - /// 3. ACCEPT established/related connections (response packets) - /// 4. LOG + REJECT all other TCP/UDP traffic (bypass attempts) - /// - /// This provides two benefits: - /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of - /// a 30-second timeout when they bypass the proxy - /// - **Diagnostics**: nftables LOG entries are picked up by the bypass - /// monitor to emit structured tracing events - /// - /// Degrades gracefully if `nft` is not available — the namespace - /// still provides isolation via routing, just without fast-fail and - /// diagnostic logging. + /// Duplicate the namespace descriptor for a retained runtime handle. + pub fn try_clone_ns_fd(&self) -> Result> { + self.ns_fd + .map(|fd| { + // SAFETY: `NetworkNamespace` owns `fd` for at least this call. + #[allow(unsafe_code)] + let borrowed = unsafe { BorrowedFd::borrow_raw(fd) }; + borrowed.try_clone_to_owned().into_diagnostic() + }) + .transpose() + } + + /// Install the legacy best-effort nftables bypass-detection rules. pub fn install_bypass_rules(&self, proxy_port: u16) -> Result<()> { - let Some(nft_path) = find_nft() else { + let Some(nft_path) = find_nft(self.helper_source) else { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) @@ -281,33 +379,25 @@ impl NetworkNamespace { ); return Ok(()); }; - - let host_ip_str = self.host_ip.to_string(); + let host_ip = self.host_ip.to_string(); let log_prefix = format!("openshell:bypass:{}:", &self.name); - - // The kernel's nf_log_syslog module suppresses log output from - // non-init network namespaces by default. Enable it so the bypass - // monitor can see log entries from the sandbox namespace. enable_nf_log_all_netns(); - let commands = - nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + nft_ruleset::generate_bypass_commands(&host_ip, proxy_port, Some(&log_prefix)); + if let Err(error) = run_nft_commands_netns(&self.name, &nft_path, &commands) { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Medium) .status(openshell_ocsf::StatusId::Failure) .state(openshell_ocsf::StateId::Disabled, "failed") .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", + "Failed to install bypass detection rules [ns:{}]: {error}", self.name )) .build() ); - return Err(e); + return Err(error); } - openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) @@ -319,195 +409,102 @@ impl NetworkNamespace { )) .build() ); - Ok(()) } - /// Replace the ordinary bypass fence with the policy-DNS and transparent - /// TCP ruleset. This is fail-closed: callers must not release workload - /// execution unless every required rule was installed. - pub fn install_transparent_tcp_rules( - &self, - proxy_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - self.validate_synthetic_pool_routes(synthetic_ipv4_cidr, synthetic_ipv6_cidr)?; - // The inner namespace has an IPv4 default route, but not an IPv6 - // default route. Install only the active synthetic IPv6 epoch so the - // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to - // the local transparent listener. - run_ip_netns( - &self.name, - &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], - )?; - let nft_path = find_nft().ok_or_else(|| { - miette::miette!( - "trusted nft helper not found; policy DNS and transparent TCP require nftables" - ) - })?; - let host_ip = self.host_ip.to_string(); - let log_prefix = format!("openshell:bypass:{}:", self.name); - let commands = nft_ruleset::generate_transparent_tcp_commands( - &host_ip, + /// Install the RFC 0012 default-deny egress ceiling inside the namespace. + /// + /// Sets up OUTPUT chain rules that: + /// 1. ACCEPT traffic destined for the proxy (`host_ip:proxy_port`) + /// 2. ACCEPT loopback traffic + /// 3. LOG + REJECT TCP/UDP bypass attempts and DROP every other packet + /// + /// This provides two benefits: + /// - **Fast-fail UX**: applications get immediate ECONNREFUSED instead of + /// a 30-second timeout when they bypass the proxy + /// - **Diagnostics**: nftables LOG entries are picked up by the bypass + /// monitor to emit structured tracing events + /// + /// Missing nftables support is fatal: without the default-deny ceiling the + /// backend cannot confirm that all workload egress reaches mediation. + pub fn install_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + let Some(nft_path) = find_nft(self.helper_source) else { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "unavailable") + .message(format!( + "nft not found; refusing to establish the egress ceiling [ns:{}]", + self.name + )) + .build() + ); + return Err(miette::miette!( + "nft not found; cannot establish default-deny egress ceiling" + )); + }; + + let host_ip_str = self.host_ip.to_string(); + let log_prefix = format!("openshell:bypass:{}:", &self.name); + + // The kernel's nf_log_syslog module suppresses log output from + // non-init network namespaces by default. Enable it so the bypass + // monitor can see log entries from the sandbox namespace. + enable_nf_log_all_netns(); + + let commands = nft_ruleset::generate_egress_ceiling_commands( + &host_ip_str, proxy_port, - POLICY_DNS_PORT, - TRANSPARENT_TCP_PORT, - synthetic_ipv4_cidr, - synthetic_ipv6_cidr, Some(&log_prefix), ); - run_nft_commands_netns(&self.name, &nft_path, &commands)?; + + if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::High) + .status(openshell_ocsf::StatusId::Failure) + .state(openshell_ocsf::StateId::Disabled, "failed") + .message(format!( + "Failed to establish egress ceiling [ns:{}]: {e}", + self.name + )) + .build() + ); + return Err(e); + } + openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(openshell_ocsf::SeverityId::Informational) .status(openshell_ocsf::StatusId::Success) .state(openshell_ocsf::StateId::Enabled, "installed") .message(format!( - "Policy DNS and transparent TCP capture installed [ns:{}]", + "Default-deny egress ceiling established [ns:{}]", self.name )) .build() ); - Ok(()) - } - fn validate_synthetic_pool_routes( - &self, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - ) -> Result<()> { - let reserved = [ - synthetic_ipv4_cidr - .parse::() - .into_diagnostic()?, - synthetic_ipv6_cidr - .parse::() - .into_diagnostic()?, - ]; - for family in ["-4", "-6"] { - let routes = - run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; - if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { - return Err(miette::miette!( - "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" - )); - } - } Ok(()) } - /// Bind IPv4 and IPv6 transparent listeners inside the workload network - /// namespace without moving an async runtime worker into that namespace. - pub async fn bind_transparent_tcp_listeners( - &self, - ) -> std::io::Result> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - let mut listeners = Vec::with_capacity(2); - for (domain, address) in [ - ( - socket2::Domain::IPV4, - format!("0.0.0.0:{TRANSPARENT_TCP_PORT}"), - ), - ( - socket2::Domain::IPV6, - format!("[::]:{TRANSPARENT_TCP_PORT}"), - ), - ] { - let socket = socket2::Socket::new( - domain, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - socket.set_reuse_address(true)?; - if domain == socket2::Domain::IPV6 { - socket.set_only_v6(true)?; - } - let address: std::net::SocketAddr = address.parse().map_err(|error| { - std::io::Error::other(format!("invalid listener address: {error}")) - })?; - socket.bind(&address.into())?; - socket.listen(128)?; - let listener: std::net::TcpListener = socket.into(); - listener.set_nonblocking(true)?; - listeners.push(listener); - } - Ok(listeners) - })(); - let _ = tx.send(result); - }); - rx.await - .map_err(|_| std::io::Error::other("netns bind thread panicked"))?? - .into_iter() - .map(tokio::net::TcpListener::from_std) - .collect() + /// Verify the live default-deny egress ceiling installed for this boundary. + /// + /// This reads the ruleset back from the kernel rather than treating a + /// successful installation attempt as proof that standing enforcement is + /// still present. + pub fn verify_egress_ceiling(&self, proxy_port: u16) -> Result<()> { + self.egress_ceiling_verifier().verify(proxy_port) } - /// Bind UDP and TCP DNS listeners inside the workload network namespace. - /// The workload keeps its image-provided resolver configuration; nftables - /// redirects port 53 to these sockets before the bypass fence runs. - pub async fn bind_policy_dns_sockets( - &self, - ) -> std::io::Result<(tokio::net::UdpSocket, tokio::net::TcpListener)> { - let ns_fd = self - .ns_fd - .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result<(std::net::UdpSocket, std::net::TcpListener)> { - #[allow(unsafe_code)] - if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { - return Err(std::io::Error::last_os_error()); - } - // Bind the exact REDIRECT destination instead of INADDR_ANY. - // For UDP this keeps replies sourced from loopback so - // conntrack can reverse the port/address translation before - // delivering them to libc in nested rootless namespaces. - let address: std::net::SocketAddr = format!("127.0.0.1:{POLICY_DNS_PORT}") - .parse() - .map_err(|error| { - std::io::Error::other(format!("invalid DNS listener address: {error}")) - })?; - - let udp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::DGRAM, - Some(socket2::Protocol::UDP), - )?; - udp.set_reuse_address(true)?; - udp.bind(&address.into())?; - udp.set_nonblocking(true)?; - - let tcp = socket2::Socket::new( - socket2::Domain::IPV4, - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - tcp.set_reuse_address(true)?; - tcp.bind(&address.into())?; - tcp.listen(128)?; - tcp.set_nonblocking(true)?; - - Ok((udp.into(), tcp.into())) - })(); - let _ = tx.send(result); - }); - let (udp, tcp) = rx - .await - .map_err(|_| std::io::Error::other("netns DNS bind thread panicked"))??; - Ok(( - tokio::net::UdpSocket::from_std(udp)?, - tokio::net::TcpListener::from_std(tcp)?, - )) + #[must_use] + pub fn egress_ceiling_verifier(&self) -> EgressCeilingVerifier { + EgressCeilingVerifier { + namespace: self.name.clone(), + host_ip: self.host_ip, + helper_source: self.helper_source, + } } /// Bind a TCP listener inside this network namespace on a dedicated thread. @@ -548,6 +545,158 @@ impl NetworkNamespace { } } +impl EgressCeilingVerifier { + fn nft_helper(&self) -> Result { + find_nft(self.helper_source) + .ok_or_else(|| miette::miette!("nft not found; cannot verify egress ceiling")) + } + + fn verify(&self, proxy_port: u16) -> Result<()> { + let nft = self.nft_helper()?; + let output = trusted_command_in_netns(&nft, &self.namespace)? + .args(["-j", "list", "chain", "inet", "openshell_bypass", "output"]) + .output() + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + /// Run a verifier helper with a hard deadline. Dropping the timed-out + /// future kills the child, so a stuck `nft` cannot retain the + /// namespace or suspend enforcement-loss detection indefinitely. + pub async fn verify_bounded( + &self, + proxy_port: u16, + timeout: std::time::Duration, + ) -> Result<()> { + let nft = self.nft_helper()?; + let mut command = trusted_tokio_command_in_netns(&nft, &self.namespace)?; + command.kill_on_drop(true).args([ + "-j", + "list", + "chain", + "inet", + "openshell_bypass", + "output", + ]); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| miette::miette!("egress ceiling verification timed out"))? + .into_diagnostic()?; + self.validate_output(proxy_port, &output) + } + + fn validate_output(&self, proxy_port: u16, output: &std::process::Output) -> Result<()> { + if !output.status.success() { + return Err(miette::miette!( + "could not read back egress ceiling in netns {}: {}", + self.namespace, + String::from_utf8_lossy(&output.stderr).trim() + )); + } + verify_egress_ceiling_json(&output.stdout, &self.host_ip.to_string(), proxy_port) + } +} + +fn verify_egress_ceiling_json(json: &[u8], host_ip: &str, proxy_port: u16) -> Result<()> { + let document: serde_json::Value = serde_json::from_slice(json).into_diagnostic()?; + let objects = document + .get("nftables") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("nft response has no nftables object list"))?; + + let chain_is_default_deny = objects.iter().any(|object| { + let Some(chain) = object.get("chain") else { + return false; + }; + chain.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && chain.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && chain.get("name").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("type").and_then(serde_json::Value::as_str) == Some("filter") + && chain.get("hook").and_then(serde_json::Value::as_str) == Some("output") + && chain.get("prio").and_then(serde_json::Value::as_i64) == Some(0) + && chain.get("policy").and_then(serde_json::Value::as_str) == Some("drop") + }); + if !chain_is_default_deny { + return Err(miette::miette!( + "egress ceiling output chain is absent or not policy drop" + )); + } + + let output_rules: Vec<&serde_json::Value> = objects + .iter() + .filter_map(|object| object.get("rule")) + .collect(); + for rule in &output_rules { + let expressions = rule + .get("expr") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| miette::miette!("egress ceiling rule has no expression list"))?; + for expression in expressions { + let keys = expression + .as_object() + .ok_or_else(|| miette::miette!("egress ceiling contains a malformed expression"))?; + if keys.len() != 1 + || !keys.keys().all(|key| { + matches!( + key.as_str(), + "match" | "counter" | "limit" | "log" | "reject" | "drop" | "accept" + ) + }) + { + return Err(miette::miette!( + "egress ceiling contains an unsupported or redirecting expression" + )); + } + } + } + let accept_rules: Vec<&serde_json::Value> = output_rules + .into_iter() + .filter(|rule| { + rule.get("family").and_then(serde_json::Value::as_str) == Some("inet") + && rule.get("table").and_then(serde_json::Value::as_str) == Some("openshell_bypass") + && rule.get("chain").and_then(serde_json::Value::as_str) == Some("output") + }) + .filter(|rule| { + rule.get("expr") + .and_then(serde_json::Value::as_array) + .is_some_and(|expressions| { + expressions + .iter() + .any(|expression| expression == &serde_json::json!({"accept": null})) + }) + }) + .collect(); + let proxy_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":host_ip}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":proxy_port}}, + {"accept":null} + ]); + let loopback_expressions = serde_json::json!([ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]); + let mut proxy_allowed = false; + let mut loopback_allowed = false; + for rule in accept_rules { + let expressions = rule.get("expr").expect("accept rule has expressions"); + if expressions == &proxy_expressions { + proxy_allowed = true; + } else if expressions == &loopback_expressions { + loopback_allowed = true; + } else { + return Err(miette::miette!( + "egress ceiling contains an unexpected accept rule: {expressions}" + )); + } + } + if !proxy_allowed || !loopback_allowed { + return Err(miette::miette!( + "egress ceiling is missing the proxy or loopback accept rule" + )); + } + Ok(()) +} + impl Drop for NetworkNamespace { fn drop(&mut self) { debug!(namespace = %self.name, "Cleaning up network namespace"); @@ -558,7 +707,9 @@ impl Drop for NetworkNamespace { } // Delete the host-side veth (this also removes the peer) - if let Err(e) = run_ip(&["link", "delete", &self.veth_host]) { + let mut cleanup_failed = false; + if let Err(e) = run_ip(self.helper_source, &["link", "delete", &self.veth_host]) { + cleanup_failed = true; warn!( error = %e, veth = %self.veth_host, @@ -567,7 +718,8 @@ impl Drop for NetworkNamespace { } // Delete the namespace - if let Err(e) = run_ip(&["netns", "delete", &self.name]) { + if let Err(e) = run_ip(self.helper_source, &["netns", "delete", &self.name]) { + cleanup_failed = true; warn!( error = %e, namespace = %self.name, @@ -575,14 +727,32 @@ impl Drop for NetworkNamespace { ); } - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Informational) - .status(openshell_ocsf::StatusId::Success) - .state(openshell_ocsf::StateId::Disabled, "cleaned_up") - .message(format!("Network namespace cleaned up [ns:{}]", self.name)) - .build() - ); + let event = openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(if cleanup_failed { + openshell_ocsf::SeverityId::High + } else { + openshell_ocsf::SeverityId::Informational + }) + .status(if cleanup_failed { + openshell_ocsf::StatusId::Failure + } else { + openshell_ocsf::StatusId::Success + }) + .state( + openshell_ocsf::StateId::Disabled, + if cleanup_failed { + "cleanup_failed" + } else { + "cleaned_up" + }, + ) + .message(if cleanup_failed { + format!("Network namespace cleanup incomplete [ns:{}]", self.name) + } else { + format!("Network namespace cleaned up [ns:{}]", self.name) + }) + .build(); + openshell_ocsf::ocsf_emit!(event); } } @@ -597,10 +767,25 @@ impl Drop for NetworkNamespace { /// /// Returns an error if proxy mode is requested but the namespace cannot be /// created (e.g., missing `CAP_NET_ADMIN` / `CAP_SYS_ADMIN` or `iproute2`). -/// Failure to install nftables bypass-detection rules is non-fatal and is -/// reported via OCSF instead. +/// Legacy bypass-rule installation remains best-effort for compatibility. pub fn create_netns_for_proxy( policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, false) +} + +/// Create a proxy namespace whose nftables policy is a mandatory RFC 0012 +/// default-deny ceiling. Unlike the legacy helper, any installation failure +/// aborts boundary establishment. +pub fn create_conformant_netns_for_proxy( + policy: &openshell_core::policy::SandboxPolicy, +) -> Result> { + create_netns(policy, true) +} + +fn create_netns( + policy: &openshell_core::policy::SandboxPolicy, + require_egress_ceiling: bool, ) -> Result> { use openshell_core::policy::NetworkMode; use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; @@ -608,7 +793,12 @@ pub fn create_netns_for_proxy( if !matches!(policy.network.mode, NetworkMode::Proxy) { return Ok(None); } - match NetworkNamespace::create() { + let namespace = if require_egress_ceiling { + NetworkNamespace::create_conformant() + } else { + NetworkNamespace::create() + }; + match namespace { Ok(ns) => { let proxy_port = policy .network @@ -616,14 +806,26 @@ pub fn create_netns_for_proxy( .as_ref() .and_then(|p| p.http_addr) .map_or(3128, |addr| addr.port()); - if let Err(e) = ns.install_bypass_rules(proxy_port) { + if require_egress_ceiling { + ns.install_egress_ceiling(proxy_port).map_err(|error| { + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Disabled, "failed") + .message(format!("Failed to establish egress ceiling: {error}")) + .build() + ); + error + })?; + } else if let Err(error) = ns.install_bypass_rules(proxy_port) { ocsf_emit!( ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) .severity(SeverityId::Medium) .status(StatusId::Failure) .state(StateId::Disabled, "degraded") .message(format!( - "Failed to install bypass detection rules (non-fatal): {e}" + "Failed to install bypass detection rules (non-fatal): {error}" )) .build() ); @@ -666,7 +868,7 @@ pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { } fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { - let nft_cmd = find_nft().ok_or_else(|| { + let nft_cmd = find_nft(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted nft helper not found; sidecar network enforcement requires nftables" ) @@ -680,14 +882,14 @@ const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { - let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { + let ipv4_filter_tool = find_iptables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" ) })?; let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { - Some(find_ip6tables_legacy().ok_or_else(|| { + Some(find_ip6tables_legacy(HelperSource::TrustedSupervisorRuntime).ok_or_else(|| { miette::miette!( "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" ) @@ -699,17 +901,14 @@ fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { None }; - cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); if let Err(e) = install_sidecar_iptables_legacy_family_rules( &ipv4_filter_tool, proxy_uid, "icmp-port-unreachable", ) { - cleanup_sidecar_iptables_legacy_rule_families( - &ipv4_filter_tool, - ipv6_fence_tool.as_deref(), - ); + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_ref()); return Err(e); } @@ -746,7 +945,7 @@ fn has_non_loopback_ipv6_interface(content: &str) -> bool { } fn install_sidecar_iptables_legacy_family_rules( - cmd: &str, + cmd: &TrustedHelper, proxy_uid: u32, udp_reject_with: &str, ) -> Result<()> { @@ -807,7 +1006,7 @@ fn install_sidecar_iptables_legacy_family_rules( Ok(()) } -fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { +fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &TrustedHelper) { while run_iptables_legacy_current_namespace( iptables_cmd, &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], @@ -818,28 +1017,70 @@ fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); } -fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { +fn cleanup_sidecar_iptables_legacy_rule_families( + ipv4_cmd: &TrustedHelper, + ipv6_cmd: Option<&TrustedHelper>, +) { cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); if let Some(ipv6_cmd) = ipv6_cmd { cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); } } +#[allow(unsafe_code)] +fn trusted_command_in_netns(helper: &TrustedHelper, netns: &str) -> Result { + use std::os::unix::process::CommandExt as _; + + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + +#[allow(unsafe_code)] +fn trusted_tokio_command_in_netns( + helper: &TrustedHelper, + netns: &str, +) -> Result { + let namespace = std::fs::File::open(format!("/var/run/netns/{netns}")).into_diagnostic()?; + let mut command = helper.tokio_command(); + // SAFETY: `setns` is async-signal-safe and the captured file remains open + // in the child until this pre-exec hook completes. + unsafe { + command.pre_exec(move || { + if libc::setns(namespace.as_raw_fd(), libc::CLONE_NEWNET) == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } + }); + } + Ok(command) +} + /// Run an `ip` command on the host. -fn run_ip(args: &[&str]) -> Result<()> { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; +fn run_ip(source: HelperSource, args: &[&str]) -> Result<()> { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; - debug!(command = %format!("{ip_path} {}", args.join(" ")), "Running ip command"); + debug!(command = %format!("{} {}", ip.executable.display(), args.join(" ")), "Running ip command"); - let output = Command::new(ip_path) - .args(args) - .output() - .into_diagnostic()?; + let output = ip.command().args(args).output().into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{ip_path} {} failed: {}", + "{} {} failed: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); @@ -848,13 +1089,17 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } -fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { +fn run_iptables_legacy_current_namespace( + iptables_cmd: &TrustedHelper, + args: &[&str], +) -> Result<()> { debug!( - command = %format!("{iptables_cmd} {}", args.join(" ")), + command = %format!("{} {}", iptables_cmd.executable.display(), args.join(" ")), "Running iptables-legacy sidecar command" ); - let output = Command::new(iptables_cmd) + let output = iptables_cmd + .command() .args(args) .output() .into_diagnostic()?; @@ -862,7 +1107,8 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{iptables_cmd} {} failed: {}", + "{} {} failed: {}", + iptables_cmd.executable.display(), args.join(" "), stderr.trim() )); @@ -880,14 +1126,15 @@ fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> R /// Commands marked as non-required are allowed to fail with a warning. /// Required commands that fail abort the sequence immediately. fn run_nft_commands_current_namespace( - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { for cmd in commands { let args_str = cmd.args.join(" "); - debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); + debug!(command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command"); - let output = Command::new(nft_cmd) + let output = nft_cmd + .command() .args(&cmd.args) .output() .into_diagnostic()?; @@ -896,7 +1143,8 @@ fn run_nft_commands_current_namespace( let stderr = String::from_utf8_lossy(&output.stderr); if cmd.required { return Err(miette::miette!( - "{nft_cmd} {args_str} failed: {}", + "{} {args_str} failed: {}", + nft_cmd.executable.display(), stderr.trim() )); } @@ -910,102 +1158,55 @@ fn run_nft_commands_current_namespace( Ok(()) } -/// Run an `ip` command inside a network namespace via `nsenter --net=`. +/// Run an `ip` command inside a network namespace. /// -/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` -/// remounts `/sys` to reflect the target namespace's sysfs entries. That -/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, -/// which is unavailable in rootless container runtimes (e.g. rootless -/// Podman). `nsenter --net=` enters only the network namespace without -/// changing the mount namespace, avoiding the sysfs remount entirely. -/// The supervisor's operations (addr add, link set, route add) are all -/// netlink-based and do not need sysfs access. -fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - run_ip_netns_output(netns, args).map(|_| ()) -} - -fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { - let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - - let mut full_args = vec![net_flag.as_str(), "--", ip_path]; - full_args.extend(args); +/// The child enters only the network namespace before exec. This avoids both +/// `ip netns exec`'s sysfs remount and a separate `nsenter` helper. +fn run_ip_netns(source: HelperSource, netns: &str, args: &[&str]) -> Result<()> { + let ip = find_binary(source, "ip", IP_SEARCH_PATHS)?; debug!( - command = %format!("{nsenter_path} {}", full_args.join(" ")), - "Running ip in namespace via nsenter" + command = %format!("{} {}", ip.executable.display(), args.join(" ")), + "Running ip in namespace" ); - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(&ip, netns)? + .args(args) .output() .into_diagnostic()?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "{nsenter_path} --net={} {ip_path} {} failed: {}", - ns_path.display(), + "{} {} failed in netns {netns}: {}", + ip.executable.display(), args.join(" "), stderr.trim() )); } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - -fn first_route_overlap( - routes: &str, - reserved: &[ipnet::IpNet], -) -> Option<(ipnet::IpNet, ipnet::IpNet)> { - routes.lines().find_map(|line| { - line.split_whitespace().find_map(|token| { - let route = token - .parse::() - .ok() - .or_else(|| token.parse::().ok().map(ipnet::IpNet::from))?; - reserved - .iter() - .copied() - .find(|pool| { - let same_family = route.addr().is_ipv4() == pool.addr().is_ipv4(); - let overlaps = - route.contains(&pool.network()) || pool.contains(&route.network()); - same_family && overlaps - }) - .map(|pool| (route, pool)) - }) - }) + Ok(()) } -/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. +/// Run a sequence of nft commands inside a network namespace. /// /// Each command is executed as a separate invocation to avoid atomic batch /// rollback. See [`run_nft_commands_current_namespace`] for rationale. fn run_nft_commands_netns( netns: &str, - nft_cmd: &str, + nft_cmd: &TrustedHelper, commands: &[nft_ruleset::NftCommand], ) -> Result<()> { - let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; - let ns_path = openshell_core::container_paths::netns_path(netns); - let net_flag = format!("--net={}", ns_path.display()); - for cmd in commands { let args_str = cmd.args.join(" "); debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), + command = %format!("{} {args_str}", nft_cmd.executable.display()), "Running nft command in namespace" ); - let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); - full_args.extend(&arg_refs); - - let output = Command::new(nsenter_path) - .args(&full_args) + let output = trusted_command_in_netns(nft_cmd, netns)? + .args(&arg_refs) .output() .into_diagnostic()?; @@ -1055,109 +1256,282 @@ fn enable_nf_log_all_netns() { } } -/// Well-known paths where nft may be installed. -const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; +/// Paths within the driver-controlled supervisor runtime. +const NFT_SEARCH_PATHS: &[&str] = &["usr/sbin/nft", "sbin/nft", "usr/bin/nft"]; const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/iptables-legacy", - "/sbin/iptables-legacy", - "/usr/bin/iptables-legacy", + "usr/sbin/iptables-legacy", + "sbin/iptables-legacy", + "usr/bin/iptables-legacy", ]; const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ - "/usr/sbin/ip6tables-legacy", - "/sbin/ip6tables-legacy", - "/usr/bin/ip6tables-legacy", + "usr/sbin/ip6tables-legacy", + "sbin/ip6tables-legacy", + "usr/bin/ip6tables-legacy", ]; -fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { - paths +fn trusted_runtime_root() -> Result { + #[cfg(test)] + if let Some(root) = std::env::var_os("OPENSHELL_TEST_TRUSTED_RUNTIME_ROOT") { + return Ok(PathBuf::from(root)); + } + if let Some(root) = TRUSTED_RUNTIME_ROOT.get() { + return Ok(root.clone()); + } + let executable = std::env::current_exe().into_diagnostic()?; + let parent = executable + .parent() + .ok_or_else(|| miette::miette!("supervisor executable has no parent directory"))?; + Ok(parent.join("openshell-runtime")) +} + +fn find_binary(source: HelperSource, name: &str, paths: &[&str]) -> Result { + match source { + HelperSource::LegacyWorkloadImage => find_legacy_binary(name, paths), + HelperSource::TrustedSupervisorRuntime => find_trusted_binary(name, paths), + } +} + +fn find_legacy_binary(name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt as _; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let executable = paths .iter() - .copied() - .find(|path| { - let path = Path::new(path); - path.is_absolute() && path.is_file() + .map(|path| Path::new("/").join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + let metadata = resolved.metadata().ok()?; + (metadata.is_file() + && metadata.uid() == trusted_uid + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) }) .ok_or_else(|| { miette::miette!( - "trusted {name} helper not found; checked {}", + "{name} helper not found in legacy workload image; checked {}", paths.join(", ") ) + })?; + Ok(TrustedHelper { + executable, + loader: None, + library_path: String::new(), + xtables_path: PathBuf::new(), + }) +} + +fn find_trusted_binary(name: &str, paths: &[&str]) -> Result { + find_trusted_binary_in(&trusted_runtime_root()?, name, paths) +} + +fn find_trusted_binary_in(root: &Path, name: &str, paths: &[&str]) -> Result { + use std::os::unix::fs::MetadataExt; + + let trusted_uid = nix::unistd::geteuid().as_raw(); + let resolved_root = root.canonicalize().map_err(|error| { + miette::miette!( + "trusted supervisor helper runtime {} is unavailable: {error}", + root.display() + ) + })?; + // Kubernetes and Podman preserve root ownership from the supervisor image. + // Docker may materialize the same image-owned runtime in a gateway-user + // cache before bind-mounting it read-only. In that case the immutable + // mount, not its namespace-visible UID, establishes provenance. + let runtime_is_read_only = nix::sys::statvfs::statvfs(&resolved_root) + .is_ok_and(|stat| stat.flags().contains(nix::sys::statvfs::FsFlags::ST_RDONLY)); + let executable = paths + .iter() + .map(|path| resolved_root.join(path)) + .find_map(|path| { + let resolved = path.canonicalize().ok()?; + if !resolved.starts_with(&resolved_root) { + return None; + } + let Ok(metadata) = resolved.metadata() else { + return None; + }; + (metadata.is_file() + && (metadata.uid() == trusted_uid || runtime_is_read_only) + && metadata.mode() & 0o111 != 0 + && metadata.mode() & 0o022 == 0) + .then_some(resolved) + }) + .ok_or_else(|| { + miette::miette!( + "trusted {name} helper not found below {}; checked {}", + resolved_root.display(), + paths.join(", ") + ) + })?; + let loader = runtime_library_directories(&resolved_root) + .into_iter() + .filter_map(|directory| std::fs::read_dir(directory).ok()) + .flatten() + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .find(|path| is_runtime_loader(path)) + .ok_or_else(|| { + miette::miette!( + "trusted dynamic loader not found below {}", + resolved_root.display() + ) + })? + .canonicalize() + .into_diagnostic()?; + if !loader.starts_with(&resolved_root) { + return Err(miette::miette!("trusted runtime loader escapes its root")); + } + let loader_metadata = loader.metadata().into_diagnostic()?; + if !loader_metadata.is_file() + || (loader_metadata.uid() != trusted_uid && !runtime_is_read_only) + || loader_metadata.mode() & 0o111 == 0 + || loader_metadata.mode() & 0o022 != 0 + { + return Err(miette::miette!( + "trusted runtime loader has unsafe ownership or mode" + )); + } + let library_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(":"); + let xtables_path = runtime_library_directories(&resolved_root) + .into_iter() + .map(|directory| directory.join("xtables")) + .find(|path| path.is_dir()) + .unwrap_or_else(|| resolved_root.join("usr/lib/xtables")); + Ok(TrustedHelper { + executable, + loader: Some(loader), + library_path, + xtables_path, + }) +} + +fn runtime_library_directories(root: &Path) -> Vec { + let mut directories = Vec::new(); + for base in ["lib", "lib64", "usr/lib", "usr/lib64"] { + let base = root.join(base); + if !base.is_dir() { + continue; + } + directories.push(base.clone()); + if let Ok(entries) = std::fs::read_dir(base) { + directories.extend( + entries + .filter_map(std::result::Result::ok) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()), + ); + } + } + directories +} + +fn is_runtime_loader(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + (name.starts_with("ld-musl-") && name.ends_with(".so.1")) + || name == "ld-linux-x86-64.so.2" + || name == "ld-linux-aarch64.so.1" }) } /// Find the nft binary path, checking well-known locations. -fn find_nft() -> Option { - find_trusted_binary("nft", NFT_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_nft(source: HelperSource) -> Option { + find_binary(source, "nft", NFT_SEARCH_PATHS).ok() } -fn find_iptables_legacy() -> Option { - find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_iptables_legacy(source: HelperSource) -> Option { + find_binary(source, "iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS).ok() } -fn find_ip6tables_legacy() -> Option { - find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) - .ok() - .map(String::from) +fn find_ip6tables_legacy(source: HelperSource) -> Option { + find_binary(source, "ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS).ok() } #[cfg(test)] mod tests { use super::*; use std::fs; + use std::os::unix::fs::PermissionsExt as _; // These tests require root and network namespace support // Run with: sudo cargo test -- --ignored #[test] - fn find_trusted_binary_uses_absolute_existing_file() { + fn find_trusted_binary_uses_only_the_supplied_runtime() { let tempdir = tempfile::tempdir().unwrap(); - let helper = tempdir.path().join("ip"); + let helper = tempdir.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); fs::write(&helper, b"test helper").unwrap(); - let helper = helper.to_str().unwrap(); - - assert_eq!( - find_trusted_binary("ip", &["relative-ip", "/missing/ip", helper]).unwrap(), - helper - ); + fs::set_permissions(&helper, fs::Permissions::from_mode(0o755)).unwrap(); + let lib = tempdir.path().join("lib"); + fs::create_dir(&lib).unwrap(); + let loader = lib.join("ld-musl-test.so.1"); + fs::write(&loader, b"test loader").unwrap(); + fs::set_permissions(loader, fs::Permissions::from_mode(0o755)).unwrap(); + + let resolved = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap(); + assert_eq!(resolved.executable, helper); } #[test] fn find_trusted_binary_rejects_missing_helpers() { - let err = - find_trusted_binary("nsenter", &["relative-nsenter", "/missing/nsenter"]).unwrap_err(); + let tempdir = tempfile::tempdir().unwrap(); + let err = find_trusted_binary_in(tempdir.path(), "ip", &["usr/sbin/ip"]).unwrap_err(); - assert!(err.to_string().contains("trusted nsenter helper not found")); + assert!(err.to_string().contains("trusted ip helper not found")); } #[test] - fn nft_search_paths_are_absolute() { + fn trusted_runtime_rejects_helper_symlink_into_workload_root() { + use std::os::unix::fs::symlink; + + let runtime = tempfile::tempdir().unwrap(); + let workload = tempfile::tempdir().unwrap(); + let malicious = workload.path().join("ip"); + fs::write(&malicious, b"malicious workload helper").unwrap(); + fs::set_permissions(&malicious, fs::Permissions::from_mode(0o755)).unwrap(); + let helper = runtime.path().join("usr/sbin/ip"); + fs::create_dir_all(helper.parent().unwrap()).unwrap(); + symlink(&malicious, &helper).unwrap(); + + let error = find_trusted_binary_in(runtime.path(), "ip", &["usr/sbin/ip"]) + .expect_err("helper escaping the trusted runtime must be rejected"); + assert!(error.to_string().contains("trusted ip helper not found")); + } + + #[test] + fn nft_search_paths_are_runtime_relative() { for path in NFT_SEARCH_PATHS { assert!( - path.starts_with('/'), - "NFT_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "NFT_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn iptables_legacy_search_paths_are_absolute() { + fn iptables_legacy_search_paths_are_runtime_relative() { for path in IPTABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IPTABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } #[test] - fn ip6tables_legacy_search_paths_are_absolute() { + fn ip6tables_legacy_search_paths_are_runtime_relative() { for path in IP6TABLES_LEGACY_SEARCH_PATHS { assert!( - path.starts_with('/'), - "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + !path.starts_with('/'), + "IP6TABLES_LEGACY_SEARCH_PATHS entry must be runtime-relative: {path}" ); } } @@ -1186,25 +1560,112 @@ fe800000000000000000000000000001 02 40 20 80 eth0 } #[test] - fn route_overlap_detects_reserved_pool_collision() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; - let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); - assert_eq!(route.to_string(), "198.18.0.0/15"); - assert_eq!(pool.to_string(), "198.18.1.0/25"); + fn egress_ceiling_verification_accepts_required_live_rules() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}}, + {"match":{"op":"==","left":{"payload":{"protocol":"tcp","field":"dport"}},"right":3128}}, + {"accept":null} + ]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {"match":{"op":"==","left":{"meta":{"key":"oifname"}},"right":"lo"}}, + {"accept":null} + ]}} + ] + }"#; + + verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).unwrap(); + } + + #[test] + fn egress_ceiling_verification_rejects_fail_open_chain() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"accept"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"10.200.0.1"}},{"match":{"right":3128}},{"accept":null}]}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_missing_required_allow() { + let ruleset = br#"{ + "nftables": [ + {"chain":{"family":"inet","table":"openshell_bypass","name":"output","hook":"output","policy":"drop"}}, + {"rule":{"family":"inet","table":"openshell_bypass","chain":"output","expr":[{"match":{"right":"lo"}},{"accept":null}]}} + ] + }"#; + + assert!(verify_egress_ceiling_json(ruleset, "10.200.0.1", 3128).is_err()); + } + + fn ruleset_with_accept_expressions(expressions: &str) -> Vec { + format!( + r#"{{"nftables":[ + {{"chain":{{"family":"inet","table":"openshell_bypass","name":"output","type":"filter","hook":"output","prio":0,"policy":"drop"}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"ip","field":"daddr"}}}},"right":"10.200.0.1"}}}}, + {{"match":{{"op":"==","left":{{"payload":{{"protocol":"tcp","field":"dport"}}}},"right":3128}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":[ + {{"match":{{"op":"==","left":{{"meta":{{"key":"oifname"}}}},"right":"lo"}}}},{{"accept":null}}]}}}}, + {{"rule":{{"family":"inet","table":"openshell_bypass","chain":"output","expr":{expressions}}}}} + ]}}"# + ) + .into_bytes() + } + + #[test] + fn egress_ceiling_verification_rejects_unconditional_accept() { + let ruleset = ruleset_with_accept_expressions(r#"[{"accept":null}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); } #[test] - fn route_overlap_ignores_default_and_unrelated_routes() { - let reserved = [ - "198.18.1.0/25".parse().unwrap(), - "fd23:6f70:656e:1::/120".parse().unwrap(), - ]; - let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; - assert_eq!(first_route_overlap(routes, &reserved), None); + fn egress_ceiling_verification_rejects_unrelated_matching_metadata() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"comment":{"address":"10.200.0.1","port":3128}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_wrong_protocol_or_operator() { + for expressions in [ + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"udp","field":"dport"}},"right":3128}},{"accept":null}]"#, + r#"[{"match":{"op":"!=","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"10.200.0.1"}},{"accept":null}]"#, + ] { + let ruleset = ruleset_with_accept_expressions(expressions); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + } + + #[test] + fn egress_ceiling_verification_rejects_extra_destination_allow() { + let ruleset = ruleset_with_accept_expressions( + r#"[{"match":{"op":"==","left":{"payload":{"protocol":"ip","field":"daddr"}},"right":"203.0.113.1"}},{"accept":null}]"#, + ); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_rejects_jump_to_unverified_chain() { + let ruleset = ruleset_with_accept_expressions(r#"[{"jump":{"target":"unverified"}}]"#); + assert!(verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).is_err()); + } + + #[test] + fn egress_ceiling_verification_ignores_accept_in_another_chain() { + let mut document: serde_json::Value = + serde_json::from_slice(&ruleset_with_accept_expressions(r#"[{"accept":null}]"#)) + .unwrap(); + document["nftables"][3]["rule"]["chain"] = serde_json::json!("other"); + let ruleset = serde_json::to_vec(&document).unwrap(); + verify_egress_ceiling_json(&ruleset, "10.200.0.1", 3128).unwrap(); } #[test] @@ -1214,8 +1675,8 @@ fe800000000000000000000000000001 02 40 20 80 eth0 let name = ns.name().to_string(); // Verify namespace exists - let ns_path = openshell_core::container_paths::netns_path(&name); - assert!(ns_path.exists(), "Namespace file should exist"); + let ns_path = format!("/var/run/netns/{name}"); + assert!(Path::new(&ns_path).exists(), "Namespace file should exist"); // Verify IPs are set correctly assert_eq!( @@ -1236,4 +1697,181 @@ fe800000000000000000000000000001 02 40 20 80 eth0 "Namespace should be cleaned up" ); } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_round_trips_through_kernel() { + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + ns.install_egress_ceiling(3128).expect("install ceiling"); + ns.verify_egress_ceiling(3128).expect("verify ceiling"); + } + + #[test] + #[ignore = "requires root privileges"] + fn installed_egress_ceiling_allows_only_proxy_tcp() { + use std::time::Duration; + + #[allow(unsafe_code)] + fn enter_namespace(ns_fd: RawFd) { + // SAFETY: the owning NetworkNamespace remains alive until every + // test thread has joined, so the descriptor stays valid. + let result = unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) }; + assert_eq!(result, 0, "enter workload network namespace"); + } + + let ns = NetworkNamespace::create_conformant().expect("create conformant namespace"); + let ns_fd = ns.ns_fd().expect("network namespace fd"); + let host_ip = ns.host_ip(); + + let alternate_host_ip: std::net::Ipv4Addr = "10.200.0.3".parse().unwrap(); + run_ip( + ns.helper_source, + &["addr", "add", "10.200.0.3/24", "dev", &ns.veth_host], + ) + .expect("add alternate routed IPv4 destination"); + let host_ipv6: std::net::Ipv6Addr = "fd00:200::1".parse().unwrap(); + run_ip( + ns.helper_source, + &[ + "-6", + "addr", + "add", + "fd00:200::1/64", + "dev", + &ns.veth_host, + "nodad", + ], + ) + .expect("add host IPv6 destination"); + run_ip_netns( + ns.helper_source, + ns.name(), + &[ + "-6", + "addr", + "add", + "fd00:200::2/64", + "dev", + &ns.veth_sandbox, + "nodad", + ], + ) + .expect("add workload IPv6 source"); + + // Positive controls prove each route/protocol works before the ceiling + // is installed, so later denial cannot pass because of broken setup. + let ipv4_control = + std::net::TcpListener::bind((alternate_host_ip, 0)).expect("bind IPv4 control"); + let ipv4_control_address = ipv4_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv4_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv4 control thread") + .is_ok(), + "alternate IPv4 route must work before enforcement" + ); + + let ipv6_control = std::net::TcpListener::bind((host_ipv6, 0)).expect("bind IPv6 control"); + let ipv6_control_address = ipv6_control.local_addr().unwrap(); + assert!( + std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_control_address, Duration::from_secs(1)) + }) + .join() + .expect("IPv6 control thread") + .is_ok(), + "IPv6 route must work before enforcement" + ); + + let udp_control = + std::net::UdpSocket::bind((host_ip, 0)).expect("bind UDP positive control"); + udp_control + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let udp_control_address = udp_control.local_addr().unwrap(); + std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind control UDP"); + socket.send_to(b"control", udp_control_address) + }) + .join() + .expect("UDP control thread") + .expect("send UDP positive control"); + let mut control = [0_u8; 7]; + udp_control + .recv_from(&mut control) + .expect("UDP route must work before enforcement"); + assert_eq!(&control, b"control"); + + let proxy = std::net::TcpListener::bind((host_ip, 0)).expect("bind proxy listener"); + let proxy_address = proxy.local_addr().expect("proxy address"); + ns.install_egress_ceiling(proxy_address.port()) + .expect("install ceiling"); + + let allowed = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&proxy_address, Duration::from_secs(1)) + }); + proxy + .set_nonblocking(true) + .expect("set proxy listener nonblocking"); + assert!(allowed.join().expect("allowed-connect thread").is_ok()); + + let direct = std::net::TcpListener::bind((host_ip, 0)).expect("bind direct listener"); + let direct_address = direct.local_addr().expect("direct address"); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&direct_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("denied-connect thread").is_err(), + "direct TCP must not bypass mediation" + ); + + let alternate = std::net::TcpListener::bind((alternate_host_ip, proxy_address.port())) + .expect("bind alternate routed listener"); + let alternate_address = alternate.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&alternate_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("alternate-connect thread").is_err(), + "the proxy port at another routed IPv4 destination must be denied" + ); + + let ipv6 = std::net::TcpListener::bind((host_ipv6, proxy_address.port())) + .expect("bind IPv6 observer"); + let ipv6_address = ipv6.local_addr().unwrap(); + let denied = std::thread::spawn(move || { + enter_namespace(ns_fd); + std::net::TcpStream::connect_timeout(&ipv6_address, Duration::from_millis(300)) + }); + assert!( + denied.join().expect("IPv6-connect thread").is_err(), + "direct IPv6 TCP must not bypass mediation" + ); + + let udp = std::net::UdpSocket::bind((host_ip, proxy_address.port())) + .expect("bind UDP observer at proxy destination"); + udp.set_read_timeout(Some(Duration::from_millis(300))) + .expect("set UDP timeout"); + let udp_address = udp.local_addr().expect("UDP address"); + let udp_send = std::thread::spawn(move || { + enter_namespace(ns_fd); + let socket = std::net::UdpSocket::bind("0.0.0.0:0").expect("bind workload UDP"); + socket.send_to(b"bypass", udp_address) + }) + .join() + .expect("UDP thread"); + let mut byte = [0_u8; 1]; + assert!( + udp_send.is_err() || udp.recv_from(&mut byte).is_err(), + "direct UDP must not bypass mediation" + ); + } } diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 2fb075b420..72b55a54e3 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -13,8 +13,6 @@ //! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the //! entire transaction including table/chain creation. -const DNS_DESTINATION_PORT: &str = "53"; - /// A single nft command with metadata about whether it is required. pub struct NftCommand { /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). @@ -235,190 +233,6 @@ fn generate_commands( cmds } -/// Generate the combined policy-DNS, transparent-TCP, and bypass fence. -/// -/// DNS may reach only the supervisor's trusted listener. TCP addressed to the -/// reserved synthetic pools is redirected before the terminal bypass reject; -/// all other direct TCP/UDP retains the existing fast-fail behavior. -pub fn generate_transparent_tcp_commands( - host_ip: &str, - proxy_port: u16, - dns_port: u16, - transparent_port: u16, - synthetic_ipv4_cidr: &str, - synthetic_ipv6_cidr: &str, - log_prefix: Option<&str>, -) -> Vec { - let mut cmds = vec![ - nft_cmd(true, &["add", "table", "inet", "openshell_transparent"]), - nft_cmd(true, &["flush", "table", "inet", "openshell_transparent"]), - nft_cmd( - true, - &[ - "add", - "chain", - "inet", - "openshell_transparent", - "output", - "{ type nat hook output priority dstnat; policy accept; }", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "udp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip", - "daddr", - synthetic_ipv4_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - // Synthetic destinations must take precedence over the generic TCP - // DNS capture. A policy endpoint may legitimately use TCP port 53; - // that connection belongs to transparent TCP, not the DNS listener. - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "meta", - "nfproto", - "ipv4", - "tcp", - "dport", - DNS_DESTINATION_PORT, - "redirect", - "to", - &format!(":{dns_port}"), - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_transparent", - "output", - "ip6", - "daddr", - synthetic_ipv6_cidr, - "tcp", - "dport", - "1-65535", - "redirect", - "to", - &format!(":{transparent_port}"), - ], - ), - ]; - let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); - // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the - // filter hook. Some kernels retain the packet's pre-REDIRECT output - // interface for filter matching, so `oifname lo accept` alone is not - // portable. Admit only packets that the kernel records as DNATed to the - // supervisor listeners. A direct dial to either port has no DNAT status - // and still reaches the terminal bypass reject. Transparent TCP - // authorization after accept remains bound by SO_ORIGINAL_DST plus the - // synthetic-address mapping. - let insertion = bypass - .iter() - .position(|command| { - command.args.iter().any(|arg| arg == "log") - || command.args.iter().any(|arg| arg == "reject") - }) - .unwrap_or(bypass.len()); - bypass.splice( - insertion..insertion, - [ - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "udp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &transparent_port.to_string(), - "accept", - ], - ), - ], - ); - cmds.extend(bypass); - cmds -} - /// Generate nft commands for Kubernetes sidecar enforcement. /// /// The network sidecar and the process supervisor share a pod network @@ -668,78 +482,6 @@ mod tests { assert!(ct_pos < reject_pos); } - #[test] - fn transparent_rules_precede_bypass_rejects_and_scope_dns() { - let commands = generate_transparent_tcp_commands( - "10.200.0.1", - 3128, - 15053, - 15001, - "198.18.0.0/24", - "fd23:6f70:656e::/48", - None, - ); - let text = all_strs(&commands); - assert!(text.contains("meta nfproto ipv4 udp dport 53 redirect to :15053")); - assert!(text.contains("meta nfproto ipv4 tcp dport 53 redirect to :15053")); - assert!(!text.contains("udp dport 53 accept")); - assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); - assert!( - text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") - ); - assert!(!text.contains("meta mark")); - assert!(text.contains("ct status dnat udp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15053 accept")); - assert!(text.contains("ct status dnat tcp dport 15001 accept")); - for (protocol, port) in [("udp", "15053"), ("tcp", "15053"), ("tcp", "15001")] { - assert!(!commands.iter().any(|command| { - command.args.ends_with(&[ - protocol.to_string(), - "dport".to_string(), - port.to_string(), - "accept".to_string(), - ]) && !command.args.windows(3).any(|window| { - window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] - }) - })); - } - assert!(text.contains("oifname lo accept")); - assert!( - text.find("ct status dnat tcp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat udp dport 15053 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto udp reject") - .unwrap() - ); - assert!( - text.find("ct status dnat tcp dport 15001 accept").unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 meta l4proto tcp reject") - .unwrap() - ); - assert!(!text.contains("meta nfproto ipv6 udp dport 53 redirect")); - assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") - .unwrap() - < text - .find("meta nfproto ipv4 tcp dport 53 redirect to :15053") - .unwrap(), - "synthetic TCP:53 must reach transparent TCP before generic DNS capture" - ); - } - #[test] fn both_ipv4_and_ipv6_reject_types_are_present() { let cmds = generate_bypass_commands("10.0.2.2", 8080, None); diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 52557493cd..c77d27a6f9 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -9,45 +9,27 @@ use crate::managed_children; #[cfg(target_os = "linux")] use crate::netns::NetworkNamespace; use crate::sandbox; -#[cfg(target_os = "linux")] -use miette::WrapErr; use miette::{IntoDiagnostic, Result}; use nix::sys::signal::{self, Signal}; use nix::unistd::{Gid, Group, Pid, Uid, User}; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use std::collections::HashMap; use std::ffi::CString; -#[cfg(unix)] -use std::os::fd::AsRawFd; #[cfg(target_os = "linux")] -use std::os::fd::RawFd; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; #[cfg(target_os = "linux")] use std::os::unix::ffi::OsStrExt; -#[cfg(unix)] -use std::os::unix::fs::{MetadataExt, PermissionsExt}; #[cfg(any(test, unix))] use std::path::Path; use std::path::PathBuf; use std::process::Stdio; +use std::sync::Arc; #[cfg(target_os = "linux")] use std::sync::OnceLock; -#[cfg(target_os = "linux")] -use std::sync::mpsc; -use tokio::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; +use std::sync::atomic::{AtomicBool, Ordering}; +use tokio::process::{Child, Command}; use tracing::{debug, info}; -// `libc::TIOCSCTTY` and the request parameter accepted by `ioctl` vary across -// glibc, musl, and BSD targets. The conversion is a no-op on some targets but -// is required on others. -#[cfg(unix)] -#[allow(unsafe_code, clippy::useless_conversion)] -fn set_controlling_tty(fd: libc::c_int) -> std::io::Result<()> { - if unsafe { libc::ioctl(fd, libc::TIOCSCTTY.into(), 0) } < 0 { - return Err(std::io::Error::last_os_error()); - } - Ok(()) -} - /// Process/filesystem enforcement performed by the process supervisor. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ProcessEnforcementMode { @@ -99,35 +81,6 @@ impl ResolvedProcessIdentity { } } -/// Resolved process workspace and its child-environment semantics. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ResolvedWorkspace { - root: Option, - use_as_home: bool, -} - -impl ResolvedWorkspace { - #[must_use] - pub fn new(root: Option, use_as_home: bool) -> Self { - Self { root, use_as_home } - } - - #[must_use] - pub fn root(&self) -> Option<&str> { - self.root.as_deref() - } - - #[must_use] - pub fn owned_root(&self) -> Option { - self.root.clone() - } - - #[must_use] - pub fn home(&self) -> Option<&str> { - self.use_as_home.then(|| self.root()).flatten() - } -} - impl ProcessEnforcementMode { #[must_use] pub const fn uses_privileged_process_setup(self) -> bool { @@ -169,7 +122,6 @@ const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::TLS_CERT, openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, - openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, ]; pub fn is_supervisor_only_env_var(key: &str) -> bool { @@ -191,67 +143,6 @@ fn inject_provider_env(cmd: &mut Command, provider_env: &HashMap } } -/// Derive the child USER and HOME from the policy's sandbox identity. -/// -/// Name-based identities use their passwd entry. Numeric identities have no -/// reliable passwd entry, so their workspace remains the portable fallback. -pub(crate) fn session_user_and_home( - policy: &SandboxPolicy, - workdir_home: Option<&str>, -) -> (String, String) { - let (user, default_home) = match policy.process.run_as_user.as_deref() { - Some(user) if !user.is_empty() => { - if user.parse::().is_ok() { - (user.to_string(), "/sandbox".to_string()) - } else { - let home = User::from_name(user).ok().flatten().map_or_else( - || format!("/home/{user}"), - |entry| entry.dir.to_string_lossy().into_owned(), - ); - (user.to_string(), home) - } - } - _ => ("sandbox".to_string(), "/sandbox".to_string()), - }; - let home = workdir_home.map_or(default_home, str::to_string); - (user, home) -} - -fn apply_canonical_process_environment( - cmd: &mut Command, - policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, - interactive: bool, - user_environment: &HashMap, -) { - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); - - for (key, value) in [ - ("HOME", session_home.as_str()), - ("USER", session_user.as_str()), - ("SHELL", "/bin/bash"), - ( - "TERM", - if interactive { - "xterm-256color" - } else { - "dumb" - }, - ), - ] { - if !user_environment.contains_key(key) { - cmd.env(key, value); - } - } -} - -fn configured_user_environment() -> HashMap { - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default() -} - #[cfg(unix)] pub fn harden_child_process() -> Result<()> { use rustix::process::{Resource, Rlimit, setrlimit}; @@ -401,13 +292,11 @@ static SUPERVISOR_IDENTITY_MOUNT_NS: OnceLock, + fd: OwnedFd, } #[cfg(target_os = "linux")] type SupervisorIdentityNsRef = &'static SupervisorIdentityMountNamespace; -#[cfg(target_os = "linux")] -type SupervisorIdentitySpawnJob = Box; #[cfg(target_os = "linux")] impl SupervisorIdentityMountNamespace { @@ -416,9 +305,13 @@ impl SupervisorIdentityMountNamespace { return Ok(None); }; Ok(Some(Self { - spawn_tx: start_supervisor_identity_spawn_worker(target)?, + fd: create_supervisor_identity_mount_namespace(&target)?, })) } + + pub fn enter_for_child(&self) -> std::io::Result<()> { + set_mount_namespace(self.fd.as_raw_fd()) + } } #[cfg(target_os = "linux")] @@ -449,100 +342,6 @@ pub fn supervisor_identity_mount_from_env() -> Result std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_tokio_command(cmd) -} - -#[cfg(target_os = "linux")] -pub fn spawn_std_command_with_supervisor_identity_namespace( - mut cmd: std::process::Command, -) -> std::io::Result { - let namespace = supervisor_identity_mount_from_env() - .map_err(|err| std::io::Error::other(err.to_string()))?; - let Some(namespace) = namespace else { - return cmd.spawn(); - }; - namespace.spawn_std_command(cmd) -} - -#[cfg(target_os = "linux")] -impl SupervisorIdentityMountNamespace { - fn spawn_tokio_command(&self, mut cmd: Command) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - let handle = tokio::runtime::Handle::current(); - self.spawn_tx - .send(Box::new(move || { - let _guard = handle.enter(); - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } - - fn spawn_std_command( - &self, - mut cmd: std::process::Command, - ) -> std::io::Result { - let (result_tx, result_rx) = mpsc::channel(); - self.spawn_tx - .send(Box::new(move || { - let _ = result_tx.send(cmd.spawn()); - })) - .map_err(|_| std::io::Error::other("supervisor identity spawn worker stopped"))?; - result_rx - .recv() - .map_err(|_| std::io::Error::other("supervisor identity spawn worker dropped result"))? - } -} - -#[cfg(target_os = "linux")] -fn start_supervisor_identity_spawn_worker( - target: PathBuf, -) -> Result> { - let (spawn_tx, spawn_rx) = mpsc::channel::(); - let (ready_tx, ready_rx) = mpsc::channel::>(); - std::thread::Builder::new() - .name("openshell-identity-spawn".into()) - .spawn(move || { - let setup = (|| -> std::io::Result<()> { - private_mount_namespace()?; - let target = - cstring_path(&target).map_err(|err| std::io::Error::other(err.to_string()))?; - mount_empty_tmpfs(&target) - })(); - let ready = match &setup { - Ok(()) => Ok(()), - Err(err) => Err(std::io::Error::new( - err.kind(), - format!("supervisor identity setup failed: {err}"), - )), - }; - let _ = ready_tx.send(ready); - if setup.is_err() { - return; - } - while let Ok(job) = spawn_rx.recv() { - job(); - } - }) - .map_err(|err| miette::miette!("failed to spawn supervisor identity worker: {err}"))?; - ready_rx - .recv() - .map_err(|err| miette::miette!("supervisor identity worker did not start: {err}"))? - .map_err(|err| miette::miette!("{err}"))?; - Ok(spawn_tx) -} - #[cfg(target_os = "linux")] fn supervisor_identity_socket_path_from_env() -> Option<(&'static str, String)> { std::env::var(openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) @@ -563,7 +362,10 @@ fn supervisor_identity_mount_target(socket_path: &str) -> Result return Ok(None); } if trimmed.starts_with("tcp:") { - return Ok(None); + return Err(miette::miette!( + "{} must be a UNIX socket path so sandbox child processes can hide it", + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET + )); } let path = trimmed.strip_prefix("unix:").unwrap_or(trimmed); let path = Path::new(path); @@ -606,12 +408,52 @@ fn cstring_path(path: &Path) -> Result { .map_err(|_| miette::miette!("path contains an interior NUL byte: {}", path.display())) } +#[cfg(target_os = "linux")] +fn create_supervisor_identity_mount_namespace(target: &Path) -> Result { + let original_ns = open_current_mount_namespace() + .map_err(|err| miette::miette!("failed to open original mount namespace: {err}"))?; + + private_mount_namespace() + .map_err(|err| miette::miette!("failed to create supervisor identity namespace: {err}"))?; + + let target = cstring_path(target)?; + let result = (|| -> Result { + mount_empty_tmpfs(&target).map_err(|err| { + miette::miette!("failed to hide supervisor identity mount from child namespace: {err}") + })?; + open_current_mount_namespace() + .map_err(|err| miette::miette!("failed to open sanitized mount namespace: {err}")) + })(); + + set_mount_namespace(original_ns.as_raw_fd()).map_err(|restore_err| { + let result_msg = result.as_ref().err().map_or_else( + || "sanitized namespace was created".to_string(), + ToString::to_string, + ); + miette::miette!( + "failed to restore original mount namespace after supervisor identity isolation setup: \ + {restore_err}; setup result: {result_msg}" + ) + })?; + + result +} + +#[cfg(target_os = "linux")] +fn open_current_mount_namespace() -> std::io::Result { + let file = std::fs::File::open("/proc/thread-self/ns/mnt")?; + Ok(file.into()) +} + #[cfg(target_os = "linux")] fn private_mount_namespace() -> std::io::Result<()> { #[allow(unsafe_code)] let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) }; if rc != 0 { - return Err(std::io::Error::last_os_error()); + return Err(std::io::Error::other(format!( + "failed to create private mount namespace: {}", + std::io::Error::last_os_error() + ))); } #[allow(unsafe_code)] @@ -626,7 +468,23 @@ fn private_mount_namespace() -> std::io::Result<()> { ) }; if rc != 0 { - return Err(std::io::Error::last_os_error()); + return Err(std::io::Error::other(format!( + "failed to mark mount namespace private: {}", + std::io::Error::last_os_error() + ))); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn set_mount_namespace(fd: RawFd) -> std::io::Result<()> { + #[allow(unsafe_code)] + let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNS) }; + if rc != 0 { + return Err(std::io::Error::other(format!( + "failed to enter mount namespace: {}", + std::io::Error::last_os_error() + ))); } Ok(()) } @@ -646,7 +504,10 @@ fn mount_empty_tmpfs(target: &CString) -> std::io::Result<()> { ) }; if rc != 0 { - return Err(std::io::Error::last_os_error()); + return Err(std::io::Error::other(format!( + "failed to hide supervisor identity mount from child process: {}", + std::io::Error::last_os_error() + ))); } Ok(()) } @@ -655,18 +516,10 @@ fn mount_empty_tmpfs(target: &CString) -> std::io::Result<()> { pub struct ProcessHandle { child: Child, pid: u32, - io: Option, -} - -/// Supervisor-owned canonical-process I/O. These handles outlive individual -/// SSH attachments and are consumed by the main-session multiplexer. -pub enum ProcessIo { - Pty(std::fs::File), - Pipes { - stdin: ChildStdin, - stdout: ChildStdout, - stderr: ChildStderr, - }, + #[cfg(target_os = "linux")] + managed_child: Option, + terminal: Arc, + signal_lock: Arc>, } impl ProcessHandle { @@ -680,8 +533,9 @@ impl ProcessHandle { pub fn spawn( program: &str, args: &[String], - workspace: &ResolvedWorkspace, + workdir: Option<&str>, interactive: bool, + dedicated_process_group: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, @@ -692,8 +546,9 @@ impl ProcessHandle { Self::spawn_impl( program, args, - workspace, + workdir, interactive, + dedicated_process_group, policy, resolved_identity, enforcement_mode, @@ -713,8 +568,9 @@ impl ProcessHandle { pub fn spawn( program: &str, args: &[String], - workspace: &ResolvedWorkspace, + workdir: Option<&str>, interactive: bool, + dedicated_process_group: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, @@ -724,8 +580,9 @@ impl ProcessHandle { Self::spawn_impl( program, args, - workspace, + workdir, interactive, + dedicated_process_group, policy, resolved_identity, enforcement_mode, @@ -739,8 +596,9 @@ impl ProcessHandle { fn spawn_impl( program: &str, args: &[String], - workspace: &ResolvedWorkspace, + workdir: Option<&str>, interactive: bool, + dedicated_process_group: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, @@ -750,32 +608,12 @@ impl ProcessHandle { ) -> Result { let mut cmd = Command::new(program); cmd.args(args) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .kill_on_drop(true) .env(openshell_core::sandbox_env::SANDBOX, "1"); - let mut pty_master = None; - let mut terminal_slave_fd = None; - if interactive { - let winsize = nix::pty::Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; - let master = std::fs::File::from(pty.master); - let slave = std::fs::File::from(pty.slave); - terminal_slave_fd = Some(slave.as_raw_fd()); - cmd.stdin(slave.try_clone().into_diagnostic()?) - .stdout(slave.try_clone().into_diagnostic()?) - .stderr(slave); - pty_master = Some(master); - } else { - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - } - // Strip supervisor-only identity material from the entrypoint's // inherited environment. The entrypoint drops to the sandbox user // before `exec`; without this strip, sandbox code could recover @@ -783,15 +621,8 @@ impl ProcessHandle { strip_supervisor_only_env(&mut cmd); inject_provider_env(&mut cmd, provider_env); - apply_canonical_process_environment( - &mut cmd, - policy, - workspace, - interactive, - &configured_user_environment(), - ); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir { cmd.current_dir(dir); } @@ -831,7 +662,7 @@ impl ProcessHandle { // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir); } // Phase 1: Prepare Landlock ruleset by opening PathFds. @@ -840,11 +671,20 @@ impl ProcessHandle { // runs as the sandbox UID, so inaccessible paths are unavailable to // the workload and best-effort compatibility skips them. #[cfg(target_os = "linux")] - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + let prepared_sandbox = prepare_child_sandbox(policy, workdir, enforcement_mode) .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; - // Set up process group for signal handling (non-interactive mode only). - // In interactive mode, we inherit the parent's process group to maintain - // proper terminal control for shells and interactive programs. + #[cfg(target_os = "linux")] + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + supervisor_identity_mount_from_env().map_err(|err| { + miette::miette!("Failed to prepare supervisor identity isolation: {err}") + })? + } else { + None + }; + + // Give the workload its own process group so boundary termination can + // signal the entrypoint and all of its descendants without touching + // the trusted supervisor. // SAFETY: pre_exec runs after fork but before exec in the child process. // setpgid and setns are async-signal-safe and safe to call in this context. { @@ -856,26 +696,23 @@ impl ProcessHandle { #[allow(unsafe_code)] unsafe { cmd.pre_exec(move || { - if let Some(slave_fd) = terminal_slave_fd { - if libc::setsid() < 0 { - return Err(std::io::Error::last_os_error()); - } - set_controlling_tty(slave_fd)?; - } else if libc::setpgid(0, 0) < 0 { + if (!interactive || dedicated_process_group) && libc::setpgid(0, 0) != 0 { return Err(std::io::Error::last_os_error()); } - // Enter network namespace before applying other restrictions. + // Enter network namespace before applying other restrictions if let Some(fd) = netns_fd { let result = libc::setns(fd, libc::CLONE_NEWNET); if result != 0 { - return Err(std::io::Error::other(format!( - "failed to enter network namespace: {}", - std::io::Error::last_os_error() - ))); + return Err(std::io::Error::last_os_error()); } } + #[cfg(target_os = "linux")] + if let Some(mount) = supervisor_identity_mount { + mount.enter_for_child()?; + } + // Drop privileges. initgroups/setgid/setuid need access to // /etc/group and /etc/passwd which would be blocked if // Landlock were already enforced. @@ -901,33 +738,20 @@ impl ProcessHandle { } #[cfg(target_os = "linux")] - let mut child = spawn_command_with_supervisor_identity_namespace(cmd) - .into_diagnostic() - .wrap_err("failed to spawn sandbox entrypoint process")?; - #[cfg(not(target_os = "linux"))] - let mut child = cmd - .spawn() - .into_diagnostic() - .wrap_err("failed to spawn sandbox entrypoint process")?; + let mut child_registry = managed_children::lock(); + let child = cmd.spawn().into_diagnostic()?; let pid = child.id().unwrap_or(0); - managed_children::register(pid); - - let io = if let Some(master) = pty_master { - ProcessIo::Pty(master) - } else { - ProcessIo::Pipes { - stdin: child.stdin.take().expect("canonical stdin must be piped"), - stdout: child.stdout.take().expect("canonical stdout must be piped"), - stderr: child.stderr.take().expect("canonical stderr must be piped"), - } - }; + let managed_child = child_registry.register(pid); + drop(child_registry); debug!(pid, program, "Process spawned"); Ok(Self { child, pid, - io: Some(io), + managed_child, + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -936,8 +760,9 @@ impl ProcessHandle { fn spawn_impl( program: &str, args: &[String], - workspace: &ResolvedWorkspace, + workdir: Option<&str>, interactive: bool, + dedicated_process_group: bool, policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, @@ -946,48 +771,19 @@ impl ProcessHandle { ) -> Result { let mut cmd = Command::new(program); cmd.args(args) + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .kill_on_drop(true) .env(openshell_core::sandbox_env::SANDBOX, "1"); - let mut pty_master = None; - let mut terminal_slave_fd = None; - #[cfg(unix)] - if interactive { - let winsize = nix::pty::Winsize { - ws_row: 24, - ws_col: 80, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(Some(&winsize), None).into_diagnostic()?; - let master = std::fs::File::from(pty.master); - let slave = std::fs::File::from(pty.slave); - terminal_slave_fd = Some(slave.as_raw_fd()); - cmd.stdin(slave.try_clone().into_diagnostic()?) - .stdout(slave.try_clone().into_diagnostic()?) - .stderr(slave); - pty_master = Some(master); - } - if !interactive { - cmd.stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - } - // Strip supervisor-only identity material from the entrypoint's // inherited environment. strip_supervisor_only_env(&mut cmd); inject_provider_env(&mut cmd, provider_env); - apply_canonical_process_environment( - &mut cmd, - policy, - workspace, - interactive, - &configured_user_environment(), - ); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir { cmd.current_dir(dir); } @@ -1012,24 +808,18 @@ impl ProcessHandle { } } - // Create a dedicated session for PTY children and a dedicated process - // group for pipe children so attachment signals target only the - // canonical workload tree. + // Give the workload its own process group so boundary termination can + // signal the entrypoint and all of its descendants. // SAFETY: pre_exec runs after fork but before exec in the child process. // setpgid is async-signal-safe and safe to call in this context. #[cfg(unix)] { let policy = policy.clone(); - let workdir = workspace.owned_root(); + let workdir = workdir.map(str::to_string); #[allow(unsafe_code)] unsafe { cmd.pre_exec(move || { - if let Some(slave_fd) = terminal_slave_fd { - if libc::setsid() < 0 { - return Err(std::io::Error::last_os_error()); - } - set_controlling_tty(slave_fd)?; - } else if libc::setpgid(0, 0) < 0 { + if (!interactive || dedicated_process_group) && libc::setpgid(0, 0) != 0 { return Err(std::io::Error::last_os_error()); } @@ -1053,27 +843,24 @@ impl ProcessHandle { } } - let mut child = cmd.spawn().into_diagnostic()?; + #[cfg(target_os = "linux")] + let mut child_registry = managed_children::lock(); + let child = cmd.spawn().into_diagnostic()?; let pid = child.id().unwrap_or(0); #[cfg(target_os = "linux")] - managed_children::register(pid); + let managed_child = child_registry.register(pid); + #[cfg(target_os = "linux")] + drop(child_registry); debug!(pid, program, "Process spawned"); - let io = if let Some(master) = pty_master { - ProcessIo::Pty(master) - } else { - ProcessIo::Pipes { - stdin: child.stdin.take().expect("canonical stdin must be piped"), - stdout: child.stdout.take().expect("canonical stdout must be piped"), - stderr: child.stderr.take().expect("canonical stderr must be piped"), - } - }; - Ok(Self { child, pid, - io: Some(io), + #[cfg(target_os = "linux")] + managed_child, + terminal: Arc::new(AtomicBool::new(false)), + signal_lock: Arc::new(std::sync::Mutex::new(())), }) } @@ -1083,9 +870,9 @@ impl ProcessHandle { self.pid } - /// Transfer retained stdio to the main-session multiplexer. - pub fn take_io(&mut self) -> ProcessIo { - self.io.take().expect("canonical process I/O already taken") + #[must_use] + pub fn signaling_state(&self) -> (Arc, Arc>) { + (self.terminal.clone(), self.signal_lock.clone()) } /// Wait for the process to exit. @@ -1094,21 +881,32 @@ impl ProcessHandle { /// /// Returns an error if waiting fails. pub async fn wait(&mut self) -> std::io::Result { - let status = self.child.wait().await; #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); - let status = status?; - Ok(ProcessStatus::from(status)) - } - - /// Observe an already-terminated child without blocking. - pub fn try_wait(&mut self) -> std::io::Result> { - let status = self.child.try_wait()?; - if status.is_some() { - #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); + let status = { + let pid = self.pid; + tokio::task::spawn_blocking(move || managed_children::wait_until_terminal(pid)) + .await + .map_err(std::io::Error::other)??; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + self.terminal.store(true, Ordering::Release); + self.child.try_wait()?.ok_or_else(|| { + std::io::Error::other("terminal child was not waitable after waitid") + })? + }; + #[cfg(not(target_os = "linux"))] + let status = { + let status = self.child.wait().await?; + self.terminal.store(true, Ordering::Release); + status + }; + #[cfg(target_os = "linux")] + if let Some(managed_child) = self.managed_child.take() { + managed_children::unregister(managed_child); } - Ok(status.map(ProcessStatus::from)) + Ok(ProcessStatus::from(status)) } /// Send a signal to the process. @@ -1117,6 +915,13 @@ impl ProcessHandle { /// /// Returns an error if the signal cannot be sent. pub fn signal(&self, sig: Signal) -> Result<()> { + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("process has exited")); + } let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); signal::kill(Pid::from_raw(pid), sig).into_diagnostic() } @@ -1156,7 +961,9 @@ impl ProcessHandle { impl Drop for ProcessHandle { fn drop(&mut self) { #[cfg(target_os = "linux")] - managed_children::unregister(self.pid); + if let Some(managed_child) = self.managed_child.take() { + managed_children::unregister(managed_child); + } } } @@ -1510,503 +1317,53 @@ fn chown_sandbox_home(root: &Path, uid: Option, gid: Option) -> Result } #[cfg(unix)] -fn prepare_oci_workspace( - root: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], -) -> Result<()> { - prepare_oci_workspace_with(root, uid, gid, supplementary_gids, &nix::unistd::chown) -} - -/// Validate that selecting an image-provided OCI workdir does not grant the -/// sandbox identity any filesystem authority it lacked in the immutable image. -/// -/// Every path component must be a real directory (never a symlink), every -/// parent must already be traversable, and the final directory must already be -/// writable and traversable. No ownership or mode bits are changed. -#[cfg(unix)] -pub fn validate_oci_workspace( - root: &Path, +fn chown_children( + dir: &Path, uid: Option, gid: Option, - supplementary_gids: &[Gid], -) -> Result<()> { - let components = validated_workspace_components(root, false)?; - let mut current = PathBuf::from("/"); - validate_workspace_component(¤t, uid, gid, supplementary_gids, false)?; - let last_component = components.len().saturating_sub(1); - for (index, component) in components.into_iter().enumerate() { - current.push(component); - validate_workspace_component( - ¤t, - uid, - gid, - supplementary_gids, - index == last_component, - )?; - } - Ok(()) -} - -/// Validate an image-provided workdir in a clean copy of the supervisor so the -/// main process retains the root authority needed for subsequent setup. -#[cfg(target_os = "linux")] -fn validate_oci_workspace_in_subprocess( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, - workdir: &Path, + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, ) -> Result<()> { - use std::os::unix::process::CommandExt; - - let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; - let uid = uid.ok_or_else(|| miette::miette!("workspace validator UID is unresolved"))?; - let gid = gid.ok_or_else(|| miette::miette!("workspace validator GID is unresolved"))?; - let groups = supplementary_gids - .iter() - .map(|group| group.as_raw()) - .collect::>(); - let executable = std::env::current_exe().into_diagnostic()?; - let mut command = std::process::Command::new(executable); - command - .arg("validate-workspace") - .arg("--workdir") - .arg(workdir) - .arg("--expected-uid") - .arg(uid.to_string()) - .arg("--expected-gid") - .arg(gid.to_string()) - .env_clear() - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::piped()); - - // `pre_exec` runs after fork and before exec. These direct credential - // syscalls are async-signal-safe and affect only the one-shot child. - #[allow(unsafe_code)] - unsafe { - command.pre_exec(move || { - if libc::setgroups(groups.len(), groups.as_ptr()) != 0 - || libc::setgid(gid.as_raw()) != 0 - || libc::setuid(uid.as_raw()) != 0 - { - return Err(std::io::Error::last_os_error()); + match std::fs::read_dir(dir) { + Ok(entries) => { + for entry in entries { + let entry = entry.into_diagnostic()?; + chown_recursive(&entry.path(), uid, gid, do_chown)?; } - Ok(()) - }); - } - - let output = command.output().into_diagnostic()?; - if output.status.success() { - return Ok(()); - } - - let diagnostic = String::from_utf8_lossy(&output.stderr); - let diagnostic = diagnostic.trim(); - if diagnostic.is_empty() { - return Err(miette::miette!( - "image workspace validation failed with status {}", - output.status - )); + } + Err(error) => { + debug!( + path = %dir.display(), + %error, + "Cannot list directory during sandbox home chown" + ); + } } - Err(miette::miette!( - "image workspace validation failed: {diagnostic}" - )) + Ok(()) } #[cfg(unix)] -fn validate_workspace_component( +fn chown_recursive( path: &Path, uid: Option, gid: Option, - supplementary_gids: &[Gid], - is_workspace: bool, + do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, ) -> Result<()> { - let metadata = std::fs::symlink_metadata(path).map_err(|error| { - if error.kind() == std::io::ErrorKind::NotFound { - miette::miette!( - "image workspace path component '{}' does not exist", - path.display() - ) - } else { - miette::miette!( - "failed to inspect image workspace path component '{}': {error}", - path.display() - ) - } - })?; - if metadata.file_type().is_symlink() { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - path.display() - )); - } - if !metadata.is_dir() { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - path.display() - )); - } - let required = if is_workspace { 0o3 } else { 0o1 }; - if !identity_has_permissions(&metadata, uid, gid, supplementary_gids, required) { - let requirement = if is_workspace { - "writable and traversable" - } else { - "traversable" - }; - return Err(miette::miette!( - "workspace path component '{}' is not {requirement} by the sandbox identity in the image", - path.display() - )); + let meta = std::fs::symlink_metadata(path).into_diagnostic()?; + if meta.file_type().is_symlink() { + debug!(path = %path.display(), "Skipping symlink during sandbox home chown"); + return Ok(()); } - Ok(()) -} - -#[cfg(target_os = "linux")] -pub fn validate_oci_workspace_as_effective_identity(root: &Path) -> Result<()> { - use rustix::fs::{Access, AtFlags, FileType, Mode, OFlags}; - - let components = validated_workspace_components(root, false)?; - let open_flags = OFlags::PATH | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC; - let mut current_path = PathBuf::from("/"); - let mut current_fd = rustix::fs::open("/", open_flags, Mode::empty()).into_diagnostic()?; - rustix::fs::accessat( - ¤t_fd, - ".", - Access::EXEC_OK, - AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, - ) - .map_err(|error| { - miette::miette!( - "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", - current_path.display() - ) - })?; - let last_component = components.len().saturating_sub(1); - for (index, component) in components.into_iter().enumerate() { - current_path.push(&component); - let stat = rustix::fs::statat(¤t_fd, &component, AtFlags::SYMLINK_NOFOLLOW).map_err( - |error| { - if error == rustix::io::Errno::NOENT { - miette::miette!( - "image workspace path component '{}' does not exist", - current_path.display() - ) - } else { - miette::miette!( - "failed to inspect image workspace path component '{}': {error}", - current_path.display() - ) - } - }, - )?; - let file_type = FileType::from_raw_mode(stat.st_mode); - if file_type.is_symlink() { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - current_path.display() - )); - } - if !file_type.is_dir() { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - current_path.display() - )); + if let Err(error) = do_chown(path, uid, gid) { + if error == nix::errno::Errno::EROFS { + debug!(path = %path.display(), "Skipping read-only path during sandbox home chown"); + return Ok(()); } + return Err(error).into_diagnostic(); + } - let is_workspace = index == last_component; - rustix::fs::accessat( - ¤t_fd, - &component, - Access::EXEC_OK, - AtFlags::EACCESS | AtFlags::SYMLINK_NOFOLLOW, - ) - .map_err(|error| { - miette::miette!( - "workspace path component '{}' is not traversable by the sandbox identity in the image: {error}", - current_path.display() - ) - })?; - - let next_fd = rustix::fs::openat(¤t_fd, &component, open_flags, Mode::empty()) - .map_err(|error| { - miette::miette!( - "failed to open image workspace path component '{}': {error}", - current_path.display() - ) - })?; - if is_workspace { - validate_effective_workspace_write(&next_fd, ¤t_path)?; - } - current_fd = next_fd; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn validate_effective_workspace_write(fd: &impl std::os::fd::AsFd, path: &Path) -> Result<()> { - use rustix::fs::{AtFlags, Mode, OFlags}; - - let mode = Mode::RUSR | Mode::WUSR; - let tmpfile_flags = OFlags::TMPFILE | OFlags::WRONLY | OFlags::CLOEXEC; - match rustix::fs::openat(fd, ".", tmpfile_flags, mode) { - Ok(_probe) => return Ok(()), - Err(rustix::io::Errno::INVAL | rustix::io::Errno::ISDIR | rustix::io::Errno::NOTSUP) => {} - Err(error) => { - return Err(miette::miette!( - "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", - path.display() - )); - } - } - - // Some filesystems do not implement O_TMPFILE. Fall back to a short-lived, - // no-follow entry. A collision fails closed after bounded retries. - let create_flags = - OFlags::CREATE | OFlags::EXCL | OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; - for attempt in 0..16 { - let name = format!(".openshell-workdir-probe-{}-{attempt}", std::process::id()); - match rustix::fs::openat(fd, &name, create_flags, mode) { - Ok(_probe) => { - rustix::fs::unlinkat(fd, &name, AtFlags::empty()).map_err(|error| { - miette::miette!( - "workspace write probe cleanup failed for '{}': {error}", - path.display() - ) - })?; - return Ok(()); - } - Err(rustix::io::Errno::EXIST) => {} - Err(error) => { - return Err(miette::miette!( - "workspace path component '{}' is not writable by the sandbox identity in the image: {error}", - path.display() - )); - } - } - } - - Err(miette::miette!( - "workspace write probe could not allocate a unique entry in '{}'", - path.display() - )) -} - -/// Prepare only the resolved `OpenShell` workspace directory itself. -/// -/// Image-provided children retain their declared ownership. This avoids -/// crossing symlinks or user-provided nested mounts. -#[cfg(unix)] -fn prepare_oci_workspace_with( - root: &Path, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - let components = validated_workspace_components(root, true)?; - - let last_component = components.len().saturating_sub(1); - let mut current = PathBuf::from("/"); - for (index, component) in components.into_iter().enumerate() { - current.push(component); - match std::fs::symlink_metadata(¤t) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(miette::miette!( - "workspace path component '{}' is a symlink — refusing to follow it", - current.display() - )); - } - Ok(metadata) if !metadata.is_dir() => { - return Err(miette::miette!( - "workspace path component '{}' is not a directory", - current.display() - )); - } - Ok(metadata) => { - if index != last_component - && !identity_can_traverse(&metadata, uid, gid, supplementary_gids) - { - return Err(miette::miette!( - "workspace parent '{}' is not traversable by the sandbox identity", - current.display() - )); - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - std::fs::create_dir(¤t).into_diagnostic()?; - std::fs::set_permissions(¤t, std::fs::Permissions::from_mode(0o755)) - .into_diagnostic()?; - } - Err(error) => return Err(error).into_diagnostic(), - } - } - - do_chown(root, uid, gid).into_diagnostic()?; - - let metadata = std::fs::symlink_metadata(root).into_diagnostic()?; - let mode = metadata.permissions().mode() & 0o7777; - if mode & 0o300 != 0o300 { - std::fs::set_permissions(root, std::fs::Permissions::from_mode(mode | 0o300)) - .into_diagnostic()?; - } - Ok(()) -} - -#[cfg(unix)] -fn validated_workspace_components( - root: &Path, - allow_managed_fallback: bool, -) -> Result> { - let root_str = root - .to_str() - .ok_or_else(|| miette::miette!("workspace path must be valid UTF-8"))?; - let validated_root = openshell_core::driver_mounts::resolve_oci_workspace_root(root_str) - .map_err(|error| miette::miette!(error))?; - if Path::new(&validated_root) != root - || (!allow_managed_fallback - && validated_root == openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) - { - return Err(miette::miette!( - "workspace path '{}' must be a normalized absolute {}path", - root.display(), - if allow_managed_fallback { - "non-root " - } else { - "non-fallback " - } - )); - } - - root.components() - .skip(1) - .map(|component| match component { - std::path::Component::Normal(component) => Ok(component.to_os_string()), - _ => Err(miette::miette!( - "workspace path '{}' must be normalized", - root.display() - )), - }) - .collect() -} - -#[cfg(unix)] -fn identity_can_traverse( - metadata: &std::fs::Metadata, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], -) -> bool { - identity_has_permissions(metadata, uid, gid, supplementary_gids, 0o1) -} - -#[cfg(unix)] -fn identity_has_permissions( - metadata: &std::fs::Metadata, - uid: Option, - gid: Option, - supplementary_gids: &[Gid], - required: u32, -) -> bool { - let user_id = uid.unwrap_or_else(nix::unistd::geteuid).as_raw(); - if user_id == 0 { - return true; - } - - let group_id = gid.unwrap_or_else(nix::unistd::getegid).as_raw(); - let mode = metadata.permissions().mode(); - if metadata.uid() == user_id { - mode & (required << 6) == required << 6 - } else if metadata.gid() == group_id - || supplementary_gids - .iter() - .any(|supplementary_gid| supplementary_gid.as_raw() == metadata.gid()) - { - mode & (required << 3) == required << 3 - } else { - mode & required == required - } -} - -#[cfg(not(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" -)))] -fn named_user_supplementary_groups(user_name: &str, primary_gid: Gid) -> Result> { - let user_name = CString::new(user_name).map_err(|_| miette::miette!("Invalid user name"))?; - nix::unistd::getgrouplist(user_name.as_c_str(), primary_gid).into_diagnostic() -} - -#[cfg(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" -))] -#[allow(clippy::unnecessary_wraps)] -fn named_user_supplementary_groups(_user_name: &str, _primary_gid: Gid) -> Result> { - // Privilege dropping does not call initgroups on these targets. - Ok(Vec::new()) -} - -#[cfg(unix)] -fn chown_children( - dir: &Path, - uid: Option, - gid: Option, - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - match std::fs::read_dir(dir) { - Ok(entries) => { - for entry in entries { - let entry = entry.into_diagnostic()?; - chown_recursive(&entry.path(), uid, gid, do_chown)?; - } - } - Err(error) => { - debug!( - path = %dir.display(), - %error, - "Cannot list directory during sandbox home chown" - ); - } - } - Ok(()) -} - -#[cfg(unix)] -fn chown_recursive( - path: &Path, - uid: Option, - gid: Option, - do_chown: &impl Fn(&Path, Option, Option) -> nix::Result<()>, -) -> Result<()> { - let meta = std::fs::symlink_metadata(path).into_diagnostic()?; - if meta.file_type().is_symlink() { - debug!(path = %path.display(), "Skipping symlink during sandbox home chown"); - return Ok(()); - } - - if let Err(error) = do_chown(path, uid, gid) { - if error == nix::errno::Errno::EROFS { - debug!(path = %path.display(), "Skipping read-only path during sandbox home chown"); - return Ok(()); - } - return Err(error).into_diagnostic(); - } - - if meta.is_dir() { - chown_children(path, uid, gid, do_chown)?; + if meta.is_dir() { + chown_children(path, uid, gid, do_chown)?; } Ok(()) @@ -2022,100 +1379,32 @@ fn chown_recursive( /// UIDs/GIDs (passed directly to `chown` without a passwd lookup). #[cfg(unix)] pub fn prepare_filesystem(policy: &SandboxPolicy) -> Result<()> { - prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default(), None, false) + prepare_filesystem_with_identity(policy, ResolvedProcessIdentity::default()) } #[cfg(unix)] pub fn prepare_filesystem_with_identity( policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, - workdir: Option<&str>, - prepare_workspace: bool, ) -> Result<()> { use nix::unistd::chown; + use nix::unistd::{Gid, Uid}; + + let user_name = match policy.process.run_as_user.as_deref() { + Some(name) if !name.is_empty() => Some(name), + _ => None, + }; + let group_name = match policy.process.run_as_group.as_deref() { + Some(name) if !name.is_empty() => Some(name), + _ => None, + }; // If no user/group configured, nothing to do - if policy - .process - .run_as_user - .as_deref() - .is_none_or(str::is_empty) - && policy - .process - .run_as_group - .as_deref() - .is_none_or(str::is_empty) - { + if user_name.is_none() && group_name.is_none() { return Ok(()); } - let (uid, gid, supplementary_gids) = resolve_filesystem_identity(policy, resolved_identity)?; - - // Docker owns workspace resolution and must make the selected root usable - // by the final effective identity, including when both policy identity - // fields were explicit. Validate it before processing any user-authored - // read-write paths so an unsafe image path fails first. Other drivers - // retain their preparation. - if prepare_workspace { - let workspace = workdir.ok_or_else(|| { - miette::miette!("local container driver did not supply a workspace workdir") - })?; - let workspace = Path::new(workspace); - if workspace == Path::new(openshell_core::driver_mounts::DEFAULT_WORKSPACE_ROOT) { - info!(path = %workspace.display(), ?uid, ?gid, "Preparing managed workspace"); - prepare_oci_workspace(workspace, uid, gid, &supplementary_gids)?; - } else { - info!(path = %workspace.display(), ?uid, ?gid, "Validating image workspace authority"); - #[cfg(target_os = "linux")] - validate_oci_workspace_in_subprocess(policy, resolved_identity, workspace)?; - #[cfg(not(target_os = "linux"))] - validate_oci_workspace(workspace, uid, gid, &supplementary_gids)?; - } - } - - // Create missing read_write paths and only chown the ones we created. - for path in &policy.filesystem.read_write { - if prepare_read_write_path(path)? { - debug!( - path = %path.display(), - ?uid, - ?gid, - "Setting ownership on newly created read_write path" - ); - chown(path, uid, gid).into_diagnostic()?; - } - } - - // Retain the existing Kubernetes/OpenShift behavior for driver-injected - // numeric identities. Docker clears this variable and does not receive - // identity-specific workspace preparation. - if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { - let sandbox_home = Path::new("/sandbox"); - if sandbox_home.exists() { - info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); - chown_sandbox_home(sandbox_home, uid, gid)?; - } - } - - Ok(()) -} - -#[cfg(unix)] -fn resolve_filesystem_identity( - policy: &SandboxPolicy, - resolved_identity: ResolvedProcessIdentity, -) -> Result<(Option, Option, Vec)> { - let user_name = policy - .process - .run_as_user - .as_deref() - .filter(|name| !name.is_empty()); - let group_name = policy - .process - .run_as_group - .as_deref() - .filter(|name| !name.is_empty()); - + // Resolve UID: numeric values are passed directly; names resolve via passwd. let uid = match resolved_identity.uid() { Some(uid) => Some(Uid::from_raw(uid)), None => match user_name { @@ -2139,31 +1428,31 @@ fn resolve_filesystem_identity( }, }; - let supplementary_gids = match user_name { - Some(name) if name.parse::().is_err() => { - let primary_gid = if let Some(gid) = gid { - gid - } else { - let uid = - uid.ok_or_else(|| miette::miette!("Failed to resolve sandbox user '{name}'"))?; - User::from_uid(uid) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("Failed to resolve user from UID {uid}"))? - .gid - }; - if resolved_identity.uid().is_some() { - crate::identity::resolve_oci_supplementary_gids(name, primary_gid.as_raw())? - .into_iter() - .map(Gid::from_raw) - .collect() - } else { - named_user_supplementary_groups(name, primary_gid)? - } + // Create missing read_write paths and only chown the ones we created. + for path in &policy.filesystem.read_write { + if prepare_read_write_path(path)? { + debug!( + path = %path.display(), + ?uid, + ?gid, + "Setting ownership on newly created read_write path" + ); + chown(path, uid, gid).into_diagnostic()?; } - _ => Vec::new(), - }; + } - Ok((uid, gid, supplementary_gids)) + // Retain the existing Kubernetes/OpenShift behavior for driver-injected + // numeric identities. Docker and Podman clear this variable and do not + // receive identity-specific workspace preparation. + if std::env::var(openshell_core::sandbox_env::SANDBOX_UID).is_ok_and(|uid| !uid.is_empty()) { + let sandbox_home = Path::new("/sandbox"); + if sandbox_home.exists() { + info!(?uid, ?gid, "Chowning /sandbox for driver-injected UID/GID"); + chown_sandbox_home(sandbox_home, uid, gid)?; + } + } + + Ok(()) } #[cfg(not(unix))] @@ -2179,6 +1468,19 @@ pub fn drop_privileges(policy: &SandboxPolicy) -> Result<()> { drop_privileges_with_identity(policy, ResolvedProcessIdentity::default()) } +#[cfg(unix)] +fn should_clear_supplementary_groups( + current_uid: Uid, + target_uid: Uid, + user_name: Option<&str>, + resolved_identity: ResolvedProcessIdentity, +) -> bool { + resolved_identity.uses_oci_user_fallback() + && target_uid != current_uid + && !(user_name.is_some_and(|name| name.parse::().is_err()) + && resolved_identity.uid().is_none()) +} + #[cfg(unix)] #[allow(clippy::similar_names)] pub fn drop_privileges_with_identity( @@ -2274,21 +1576,23 @@ pub fn drop_privileges_with_identity( }; if target_uid != nix::unistd::geteuid() { - if resolved_identity.uses_oci_user_fallback() { - // OCI named users use the bounded /etc/group parser shared with - // workspace validation. Numeric OCI users resolve to an empty - // list. Never retain the root supervisor's inherited groups. + if should_clear_supplementary_groups( + nix::unistd::geteuid(), + target_uid, + user_name, + resolved_identity, + ) { + // OCI-derived users do not have a trustworthy NSS + // supplementary-group source. Clear the root supervisor's + // inherited groups before changing UID/GID. Platform-resolved and + // explicit numeric identities retain their pre-OCI behavior. #[cfg(not(any( target_os = "macos", target_os = "ios", target_os = "haiku", target_os = "redox" )))] - { - let (_, _, supplementary_gids) = - resolve_filesystem_identity(policy, resolved_identity)?; - nix::unistd::setgroups(&supplementary_gids).into_diagnostic()?; - } + nix::unistd::setgroups(&[]).into_diagnostic()?; } else if let Some(ref user_name) = initgroups_name { let user_cstr = CString::new(user_name.as_str()) .map_err(|_| miette::miette!("Invalid user name"))?; @@ -2370,12 +1674,14 @@ pub struct ProcessStatus { } impl ProcessStatus { - /// Get the conventional exit code when the process exited normally. + /// Construct a normal exit status. #[must_use] - pub const fn exit_code(&self) -> Option { - self.code + pub const fn exited(code: i32) -> Self { + Self { + code: Some(code), + signal: None, + } } - /// Get the exit code, or 128 + signal number if killed by signal. #[must_use] pub fn code(&self) -> i32 { @@ -2443,42 +1749,6 @@ mod tests { } } - #[cfg(unix)] - #[tokio::test] - async fn canonical_tty_environment_replaces_supervisor_identity_defaults() { - let current_user = User::from_uid(nix::unistd::geteuid()) - .expect("look up current user") - .expect("current user entry"); - let policy = policy_with_process(ProcessPolicy { - run_as_user: Some(current_user.name.clone()), - run_as_group: None, - }); - let workspace = ResolvedWorkspace::default(); - let mut cmd = Command::new("/usr/bin/env"); - cmd.env_clear() - .env("HOME", "/root") - .env("TERM", "dumb") - .stdout(StdStdio::piped()); - - apply_canonical_process_environment(&mut cmd, &policy, &workspace, true, &HashMap::new()); - - let output = cmd.output().await.expect("run environment probe"); - assert!(output.status.success()); - let environment = String::from_utf8(output.stdout).expect("environment is UTF-8"); - let variables: HashMap<_, _> = environment - .lines() - .filter_map(|line| line.split_once('=')) - .collect(); - - assert_eq!( - variables.get("HOME"), - Some(¤t_user.dir.to_string_lossy().as_ref()) - ); - assert_eq!(variables.get("USER"), Some(¤t_user.name.as_str())); - assert_eq!(variables.get("SHELL"), Some(&"/bin/bash")); - assert_eq!(variables.get("TERM"), Some(&"xterm-256color")); - } - /// Unknown names may yield `Ok(None)` (`… not found …`) or `Err` when NSS fails first /// (e.g. `ENOENT: No such file or directory`). fn assert_unknown_identity_lookup_failed(msg: &str) { @@ -2492,14 +1762,14 @@ mod tests { #[test] #[cfg(unix)] - fn explicit_identity_accepts_non_root_system_ids() { + fn explicit_identity_rejects_non_root_system_ids() { let policy = policy_with_process(ProcessPolicy { run_as_user: Some("101".into()), run_as_group: Some("102".into()), }); - assert!(validate_sandbox_user(&policy).is_ok()); - assert!(validate_sandbox_group(&policy).is_ok()); + assert!(validate_sandbox_user(&policy).is_err()); + assert!(validate_sandbox_group(&policy).is_err()); } #[test] @@ -2557,6 +1827,43 @@ mod tests { assert!(validate_sandbox_group_with_identity(&policy, resolved).is_ok()); } + #[test] + #[cfg(unix)] + fn only_oci_numeric_user_paths_clear_supplementary_groups_before_uid_drop() { + let current_uid = Uid::from_raw(0); + let target_uid = Uid::from_raw(1234); + + assert!(!should_clear_supplementary_groups( + current_uid, + target_uid, + Some("1234"), + ResolvedProcessIdentity::default(), + )); + assert!(should_clear_supplementary_groups( + current_uid, + target_uid, + Some("1234"), + ResolvedProcessIdentity::new(None, Some(1235)), + )); + } + + #[test] + #[cfg(unix)] + fn supplementary_group_clearing_preserves_explicit_named_user_behavior() { + assert!(!should_clear_supplementary_groups( + Uid::from_raw(0), + Uid::from_raw(1234), + Some("app"), + ResolvedProcessIdentity::default(), + )); + assert!(!should_clear_supplementary_groups( + Uid::from_raw(1234), + Uid::from_raw(1234), + Some("1234"), + ResolvedProcessIdentity::new(None, Some(1235)), + )); + } + #[test] fn full_enforcement_uses_privileged_setup_and_child_sandbox() { assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); @@ -3202,518 +2509,6 @@ mod tests { assert!(result.is_err(), "non-EROFS errors should propagate"); } - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_chowns_only_root() { - use std::sync::{Arc, Mutex}; - - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let child = root.join("image-content.txt"); - std::fs::write(&child, "image-owned").unwrap(); - - let chowned = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&chowned); - let fake_chown = - move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - observed.lock().unwrap().push(path.to_path_buf()); - Ok(()) - }; - - prepare_oci_workspace_with( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .expect("workspace root should be prepared"); - - assert_eq!(*chowned.lock().unwrap(), vec![root]); - assert!(child.exists(), "image-provided child should be untouched"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_accepts_existing_owner_writable_directory() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - validate_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .expect("image owner already has write and traverse authority"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_accepts_supplementary_group_write_authority() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o070)).unwrap(); - let metadata = std::fs::symlink_metadata(&root).unwrap(); - - validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[Gid::from_raw(metadata.gid())], - ) - .expect("supplementary group already has write and traverse authority"); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_unwritable_directory() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)).unwrap(); - let metadata = std::fs::symlink_metadata(&root).unwrap(); - - let error = validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("not writable and traversable")); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_missing_path() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("missing"); - - let error = validate_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("does not exist")); - } - - #[cfg(target_os = "linux")] - #[test] - #[allow(unsafe_code)] - fn effective_identity_validation_honors_named_user_acl() { - const TEST_UID: u32 = 42_234; - const TEST_GID: u32 = 42_235; - const ACL_XATTR_VERSION: u32 = 2; - const ACL_USER_OBJ: u16 = 0x01; - const ACL_USER: u16 = 0x02; - const ACL_GROUP_OBJ: u16 = 0x04; - const ACL_MASK: u16 = 0x10; - const ACL_OTHER: u16 = 0x20; - const ACL_UNDEFINED_ID: u32 = u32::MAX; - - if !nix::unistd::geteuid().is_root() { - return; - } - - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - let mut acl = ACL_XATTR_VERSION.to_ne_bytes().to_vec(); - for (tag, permissions, id) in [ - (ACL_USER_OBJ, 0o7_u16, ACL_UNDEFINED_ID), - (ACL_USER, 0o7_u16, TEST_UID), - (ACL_GROUP_OBJ, 0o0_u16, ACL_UNDEFINED_ID), - (ACL_MASK, 0o7_u16, ACL_UNDEFINED_ID), - (ACL_OTHER, 0o0_u16, ACL_UNDEFINED_ID), - ] { - acl.extend_from_slice(&tag.to_ne_bytes()); - acl.extend_from_slice(&permissions.to_ne_bytes()); - acl.extend_from_slice(&id.to_ne_bytes()); - } - let path = CString::new(root.as_os_str().as_encoded_bytes()).unwrap(); - let name = c"system.posix_acl_access"; - let result = unsafe { - libc::setxattr( - path.as_ptr(), - name.as_ptr(), - acl.as_ptr().cast(), - acl.len(), - 0, - ) - }; - assert_eq!( - result, - 0, - "setxattr failed: {}", - std::io::Error::last_os_error() - ); - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - let credentials_dropped = unsafe { - libc::setgroups(0, std::ptr::null()) == 0 - && libc::setgid(TEST_GID) == 0 - && libc::setuid(TEST_UID) == 0 - }; - let valid = credentials_dropped - && validate_oci_workspace_as_effective_identity(&root).is_ok(); - unsafe { libc::_exit(i32::from(!valid)) }; - } - ForkResult::Parent { child } => { - assert_eq!( - waitpid(child, None).expect("waitpid should succeed"), - WaitStatus::Exited(child, 0), - "named ACL user should retain workspace authority" - ); - } - } - } - - #[cfg(target_os = "linux")] - #[test] - #[allow(unsafe_code)] - fn effective_identity_validation_honors_landlock_denial() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - let root = dir.path().canonicalize().unwrap().join("project"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - - let mut policy = policy_with_process(ProcessPolicy::default()); - policy.filesystem = FilesystemPolicy { - read_only: vec![root.clone()], - read_write: Vec::new(), - include_workdir: false, - }; - policy.landlock = LandlockPolicy { - compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, - }; - let Ok(prepared) = sandbox::linux::prepare_current_user(&policy, None) else { - return; - }; - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - let denied = sandbox::linux::enforce(prepared).is_ok() - && validate_oci_workspace_as_effective_identity(&root).is_err(); - unsafe { libc::_exit(i32::from(!denied)) }; - } - ForkResult::Parent { child } => { - assert_eq!( - waitpid(child, None).expect("waitpid should succeed"), - WaitStatus::Exited(child, 0), - "kernel-effective validation should honor an enforced LSM denial" - ); - } - } - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_restrictive_parent() { - let dir = tempfile::tempdir().unwrap(); - let parent = dir.path().canonicalize().unwrap().join("private"); - let root = parent.join("project"); - std::fs::create_dir_all(&root).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o777)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - - let error = validate_oci_workspace( - &root, - Some(Uid::from_raw(metadata.uid().wrapping_add(1))), - Some(Gid::from_raw(metadata.gid().wrapping_add(1))), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("not traversable")); - } - - #[cfg(unix)] - #[test] - fn validate_oci_workspace_rejects_symlink_component() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("target"); - let link = base.join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let error = validate_oci_workspace( - &link, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!(error.to_string().contains("symlink")); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_makes_existing_root_owner_writable() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o555)).unwrap(); - - prepare_oci_workspace_with(&root, None, None, &[], &|_, _, _| Ok(())) - .expect("read-only workspace root should be prepared"); - - let mode = std::fs::symlink_metadata(&root) - .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o755); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_symlink_root() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("real"); - let link = base.join("link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &link).unwrap(); - - let err = prepare_oci_workspace( - &link, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("symlink"), - "expected symlink rejection: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_symlink_parent() { - use std::os::unix::fs::symlink; - - let dir = tempfile::tempdir().unwrap(); - let base = dir.path().canonicalize().unwrap(); - let target = base.join("real"); - let parent_link = base.join("parent-link"); - std::fs::create_dir(&target).unwrap(); - symlink(&target, &parent_link).unwrap(); - - let err = prepare_oci_workspace( - &parent_link.join("workspace"), - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("symlink"), - "expected parent symlink rejection: {err}" - ); - assert!( - !target.join("workspace").exists(), - "workspace must not be created through a symlink parent" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_parent_traversal() { - let err = prepare_oci_workspace( - Path::new("/tmp/workspace/../escape"), - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - err.to_string().contains("must be normalized"), - "expected traversal rejection: {err}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_inaccessible_existing_parent() { - let dir = tempfile::tempdir().unwrap(); - let parent = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&parent).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); - let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); - let root = parent.join("project"); - - let error = prepare_oci_workspace_with( - &root, - Some(different_user), - Some(different_group), - &[], - &|_, _, _| Ok(()), - ) - .unwrap_err(); - - assert!( - error.to_string().contains("is not traversable"), - "unexpected error: {error}" - ); - assert!( - !root.exists(), - "workspace must not be created below an inaccessible parent" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_accepts_supplementary_group_parent() { - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o710)).unwrap(); - let parent = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&parent).unwrap(); - std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o710)).unwrap(); - let metadata = std::fs::symlink_metadata(&parent).unwrap(); - let different_user = Uid::from_raw(metadata.uid().wrapping_add(1)); - let different_group = Gid::from_raw(metadata.gid().wrapping_add(1)); - let supplementary_group = Gid::from_raw(metadata.gid()); - let root = parent.join("project"); - - prepare_oci_workspace_with( - &root, - Some(different_user), - Some(different_group), - &[supplementary_group], - &|_, _, _| Ok(()), - ) - .expect("supplementary group execute permission should allow traversal"); - - assert!(root.is_dir()); - } - - #[cfg(not(any( - target_os = "aix", - target_os = "haiku", - target_os = "illumos", - target_os = "ios", - target_os = "macos", - target_os = "redox", - target_os = "solaris" - )))] - #[test] - fn named_user_supplementary_groups_include_primary_group() { - let user = User::from_uid(nix::unistd::geteuid()) - .expect("resolve current UID") - .expect("current user exists"); - - let groups = named_user_supplementary_groups(&user.name, user.gid) - .expect("resolve named-user supplementary groups"); - - assert!(groups.contains(&user.gid)); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_rejects_non_directory_root() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::write(&root, "not a directory").unwrap(); - - let error = prepare_oci_workspace( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - ) - .unwrap_err(); - assert!( - error.to_string().contains("is not a directory"), - "unexpected error: {error}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_propagates_root_chown_error() { - let dir = tempfile::tempdir().unwrap(); - let root = dir.path().canonicalize().unwrap().join("sandbox"); - std::fs::create_dir(&root).unwrap(); - let fake_chown = |_path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - Err(nix::errno::Errno::EROFS) - }; - - let error = prepare_oci_workspace_with( - &root, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .unwrap_err(); - - assert!( - error.to_string().contains("Read-only file system"), - "unexpected error: {error}" - ); - } - - #[cfg(unix)] - #[test] - fn prepare_oci_workspace_creates_missing_root() { - use std::sync::{Arc, Mutex}; - - let dir = tempfile::tempdir().unwrap(); - let missing = dir - .path() - .canonicalize() - .unwrap() - .join("missing") - .join("sandbox"); - let chowned = Arc::new(Mutex::new(Vec::new())); - let observed = Arc::clone(&chowned); - let fake_chown = - move |path: &Path, _uid: Option, _gid: Option| -> nix::Result<()> { - observed.lock().unwrap().push(path.to_path_buf()); - Ok(()) - }; - - prepare_oci_workspace_with( - &missing, - Some(nix::unistd::geteuid()), - Some(nix::unistd::getegid()), - &[], - &fake_chown, - ) - .expect("missing OCI workspace should be created"); - - assert!(missing.is_dir()); - assert_eq!( - std::fs::symlink_metadata(missing.parent().unwrap()) - .unwrap() - .permissions() - .mode() - & 0o777, - 0o755 - ); - assert_eq!(*chowned.lock().unwrap(), vec![missing]); - } - #[cfg(unix)] #[test] fn rewrite_passwd_modifies_existing_sandbox_entry() { @@ -3904,11 +2699,7 @@ mod tests { #[test] fn supervisor_identity_mount_target_rejects_unhideable_endpoints() { - assert_eq!( - supervisor_identity_mount_target("tcp:127.0.0.1:8081") - .expect("tcp endpoint should not require mount hiding"), - None - ); + assert!(supervisor_identity_mount_target("tcp:127.0.0.1:8081").is_err()); assert!(supervisor_identity_mount_target("spiffe-workload-api/spire-agent.sock").is_err()); assert!(supervisor_identity_mount_target("/spire-agent.sock").is_err()); } diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index f0e792306d..dfd9cd6b38 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -10,6 +10,8 @@ //! orchestrator, not here. use miette::{IntoDiagnostic, Result}; +#[cfg(not(target_os = "linux"))] +use std::os::fd::OwnedFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::time::Duration; @@ -26,6 +28,7 @@ use crate::netns::NetworkNamespace; use openshell_core::policy::{NetworkMode, SandboxPolicy}; use openshell_core::proposals::AgentProposals; use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation::contract::{BoundaryExec, BoundaryPortForward}; #[cfg(target_os = "linux")] use openshell_core::activity::ActivitySender; @@ -36,44 +39,159 @@ use openshell_core::denial::DenialEvent; use crate::managed_children; use crate::process::{ ProcessEnforcementMode, ProcessHandle, ProcessStatus, ResolvedProcessIdentity, - ResolvedWorkspace, }; -pub type SidecarExitReport = ( - String, - i32, - tokio::sync::oneshot::Sender>, -); - fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { openshell_ocsf::ctx::ctx() } -/// Spawn the workload entrypoint, wire up SSH and supervisor session, and -/// wait for the entrypoint child to exit. +/// Host-side SSH and gateway-session tasks for an already-running boundary. +/// +/// Delegated backends start the workload themselves, but the logical +/// supervisor still owns the user access plane. Keeping these tasks in the +/// process-supervisor crate lets local and VM boundaries share the same SSH +/// and `ConnectSupervisor` implementations. +pub struct BoundaryAccess { + terminating: Arc, + ssh_task: Option>, + session_task: Option>, +} + +impl Drop for BoundaryAccess { + fn drop(&mut self) { + self.terminating.store(true, Ordering::Release); + if let Some(task) = self.ssh_task.take() { + task.abort(); + } + if let Some(task) = self.session_task.take() { + task.abort(); + } + } +} + +/// Start the host-owned access plane for a delegated isolation boundary. +/// +/// The SSH server executes commands and opens loopback connections through +/// the boundary interfaces supplied by the driver. The persistent supervisor +/// session then registers the sandbox with the gateway and relays requests to +/// that local SSH endpoint. +#[allow(clippy::too_many_arguments)] +pub async fn start_boundary_access( + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option<&str>, + shared_ssh_socket: bool, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + enforcement_mode: ProcessEnforcementMode, + boundary_exec: Arc, + port_forward: Arc, +) -> Result { + let terminating = Arc::new(AtomicBool::new(false)); + let Some(ssh_socket_path) = ssh_socket_path.map(std::path::PathBuf::from) else { + return Ok(BoundaryAccess { + terminating, + ssh_task: None, + session_task: None, + }); + }; + + let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + let listen_path = ssh_socket_path.clone(); + let ssh_port_forward = port_forward.clone(); + let ssh_task = tokio::spawn(async move { + if let Err(err) = crate::ssh::run_ssh_server( + listen_path, + ssh_ready_tx, + ca_file_paths, + enforcement_mode, + shared_ssh_socket, + ssh_port_forward, + boundary_exec, + ) + .await + { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message(format!("SSH server failed: {err}")) + .build() + ); + } + }); + + match timeout(Duration::from_secs(10), ssh_ready_rx).await { + Ok(Ok(Ok(()))) => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message("SSH server is ready to accept connections") + .build() + ); + } + Ok(Ok(Err(err))) => { + ssh_task.abort(); + return Err(err.context("SSH server failed during startup")); + } + Ok(Err(_)) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server task panicked before signaling ready" + )); + } + Err(_) => { + ssh_task.abort(); + return Err(miette::miette!( + "SSH server did not start within 10 seconds" + )); + } + } + + let session_task = match (openshell_endpoint, sandbox_id) { + (Some(endpoint), Some(id)) => Some(crate::supervisor_session::spawn( + endpoint.to_string(), + id.to_string(), + ssh_socket_path, + port_forward, + None, + Arc::clone(&terminating), + )), + _ => None, + }; + + Ok(BoundaryAccess { + terminating, + ssh_task: Some(ssh_task), + session_task, + }) +} + +/// Run the workload entrypoint to completion using the legacy orchestration +/// surface. New isolation-backend callers retain the [`SpawnedAgent`] returned +/// by [`spawn_workload`] instead. /// /// # Errors /// -/// Returns an error if SSH server startup fails, if the entrypoint child -/// fails to spawn, or if waiting for the child returns an OS error. +/// Returns an error if the workload cannot be spawned or waited. #[allow(clippy::too_many_arguments, clippy::implicit_hasher)] pub async fn run_process( program: &str, args: &[String], - workspace: ResolvedWorkspace, + workdir: Option<&str>, timeout_secs: u64, interactive: bool, sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, ssh_socket_path: Option, shared_ssh_socket: bool, - ssh_exit_tx: Option>, policy: &SandboxPolicy, resolved_process_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, entrypoint_pid: Arc, - entrypoint_started_tx: Option>, - sidecar_exit_tx: Option>, + entrypoint_started_tx: Option>, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -84,6 +202,76 @@ pub async fn run_process( >, #[cfg(target_os = "linux")] bypass_activity_tx: Option, ) -> Result { + let mut agent = spawn_workload( + program, + args, + workdir, + timeout_secs, + interactive, + sandbox_id, + openshell_endpoint, + ssh_socket_path, + shared_ssh_socket, + policy, + resolved_process_identity, + enforcement_mode, + entrypoint_pid, + entrypoint_started_tx, + provider_credentials, + provider_env, + ca_file_paths, + agent_proposals, + #[cfg(target_os = "linux")] + netns, + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + None, + ) + .await?; + agent.wait().await.map(|status| status.code()) +} + +/// Spawn the workload entrypoint behind the boundary, wire up SSH and the +/// supervisor session, and return an owned [`SpawnedAgent`] handle. +/// +/// The agent keeps running after this returns; the caller drives it through +/// [`SpawnedAgent::wait`]/[`SpawnedAgent::signal`]. This is the placement- +/// sensitive spawn the in-pod backend's `RunningBoundary` owns (RFC 0012): +/// the returned handle, not an exit code, is the process control surface. +/// +/// # Errors +/// +/// Returns an error if SSH server startup fails or if the entrypoint child +/// fails to spawn. +#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] +pub async fn spawn_workload( + program: &str, + args: &[String], + workdir: Option<&str>, + timeout_secs: u64, + interactive: bool, + sandbox_id: Option<&str>, + openshell_endpoint: Option<&str>, + ssh_socket_path: Option, + shared_ssh_socket: bool, + policy: &SandboxPolicy, + resolved_process_identity: ResolvedProcessIdentity, + enforcement_mode: ProcessEnforcementMode, + entrypoint_pid: Arc, + entrypoint_started_tx: Option>, + provider_credentials: ProviderCredentialState, + provider_env: std::collections::HashMap, + ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, + agent_proposals: AgentProposals, + #[cfg(target_os = "linux")] netns: Option<&NetworkNamespace>, + #[cfg(target_os = "linux")] bypass_denial_tx: Option< + tokio::sync::mpsc::UnboundedSender, + >, + #[cfg(target_os = "linux")] bypass_activity_tx: Option, + boundary_runtime: Option>, +) -> Result { // Platform drivers with a resolved numeric UID/GID retain the legacy // account-file update. OCI-image identity leaves those environment values // empty, so the image's account files remain unchanged. @@ -104,12 +292,7 @@ pub async fn run_process( // is forked so the workload sees writable paths it owns. #[cfg(unix)] if enforcement_mode.uses_privileged_process_setup() { - crate::process::prepare_filesystem_with_identity( - policy, - resolved_process_identity, - workspace.root(), - workspace.home().is_some(), - )?; + crate::process::prepare_filesystem_with_identity(policy, resolved_process_identity)?; } // Eagerly fetch initial settings and install the agent skill if the @@ -118,18 +301,10 @@ pub async fn run_process( // the flag stays at its default (false) and no skill is installed. install_initial_agent_skill(sandbox_id, openshell_endpoint, &agent_proposals).await; - // Provider token grants may mount supervisor-only identity sockets such as - // the SPIFFE Workload API. Prepare the child mount namespace that hides - // those mounts before supervisor seccomp hardening removes the needed - // namespace syscalls. - #[cfg(target_os = "linux")] - crate::process::prepare_supervisor_identity_mount_namespace_from_env()?; - // Install the supervisor seccomp prelude before spawning any workload-side // tasks. By this point the orchestrator has finished privileged startup - // helpers (network namespace setup, identity mount namespace setup, - // nftables probes via run_networking), and the SSH listener and entrypoint - // child have not been exposed yet. + // helpers (network namespace setup, nftables probes via run_networking), + // and the SSH listener and entrypoint child have not been exposed yet. crate::sandbox::apply_supervisor_startup_hardening()?; // Spawn the bypass detection monitor. It tails dmesg for nftables LOG @@ -138,7 +313,7 @@ pub async fn run_process( // proxy. Spawn it before the entrypoint child so the first packets are // not missed. Best-effort: returns None when dmesg is unavailable. #[cfg(target_os = "linux")] - let _bypass_handle = netns.and_then(|ns| { + let bypass_handle = netns.and_then(|ns| { crate::bypass_monitor::spawn( ns.name().to_string(), entrypoint_pid.clone(), @@ -205,7 +380,11 @@ pub async fn run_process( break; }; - if managed_children::is_managed(pid.as_raw()) { + // Serialize the managed-child check and reap with every + // spawn-and-register operation. A fast child must not be + // mistaken for an orphan between `spawn()` and registration. + let registry = managed_children::lock(); + if registry.contains(pid.as_raw()) { // Let the explicit waiter own this child status. break; } @@ -231,40 +410,13 @@ pub async fn run_process( // Without this, SSH-spawned shells run in the host namespace and bypass // the proxy entirely. #[cfg(target_os = "linux")] - let ssh_netns_fd = netns.and_then(NetworkNamespace::ns_fd); + let ssh_netns_fd = netns + .map(NetworkNamespace::try_clone_ns_fd) + .transpose()? + .flatten() + .map(Arc::new); #[cfg(not(target_os = "linux"))] - let ssh_netns_fd: Option = None; - - #[cfg(target_os = "linux")] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - netns, - ca_file_paths.as_ref(), - &provider_env, - )?; - - #[cfg(not(target_os = "linux"))] - let mut handle = ProcessHandle::spawn( - program, - args, - &workspace, - interactive, - policy, - resolved_process_identity, - enforcement_mode, - ca_file_paths.as_ref(), - &provider_env, - )?; - - let main_pid = handle.pid(); - let main_session = crate::main_session::MainSession::new(handle.take_io(), main_pid); - let main_instance_id = uuid::Uuid::new_v4().to_string(); + let ssh_netns_fd: Option> = None; // SSH-spawned shells get http_proxy=http://: exported into // their env so cooperative tools (curl, npm, Node) route through the @@ -275,39 +427,54 @@ pub async fn run_process( #[cfg(not(target_os = "linux"))] let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); + let user_environment: std::collections::HashMap = + std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) + .ok() + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_default(); + let boundary_runtime = + boundary_runtime.unwrap_or_else(crate::boundary_io::BoundaryRuntimeState::new); + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new( + ssh_netns_fd.clone(), + Some(boundary_runtime.clone()), + )); + let boundary_exec: Arc = + Arc::new(crate::boundary_exec::LocalBoundaryExec::new( + policy.clone(), + workdir.map(str::to_string), + ssh_netns_fd, + ssh_proxy_url.clone(), + ca_file_paths.clone().map(Arc::new), + provider_credentials.clone(), + user_environment.clone(), + resolved_process_identity, + enforcement_mode, + boundary_runtime.clone(), + )); + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { - let policy_clone = policy.clone(); - let workspace_clone = workspace.clone(); - let proxy_url = ssh_proxy_url; - let netns_fd = ssh_netns_fd; let ca_paths = ca_file_paths.clone(); - let provider_credentials_clone = provider_credentials.clone(); - let main_session_clone = Arc::clone(&main_session); - let user_env_clone: std::collections::HashMap = - std::env::var(openshell_core::sandbox_env::USER_ENVIRONMENT) - .ok() - .and_then(|json| serde_json::from_str(&json).ok()) - .unwrap_or_default(); let (ssh_ready_tx, ssh_ready_rx) = tokio::sync::oneshot::channel(); + // Inject the in-pod loopback port-forward (RFC 0012). The SSH server + // drives it through the `BoundaryPortForward` interface, so a delegated + // backend would supply a different implementation without the SSH + // server changing. + let ssh_port_forward = port_forward.clone(); + let ssh_boundary_exec = boundary_exec.clone(); + tokio::spawn(async move { - let _ssh_exit_guard = ssh_exit_tx; if let Err(err) = crate::ssh::run_ssh_server( listen_path, ssh_ready_tx, - policy_clone, - workspace_clone, - netns_fd, - proxy_url, ca_paths, - provider_credentials_clone, - user_env_clone, - resolved_process_identity, enforcement_mode, shared_ssh_socket, - main_session_clone, + ssh_port_forward, + ssh_boundary_exec, ) .await { @@ -353,39 +520,61 @@ pub async fn run_process( } let supervisor_terminating = Arc::new(AtomicBool::new(false)); - // A canonical process may have completed while the SSH socket was being - // prepared. Never open a readiness-bearing supervisor session for a child - // that is already terminal. - let early_exit = handle.try_wait().into_diagnostic()?; // Spawn the persistent supervisor session if we have a gateway endpoint // and sandbox identity. The session provides relay channels for SSH // connect and ExecSandbox through the gateway. - let supervisor_session_task = if early_exit.is_none() - && let (Some(endpoint), Some(id), Some(socket)) = - (openshell_endpoint, sandbox_id, ssh_socket_path.as_ref()) + if let (Some(endpoint), Some(id), Some(socket)) = + (openshell_endpoint, sandbox_id, ssh_socket_path.as_ref()) { - let task = crate::supervisor_session::spawn( + crate::supervisor_session::spawn( endpoint.to_string(), id.to_string(), socket.clone(), - ssh_netns_fd, + port_forward.clone(), None, Arc::clone(&supervisor_terminating), - main_instance_id.clone(), ); info!("supervisor session task spawned"); - Some(task) - } else { - None - }; + } + + #[cfg(target_os = "linux")] + let handle = ProcessHandle::spawn( + program, + args, + workdir, + interactive, + boundary_runtime.requires_dedicated_process_group(), + policy, + resolved_process_identity, + enforcement_mode, + netns, + ca_file_paths.as_ref(), + &provider_env, + )?; + + #[cfg(not(target_os = "linux"))] + let handle = ProcessHandle::spawn( + program, + args, + workdir, + interactive, + boundary_runtime.requires_dedicated_process_group(), + policy, + resolved_process_identity, + enforcement_mode, + ca_file_paths.as_ref(), + &provider_env, + )?; // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); - if early_exit.is_none() - && let Some(tx) = entrypoint_started_tx - { - let _ = tx.send((handle.pid(), main_instance_id.clone())); + let (terminal, signal_lock) = handle.signaling_state(); + boundary_runtime + .register_process_group(handle.pid(), terminal, signal_lock) + .map_err(|error| miette::miette!(error.to_string()))?; + if let Some(tx) = entrypoint_started_tx { + let _ = tx.send(handle.pid()); } ocsf_emit!( ProcessActivityBuilder::new(ocsf_ctx()) @@ -400,95 +589,224 @@ pub async fn run_process( .build() ); - let outcome = if let Some(status) = early_exit { - ProcessWaitOutcome::Exited(status) - } else { - wait_for_process_exit_or_shutdown(&mut handle, timeout_secs, &supervisor_terminating) - .await? - }; + Ok(SpawnedAgent { + handle, + timeout_secs, + supervisor_terminating, + #[cfg(target_os = "linux")] + _bypass_handle: bypass_handle, + boundary_exec, + port_forward, + boundary_runtime, + }) +} + +/// An owned, running workload entrypoint plus the background guards whose +/// lifetime is tied to it (the bypass monitor). +/// +/// The in-pod `RunningBoundary` owns this; dropping it kills the child via the +/// handle's `kill_on_drop`. +pub struct SpawnedAgent { + handle: ProcessHandle, + timeout_secs: u64, + supervisor_terminating: Arc, + #[cfg(target_os = "linux")] + _bypass_handle: Option>, + boundary_exec: Arc, + port_forward: Arc, + boundary_runtime: Arc, +} + +impl SpawnedAgent { + /// The host PID of the entrypoint, for diagnostics only. + #[must_use] + pub fn pid(&self) -> u32 { + self.handle.pid() + } + + /// A lock-free signaling handle derived from the entrypoint's pid. + /// + /// Separated from the waitable [`SpawnedAgent`] so a signal can be delivered + /// while another task holds the agent to await it: the running boundary + /// keeps the agent behind a mutex for `wait`, but signals go through this + /// pid-based handle and never contend for that lock. + #[must_use] + pub fn signaler(&self) -> AgentSignaler { + let (terminal, signal_lock) = self.handle.signaling_state(); + AgentSignaler { + pid: self.handle.pid(), + terminal, + signal_lock, + } + } - let rendered_code = match outcome { - ProcessWaitOutcome::Exited(status) => status.code(), - ProcessWaitOutcome::TimedOut => { + /// The executor used by SSH and exposed by the active boundary. + #[must_use] + pub fn boundary_exec(&self) -> Arc { + self.boundary_exec.clone() + } + + /// The port-forward implementation used by sessions and exposed by the + /// active boundary. + #[must_use] + pub fn port_forward(&self) -> Arc { + self.port_forward.clone() + } + + #[must_use] + pub fn boundary_runtime(&self) -> Arc { + self.boundary_runtime.clone() + } + + /// Wait for the entrypoint to exit, applying the policy timeout. + /// + /// # Errors + /// + /// Returns an error if waiting on the child returns an OS error. + pub async fn wait(&mut self) -> Result { + let pid = self.handle.pid(); + let (registration_terminal, _) = self.handle.signaling_state(); + let outcome = wait_for_process_exit_or_shutdown( + &mut self.handle, + self.timeout_secs, + &self.supervisor_terminating, + ) + .await; + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + self.boundary_runtime + .unregister_process_group(pid, ®istration_terminal); + self.boundary_runtime.deactivate(); + return Err(error); + } + }; + + let (status, emit_normal_exit) = match outcome { + ProcessWaitOutcome::Exited(status) => (status, true), + ProcessWaitOutcome::TimedOut => { + ocsf_emit!( + ProcessActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Close) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Critical) + .status(StatusId::Failure) + .message("Process timed out, killing") + .build() + ); + (ProcessStatus::exited(124), false) + } + ProcessWaitOutcome::ShutdownSignal { signal, status } => { + info!( + signal, + exit_code = status.code(), + "Entrypoint exited after supervisor shutdown signal" + ); + (status, true) + } + }; + + self.boundary_runtime + .unregister_process_group(pid, ®istration_terminal); + self.boundary_runtime.deactivate(); + + if emit_normal_exit { ocsf_emit!( ProcessActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Close) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Critical) - .status(StatusId::Failure) - .message("Process timed out, killing") + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .exit_code(status.code()) + .message(format!("Process exited with code {}", status.code())) .build() ); - 124 } - ProcessWaitOutcome::ShutdownSignal { signal, status } => { - info!( - signal, - exit_code = status.code(), - "Entrypoint exited after supervisor shutdown signal" - ); - status.code() - } - }; - supervisor_terminating.store(true, Ordering::Release); - main_session.finish(rendered_code).await; - ocsf_emit!( - ProcessActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Close) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .exit_code(rendered_code) - .message(format!("Process exited with code {rendered_code}")) - .build() - ); + Ok(status) + } - if let Some(task) = supervisor_session_task { - task.abort(); + /// Send a signal to the entrypoint process. + /// + /// # Errors + /// + /// Returns an error if the signal cannot be delivered. + #[cfg(unix)] + pub fn signal(&self, sig: nix::sys::signal::Signal) -> Result<()> { + self.handle.signal(sig) } - if let Some(tx) = sidecar_exit_tx { - let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); - tx.send((main_instance_id.clone(), rendered_code, ack_tx)) - .await - .map_err(|_| miette::miette!("sidecar exit reporter closed"))?; - ack_rx - .await - .map_err(|_| miette::miette!("sidecar exit reporter dropped acknowledgement"))? - .map_err(|error| miette::miette!(error))?; - } else if let (Some(endpoint), Some(id)) = (openshell_endpoint, sandbox_id) { - report_main_process_exit_until_ack(endpoint, id, &main_instance_id, rendered_code).await; - info!(instance_id = %main_instance_id, "main-process exit acknowledged"); + + /// Terminate the entrypoint (SIGTERM, then SIGKILL). + /// + /// # Errors + /// + /// Returns an error if the process cannot be killed. + pub fn kill(&mut self) -> Result<()> { + self.handle.kill() } +} - Ok(rendered_code) +/// A lock-free, pid-based signaling handle to a spawned agent. +/// +/// Delivers signals to the entrypoint process without holding the waitable +/// handle's lock, so a signal and an in-flight `wait` never deadlock. +/// Placement-neutral signal mapping (e.g. RFC 0012's `BoundarySignal`) is the +/// caller's job; this handle exposes only the concrete deliveries so `nix` +/// stays in this crate. +#[derive(Clone)] +pub struct AgentSignaler { + pid: u32, + terminal: Arc, + signal_lock: Arc>, } -async fn report_main_process_exit_until_ack( - endpoint: &str, - sandbox_id: &str, - instance_id: &str, - exit_code: i32, -) { - let mut retry_delay = Duration::from_millis(250); - loop { - match crate::supervisor_session::report_main_process_exit( - endpoint, - sandbox_id, - instance_id, - exit_code, - ) - .await - { - Ok(()) => return, - Err(error) => { - tracing::warn!(%error, "main-process exit report failed; retrying"); - tokio::time::sleep(retry_delay).await; - retry_delay = (retry_delay * 2).min(Duration::from_secs(2)); - } +#[cfg(unix)] +impl AgentSignaler { + fn deliver(&self, sig: nix::sys::signal::Signal) -> Result<()> { + use nix::unistd::Pid; + let _signal_guard = self + .signal_lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.terminal.load(Ordering::Acquire) { + return Err(miette::miette!("agent has exited")); } + let pid = i32::try_from(self.pid).unwrap_or(i32::MAX); + nix::sys::signal::killpg(Pid::from_raw(pid), sig).into_diagnostic() + } + + /// Send `SIGTERM`. + /// + /// # Errors + /// Returns an error if the signal cannot be delivered. + pub fn term(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGTERM) + } + + /// Send `SIGKILL`. + /// + /// # Errors + /// Returns an error if the signal cannot be delivered. + pub fn kill(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGKILL) + } + + /// Send `SIGINT`. + /// + /// # Errors + /// Returns an error if the signal cannot be delivered. + pub fn interrupt(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGINT) + } + + /// Send `SIGHUP`. + /// + /// # Errors + /// Returns an error if the signal cannot be delivered. + pub fn hangup(&self) -> Result<()> { + self.deliver(nix::sys::signal::Signal::SIGHUP) } } @@ -521,6 +839,10 @@ async fn wait_for_process_exit_or_shutdown( () = &mut deadline => { terminating.store(true, Ordering::Release); terminate_then_kill_pid(pid).await; + // Finish the owned wait after terminating the process. Dropping + // it here would leave terminal publication, reaping, and + // managed-child cleanup pending until supervisor exit. + let _ = (&mut wait).await.into_diagnostic()?; Ok(ProcessWaitOutcome::TimedOut) } signal = wait_for_supervisor_shutdown_signal() => { @@ -567,13 +889,16 @@ fn signal_entrypoint_for_shutdown(_pid: u32, _signal: &'static str) {} #[cfg(unix)] fn signal_pid(pid: u32, signal: nix::sys::signal::Signal, reason: &'static str) { let raw_pid = i32::try_from(pid).unwrap_or(i32::MAX); - if let Err(error) = nix::sys::signal::kill(nix::unistd::Pid::from_raw(-raw_pid), signal) { + let target = nix::unistd::Pid::from_raw(raw_pid); + let result = nix::sys::signal::killpg(target, signal) + .or_else(|_| nix::sys::signal::kill(target, signal)); + if let Err(error) = result { tracing::warn!( pid, signal = ?signal, reason, error = %error, - "failed to signal entrypoint process group" + "failed to signal entrypoint process" ); } } @@ -720,4 +1045,42 @@ mod tests { assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn timeout_reaps_child_and_clears_managed_registration() { + let args = vec!["30".to_string()]; + let mut handle = ProcessHandle::spawn( + "/bin/sleep", + &args, + None, + false, + true, + &policy(NetworkMode::Allow, None), + ResolvedProcessIdentity::default(), + ProcessEnforcementMode::NetworkOnly, + None, + None, + &std::collections::HashMap::new(), + ) + .expect("spawn timeout child"); + let pid = handle.pid(); + let terminating = AtomicBool::new(false); + + let outcome = wait_for_process_exit_or_shutdown(&mut handle, 1, &terminating) + .await + .expect("timeout wait"); + + assert!(matches!(outcome, ProcessWaitOutcome::TimedOut)); + assert!(!managed_children::is_managed( + i32::try_from(pid).expect("valid pid") + )); + assert!(matches!( + nix::sys::wait::waitpid( + nix::unistd::Pid::from_raw(i32::try_from(pid).expect("valid pid")), + Some(nix::sys::wait::WaitPidFlag::WNOHANG) + ), + Err(nix::errno::Errno::ECHILD) + )); + } } diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index c0b5a2c30f..20425c9e93 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -4,31 +4,27 @@ //! Embedded SSH server for sandbox access. use crate::child_env; -use crate::main_session::{MainOutput, MainSession}; #[cfg(target_os = "linux")] use crate::managed_children; use crate::process::{ - ProcessEnforcementMode, ResolvedProcessIdentity, ResolvedWorkspace, - drop_privileges_with_identity, is_supervisor_only_env_var, session_user_and_home, + ProcessEnforcementMode, ResolvedProcessIdentity, drop_privileges_with_identity, + is_supervisor_only_env_var, }; use crate::sandbox; -#[cfg(unix)] -use libc; use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::policy::SandboxPolicy; -use openshell_core::provider_credentials::ProviderCredentialState; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; +use russh::ChannelId; +use russh::ChannelOpenFailure; use russh::keys::{Algorithm, PrivateKey}; use russh::server::{Auth, ChannelOpenHandle, Handle, Session}; -use russh::{ChannelId, ChannelOpenFailure, Sig}; use std::collections::HashMap; use std::io::{Read, Write}; -use std::os::fd::{AsRawFd, RawFd}; +use std::os::fd::{AsRawFd, OwnedFd, RawFd}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::{Arc, mpsc}; @@ -113,19 +109,13 @@ fn ssh_server_init( pub async fn run_ssh_server( listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, shared_socket: bool, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init( + let (listener, config, _ca_paths) = match ssh_server_init( &listen_path, &ca_file_paths, enforcement_mode, @@ -145,186 +135,24 @@ pub async fn run_ssh_server( } }; - let mut consecutive_resource_errors: u32 = 0; - let mut consecutive_unknown_errors: u32 = 0; - loop { - match listener.accept().await { - Ok((stream, _peer)) => { - consecutive_resource_errors = 0; - consecutive_unknown_errors = 0; - let config = config.clone(); - let policy = policy.clone(); - let workspace = workspace.clone(); - let proxy_url = proxy_url.clone(); - let ca_paths = ca_paths.clone(); - let provider_credentials = provider_credentials.clone(); - let user_environment = user_environment.clone(); - let main_session = Arc::clone(&main_session); - - tokio::spawn(async move { - if let Err(err) = handle_connection( - stream, - config, - policy, - workspace, - netns_fd, - proxy_url, - ca_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ) - .await - { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .message(format!("SSH connection failed: {err}")) - .build() - ); - } - }); - } - Err(err) => { - match classify_ssh_accept_error( - &err, - &mut consecutive_resource_errors, - &mut consecutive_unknown_errors, - ) { - SshAcceptAction::Terminal => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message(format!( - "SSH accept loop exiting on terminal error: {err}" - )) - .build() - ); - break; - } - SshAcceptAction::Retry { backoff, severity } => { - ocsf_emit!( - SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(severity) - .status(StatusId::Failure) - .message(format!( - "SSH accept error (retrying in {}ms): {err}", - backoff.as_millis(), - )) - .build() - ); - tokio::time::sleep(backoff).await; - } - } - } - } - } + let (stream, _peer) = listener.accept().await.into_diagnostic()?; + let config = config.clone(); + let port_forward = port_forward.clone(); + let boundary_exec = boundary_exec.clone(); - Ok(()) -} - -const MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS: u32 = 10; - -#[derive(Debug, PartialEq)] -enum SshAcceptAction { - Terminal, - Retry { - backoff: Duration, - severity: SeverityId, - }, -} - -fn classify_ssh_accept_error( - err: &std::io::Error, - consecutive_resource_errors: &mut u32, - consecutive_unknown_errors: &mut u32, -) -> SshAcceptAction { - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some(libc::EBADF | libc::EINVAL | libc::ENOTSOCK) - ) { - return SshAcceptAction::Terminal; - } - - #[cfg(unix)] - if matches!( - err.raw_os_error(), - Some( - libc::EMFILE - | libc::ENFILE - | libc::ENOBUFS - | libc::ENOMEM - | libc::ECONNABORTED - | libc::ECONNRESET - | libc::EINTR - | libc::ENETDOWN - | libc::EPROTO - | libc::ENOPROTOOPT - | libc::EHOSTDOWN - | libc::EHOSTUNREACH - | libc::EOPNOTSUPP - | libc::ENETUNREACH - | libc::ENOSR - | libc::ESOCKTNOSUPPORT - | libc::EPROTONOSUPPORT - | libc::ETIMEDOUT - ) - ) { - *consecutive_unknown_errors = 0; - - #[cfg(unix)] - let is_resource_pressure = matches!( - err.raw_os_error(), - Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM | libc::ENOSR) - ); - #[cfg(not(unix))] - let is_resource_pressure = false; - - if is_resource_pressure { - *consecutive_resource_errors = consecutive_resource_errors.saturating_add(1); - let backoff_ms = 100u64 - .saturating_mul(1u64 << (*consecutive_resource_errors).min(7).saturating_sub(1)) - .min(5_000); - return SshAcceptAction::Retry { - backoff: Duration::from_millis(backoff_ms), - severity: SeverityId::Medium, - }; - } - - *consecutive_resource_errors = 0; - return SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - #[cfg(unix)] - #[cfg(target_os = "linux")] - if matches!(err.raw_os_error(), Some(libc::ENONET)) { - *consecutive_unknown_errors = 0; - *consecutive_resource_errors = 0; - return SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, - }; - } - - *consecutive_unknown_errors = consecutive_unknown_errors.saturating_add(1); - if *consecutive_unknown_errors >= MAX_CONSECUTIVE_UNKNOWN_SSH_ACCEPT_ERRORS { - return SshAcceptAction::Terminal; - } - SshAcceptAction::Retry { - backoff: Duration::from_millis(100), - severity: SeverityId::Low, + tokio::spawn(async move { + if let Err(err) = handle_connection(stream, config, port_forward, boundary_exec).await { + ocsf_emit!( + SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!("SSH connection failed: {err}")) + .build() + ); + } + }); } } @@ -332,16 +160,8 @@ fn classify_ssh_accept_error( async fn handle_connection( stream: tokio::net::UnixStream, config: Arc, - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -357,18 +177,7 @@ async fn handle_connection( .build() ); - let handler = SshHandler::new( - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, - ); + let handler = SshHandler::new(port_forward, boundary_exec); russh::server::run_stream(config, stream, handler) .await .map_err(|err| miette::miette!("ssh stream error: {err}"))?; @@ -383,118 +192,28 @@ async fn handle_connection( /// sftp, etc.). #[derive(Default)] struct ChannelState { - input_sender: Option, - pty_master: Option, + input_sender: Option>>, + terminal: Option>, pty_request: Option, - main_input_owner: Option, - main_attached: bool, - main_read_only: bool, - main_detach_prefix_pending: bool, - main_output_task: Option, -} - -const MAIN_DETACH_PREFIX: u8 = 0x10; // Ctrl-P -const MAIN_DETACH_KEY: u8 = 0x11; // Ctrl-Q - -/// Remove the `OpenShell` detach sequence from canonical-main input. -/// -/// A trailing Ctrl-P remains pending across SSH data frames. If the following -/// byte is not Ctrl-Q, both bytes are forwarded unchanged. Bytes after a -/// completed detach sequence are discarded because the attachment is closing. -fn filter_main_detach_sequence(prefix_pending: &mut bool, data: &[u8]) -> (Vec, bool) { - let mut forward = Vec::with_capacity(data.len() + usize::from(*prefix_pending)); - - for &byte in data { - if *prefix_pending { - if byte == MAIN_DETACH_KEY { - *prefix_pending = false; - return (forward, true); - } - forward.push(MAIN_DETACH_PREFIX); - *prefix_pending = false; - } - - if byte == MAIN_DETACH_PREFIX { - *prefix_pending = true; - } else { - forward.push(byte); - } - } - - (forward, false) -} - -enum InputSender { - Process(mpsc::Sender>), - Main(tokio::sync::mpsc::Sender>), -} - -impl InputSender { - fn send(&self, data: Vec) -> Result<(), &'static str> { - match self { - Self::Process(sender) => sender.send(data).map_err(|_| "process stdin closed"), - Self::Main(sender) => sender.try_send(data).map_err(|error| match error { - tokio::sync::mpsc::error::TrySendError::Full(_) => "canonical stdin buffer is full", - tokio::sync::mpsc::error::TrySendError::Closed(_) => { - "canonical process stdin closed" - } - }), - } - } } struct SshHandler { - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + /// Loopback port-forward, injected by the orchestrator (RFC 0012). In-pod + /// this connects from inside the workload netns; a delegated backend + /// tunnels into its guest. The handler does not know which. + port_forward: Arc, + boundary_exec: Arc, channels: HashMap, } -impl Drop for SshHandler { - fn drop(&mut self) { - for state in self.channels.values_mut() { - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); - } - if let Some(task) = state.main_output_task.take() { - task.abort(); - } - } - } -} - impl SshHandler { - #[allow(clippy::too_many_arguments)] fn new( - policy: SandboxPolicy, - workspace: ResolvedWorkspace, - netns_fd: Option, - proxy_url: Option, - ca_file_paths: Option>, - provider_credentials: ProviderCredentialState, - user_environment: HashMap, - resolved_identity: ResolvedProcessIdentity, - enforcement_mode: ProcessEnforcementMode, - main_session: Arc, + port_forward: Arc, + boundary_exec: Arc, ) -> Self { Self { - policy, - workspace, - netns_fd, - proxy_url, - ca_file_paths, - provider_credentials, - user_environment, - resolved_identity, - enforcement_mode, - main_session, + port_forward, + boundary_exec, channels: HashMap::new(), } } @@ -536,14 +255,7 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, _session: &mut Session, ) -> Result<(), Self::Error> { - if let Some(state) = self.channels.remove(&channel) { - if let Some(owner) = state.main_input_owner { - self.main_session.release_input(owner); - } - if let Some(task) = state.main_output_task { - task.abort(); - } - } + self.channels.remove(&channel); Ok(()) } @@ -598,15 +310,23 @@ impl russh::server::Handler for SshHandler { // SSH protocol port is bounded by u32 but only u16 is meaningful; // saturate as a guard for malformed clients. let port = u16::try_from(port_to_connect).unwrap_or(u16::MAX); - let netns_fd = self.netns_fd; - // Confirm the channel before spawning: the task below writes to it, and - // the peer must see the open-confirmation first. + // Build the loopback target up front. The host already passed + // `is_loopback_host`, and `LoopbackTarget::new` re-validates the parsed + // address (defense in depth) before the connect. + let Some(target) = loopback_ip(&host) + .and_then(|ip| openshell_isolation::contract::LoopbackTarget::new(ip, port).ok()) + else { + reply + .reject(ChannelOpenFailure::AdministrativelyProhibited) + .await; + return Ok(()); + }; + let port_forward = self.port_forward.clone(); reply.accept().await; tokio::spawn(async move { - let addr = format!("{host}:{port}"); - let tcp = match connect_in_netns(&addr, netns_fd).await { + let mut tcp_stream = match port_forward.connect(target).await { Ok(stream) => stream, Err(err) => { ocsf_emit!( @@ -614,7 +334,9 @@ impl russh::server::Handler for SshHandler { .activity(ActivityId::Fail) .severity(SeverityId::Low) .status(StatusId::Failure) - .message(format!("direct-tcpip: failed to connect to {addr}: {err}")) + .message(format!( + "direct-tcpip: failed to connect to {host}:{port}: {err}" + )) .build() ); let _ = channel.close().await; @@ -623,7 +345,6 @@ impl russh::server::Handler for SshHandler { }; let mut channel_stream = channel.into_stream(); - let mut tcp_stream = tcp; let _ = tokio::io::copy_bidirectional(&mut channel_stream, &mut tcp_stream).await; }); @@ -662,27 +383,20 @@ impl russh::server::Handler for SshHandler { channel: ChannelId, col_width: u32, row_height: u32, - pixel_width: u32, - pixel_height: u32, + _pixel_width: u32, + _pixel_height: u32, _session: &mut Session, ) -> Result<(), Self::Error> { let Some(state) = self.channels.get(&channel) else { warn!("window_change_request on unknown channel {channel:?}"); return Ok(()); }; - if state.main_attached { - self.main_session - .resize(col_width, row_height, pixel_width, pixel_height); - } else if let Some(master) = state.pty_master.as_ref() { - let winsize = Winsize { - ws_row: to_u16(row_height.max(1)), - ws_col: to_u16(col_width.max(1)), - ws_xpixel: to_u16(pixel_width), - ws_ypixel: to_u16(pixel_height), - }; - if let Err(e) = unsafe_pty::set_winsize(master.as_raw_fd(), winsize) { - warn!("failed to resize PTY for channel {channel:?}: {e}"); - } + if let Some(terminal) = state.terminal.as_ref() + && let Err(e) = terminal + .resize(to_u16(col_width.max(1)), to_u16(row_height.max(1))) + .await + { + warn!("failed to resize PTY for channel {channel:?}: {e}"); } Ok(()) } @@ -699,7 +413,7 @@ impl russh::server::Handler for SshHandler { // endings. Forcing a PTY here caused CRLF translation which made // VS Code misdetect the platform as Windows (and then try to run // `powershell`). - self.start_shell(channel, session.handle(), None)?; + self.start_shell(channel, session.handle(), None).await?; Ok(()) } @@ -714,7 +428,8 @@ impl russh::server::Handler for SshHandler { if command.is_empty() { return Ok(()); } - self.start_shell(channel, session.handle(), Some(command))?; + self.start_shell(channel, session.handle(), Some(command)) + .await?; Ok(()) } @@ -724,102 +439,24 @@ impl russh::server::Handler for SshHandler { name: &str, session: &mut Session, ) -> Result<(), Self::Error> { - if name == "openshell-main" { - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - if let Some(pty) = state.pty_request.take() { - self.main_session.resize( - pty.col_width, - pty.row_height, - pty.pixel_width, - pty.pixel_height, - ); - } - let (input, input_warning) = if state.main_read_only { - (None, None) - } else { - match self.main_session.acquire_input() { - Ok((owner, input)) => { - state.main_input_owner = Some(owner); - (Some(InputSender::Main(input)), None) - } - Err(error) => { - warn!(%error, "main process input lease unavailable; attaching read-only"); - (None, Some(error)) - } - } - }; - state.main_attached = true; - state.main_detach_prefix_pending = false; - state.input_sender = input; - let mut output = self.main_session.subscribe(); - let handle = session.handle(); - session.channel_success(channel)?; - if let Some(error) = input_warning { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; attached read-only\n").into_bytes(), - ) - .await; - } - let output_task = tokio::spawn(async move { - loop { - match output.recv().await { - Ok(event) => { - let exited = matches!(event, MainOutput::Exit(_)); - send_main_output(&handle, channel, event).await; - if exited { - break; - } - } - Err(error) => { - let _ = handle - .extended_data( - channel, - 1, - format!( - "openshell: attachment fell behind by {} output chunks; reconnect for buffered output\n", - error.skipped - ) - .into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - break; - } - } - } - }); - if let Some(state) = self.channels.get_mut(&channel) { - state.main_output_task = Some(output_task.abort_handle()); - } - } else if name == "sftp" { + if name == "sftp" { session.channel_success(channel)?; // sftp-server speaks the SFTP binary protocol over stdin/stdout, - // which is exactly what spawn_pipe_exec wires up. This enables + // which the boundary executor preserves as separate pipes. This enables // modern scp (SFTP-based, OpenSSH 9.0+) and SFTP clients to // transfer files into and out of the sandbox. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - Some("/usr/lib/openssh/sftp-server".to_string()), - session.handle(), + self.start_exec_spec( channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &self.provider_credentials.child_env_with_gcp_resolved(), - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - let state = self.channels.get_mut(&channel).ok_or_else(|| { - anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") - })?; - state.input_sender = Some(InputSender::Process(input_sender)); + session.handle(), + openshell_isolation::contract::ExecSpec { + program: "/usr/lib/openssh/sftp-server".to_string(), + args: vec![], + env: vec![], + workdir: None, + pty: false, + }, + ) + .await?; } else { ocsf_emit!( SshActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -845,12 +482,7 @@ impl russh::server::Handler for SshHandler { // Accept the env request so the client knows we handled it, but we // don't actually propagate the variables — the sandbox environment is // controlled via policy. We must reply so VSCode doesn't stall. - if variable_name == "OPENSHELL_MAIN_READ_ONLY" - && variable_value == "1" - && let Some(state) = self.channels.get_mut(&channel) - { - state.main_read_only = true; - } + let _ = (variable_name, variable_value); session.channel_success(channel)?; Ok(()) } @@ -859,44 +491,14 @@ impl russh::server::Handler for SshHandler { &mut self, channel: ChannelId, data: &[u8], - session: &mut Session, + _session: &mut Session, ) -> Result<(), Self::Error> { - let Some(state) = self.channels.get_mut(&channel) else { + let Some(state) = self.channels.get(&channel) else { warn!("data on unknown channel {channel:?}"); return Ok(()); }; - - let main_attached = state.main_attached; - let (forward, detach) = if main_attached { - filter_main_detach_sequence(&mut state.main_detach_prefix_pending, data) - } else { - (data.to_vec(), false) - }; - let send_error = (!forward.is_empty()) - .then(|| state.input_sender.as_ref()?.send(forward).err()) - .flatten(); - - if let Some(error) = send_error { - let handle = session.handle(); - if main_attached { - self.close_main_attachment(channel, handle, Some(error)) - .await; - } else { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; - let _ = handle.close(channel).await; - } - return Ok(()); - } - if detach { - self.close_main_attachment(channel, session.handle(), None) - .await; - return Ok(()); + if let Some(sender) = state.input_sender.as_ref() { + let _ = sender.send(data.to_vec()); } Ok(()) } @@ -911,150 +513,148 @@ impl russh::server::Handler for SshHandler { // is essential for commands like `cat | tar xf -` which need // stdin EOF to know the input stream is complete. if let Some(state) = self.channels.get_mut(&channel) { - if state.main_attached - && let Some(owner) = state.main_input_owner.take() - { - self.main_session.release_input(owner); - } state.input_sender.take(); - state.main_detach_prefix_pending = false; } else { warn!("channel_eof on unknown channel {channel:?}"); } Ok(()) } - - async fn signal( - &mut self, - channel: ChannelId, - signal: Sig, - _session: &mut Session, - ) -> Result<(), Self::Error> { - if !self - .channels - .get(&channel) - .is_some_and(|state| state.main_attached) - { - return Ok(()); - } - let signal = match signal { - Sig::HUP => Some(nix::sys::signal::Signal::SIGHUP), - Sig::INT => Some(nix::sys::signal::Signal::SIGINT), - Sig::KILL => Some(nix::sys::signal::Signal::SIGKILL), - Sig::QUIT => Some(nix::sys::signal::Signal::SIGQUIT), - Sig::TERM => Some(nix::sys::signal::Signal::SIGTERM), - _ => None, - }; - if let Some(signal) = signal - && let Err(error) = self.main_session.signal_group(signal) - { - warn!(%error, ?signal, "failed to signal canonical main process group"); - } - Ok(()) - } -} - -async fn send_main_output(handle: &Handle, channel: ChannelId, event: MainOutput) { - match event { - MainOutput::Stdout(data) => { - let _ = handle.data(channel, data).await; - } - MainOutput::Stderr(data) => { - let _ = handle.extended_data(channel, 1, data).await; - } - MainOutput::Exit(code) => { - let _ = handle.eof(channel).await; - let _ = handle - .exit_status_request(channel, code.max(0).unsigned_abs()) - .await; - let _ = handle.close(channel).await; - } - } } impl SshHandler { - async fn close_main_attachment( + async fn start_shell( &mut self, channel: ChannelId, handle: Handle, - error: Option<&str>, - ) { - if let Some(state) = self.channels.get_mut(&channel) { - if let Some(owner) = state.main_input_owner.take() { - self.main_session.release_input(owner); - } - state.input_sender.take(); - state.main_detach_prefix_pending = false; - if let Some(task) = state.main_output_task.take() { - task.abort(); - } - state.main_attached = false; - } - if let Some(error) = error { - let _ = handle - .extended_data( - channel, - 1, - format!("openshell: {error}; closing attachment\n").into_bytes(), - ) - .await; + command: Option, + ) -> anyhow::Result<()> { + let pty = self + .channels + .get_mut(&channel) + .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))? + .pty_request + .take(); + let pty_requested = pty.is_some(); + let (program, args) = command.map_or_else( + || { + if pty_requested { + ("/bin/bash".to_string(), vec!["-i".to_string()]) + } else { + ("/bin/bash".to_string(), vec![]) + } + }, + |command| ("/bin/bash".to_string(), vec!["-lc".to_string(), command]), + ); + let env = pty + .as_ref() + .map(|request| vec![("TERM".to_string(), request.term.clone())]) + .unwrap_or_default(); + self.start_exec_spec( + channel, + handle, + openshell_isolation::contract::ExecSpec { + program, + args, + env, + workdir: None, + pty: pty_requested, + }, + ) + .await?; + if let (Some(pty), Some(terminal)) = ( + pty, + self.channels + .get(&channel) + .and_then(|state| state.terminal.as_ref()), + ) { + terminal + .resize(to_u16(pty.col_width.max(1)), to_u16(pty.row_height.max(1))) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; } - let _ = handle.eof(channel).await; - let _ = handle.exit_status_request(channel, 0).await; - let _ = handle.close(channel).await; + Ok(()) } - fn start_shell( + async fn start_exec_spec( &mut self, channel: ChannelId, handle: Handle, - command: Option, + spec: openshell_isolation::contract::ExecSpec, ) -> anyhow::Result<()> { - let provider_env = self.provider_credentials.child_env_with_gcp_resolved(); + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut exec = self + .boundary_exec + .exec(spec) + .await + .map_err(|error| anyhow::anyhow!(error.to_string()))?; let state = self .channels .get_mut(&channel) - .ok_or_else(|| anyhow::anyhow!("start_shell on unknown channel {channel:?}"))?; - if let Some(pty) = state.pty_request.take() { - // PTY was requested — allocate a real PTY (interactive shell or - // exec that explicitly asked for a terminal). - let (pty_master, input_sender) = spawn_pty_shell( - &self.policy, - &self.workspace, - command, - &pty, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.pty_master = Some(pty_master); - state.input_sender = Some(InputSender::Process(input_sender)); - } else { - // No PTY requested — use plain pipes so stdout/stderr are - // separate and output has clean LF line endings. This is the - // path VSCode Remote-SSH exec commands take. - let input_sender = spawn_pipe_exec( - &self.policy, - &self.workspace, - command, - handle, - channel, - self.netns_fd, - self.proxy_url.clone(), - self.ca_file_paths.clone(), - &provider_env, - &self.user_environment, - self.resolved_identity, - self.enforcement_mode, - )?; - state.input_sender = Some(InputSender::Process(input_sender)); + .ok_or_else(|| anyhow::anyhow!("exec on unknown channel {channel:?}"))?; + state.terminal = exec.terminal.take(); + + if let Some(mut stdin) = exec.stdin.take() { + let (sender, receiver) = mpsc::channel::>(); + let runtime = tokio::runtime::Handle::current(); + std::thread::spawn(move || { + while let Ok(bytes) = receiver.recv() { + if runtime.block_on(stdin.write_all(&bytes)).is_err() { + break; + } + } + }); + state.input_sender = Some(sender); } + + let mut stdout = exec.stdout; + let stdout_handle = handle.clone(); + let stdout_task = tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stdout.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stdout_handle.data(channel, buffer[..size].to_vec()).await; + } + } + } + }); + let stderr_task = exec.stderr.map(|mut stderr| { + let stderr_handle = handle.clone(); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + match stderr.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(size) => { + let _ = stderr_handle + .extended_data(channel, 1, buffer[..size].to_vec()) + .await; + } + } + } + }) + }); + tokio::spawn(async move { + let status = exec.process.wait().await; + let _ = stdout_task.await; + if let Some(task) = stderr_task { + let _ = task.await; + } + let code = match status { + Ok(openshell_isolation::contract::BoundaryExitStatus::Exited(code)) => { + code.max(0).cast_unsigned() + } + Ok(openshell_isolation::contract::BoundaryExitStatus::Signaled(signal)) => { + (128_i32.saturating_add(signal)).max(0).cast_unsigned() + } + Err(_) => 1, + }; + let _ = handle.eof(channel).await; + let _ = handle.exit_status_request(channel, code).await; + let _ = handle.close(channel).await; + }); Ok(()) } } @@ -1073,12 +673,11 @@ impl SshHandler { /// thread could be reused for unrelated tasks and must not be contaminated. /// On non-Linux platforms (no network namespace support), we connect directly. pub async fn connect_in_netns( - addr: &str, - netns_fd: Option, + addr: std::net::SocketAddr, + netns_fd: Option>, ) -> std::io::Result { #[cfg(target_os = "linux")] if let Some(fd) = netns_fd { - let addr = addr.to_string(); let (tx, rx) = tokio::sync::oneshot::channel(); std::thread::spawn(move || { let result = (|| -> std::io::Result { @@ -1086,11 +685,11 @@ pub async fn connect_in_netns( // SAFETY: setns is safe to call; this is a dedicated thread that // will exit after the connection is established. #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; + let rc = unsafe { libc::setns(fd.as_raw_fd(), libc::CLONE_NEWNET) }; if rc != 0 { return Err(std::io::Error::last_os_error()); } - std::net::TcpStream::connect(&addr) + std::net::TcpStream::connect_timeout(&addr, Duration::from_secs(5)) })(); let _ = tx.send(result); }); @@ -1099,19 +698,18 @@ pub async fn connect_in_netns( .await .map_err(|_| std::io::Error::other("netns connect thread panicked"))??; std_stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(std_stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); + return tokio::net::TcpStream::from_std(std_stream); } #[cfg(not(target_os = "linux"))] let _ = netns_fd; - let stream = tokio::net::TcpStream::connect(addr).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) + tokio::time::timeout(Duration::from_secs(5), tokio::net::TcpStream::connect(addr)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))? } +#[allow(dead_code)] #[derive(Clone)] struct PtyRequest { term: String, @@ -1133,6 +731,35 @@ impl Default for PtyRequest { } } +/// Derive the session USER and HOME from the policy's `run_as_user`. +/// +/// For name-based identities, looks up the home directory via `/etc/passwd` +/// (or defaults to `/home/{user}`). +/// +/// For numeric UIDs, there is no passwd entry — falls back to +/// `("{uid}", "/sandbox")` so the agent session still has a meaningful +/// USER identifier. +pub(crate) fn session_user_and_home(policy: &SandboxPolicy) -> (String, String) { + match policy.process.run_as_user.as_deref() { + Some(user) if !user.is_empty() => { + // Numeric UID — no passwd entry expected; use default HOME. + if user.parse::().is_ok() { + return (user.to_string(), "/sandbox".to_string()); + } + // Name-based identity — look up home from /etc/passwd. + let home = nix::unistd::User::from_name(user) + .ok() + .flatten() + .map_or_else( + || format!("/home/{user}"), + |u| u.dir.to_string_lossy().into_owned(), + ); + (user.to_string(), home) + } + _ => ("sandbox".to_string(), "/sandbox".to_string()), + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn apply_child_env( cmd: &mut Command, @@ -1181,9 +808,10 @@ pub(crate) fn apply_child_env( } #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pty_shell( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, pty: &PtyRequest, handle: Handle, @@ -1234,7 +862,7 @@ fn spawn_pty_shell( // Derive USER and HOME from the policy's run_as_user when available, // falling back to "sandbox" / "/sandbox" for backward compatibility. - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1247,20 +875,20 @@ fn spawn_pty_shell( ); cmd.stdin(stdin).stdout(stdout).stderr(stderr); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] @@ -1268,24 +896,25 @@ fn spawn_pty_shell( unsafe_pty::install_pre_exec( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), slave_fd, netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] + let mut child_registry = managed_children::lock(); let mut child = cmd.spawn()?; #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let master_file = master; let (sender, receiver) = mpsc::channel::>(); @@ -1331,7 +960,9 @@ fn spawn_pty_shell( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - managed_children::unregister(child_pid); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); + } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for the reader thread to finish forwarding all output before // sending exit-status and closing the channel. This prevents the @@ -1357,9 +988,10 @@ fn spawn_pty_shell( /// (type 1), preserving the separation that clients like `VSCode` Remote-SSH /// expect. Output retains clean LF line endings (no CRLF translation). #[allow(clippy::too_many_arguments)] +#[allow(dead_code)] fn spawn_pipe_exec( policy: &SandboxPolicy, - workspace: &ResolvedWorkspace, + workdir: Option, command: Option, handle: Handle, channel: ChannelId, @@ -1391,7 +1023,7 @@ fn spawn_pipe_exec( }, ); - let (session_user, session_home) = session_user_and_home(policy, workspace.home()); + let (session_user, session_home) = session_user_and_home(policy); apply_child_env( &mut cmd, &session_home, @@ -1406,20 +1038,20 @@ fn spawn_pipe_exec( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - if let Some(dir) = workspace.root() { + if let Some(dir) = workdir.as_deref() { cmd.current_dir(dir); } // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] if enforcement_mode.enforces_child_sandbox() { - sandbox::linux::log_sandbox_readiness(policy, workspace.root()); + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); } // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] let prepared_sandbox = - crate::process::prepare_child_sandbox(policy, workspace.root(), enforcement_mode) + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] @@ -1427,23 +1059,24 @@ fn spawn_pipe_exec( unsafe_pty::install_pre_exec_no_pty( &mut cmd, policy.clone(), - workspace.owned_root(), + workdir.clone(), netns_fd, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, - ); + )?; } #[cfg(target_os = "linux")] - let mut child = crate::process::spawn_std_command_with_supervisor_identity_namespace(cmd)?; - #[cfg(not(target_os = "linux"))] + let mut child_registry = managed_children::lock(); let mut child = cmd.spawn()?; #[cfg(target_os = "linux")] let child_pid = child.id(); #[cfg(target_os = "linux")] - managed_children::register(child_pid); + let managed_child = child_registry.register(child_pid); + #[cfg(target_os = "linux")] + drop(child_registry); let child_stdin = child.stdin.take(); let child_stdout = child.stdout.take().expect("stdout must be piped"); @@ -1515,7 +1148,9 @@ fn spawn_pipe_exec( std::thread::spawn(move || { let status = child.wait().ok(); #[cfg(target_os = "linux")] - managed_children::unregister(child_pid); + if let Some(managed_child) = managed_child { + managed_children::unregister(managed_child); + } let code = status.and_then(|s| s.code()).unwrap_or(1).unsigned_abs(); // Wait for both reader threads. let _ = reader_done_rx.recv_timeout(Duration::from_secs(2)); @@ -1579,11 +1214,19 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] let mut prepared = prepared; + #[cfg(target_os = "linux")] + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { + anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { setsid().map_err(|err| std::io::Error::other(err.to_string()))?; @@ -1595,10 +1238,13 @@ pub(crate) mod unsafe_pty { resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] + supervisor_identity_mount, + #[cfg(target_os = "linux")] prepared.take(), ) }); } + Ok(()) } /// Pre-exec hook for pipe-based (non-PTY) exec. @@ -1620,21 +1266,35 @@ pub(crate) mod unsafe_pty { resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] prepared: Option, - ) { + ) -> anyhow::Result<()> { #[cfg(target_os = "linux")] let mut prepared = prepared; + #[cfg(target_os = "linux")] + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { + anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { + if libc::setpgid(0, 0) != 0 { + return Err(std::io::Error::last_os_error()); + } enter_netns_and_sandbox( netns_fd, &policy, resolved_identity, enforcement_mode, #[cfg(target_os = "linux")] + supervisor_identity_mount, + #[cfg(target_os = "linux")] prepared.take(), ) }); } + Ok(()) } fn enter_netns_and_sandbox( @@ -1642,6 +1302,9 @@ pub(crate) mod unsafe_pty { policy: &SandboxPolicy, resolved_identity: ResolvedProcessIdentity, enforcement_mode: ProcessEnforcementMode, + #[cfg(target_os = "linux")] supervisor_identity_mount: Option< + &crate::process::SupervisorIdentityMountNamespace, + >, #[cfg(target_os = "linux")] prepared: Option, ) -> std::io::Result<()> { // Enter network namespace before dropping privileges. @@ -1660,6 +1323,11 @@ pub(crate) mod unsafe_pty { #[cfg(not(target_os = "linux"))] let _ = netns_fd; + #[cfg(target_os = "linux")] + if let Some(mount) = supervisor_identity_mount { + mount.enter_for_child()?; + } + // Drop privileges. initgroups/setgid/setuid need /etc/group and // /etc/passwd which would be blocked if Landlock were already enforced. if enforcement_mode.uses_privileged_process_setup() { @@ -1725,6 +1393,22 @@ fn is_loopback_host(host: &str) -> bool { } } +/// Resolve a (loopback-validated) destination host string to an `IpAddr`, +/// mapping `localhost` to `127.0.0.1`. +/// +/// Returns `None` for anything that does not parse to an IP, so +/// [`LoopbackTarget::new`] never sees a hostname. +fn loopback_ip(host: &str) -> Option { + let host = host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(host); + if host.eq_ignore_ascii_case("localhost") { + return Some(std::net::Ipv4Addr::LOCALHOST.into()); + } + host.parse().ok() +} + #[cfg(test)] #[allow( clippy::doc_markdown, @@ -1735,20 +1419,6 @@ mod tests { use super::*; use std::process::Stdio; - /// Regression test: the direct-tcpip connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_in_netns_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_in_netns(&addr.to_string(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[cfg(unix)] fn file_mode(path: &Path) -> u32 { use std::os::unix::fs::PermissionsExt; @@ -2031,11 +1701,11 @@ mod tests { let (tx_b, rx_b) = mpsc::channel::>(); let mut state_a = ChannelState { - input_sender: Some(InputSender::Process(tx_a)), + input_sender: Some(tx_a), ..Default::default() }; let state_b = ChannelState { - input_sender: Some(InputSender::Process(tx_b)), + input_sender: Some(tx_b), ..Default::default() }; @@ -2074,56 +1744,6 @@ mod tests { assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } - #[test] - fn main_detach_filter_forwards_ctrl_c_unchanged() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x03after"); - - assert_eq!(forward, b"before\x03after"); - assert!(!detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_removes_sequence_and_trailing_input() { - let mut prefix_pending = false; - let (forward, detach) = - filter_main_detach_sequence(&mut prefix_pending, b"before\x10\x11after"); - - assert_eq!(forward, b"before"); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_recognizes_sequence_across_frames() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"before\x10"); - assert_eq!(forward, b"before"); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x11"); - assert!(forward.is_empty()); - assert!(detach); - assert!(!prefix_pending); - } - - #[test] - fn main_detach_filter_forwards_unmatched_prefix() { - let mut prefix_pending = false; - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"\x10"); - assert!(forward.is_empty()); - assert!(!detach); - assert!(prefix_pending); - - let (forward, detach) = filter_main_detach_sequence(&mut prefix_pending, b"x"); - assert_eq!(forward, b"\x10x"); - assert!(!detach); - assert!(!prefix_pending); - } - // ----------------------------------------------------------------------- // session_user_and_home tests (Phase 2: numeric UID support) // ----------------------------------------------------------------------- @@ -2143,33 +1763,12 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000"); // Numeric UID has no passwd entry — defaults to /sandbox. assert_eq!(home, "/sandbox"); } - #[test] - fn session_user_and_home_uses_driver_workspace_when_supplied() { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - let policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: Some("1234".into()), - run_as_group: Some("1235".into()), - }, - }; - - let (user, home) = session_user_and_home(&policy, Some("/workspace/project")); - assert_eq!(user, "1234"); - assert_eq!(home, "/workspace/project"); - } - #[test] fn session_user_and_home_returns_name_from_passwd() { use openshell_core::policy::{ @@ -2185,7 +1784,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); // Name-based — should resolve via passwd (or /home/{user}). assert!(!home.is_empty()); @@ -2206,7 +1805,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2226,7 +1825,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "sandbox"); assert_eq!(home, "/sandbox"); } @@ -2246,7 +1845,7 @@ mod tests { run_as_group: None, }, }; - let (user, home) = session_user_and_home(&policy, None); + let (user, home) = session_user_and_home(&policy); assert_eq!(user, "1000660000"); assert_eq!(home, "/sandbox"); } @@ -2310,7 +1909,8 @@ mod tests { ) .expect("prepare should succeed in test environment"), ), - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2365,7 +1965,8 @@ mod tests { ProcessEnforcementMode::Full, #[cfg(target_os = "linux")] None, - ); + ) + .expect("install pre_exec should succeed"); let output = cmd .spawn() @@ -2378,246 +1979,4 @@ mod tests { "resolved-identity-ok" ); } - - // ----------------------------------------------------------------------- - // direct-tcpip authorization wiring (SEC-007) - // - // The `loopback_host_*` tests above cover the predicate in isolation. - // These drive the real `russh::server::Handler` over an in-memory duplex - // so the deny path itself is covered: channel-open authorization travels - // through a reply handle rather than the handler's return value, so a - // handler that never rejects anything still type-checks and still passes - // every predicate test. - // ----------------------------------------------------------------------- - - struct AcceptAnyServerKey; - - impl russh::client::Handler for AcceptAnyServerKey { - type Error = russh::Error; - - async fn check_server_key( - &mut self, - _server_public_key: &russh::keys::PublicKey, - ) -> Result { - Ok(true) - } - } - - fn forwarding_test_policy() -> SandboxPolicy { - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, - }; - - SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, - }, - } - } - - /// Serve `SshHandler` on one end of an in-memory duplex and return an - /// authenticated client handle for the other end. - /// - /// The handler gets `netns_fd: None` so `connect_in_netns` performs a plain - /// TCP connect, making the forwarding path reachable without a network - /// namespace. - async fn authenticated_test_client_with_main( - main_session: Arc, - ) -> russh::client::Handle { - // Scoped so the `!Send` ThreadRng is dropped before the first await. - let host_key = { - let mut rng = rand::rng(); - PrivateKey::random(&mut rng, Algorithm::Ed25519).expect("host key") - }; - let mut server_config = russh::server::Config { - auth_rejection_time: Duration::from_millis(1), - ..Default::default() - }; - server_config.keys.push(host_key); - - let handler = SshHandler::new( - forwarding_test_policy(), - ResolvedWorkspace::default(), - None, - None, - None, - ProviderCredentialState::from_child_env_snapshot(0, HashMap::new()), - HashMap::new(), - ResolvedProcessIdentity::default(), - ProcessEnforcementMode::NetworkOnly, - main_session, - ); - - let (server_stream, client_stream) = tokio::io::duplex(64 * 1024); - tokio::spawn(async move { - if let Ok(session) = - russh::server::run_stream(Arc::new(server_config), server_stream, handler).await - { - let _ = session.await; - } - }); - - let mut client = russh::client::connect_stream( - Arc::new(russh::client::Config::default()), - client_stream, - AcceptAnyServerKey, - ) - .await - .expect("SSH handshake should complete over the duplex"); - - let auth = client - .authenticate_none("sandbox") - .await - .expect("auth_none should not error"); - assert!( - matches!(auth, russh::client::AuthResult::Success), - "sandbox SSH server accepts the none auth method" - ); - - client - } - - async fn authenticated_test_client() -> russh::client::Handle { - authenticated_test_client_with_main(MainSession::inert()).await - } - - #[tokio::test] - async fn abrupt_transport_drop_releases_main_input_lease() { - let main_session = MainSession::inert(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - match main_session.acquire_input() { - Err(_) => break, - Ok((owner, _)) => main_session.release_input(owner), - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should acquire canonical input lease"); - - drop(channel); - drop(client); - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.acquire_input().is_ok() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("handler drop should release canonical input lease"); - } - - #[tokio::test] - async fn main_subsystem_applies_initial_pty_dimensions() { - let (main_session, _slave) = MainSession::terminal_for_test(); - let client = authenticated_test_client_with_main(Arc::clone(&main_session)).await; - let channel = client.channel_open_session().await.expect("open session"); - channel - .request_pty(true, "xterm-256color", 200, 60, 1600, 900, &[]) - .await - .expect("request PTY"); - channel - .request_subsystem(true, "openshell-main") - .await - .expect("attach main subsystem"); - - tokio::time::timeout(Duration::from_secs(1), async { - loop { - if main_session.terminal_size_for_test() == (200, 60) { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .expect("main subsystem should apply the initial PTY dimensions"); - } - - #[tokio::test] - async fn direct_tcpip_rejects_non_loopback_destination() { - let client = authenticated_test_client().await; - - let err = client - .channel_open_direct_tcpip("10.0.0.1", 80, "127.0.0.1", 0) - .await - .expect_err("forwarding to a non-loopback host must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_rejects_port_above_tcp_range() { - let client = authenticated_test_client().await; - - // 65_537 truncates to port 1 when cast to u16, so the guard has to - // reject it before the cast rather than forward to a privileged port. - let err = client - .channel_open_direct_tcpip("127.0.0.1", 65_537, "127.0.0.1", 0) - .await - .expect_err("a port outside the TCP range must be refused"); - - assert!( - matches!( - err, - russh::Error::ChannelOpenFailure(ChannelOpenFailure::AdministrativelyProhibited) - ), - "expected AdministrativelyProhibited, got {err:?}" - ); - } - - #[tokio::test] - async fn direct_tcpip_forwards_to_loopback_listener() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind loopback echo listener"); - let port = listener.local_addr().expect("listener address").port(); - tokio::spawn(async move { - if let Ok((mut socket, _)) = listener.accept().await { - let mut buf = [0u8; 64]; - if let Ok(n) = socket.read(&mut buf).await - && n > 0 - { - let _ = socket.write_all(&buf[..n]).await; - } - } - }); - - let client = authenticated_test_client().await; - let channel = client - .channel_open_direct_tcpip("127.0.0.1", u32::from(port), "127.0.0.1", 0) - .await - .expect("forwarding to a loopback listener must be allowed"); - - let mut stream = channel.into_stream(); - stream.write_all(b"ping").await.expect("write to channel"); - - let mut echoed = [0u8; 4]; - tokio::time::timeout(Duration::from_secs(10), stream.read_exact(&mut echoed)) - .await - .expect("relayed response should arrive before the timeout") - .expect("read from channel"); - assert_eq!(&echoed, b"ping", "bytes round-trip through the tunnel"); - } } diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index e8a140e483..aae7db9630 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -11,18 +11,17 @@ //! selection — it has no protocol awareness of the bytes flowing through. use std::net::IpAddr; -#[cfg(target_os = "linux")] -use std::os::fd::RawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use openshell_core::proto::open_shell_client::OpenShellClient; use openshell_core::proto::{ - GatewayMessage, RelayFrame, RelayInit, RelayOpen, RelayOpenResult, - ReportMainProcessExitRequest, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, - TcpRelayTarget, gateway_message, relay_open, supervisor_message, + GatewayMessage, RelayFrame, RelayInit, RelayOpen, RelayOpenResult, SupervisorHeartbeat, + SupervisorHello, SupervisorMessage, TcpRelayTarget, gateway_message, relay_open, + supervisor_message, }; +use openshell_isolation::contract::{BoundaryPortForward, LoopbackTarget}; use openshell_ocsf::{ ActivityId, ConnectionInfo, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, ocsf_emit, @@ -33,7 +32,6 @@ use tokio_stream::StreamExt; use tracing::{debug, warn}; use openshell_core::grpc_client; -use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::transport_errors::is_expected_transport_close_status; const INITIAL_BACKOFF: Duration = Duration::from_secs(1); @@ -278,19 +276,17 @@ pub fn spawn( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, - instance_id: String, ) -> tokio::task::JoinHandle<()> { tokio::spawn(run_session_loop( endpoint, sandbox_id, ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, terminating, - instance_id, )) } @@ -298,10 +294,9 @@ async fn run_session_loop( endpoint: String, sandbox_id: String, ssh_socket_path: std::path::PathBuf, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, - instance_id: String, ) { let mut backoff = INITIAL_BACKOFF; let mut attempt: u64 = 0; @@ -313,10 +308,9 @@ async fn run_session_loop( &endpoint, &sandbox_id, &ssh_socket_path, - netns_fd, + port_forward.clone(), expected_ssh_peer_pid, Arc::clone(&terminating), - &instance_id, ) .await { @@ -345,10 +339,9 @@ async fn run_single_session( endpoint: &str, sandbox_id: &str, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, terminating: Arc, - instance_id: &str, ) -> Result<(), Box> { // Connect to the gateway. The same `Channel` is used for both the // long-lived control stream and all data-plane `RelayStream` calls, so @@ -364,10 +357,11 @@ async fn run_single_session( let outbound = tokio_stream::wrappers::ReceiverStream::new(rx); // Send hello as the first message. + let instance_id = uuid::Uuid::new_v4().to_string(); tx.send(SupervisorMessage { payload: Some(supervisor_message::Payload::Hello(SupervisorHello { sandbox_id: sandbox_id.to_string(), - instance_id: instance_id.to_string(), + instance_id: instance_id.clone(), })), }) .await @@ -422,7 +416,7 @@ async fn run_single_session( let context = GatewayMessageContext { sandbox_id, ssh_socket_path, - netns_fd, + port_forward: &port_forward, expected_ssh_peer_pid, channel: &channel, tx: &tx, @@ -447,31 +441,10 @@ async fn run_single_session( } } -/// Report the canonical process result and wait for durable handling. -pub async fn report_main_process_exit( - endpoint: &str, - sandbox_id: &str, - instance_id: &str, - exit_code: i32, -) -> Result<(), Box> { - let channel = grpc_client::connect_channel_pub(endpoint) - .await - .map_err(|error| format!("connect failed: {error}"))?; - let mut client = OpenShellClient::new(channel); - client - .report_main_process_exit(ReportMainProcessExitRequest { - sandbox_id: sandbox_id.to_string(), - instance_id: instance_id.to_string(), - exit_code, - }) - .await?; - Ok(()) -} - struct GatewayMessageContext<'a> { sandbox_id: &'a str, ssh_socket_path: &'a std::path::Path, - netns_fd: Option, + port_forward: &'a Arc, expected_ssh_peer_pid: Option, channel: &'a grpc_client::AuthedChannel, tx: &'a mpsc::Sender, @@ -490,7 +463,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< let channel = context.channel.clone(); let ssh_socket_path = context.ssh_socket_path.to_path_buf(); let tx = context.tx.clone(); - let netns_fd = context.netns_fd; + let port_forward = context.port_forward.clone(); let expected_ssh_peer_pid = context.expected_ssh_peer_pid; let terminating = Arc::clone(context.terminating); @@ -502,7 +475,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< match handle_relay_open( relay_open, &ssh_socket_path, - netns_fd, + port_forward, expected_ssh_peer_pid, channel, tx, @@ -559,7 +532,7 @@ fn handle_gateway_message(msg: &GatewayMessage, context: &GatewayMessageContext< async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: Arc, expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, @@ -569,7 +542,7 @@ async fn handle_relay_open( let target = match open_target( &relay_open, ssh_socket_path, - netns_fd, + &port_forward, expected_ssh_peer_pid, ) .await @@ -714,11 +687,11 @@ async fn send_relay_open_result( async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, - netns_fd: Option, + port_forward: &Arc, expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { - Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, + Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, port_forward).await, Some(relay_open::Target::Ssh(_)) | None => { let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; @@ -739,59 +712,27 @@ async fn open_target( async fn open_tcp_target( target: &TcpRelayTarget, - netns_fd: Option, + port_forward: &Arc, ) -> Result, Box> { let host = normalize_tcp_target_host(target)?; let port = u16::try_from(target.port).map_err(|_| "tcp target port must fit in u16")?; - let stream = connect_tcp_target(host, port, netns_fd).await?; + // `normalize_tcp_target_host` returns a loopback IP string; parse it and let + // `LoopbackTarget::new` re-validate before connecting. + let ip: IpAddr = host + .parse() + .map_err(|_| "tcp target host must be a loopback IP")?; + let target = LoopbackTarget::new(ip, port) + .map_err(|e| -> Box { e.to_string().into() })?; + // Connect inside the boundary through the injected port-forward interface + // (RFC 0012). In-pod this enters the workload netns, a delegated backend + // tunnels into its guest. + let stream = port_forward + .connect(target) + .await + .map_err(|e| -> Box { e.to_string().into() })?; Ok(Box::new(stream)) } -#[cfg(target_os = "linux")] -async fn connect_tcp_target( - host: String, - port: u16, - netns_fd: Option, -) -> Result> { - if let Some(fd) = netns_fd { - let (tx, rx) = tokio::sync::oneshot::channel(); - std::thread::spawn(move || { - let result = (|| -> std::io::Result { - #[allow(unsafe_code)] - let rc = unsafe { libc::setns(fd, libc::CLONE_NEWNET) }; - if rc != 0 { - return Err(std::io::Error::last_os_error()); - } - std::net::TcpStream::connect((host.as_str(), port)) - })(); - let _ = tx.send(result); - }); - - let stream = rx - .await - .map_err(|_| "netns tcp connect thread panicked")??; - stream.set_nonblocking(true)?; - let stream = tokio::net::TcpStream::from_std(stream)?; - set_tcp_nodelay_best_effort(&stream); - return Ok(stream); - } - - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - -#[cfg(not(target_os = "linux"))] -async fn connect_tcp_target( - host: String, - port: u16, - _netns_fd: Option, -) -> Result> { - let stream = tokio::net::TcpStream::connect((host.as_str(), port)).await?; - set_tcp_nodelay_best_effort(&stream); - Ok(stream) -} - #[cfg(test)] fn validate_tcp_target(target: &TcpRelayTarget) -> Result<(), String> { normalize_tcp_target_host(target).map(|_| ()) @@ -831,20 +772,6 @@ mod target_tests { } } - /// Regression test: the TCP relay connect path sets `TCP_NODELAY`. - #[tokio::test] - async fn connect_tcp_target_sets_tcp_nodelay() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind listener"); - let addr = listener.local_addr().expect("local addr"); - - let stream = connect_tcp_target(addr.ip().to_string(), addr.port(), None) - .await - .expect("connect"); - assert!(stream.nodelay().expect("query TCP_NODELAY")); - } - #[test] fn tcp_target_allows_loopback_hosts() { validate_tcp_target(&tcp("127.0.0.1", 8080)).expect("ipv4 loopback"); @@ -1127,7 +1054,12 @@ mod ocsf_event_tests { }); let relay = ssh_relay_open("peer-check"); - let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + // The SSH relay path does not use the port-forward (that is the TCP + // target path); connect from the supervisor's own namespace. + let port_forward: Arc = + Arc::new(crate::boundary_io::NetnsPortForward::new(None, None)); + + let trusted = open_target(&relay, &socket, &port_forward, Some(std::process::id())) .await .expect("matching peer PID should be accepted"); drop(trusted); @@ -1135,7 +1067,7 @@ mod ocsf_event_tests { let Err(err) = open_target( &relay, &socket, - None, + &port_forward, Some(std::process::id().saturating_add(1)), ) .await diff --git a/deploy/docker/Dockerfile.driver-vm-macos b/deploy/docker/Dockerfile.driver-vm-macos index 58317a52d8..1ab0a45a6f 100644 --- a/deploy/docker/Dockerfile.driver-vm-macos +++ b/deploy/docker/Dockerfile.driver-vm-macos @@ -3,14 +3,14 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Cross-compile the openshell-driver-vm binary for macOS aarch64 (Apple -# Silicon) using the osxcross toolchain. +# Cross-compile the openshell-driver-vm binary and its native host supervisor +# for macOS aarch64 (Apple Silicon) using the osxcross toolchain. # # openshell-driver-vm loads libkrun/libkrunfw at runtime via dlopen, so it # does NOT need Hypervisor.framework headers at build time. Pre-compressed -# runtime artifacts (libkrun, libkrunfw, gvproxy, bundled supervisor) are injected via -# the vm-runtime-compressed build context and embedded into the binary via -# include_bytes!(). +# runtime artifacts (libkrun, libkrunfw, gvproxy, guest process leaf) are +# injected via the vm-runtime-compressed build context and embedded into the +# driver via include_bytes!(). # # Usage: # docker buildx build -f deploy/docker/Dockerfile.driver-vm-macos \ @@ -56,52 +56,18 @@ ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_AR=aarch64-apple-darwin25.1-ar # aws-lc-sys workaround (in case it ends up in the dep tree via feature unification) RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld -# --------------------------------------------------------------------------- -# Stage 1: dependency caching — copy only manifests, create dummy sources, -# build dependencies. This layer is cached unless Cargo.toml/lock changes. -# --------------------------------------------------------------------------- +# The VM driver now consumes the shared VM isolation transport and the native +# host supervisor consumes the same contract. Copy the complete workspace so +# Cargo can resolve those transitive workspace crates without maintaining a +# second, release-only member list here. COPY Cargo.toml Cargo.lock ./ -COPY crates/openshell-driver-vm/Cargo.toml crates/openshell-driver-vm/Cargo.toml -COPY crates/openshell-driver-vm/build.rs crates/openshell-driver-vm/build.rs -COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml -COPY crates/openshell-core/build.rs crates/openshell-core/build.rs COPY proto/ proto/ - -# Scope workspace to the driver + its only internal dep. -RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml - -RUN mkdir -p crates/openshell-driver-vm/src \ - crates/openshell-core/src && \ - echo "fn main() {}" > crates/openshell-driver-vm/src/main.rs && \ - touch crates/openshell-driver-vm/src/lib.rs && \ - touch crates/openshell-core/src/lib.rs - -# Build deps only (cached layer). The 2>/dev/null || true is a warm-cache -# technique; real source is copied in stage 2. -RUN --mount=type=cache,id=cargo-registry-driver-vm-macos,sharing=locked,target=/root/.cargo/registry \ - --mount=type=cache,id=cargo-git-driver-vm-macos,sharing=locked,target=/root/.cargo/git \ - --mount=type=cache,id=cargo-target-driver-vm-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-driver-vm 2>/dev/null || true - -# --------------------------------------------------------------------------- -# Stage 2: real build with compressed runtime artifacts -# --------------------------------------------------------------------------- COPY crates/ crates/ # Copy compressed VM runtime artifacts for embedding. # These are passed in via --build-context vm-runtime-compressed=... COPY --from=vm-runtime-compressed / /build/vm-runtime-compressed/ -# Touch source files to ensure they're rebuilt (not the cached dummy). -RUN touch crates/openshell-driver-vm/src/main.rs \ - crates/openshell-driver-vm/src/lib.rs \ - crates/openshell-driver-vm/build.rs \ - crates/openshell-core/src/lib.rs \ - crates/openshell-core/build.rs \ - proto/*.proto - -# Declare version ARGs here (not earlier) so the git-hash-bearing values do not -# invalidate the expensive dependency-build layers above on every commit. ARG OPENSHELL_CARGO_VERSION ARG OPENSHELL_IMAGE_TAG RUN --mount=type=cache,id=cargo-registry-driver-vm-macos,sharing=locked,target=/root/.cargo/registry \ @@ -112,8 +78,11 @@ RUN --mount=type=cache,id=cargo-registry-driver-vm-macos,sharing=locked,target=/ fi && \ OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=/build/vm-runtime-compressed \ OPENSHELL_IMAGE_TAG="${OPENSHELL_IMAGE_TAG:-dev}" \ - cargo build --release --target aarch64-apple-darwin -p openshell-driver-vm && \ - cp target/aarch64-apple-darwin/release/openshell-driver-vm /openshell-driver-vm + cargo build --release --target aarch64-apple-darwin \ + -p openshell-driver-vm -p openshell-sandbox && \ + cp target/aarch64-apple-darwin/release/openshell-driver-vm /openshell-driver-vm && \ + cp target/aarch64-apple-darwin/release/openshell-sandbox /openshell-sandbox FROM scratch AS binary COPY --from=builder /openshell-driver-vm /openshell-driver-vm +COPY --from=builder /openshell-sandbox /openshell-sandbox diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c77c5c0aff..e83a28692b 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -6,9 +6,9 @@ # Supervisor image build. # # The final image carries the static `openshell-sandbox` binary used by Docker -# extraction, Podman image volumes, and the Kubernetes init container copy-self -# path. It also includes nftables so the Kubernetes supervisor sidecar can -# install pod-namespace egress enforcement rules. +# extraction, Podman image volumes, and Kubernetes side-loading. It also carries +# a materialized helper runtime used by the VM prototype for trusted network +# setup and enforcement. # # The Rust binary is built natively before this image build runs and staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox @@ -26,7 +26,23 @@ FROM alpine:3.22 AS supervisor ARG TARGETARCH -RUN apk add --no-cache nftables iptables iptables-legacy +RUN apk add --no-cache iproute2 nftables iptables iptables-legacy \ + && mkdir -p /openshell-runtime/usr /openshell-runtime/etc \ + && cp -aL /bin /sbin /lib /openshell-runtime/ \ + && cp -aL /usr/bin /usr/sbin /usr/lib /openshell-runtime/usr/ \ + && if [ -d /etc/iproute2 ]; then \ + cp -aL /etc/iproute2 /openshell-runtime/etc/; \ + fi \ + && if [ -d /usr/share/nftables ]; then \ + mkdir -p /openshell-runtime/usr/share; \ + cp -aL /usr/share/nftables /openshell-runtime/usr/share/; \ + fi \ + && loader="$(find /openshell-runtime/lib -maxdepth 1 -type f -name 'ld-musl-*.so.1' | head -n 1)" \ + && test -n "$loader" \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/sbin/ip -Version \ + && "$loader" --library-path /openshell-runtime/lib:/openshell-runtime/usr/lib \ + /openshell-runtime/usr/sbin/nft --version # --chmod=0555 restores execute bits after the actions/upload-artifact + # download-artifact roundtrip strips them. Ownership stays root (0:0) for diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d1fed9ae44..655851e156 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -737,7 +737,7 @@ health_check_interval_secs = 10 ### MicroVM -Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. Use this driver when you want stronger isolation than container namespaces alone. +Each sandbox runs inside its own libkrun microVM managed by the standalone `openshell-driver-vm` subprocess. The logical supervisor runs natively on the host and drives an authenticated RFC 0012 boundary transport to a portable process leaf in the guest. The guest receives no gateway token or mTLS private key. The existing custom kernel and driver-controlled guest helper runtime remain part of the VM bootstrap. Use this driver when you want stronger isolation than container namespaces alone. ```toml [openshell] @@ -754,22 +754,30 @@ state_dir = "/var/lib/openshell/vm" # Where the gateway looks for the openshell-driver-vm subprocess binary. driver_dir = "/usr/local/libexec/openshell" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -grpc_endpoint = "https://host.containers.internal:17670" +# Host-reachable endpoint used by the native supervisor. Loopback is valid. +grpc_endpoint = "https://127.0.0.1:17670" # Empty falls back to default_image. bootstrap_image = "ghcr.io/nvidia/openshell/sandbox:latest" krun_log_level = 1 vcpus = 2 mem_mib = 2048 overlay_disk_mib = 4096 -guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" -guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" -guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" +# Historical field names: these files are consumed by the host supervisor and +# are never copied into the guest. +guest_tls_ca = "/var/lib/openshell/supervisor-tls/ca.pem" +guest_tls_cert = "/var/lib/openshell/supervisor-tls/client.pem" +guest_tls_key = "/var/lib/openshell/supervisor-tls/client-key.pem" # Resolved sandbox UID/GID for the rootfs /etc/passwd entry. # Defaults to 10001 when unset; matching GID is used if sandbox_gid is empty. # Any non-root Linux UID/GID is valid. # sandbox_uid = 20001 ``` +`openshell-sandbox` must be installed beside `openshell-driver-vm` or selected +with `OPENSHELL_VM_SUPERVISOR_BIN`. Linux drivers can fall back to extracting +the same-target embedded guest binary for host execution. macOS packages ship a +native supervisor sibling because the embedded process leaf targets Linux. + ### Extension Driver Extension drivers run outside the gateway and expose the diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 1960f83588..ec757bdd37 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -5,15 +5,12 @@ # Run the Rust e2e smoke test against an openshell-gateway running the # standalone VM compute driver (`openshell-driver-vm`). # -# Architecture (post supervisor-initiated relay, PR #867): -# * The gateway never dials the sandbox. Instead, the in-guest -# supervisor opens an outbound `ConnectSupervisor` gRPC stream to -# the gateway on startup and keeps it alive for the sandbox -# lifetime. SSH (`/connect/ssh`) and `ExecSandbox` traffic ride the -# same TCP+TLS+HTTP/2 connection as multiplexed HTTP/2 streams. -# * There is no host-side SSH port forward. gvproxy still provides -# guest egress so the supervisor can reach the gateway, but it no -# longer forwards any TCP port back to the guest. +# Architecture (RFC 0012 VM topology): +# * The logical openshell-sandbox supervisor runs on the host and opens +# `ConnectSupervisor` to the gateway. +# * An authenticated guest process leaf receives lifecycle, exec/PTY, +# loopback-forward, and mediated network streams over virtio-vsock. +# * Gateway credentials and the network-policy engine stay outside the VM. # * Readiness is authoritative on the gateway: a sandbox's phase # flips to `Ready` the moment `ConnectSupervisor` registers, and # back to `Provisioning` when the session drops. The VM driver @@ -24,9 +21,10 @@ # # What the script does: # 1. When no prebuilt VM driver is supplied, ensures the VM runtime -# (libkrun + gvproxy) and bundled supervisor are staged. -# 2. Builds `openshell-gateway`, `openshell-driver-vm`, and the -# `openshell` CLI with the embedded runtime as needed. When CI supplies +# (libkrun + gvproxy) and guest process leaf are staged. +# 2. Builds `openshell-gateway`, `openshell-sandbox`, +# `openshell-driver-vm`, and the `openshell` CLI with the embedded +# runtime as needed. When CI supplies # OPENSHELL_GATEWAY_BIN, OPENSHELL_VM_DRIVER_BIN, or OPENSHELL_BIN, the # matching prebuilt binary is reused instead of rebuilt. # 3. On macOS, codesigns the VM driver (libkrun needs the @@ -40,7 +38,7 @@ # # Prerequisites (handled automatically by this script for local VM-driver builds): # - `mise run vm:setup` — downloads / builds the libkrun runtime. -# - `mise run vm:supervisor` — builds the bundled sandbox supervisor. +# - `mise run vm:supervisor` — builds the portable guest process leaf. set -euo pipefail @@ -51,9 +49,13 @@ COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}" CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" +HOST_SUPERVISOR_BIN="${OPENSHELL_VM_SUPERVISOR_BIN:-${ROOT}/target/debug/openshell-sandbox}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" +BOOTSTRAP_IMAGE="${OPENSHELL_VM_BOOTSTRAP_IMAGE:-}" +IMAGE_CACHE_DIR="${OPENSHELL_E2E_VM_IMAGE_CACHE_DIR:-}" +ORIGINAL_ARGS=("$@") # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. # AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored @@ -66,18 +68,63 @@ STATE_DIR_ROOT="/tmp" # Smoke test timeouts. First boot extracts the embedded libkrun runtime # (~60-90MB of zstd per architecture) and prepares an ext4 root disk from the -# configured image. The guest then starts the sandbox supervisor directly; a cold -# microVM is typically ready within ~15s after image preparation. +# configured image. The guest then starts the process leaf while the logical +# supervisor stays on the host; a cold microVM is typically ready within ~15s +# after image preparation. GATEWAY_READY_TIMEOUT=60 SANDBOX_PROVISION_TIMEOUT=180 # ── Build prerequisites ────────────────────────────────────────────── +configure_bindgen_include() { + local gcc_include + command -v gcc >/dev/null 2>&1 || return 0 + gcc_include="$(gcc -print-file-name=include)" + [ -f "${gcc_include}/stdbool.h" ] || return 0 + case " ${BINDGEN_EXTRA_CLANG_ARGS:-} " in + *" -isystem ${gcc_include} "*) ;; + *) + export BINDGEN_EXTRA_CLANG_ARGS="${BINDGEN_EXTRA_CLANG_ARGS:+${BINDGEN_EXTRA_CLANG_ARGS} }-isystem ${gcc_include}" + ;; + esac +} + +ensure_kvm_access() { + [ "$(uname -s)" = "Linux" ] || return 0 + [ -e /dev/kvm ] || { + echo "ERROR: /dev/kvm does not exist; enable KVM on this host" >&2 + exit 1 + } + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + return 0 + fi + + local kvm_group command arg + kvm_group="$(stat -c %G /dev/kvm)" + if [ "${OPENSHELL_E2E_VM_KVM_REEXEC:-0}" != "1" ] \ + && command -v sg >/dev/null 2>&1 \ + && [[ " $(id -nG "$(id -un)") " == *" ${kvm_group} "* ]]; then + echo "==> Entering the configured ${kvm_group} group for VM e2e" + export OPENSHELL_E2E_VM_KVM_REEXEC=1 + printf -v command 'exec %q' "${ROOT}/e2e/rust/e2e-vm.sh" + for arg in "${ORIGINAL_ARGS[@]}"; do + printf -v command '%s %q' "${command}" "${arg}" + done + exec sg "${kvm_group}" -c "${command}" + fi + + echo "ERROR: /dev/kvm is not readable and writable; add $(id -un) to ${kvm_group} and start a new login session" >&2 + exit 1 +} + if [ -n "${RUSTC_WRAPPER:-}" ] && [ "${OPENSHELL_E2E_VM_ALLOW_RUSTC_WRAPPER:-0}" != "1" ]; then echo "==> Building without RUSTC_WRAPPER=${RUSTC_WRAPPER} (set OPENSHELL_E2E_VM_ALLOW_RUSTC_WRAPPER=1 to keep it)" unset RUSTC_WRAPPER fi +configure_bindgen_include +ensure_kvm_access + if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then mkdir -p "${COMPRESSED_DIR}" @@ -86,8 +133,9 @@ if [ -z "${OPENSHELL_VM_DRIVER_BIN:-}" ]; then mise run vm:setup fi - if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then - echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ] \ + || [ ! -f "${COMPRESSED_DIR}/openshell-runtime.tar.zst" ]; then + echo "==> Building portable VM guest process leaf (mise run vm:supervisor)" mise run vm:supervisor fi @@ -97,6 +145,11 @@ else fi build_packages=() +if [ -z "${OPENSHELL_VM_SUPERVISOR_BIN:-}" ]; then + build_packages+=(-p openshell-sandbox) +else + echo "==> Using prebuilt host supervisor at ${HOST_SUPERVISOR_BIN}" +fi if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then echo "==> Building driver-free openshell-gateway" @@ -128,7 +181,8 @@ fi for pair in \ "openshell-gateway:${GATEWAY_BIN}" \ "openshell-driver-vm:${DRIVER_BIN}" \ - "openshell CLI:${CLI_BIN}"; do + "openshell CLI:${CLI_BIN}" \ + "host supervisor:${HOST_SUPERVISOR_BIN}"; do label="${pair%%:*}" path="${pair#*:}" if [ ! -x "${path}" ]; then @@ -137,6 +191,7 @@ for pair in \ fi done export OPENSHELL_BIN="${CLI_BIN}" +export OPENSHELL_VM_SUPERVISOR_BIN="${HOST_SUPERVISOR_BIN}" DRIVER_DIR="$(dirname "${DRIVER_BIN}")" if [ "$(uname -s)" = "Darwin" ]; then @@ -162,6 +217,11 @@ s.close()')" # basename short — see the SUN_LEN comment above. RUN_STATE_DIR="${STATE_DIR_ROOT}/os-vm-e2e-${HOST_PORT}-$$" mkdir -p "${RUN_STATE_DIR}" +if [ -n "${IMAGE_CACHE_DIR}" ]; then + mkdir -p "${IMAGE_CACHE_DIR}" + ln -s "${IMAGE_CACHE_DIR}" "${RUN_STATE_DIR}/images" + echo "==> Reusing VM image cache at ${IMAGE_CACHE_DIR}" +fi export XDG_CONFIG_HOME="${RUN_STATE_DIR}/config" export XDG_DATA_HOME="${RUN_STATE_DIR}/data" @@ -243,12 +303,7 @@ echo "==> Starting openshell-gateway on 127.0.0.1:${HOST_PORT} (state: ${RUN_STA # `~/.local/libexec/openshell/openshell-driver-vm` when present, # which silently shadows development builds — a subtle source of # stale-binary bugs in e2e runs. -# `grpc_endpoint` is the URL the VM driver passes into each guest as -# OPENSHELL_ENDPOINT. The supervisor inside the VM dials this address. -# Use `host.openshell.internal` rather than `127.0.0.1` so gvproxy's -# host-loopback proxy carries the connection while keeping the endpoint aligned -# with package-managed gateway certificates. gvproxy's bare gateway IP -# (192.168.127.1) does NOT forward arbitrary host ports. +# `grpc_endpoint` is consumed by the host supervisor, so use host loopback. e2e_generate_gateway_jwt "${JWT_DIR}" e2e_generate_pki "${GATEWAY_BIN}" "${PKI_DIR}" @@ -284,11 +339,14 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then else cat >>"${GATEWAY_CONFIG}" </dev/null 2>&1 || return 0 + gcc_include="$(gcc -print-file-name=include)" + [ -f "${gcc_include}/stdbool.h" ] || return 0 + case " ${BINDGEN_EXTRA_CLANG_ARGS:-} " in + *" -isystem ${gcc_include} "*) ;; + *) + export BINDGEN_EXTRA_CLANG_ARGS="${BINDGEN_EXTRA_CLANG_ARGS:+${BINDGEN_EXTRA_CLANG_ARGS} }-isystem ${gcc_include}" + ;; + esac +} + +ensure_kvm_access() { + [ "$(uname -s)" = "Linux" ] || return 0 + [ -e /dev/kvm ] || { + echo "ERROR: /dev/kvm does not exist; enable KVM on this host" >&2 + exit 1 + } + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + return 0 + fi + + local kvm_group command arg + kvm_group="$(stat -c %G /dev/kvm)" + if [ "${OPENSHELL_VM_KVM_REEXEC:-0}" != "1" ] \ + && command -v sg >/dev/null 2>&1 \ + && [[ " $(id -nG "$(id -un)") " == *" ${kvm_group} "* ]]; then + echo "==> Entering the configured ${kvm_group} group for the VM gateway" + export OPENSHELL_VM_KVM_REEXEC=1 + printf -v command 'exec %q' "${ROOT}/tasks/scripts/gateway-vm.sh" + for arg in "${ORIGINAL_ARGS[@]}"; do + printf -v command '%s %q' "${command}" "${arg}" + done + exec sg "${kvm_group}" -c "${command}" + fi + + echo "ERROR: /dev/kvm is not readable and writable; add $(id -un) to ${kvm_group} and start a new login session" >&2 + exit 1 +} + port_is_in_use() { local port=$1 if command -v lsof >/dev/null 2>&1; then @@ -194,7 +235,7 @@ check_supervisor_cross_toolchain() { fi local missing=0 if ! command -v cargo-zigbuild >/dev/null 2>&1; then - echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2 + echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest process leaf)." >&2 echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2 missing=1 fi @@ -210,6 +251,8 @@ check_supervisor_cross_toolchain() { VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")" +ensure_kvm_access + while [ "$#" -gt 0 ]; do case "$1" in --gpu) @@ -271,8 +314,9 @@ VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-v VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}" DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" +configure_bindgen_include -# Build prerequisites: VM runtime artifacts + bundled supervisor. +# Build prerequisites: VM runtime artifacts + portable guest process leaf. if [ ! -d "${COMPRESSED_DIR}" ] \ || ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q . \ || [ ! -f "${COMPRESSED_DIR}/gvproxy.zst" ] \ @@ -281,9 +325,10 @@ if [ ! -d "${COMPRESSED_DIR}" ] \ mise run vm:setup fi -if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then +if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ] \ + || [ ! -f "${COMPRESSED_DIR}/openshell-runtime.tar.zst" ]; then check_supervisor_cross_toolchain - echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + echo "==> Building portable VM guest process leaf (mise run vm:supervisor)" mise run vm:supervisor fi @@ -294,9 +339,9 @@ if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}") fi -echo "==> Building openshell-gateway and openshell-driver-vm" +echo "==> Building openshell-gateway, host supervisor, and openshell-driver-vm" cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-server -p openshell-driver-vm + -p openshell-server -p openshell-sandbox -p openshell-driver-vm if [ "$(uname -s)" = "Darwin" ]; then echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index dcf1c50119..fe7cc2fd93 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -283,6 +283,7 @@ def install resource("openshell-driver-vm").stage do libexec.install "openshell-driver-vm" + libexec.install "openshell-sandbox" end (libexec/"openshell-gateway-homebrew-service").write <<~SH diff --git a/tasks/scripts/vm/build-supervisor-bundle.sh b/tasks/scripts/vm/build-supervisor-bundle.sh index 0085c0619d..a31e999603 100755 --- a/tasks/scripts/vm/build-supervisor-bundle.sh +++ b/tasks/scripts/vm/build-supervisor-bundle.sh @@ -60,6 +60,7 @@ esac SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox" SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst" +SUPERVISOR_RUNTIME_OUTPUT="${OUTPUT_DIR}/openshell-runtime.tar.zst" echo "==> Building openshell-sandbox supervisor bundle" echo " Guest arch: ${GUEST_ARCH}" @@ -123,6 +124,51 @@ fi zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}" +case "${GUEST_ARCH}" in + aarch64|arm64) DOCKER_ARCH="arm64" ;; + x86_64|amd64) DOCKER_ARCH="amd64" ;; +esac + +echo "==> Building trusted supervisor helper runtime" +STAGED_SUPERVISOR="${ROOT}/deploy/docker/.build/prebuilt-binaries/${DOCKER_ARCH}/openshell-sandbox" +RUNTIME_IMAGE="openshell-vm-helper-runtime:${DOCKER_ARCH}-$$" +mkdir -p "$(dirname "${STAGED_SUPERVISOR}")" +cp "${SUPERVISOR_BIN}" "${STAGED_SUPERVISOR}" + +case "$(uname -m)" in + aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; + x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; + *) HOST_DOCKER_ARCH="" ;; +esac + +if [ "${HOST_DOCKER_ARCH}" = "${DOCKER_ARCH}" ]; then + docker build \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +else + docker buildx build \ + --load \ + --platform "linux/${DOCKER_ARCH}" \ + --build-arg "TARGETARCH=${DOCKER_ARCH}" \ + --file "${ROOT}/deploy/docker/Dockerfile.supervisor" \ + --tag "${RUNTIME_IMAGE}" \ + "${ROOT}" +fi + +RUNTIME_CONTAINER="$(docker create "${RUNTIME_IMAGE}")" +cleanup_runtime_image() { + docker rm -f "${RUNTIME_CONTAINER}" >/dev/null 2>&1 || true + docker image rm "${RUNTIME_IMAGE}" >/dev/null 2>&1 || true +} +trap cleanup_runtime_image EXIT +docker cp "${RUNTIME_CONTAINER}:/openshell-runtime" - \ + | zstd -19 -T0 -f -o "${SUPERVISOR_RUNTIME_OUTPUT}" +cleanup_runtime_image +trap - EXIT + echo "==> Bundled supervisor ready" echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)" echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)" +echo " Helper runtime: $(du -sh "${SUPERVISOR_RUNTIME_OUTPUT}" | cut -f1)"