diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index c379bb8742..18b83d93b5 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -184,6 +184,31 @@ For source checkout development, restart the local gateway with: mise run gateway:docker ``` +For the experimental host-supervised Firecracker path, use: + +```bash +mise run gateway:firecracker +``` + +This task requires Linux, read/write `/dev/kvm`, `debugfs`, and the Firecracker, +kernel, and ext4 fixtures documented under `e2e/firecracker/`. It starts an +operator-managed `firecracker` compute-driver socket and a plaintext gateway. +The driver log defaults below `/tmp/openshell-firecracker--/`. +The prototype creates no TAP device or guest NIC and needs neither `sudo` nor +`CAP_NET_ADMIN`. + +If an older checkout fails while compiling `z3-sys` or another bindgen consumer +with `fatal error: 'stdbool.h' file not found`, set the GCC architecture header +path before retrying: + +```bash +export BINDGEN_EXTRA_CLANG_ARGS="${BINDGEN_EXTRA_CLANG_ARGS:+${BINDGEN_EXTRA_CLANG_ARGS} }-isystem $(gcc -print-file-name=include)" +mise run gateway:firecracker +``` + +The current Firecracker gateway task applies this fallback automatically after +its optional `sg kvm` re-exec. + ### Step 5: Check Podman-Backed Gateways ```bash @@ -429,6 +454,14 @@ Use the VM driver logs and host diagnostics available in the user's environment. - Host virtualization support is enabled. - The sandbox supervisor can establish its callback connection to the gateway. +For Firecracker, also verify the configured external socket and driver log: + +```bash +rg -n 'firecracker|socket_path' .cache/gateway-firecracker/gateway.toml +stat /tmp/openshell-firecracker-*/compute-driver.sock +tail -n 200 /tmp/openshell-firecracker-*/driver.log +``` + Then run: ```bash diff --git a/AGENTS.md b/AGENTS.md index 487ffecade..a634e88041 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,6 +47,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers | | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers | | `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | +| `crates/openshell-driver-firecracker/` | Firecracker isolation driver | Experimental host-side RFC 0012 backend with a private guest process-supervisor leaf mode | | `crates/openshell-prover/` | Policy prover | Policy verification and proof generation | | `crates/openshell-server-macros/` | Server macros | Compile-time helpers for gateway RPC authorization | | `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution | diff --git a/Cargo.lock b/Cargo.lock index a7fccdc24b..8a7033507e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3920,6 +3920,31 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-driver-firecracker" +version = "0.0.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "clap", + "futures", + "libc", + "nix 0.29.0", + "openshell-core", + "openshell-isolation", + "openshell-supervisor-process", + "rand 0.9.4", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-subscriber", +] + [[package]] name = "openshell-driver-kubernetes" version = "0.0.0" @@ -4133,6 +4158,7 @@ dependencies = [ "miette", "nix 0.29.0", "openshell-core", + "openshell-driver-firecracker", "openshell-isolation", "openshell-ocsf", "openshell-policy", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 55d7e7a29d..46d9cdeb33 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -120,6 +120,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Firecracker | Experimental host-supervised microVM isolation. | Per-sandbox no-NIC Firecracker VM with the logical supervisor on the host. | Operator-managed endpoint driver started by `mise run gateway:firecracker`. The prototype clones a configured ext4 fixture, injects the private guest mode, and passes an authenticated RFC 0012 topology descriptor to the host supervisor. OCI image materialization and the full exec/forwarding surfaces remain follow-up work. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through @@ -158,6 +159,7 @@ Runtime-specific implementation notes belong in the driver crate README: - `crates/openshell-driver-podman/README.md` - `crates/openshell-driver-kubernetes/README.md` - `crates/openshell-driver-vm/README.md` +- `crates/openshell-driver-firecracker/README.md` The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 72814b1963..d9829cc706 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -70,6 +70,14 @@ by default. The co-located backend requires the supervisor to own the execution environment's PID namespace so boundary teardown can terminate every remaining workload process. +The experimental Firecracker backend exercises a delegated placement. The +logical supervisor stays on the host and sends the admitted policy and workload +spec over an authenticated, backend-private virtio-vsock channel only after the +RFC lifecycle reaches `start_agent`. A private guest mode of the Firecracker +driver invokes the existing process-supervisor implementation as the in-VM +leaf. The prototype attaches no guest NIC, so network access is blocked by the +VM topology while mediated egress remains unimplemented. + For proxy-mode boundaries, the co-located backend verifies its default-deny kernel egress ceiling before exposing any workload execution surface and then rechecks it every 250 milliseconds. Each check has a two-second deadline. diff --git a/crates/openshell-driver-firecracker/Cargo.toml b/crates/openshell-driver-firecracker/Cargo.toml new file mode 100644 index 0000000000..1a6e66aebc --- /dev/null +++ b/crates/openshell-driver-firecracker/Cargo.toml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-driver-firecracker" +description = "Experimental Firecracker isolation driver for OpenShell" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "openshell_driver_firecracker" +path = "src/lib.rs" + +[[bin]] +name = "openshell-driver-firecracker" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-isolation = { path = "../openshell-isolation" } +openshell-supervisor-process = { path = "../openshell-supervisor-process" } + +async-trait = "0.1" +base64 = { workspace = true } +clap = { workspace = true } +futures = { workspace = true } +libc = "0.2" +nix = { workspace = true } +rand = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true, features = ["net"] } +tonic = { workspace = true, features = ["transport"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/openshell-driver-firecracker/README.md b/crates/openshell-driver-firecracker/README.md new file mode 100644 index 0000000000..af6a5c40d7 --- /dev/null +++ b/crates/openshell-driver-firecracker/README.md @@ -0,0 +1,56 @@ +# OpenShell Firecracker driver + +`openshell-driver-firecracker` is an experimental host-side implementation of +the RFC 0012 Isolation Backend contract. It deliberately lives outside the +existing libkrun VM driver so the two runtimes can evolve independently. + +The main OpenShell supervisor remains on the host. A private mode of the same +driver binary listens on the guest's virtio-vsock device and invokes the current +`openshell-supervisor-process` implementation only after the host advances the +boundary through `attach -> confirm -> start_agent`. The admitted policy crosses +that authenticated channel with the workload spec. The private protocol is not +a second public supervisor or agent API. + +The prototype has no virtual NIC. This makes the network ceiling structurally +fail closed without TAP devices, nftables, `CAP_NET_ADMIN`, or `sudo`. Host +requirements are a Linux Firecracker binary and read/write access to `/dev/kvm`. + +Current scope: + +- boots an existing ext4 guest image with Firecracker; +- authenticates host-to-guest control over virtio-vsock; +- implements the RFC lifecycle and agent wait/signal operations; +- delegates guest process enforcement to the existing process supervisor leaf; +- serves the gateway compute-driver contract over a private Unix socket; +- provides an unprivileged KVM end-to-end smoke runner. + +Exec, PTY, port forwarding, mediated guest egress, and per-connection binary +identity are intentionally deferred. The contract surfaces fail closed for +those operations. Code that might later become a shared VM helper is duplicated +here until a second consumer establishes a small, stable abstraction. + +Run the smoke test with: + +```shell +mise run e2e:firecracker +``` + +Start a plaintext local gateway backed by the driver with: + +```shell +mise run gateway:firecracker +``` + +If the account is configured in the `kvm` group but the current process has +stale supplementary groups, the launcher re-enters that group with `sg`. It +does not use `sudo`. + +On Linux toolchains where bindgen does not discover GCC's architecture-specific +headers, the task derives the include directory from +`gcc -print-file-name=include` and appends it to `BINDGEN_EXTRA_CLANG_ARGS`. + +The gateway mode currently boots the configured rootfs fixture rather than +materializing the requested OCI image. Set `driver_config.command` to a string +array to override its default long-running shell workload. + +See `e2e/firecracker/README.md` for fixture overrides. diff --git a/crates/openshell-driver-firecracker/src/backend.rs b/crates/openshell-driver-firecracker/src/backend.rs new file mode 100644 index 0000000000..5f0231a293 --- /dev/null +++ b/crates/openshell-driver-firecracker/src/backend.rs @@ -0,0 +1,495 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Host-side RFC 0012 backend for an already-provisioned Firecracker VM. + +use std::fmt; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use openshell_isolation::AgentSpec; +use openshell_isolation::contract::{ + BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus, + BoundaryPortForward, BoundaryProcess, BoundarySignal, ExecSession, ExecSpec, INTERFACE_VERSION, + IsolationBackend, LoopbackTarget, MediatedConnection, NetworkMediationSource, ReadyBoundary, + RunningBoundary, SandboxContext, VerifiedTopologyDescriptor, +}; +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::UnixStream; + +use crate::protocol::{ + AgentSpecWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope, + SandboxPolicyWire, SignalWire, decode_frame, encode_frame, +}; + +pub const BACKEND_NAME: &str = "firecracker"; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32; +const MAX_VSOCK_ACK_BYTES: usize = 64; + +/// Backend-private provisioned topology payload. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FirecrackerTopology { + pub boundary_id: String, + pub vsock_uds_path: PathBuf, + pub control_port: u32, + pub bootstrap_token: String, +} + +impl fmt::Debug for FirecrackerTopology { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FirecrackerTopology") + .field("boundary_id", &self.boundary_id) + .field("vsock_uds_path", &self.vsock_uds_path) + .field("control_port", &self.control_port) + .field("bootstrap_token", &"") + .finish() + } +} + +impl FirecrackerTopology { + pub fn encode(&self) -> Result, BackendError> { + serde_json::to_vec(self) + .map_err(|error| BackendError::Descriptor(format!("encode topology: {error}"))) + } +} + +/// Host-side Firecracker implementation registered with the supervisor. +#[derive(Debug, Default)] +pub struct FirecrackerHostBackend; + +#[async_trait] +impl IsolationBackend for FirecrackerHostBackend { + fn backend_name(&self) -> &str { + BACKEND_NAME + } + + fn version(&self) -> u32 { + INTERFACE_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + let topology: FirecrackerTopology = serde_json::from_slice(descriptor.payload()) + .map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?; + validate_topology(&topology, &sandbox)?; + let client = Arc::new(GuestClient::new(topology)); + expect_response(client.call(Request::Attach).await?, "attached")?; + Ok(Box::new(FirecrackerBound { + client, + agent: sandbox.agent, + policy: sandbox.policy, + sandbox_id: sandbox.sandbox_id, + mediation: Arc::new(NoGuestNetwork), + })) + } +} + +fn validate_topology( + topology: &FirecrackerTopology, + sandbox: &SandboxContext, +) -> Result<(), BackendError> { + if topology.boundary_id != sandbox.sandbox_id { + return Err(BackendError::Descriptor(format!( + "Firecracker boundary {:?} does not match sandbox {:?}", + topology.boundary_id, sandbox.sandbox_id + ))); + } + if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES { + return Err(BackendError::Descriptor(format!( + "Firecracker bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes" + ))); + } + if !topology.vsock_uds_path.is_absolute() { + return Err(BackendError::Descriptor( + "Firecracker vsock UDS path must be absolute".to_string(), + )); + } + if topology.control_port == 0 { + return Err(BackendError::Descriptor( + "Firecracker guest control port must be nonzero".to_string(), + )); + } + Ok(()) +} + +struct FirecrackerBound { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, + mediation: Arc, +} + +#[async_trait] +impl BoundBoundary for FirecrackerBound { + fn network_mediation_source(&self) -> Arc { + self.mediation.clone() + } + + async fn confirm(self: Box) -> Result, BackendError> { + expect_response(self.client.call(Request::Confirm).await?, "confirmed")?; + Ok(Box::new(FirecrackerReady { + client: self.client, + agent: self.agent, + policy: self.policy, + sandbox_id: self.sandbox_id, + })) + } +} + +struct FirecrackerReady { + client: Arc, + agent: AgentSpec, + policy: openshell_core::policy::SandboxPolicy, + sandbox_id: String, +} + +#[async_trait] +impl ReadyBoundary for FirecrackerReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let response = self + .client + .call(Request::StartAgent { + sandbox_id: self.sandbox_id, + spec: AgentSpecWire::from(self.agent), + policy: Box::new(SandboxPolicyWire::from(self.policy)), + }) + .await?; + let Response::Started { process_id } = response else { + return Err(unexpected_response("started", &response)); + }; + let process = Arc::new(FirecrackerProcess { + client: self.client, + process_id, + }); + Ok(Box::new(FirecrackerRunning { + process, + exec: Arc::new(UnsupportedExec), + port_forward: Arc::new(UnsupportedPortForward), + })) + } +} + +struct FirecrackerRunning { + process: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for FirecrackerRunning { + fn agent(&self) -> Arc { + self.process.clone() + } + + fn exec(&self) -> Arc { + self.exec.clone() + } + + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +struct FirecrackerProcess { + client: Arc, + process_id: String, +} + +#[async_trait] +impl BoundaryProcess for FirecrackerProcess { + async fn wait(&self) -> Result { + let response = self + .client + .call_wait(Request::Wait { + process_id: self.process_id.clone(), + }) + .await?; + let Response::Exited { status } = response else { + return Err(unexpected_response("exited", &response)); + }; + Ok(status.into()) + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Signal { + process_id: self.process_id.clone(), + signal: SignalWire::from(signal), + }) + .await?; + expect_response(response, "signaled") + } + + async fn terminate(&self) -> Result<(), BackendError> { + let response = self + .client + .call(Request::Terminate { + process_id: self.process_id.clone(), + }) + .await?; + expect_response(response, "terminated") + } +} + +struct UnsupportedExec; + +#[async_trait] +impl BoundaryExec for UnsupportedExec { + async fn exec(&self, _spec: ExecSpec) -> Result { + Err(BackendError::Unavailable( + "Firecracker exec and PTY transport are not implemented in this prototype".to_string(), + )) + } +} + +struct UnsupportedPortForward; + +#[async_trait] +impl BoundaryPortForward for UnsupportedPortForward { + async fn connect(&self, _target: LoopbackTarget) -> Result { + Err(BackendError::Unavailable( + "Firecracker loopback forwarding is not implemented in this prototype".to_string(), + )) + } +} + +/// The prototype provisions no guest NIC. There can be no workload egress to +/// accept, and attempting to consume the source fails the boundary closed. +struct NoGuestNetwork; + +#[async_trait] +impl NetworkMediationSource for NoGuestNetwork { + async fn accept(&self) -> Result { + // No NIC means there is no connection source to drain. Keeping the + // accept future pending lets the host network supervisor remain alive + // without treating the structurally closed boundary as a transport + // failure. + std::future::pending().await + } +} + +struct GuestClient { + topology: FirecrackerTopology, + next_request_id: AtomicU64, +} + +impl GuestClient { + fn new(topology: FirecrackerTopology) -> Self { + Self { + topology, + next_request_id: AtomicU64::new(1), + } + } + + async fn call(&self, request: Request) -> Result { + tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request)) + .await + .map_err(|_| BackendError::Unavailable("guest control request timed out".to_string()))? + } + + async fn call_wait(&self, request: Request) -> Result { + self.exchange(request).await + } + + async fn exchange(&self, request: Request) -> Result { + let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed); + let envelope = RequestEnvelope { + request_id, + boundary_id: self.topology.boundary_id.clone(), + bootstrap_token: self.topology.bootstrap_token.clone(), + request, + }; + let mut stream = self.connect_vsock().await?; + let frame = encode_frame(&envelope) + .map_err(|error| BackendError::Process(format!("encode control request: {error}")))?; + stream.write_all(&frame).await.map_err(|error| { + BackendError::Unavailable(format!("write guest control request: {error}")) + })?; + let mut header = [0_u8; 4]; + stream.read_exact(&mut header).await.map_err(|error| { + BackendError::Unavailable(format!("read guest control response header: {error}")) + })?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(BackendError::Process(format!( + "guest control response is too large: {declared} bytes" + ))); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + stream.read_exact(&mut frame[4..]).await.map_err(|error| { + BackendError::Unavailable(format!("read guest control response: {error}")) + })?; + let response: ResponseEnvelope = decode_frame(&frame) + .map_err(|error| BackendError::Process(format!("decode control response: {error}")))?; + if response.request_id != request_id { + return Err(BackendError::Process(format!( + "guest response ID {} did not match request ID {request_id}", + response.request_id + ))); + } + match response.response { + Response::Error { kind, message } => Err(guest_error(&kind, message)), + response => Ok(response), + } + } + + async fn connect_vsock(&self) -> Result { + let mut stream = UnixStream::connect(&self.topology.vsock_uds_path) + .await + .map_err(|error| { + BackendError::Unavailable(format!("connect to Firecracker vsock UDS: {error}")) + })?; + stream + .write_all(format!("CONNECT {}\n", self.topology.control_port).as_bytes()) + .await + .map_err(|error| { + BackendError::Unavailable(format!("write Firecracker vsock handshake: {error}")) + })?; + let acknowledgment = read_vsock_acknowledgment(&mut stream).await?; + validate_vsock_acknowledgment(&acknowledgment)?; + Ok(stream) + } +} + +async fn read_vsock_acknowledgment(stream: &mut UnixStream) -> Result, BackendError> { + let mut acknowledgment = Vec::with_capacity(24); + loop { + if acknowledgment.len() == MAX_VSOCK_ACK_BYTES { + return Err(BackendError::Process( + "Firecracker vsock acknowledgment is too long".to_string(), + )); + } + let byte = stream.read_u8().await.map_err(|error| { + BackendError::Unavailable(format!("read Firecracker vsock acknowledgment: {error}")) + })?; + acknowledgment.push(byte); + if byte == b'\n' { + return Ok(acknowledgment); + } + } +} + +fn validate_vsock_acknowledgment(acknowledgment: &[u8]) -> Result<(), BackendError> { + let acknowledgment = std::str::from_utf8(acknowledgment).map_err(|_| { + BackendError::Process("Firecracker vsock acknowledgment is not UTF-8".to_string()) + })?; + let mut fields = acknowledgment + .trim_end_matches('\n') + .split_ascii_whitespace(); + let status = fields.next(); + let assigned_port = fields.next().and_then(|port| port.parse::().ok()); + if status != Some("OK") || assigned_port.is_none() || fields.next().is_some() { + return Err(BackendError::Process(format!( + "invalid Firecracker vsock acknowledgment: {acknowledgment:?}" + ))); + } + Ok(()) +} + +fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> { + let matches = matches!( + (&response, expected), + (Response::Attached, "attached") + | (Response::Confirmed, "confirmed") + | (Response::Signaled, "signaled") + | (Response::Terminated, "terminated") + ); + if matches { + Ok(()) + } else { + Err(unexpected_response(expected, &response)) + } +} + +fn unexpected_response(expected: &str, response: &Response) -> BackendError { + BackendError::Process(format!( + "expected guest response {expected:?}, received {response:?}" + )) +} + +fn guest_error(kind: &str, message: String) -> BackendError { + let message = format!("Firecracker guest process leaf: {message}"); + match kind { + "invalid" => BackendError::Descriptor(message), + "denied" => BackendError::Denied(message), + "unavailable" => BackendError::Unavailable(message), + "terminated" => BackendError::Terminated(message), + _ => BackendError::Process(message), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + + use super::*; + + fn sandbox() -> SandboxContext { + SandboxContext { + sandbox_id: "sandbox-1".to_string(), + policy: SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }, + agent: AgentSpec { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + } + } + + #[test] + fn topology_debug_redacts_token() { + let topology = FirecrackerTopology { + boundary_id: "sandbox-1".to_string(), + vsock_uds_path: PathBuf::from("/tmp/vsock.sock"), + control_port: 5500, + bootstrap_token: "never-log-this-never-log-this".to_string(), + }; + let debug = format!("{topology:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn topology_must_match_sandbox() { + let topology = FirecrackerTopology { + boundary_id: "other".to_string(), + vsock_uds_path: PathBuf::from("/tmp/vsock.sock"), + control_port: 5500, + bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(), + }; + assert!(matches!( + validate_topology(&topology, &sandbox()), + Err(BackendError::Descriptor(_)) + )); + } + + #[test] + fn validates_firecracker_acknowledgment() { + assert!(validate_vsock_acknowledgment(b"OK 1234\n").is_ok()); + assert!(validate_vsock_acknowledgment(b"ERR\n").is_err()); + } +} diff --git a/crates/openshell-driver-firecracker/src/compute.rs b/crates/openshell-driver-firecracker/src/compute.rs new file mode 100644 index 0000000000..9dd7ea0afc --- /dev/null +++ b/crates/openshell-driver-firecracker/src/compute.rs @@ -0,0 +1,599 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental gateway compute-driver adapter for the Firecracker backend. + +#![allow(unsafe_code)] + +use std::collections::HashMap; +use std::fmt::Write as _; +use std::fs::Permissions; +use std::os::unix::fs::PermissionsExt as _; +use std::os::unix::process::CommandExt as _; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::process::{Child, Command, Stdio}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; + +use base64::Engine as _; +use futures::Stream; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + DriverCondition, DriverSandbox, DriverSandboxStatus, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetGatewayListenerRequirementsRequest, + GetGatewayListenerRequirementsResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, watch_sandboxes_event, +}; +use openshell_core::proto_struct::struct_to_json_value; +use openshell_isolation::contract::INTERFACE_VERSION; +use serde::Deserialize; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; + +use crate::{FirecrackerLaunchConfig, FirecrackerTopology, FirecrackerVm, GuestConfig}; + +const WATCH_BUFFER: usize = 128; +const DEFAULT_AGENT_UID: u32 = 10_001; +const DEFAULT_AGENT_GID: u32 = 10_001; + +#[derive(Debug, Clone)] +pub struct FirecrackerComputeConfig { + pub gateway_endpoint: String, + pub state_dir: PathBuf, + pub firecracker_binary: PathBuf, + pub kernel_image: PathBuf, + pub root_disk: PathBuf, + pub supervisor_binary: PathBuf, + pub driver_binary: PathBuf, + pub default_image: String, + pub vcpus: u8, + pub mem_mib: u32, +} + +#[derive(Clone)] +pub struct FirecrackerComputeDriver { + config: Arc, + records: Arc>>, + events: broadcast::Sender, + next_cid: Arc, +} + +struct SandboxRecord { + snapshot: DriverSandbox, + vm: FirecrackerVm, + supervisor: Child, + run_dir: PathBuf, +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct FirecrackerSandboxConfig { + command: Option>, + workdir: Option, +} + +impl FirecrackerComputeDriver { + pub fn new(config: FirecrackerComputeConfig) -> Result { + for (label, path) in [ + ("Firecracker binary", &config.firecracker_binary), + ("kernel image", &config.kernel_image), + ("root disk", &config.root_disk), + ("sandbox supervisor", &config.supervisor_binary), + ("Firecracker driver", &config.driver_binary), + ] { + if !path.is_file() { + return Err(format!("{label} not found: {}", path.display())); + } + } + std::fs::create_dir_all(&config.state_dir) + .map_err(|error| format!("create state directory: {error}"))?; + std::fs::set_permissions(&config.state_dir, Permissions::from_mode(0o700)) + .map_err(|error| format!("restrict state directory: {error}"))?; + let (events, _) = broadcast::channel(WATCH_BUFFER); + Ok(Self { + config: Arc::new(config), + records: Arc::new(Mutex::new(HashMap::new())), + events, + next_cid: Arc::new(AtomicU32::new(3)), + }) + } + + fn validate(sandbox: &DriverSandbox) -> Result { + validate_id(&sandbox.id)?; + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox spec is required"))?; + if spec.sandbox_token.trim().is_empty() { + return Err(Status::failed_precondition( + "firecracker sandboxes require gateway JWT auth", + )); + } + if spec + .resource_requirements + .as_ref() + .and_then(|value| value.gpu.as_ref()) + .is_some() + { + return Err(Status::failed_precondition( + "the Firecracker prototype does not support GPUs", + )); + } + let template = spec + .template + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox template is required"))?; + if !template.agent_socket_path.is_empty() { + return Err(Status::failed_precondition( + "the Firecracker prototype does not support agent_socket_path", + )); + } + if template + .platform_config + .as_ref() + .is_some_and(|value| !value.fields.is_empty()) + { + return Err(Status::failed_precondition( + "the Firecracker prototype does not support platform_config", + )); + } + let driver = template.driver_config.as_ref().map_or_else( + || Ok(FirecrackerSandboxConfig::default()), + |value| { + serde_json::from_value(struct_to_json_value(value)).map_err(|error| { + Status::invalid_argument(format!("invalid firecracker driver_config: {error}")) + }) + }, + )?; + if driver.command.as_ref().is_some_and(Vec::is_empty) { + return Err(Status::invalid_argument( + "firecracker driver_config.command must not be empty", + )); + } + Ok(driver) + } + + async fn snapshots(&self) -> Vec { + let records = self.records.lock().await; + let mut snapshots = records + .values() + .map(|record| record.snapshot.clone()) + .collect::>(); + snapshots.sort_by(|left, right| left.id.cmp(&right.id)); + snapshots + } + + fn publish_snapshot(&self, snapshot: DriverSandbox) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(snapshot), + }, + )), + }); + } + + fn provision( + &self, + sandbox: &DriverSandbox, + driver: FirecrackerSandboxConfig, + ) -> Result { + let run_dir = self.config.state_dir.join(&sandbox.id); + std::fs::create_dir(&run_dir) + .map_err(|error| Status::internal(format!("create sandbox state: {error}")))?; + std::fs::set_permissions(&run_dir, Permissions::from_mode(0o700)) + .map_err(|error| Status::internal(format!("restrict sandbox state: {error}")))?; + + let root_disk = run_dir.join("root.ext4"); + run_checked( + "cp", + &[ + "--reflink=auto".as_ref(), + self.config.root_disk.as_os_str(), + root_disk.as_os_str(), + ], + )?; + let bootstrap_token = random_token(); + let guest_config_path = run_dir.join("firecracker.json"); + let guest_config = GuestConfig { + boundary_id: sandbox.id.clone(), + bootstrap_token: bootstrap_token.clone(), + control_port: 5500, + agent_uid: DEFAULT_AGENT_UID, + agent_gid: DEFAULT_AGENT_GID, + }; + let bytes = serde_json::to_vec(&guest_config) + .map_err(|error| Status::internal(format!("encode guest config: {error}")))?; + std::fs::write(&guest_config_path, bytes) + .map_err(|error| Status::internal(format!("write guest config: {error}")))?; + inject_guest_file( + &root_disk, + &guest_config_path, + "/etc/openshell/firecracker.json", + )?; + inject_guest_file( + &root_disk, + &self.config.driver_binary, + "/opt/openshell/bin/openshell-driver-firecracker", + )?; + run_debugfs( + &root_disk, + "set_inode_field /opt/openshell/bin/openshell-driver-firecracker mode 0100755", + )?; + + let console = run_dir.join("console.log"); + let cid = self.next_cid.fetch_add(1, Ordering::Relaxed); + let vm = FirecrackerVm::launch(&FirecrackerLaunchConfig { + firecracker_binary: self.config.firecracker_binary.clone(), + kernel_image: self.config.kernel_image.clone(), + root_disk, + run_dir: run_dir.clone(), + console_output: console, + guest_init: "/opt/openshell/bin/openshell-driver-firecracker".to_string(), + vcpus: self.config.vcpus, + mem_mib: self.config.mem_mib, + vsock_cid: cid, + }) + .map_err(Status::internal)?; + + let sandbox_token_path = run_dir.join("sandbox.jwt"); + let sandbox_token = &sandbox.spec.as_ref().expect("validated spec").sandbox_token; + std::fs::write(&sandbox_token_path, format!("{sandbox_token}\n")) + .map_err(|error| Status::internal(format!("write sandbox token: {error}")))?; + std::fs::set_permissions(&sandbox_token_path, Permissions::from_mode(0o600)) + .map_err(|error| Status::internal(format!("restrict sandbox token: {error}")))?; + let topology = FirecrackerTopology { + boundary_id: sandbox.id.clone(), + vsock_uds_path: vm.vsock_uds_path().to_path_buf(), + control_port: 5500, + bootstrap_token, + }; + let payload = topology + .encode() + .map_err(|error| Status::internal(error.to_string()))?; + let command = driver.command.unwrap_or_else(|| { + vec![ + "/bin/sh".to_string(), + "-lc".to_string(), + "while :; do sleep 3600; done".to_string(), + ] + }); + let mut supervisor_command = Command::new(&self.config.supervisor_binary); + supervisor_command + .arg(format!("--topology-backend-name={}", crate::BACKEND_NAME)) + .arg(format!("--topology-version={INTERFACE_VERSION}")) + .arg(format!( + "--topology-payload-base64={}", + base64::engine::general_purpose::STANDARD.encode(payload) + )) + .arg("--workdir") + .arg(driver.workdir.unwrap_or_else(|| "/sandbox".to_string())) + .arg("--") + .args(command) + .env( + openshell_core::sandbox_env::ENDPOINT, + &self.config.gateway_endpoint, + ) + .env(openshell_core::sandbox_env::SANDBOX_ID, &sandbox.id) + .env(openshell_core::sandbox_env::SANDBOX, &sandbox.name) + .env( + openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, + &sandbox_token_path, + ) + .env( + openshell_core::sandbox_env::SSH_SOCKET_PATH, + run_dir.join("ssh.sock"), + ) + .env( + openshell_core::sandbox_env::SANDBOX_UID, + DEFAULT_AGENT_UID.to_string(), + ) + .env( + openshell_core::sandbox_env::SANDBOX_GID, + DEFAULT_AGENT_GID.to_string(), + ) + .env(openshell_core::sandbox_env::OCI_IMAGE_USER, "") + .env( + openshell_core::sandbox_env::LOG_LEVEL, + sandbox + .spec + .as_ref() + .expect("validated spec") + .log_level + .clone(), + ) + .stdout(Stdio::from( + std::fs::File::create(run_dir.join("supervisor.log")) + .map_err(|error| Status::internal(error.to_string()))?, + )) + .stderr(Stdio::from( + std::fs::File::create(run_dir.join("supervisor.err.log")) + .map_err(|error| Status::internal(error.to_string()))?, + )); + unsafe { + supervisor_command.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(nix::sys::signal::Signal::SIGKILL) + .map_err(|error| std::io::Error::other(error.to_string())) + }); + } + let supervisor = supervisor_command + .spawn() + .map_err(|error| Status::internal(format!("start host supervisor: {error}")))?; + Ok(SandboxRecord { + snapshot: ready_snapshot(sandbox), + vm, + supervisor, + run_dir, + }) + } +} + +#[tonic::async_trait] +impl ComputeDriver for FirecrackerComputeDriver { + async fn get_capabilities( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(GetCapabilitiesResponse { + driver_name: crate::BACKEND_NAME.to_string(), + driver_version: openshell_core::VERSION.to_string(), + default_image: self.config.default_image.clone(), + })) + } + + async fn get_gateway_listener_requirements( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(GetGatewayListenerRequirementsResponse { + requirements: Vec::new(), + })) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + Self::validate(&sandbox)?; + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + let driver = Self::validate(&sandbox)?; + let mut records = self.records.lock().await; + if records.contains_key(&sandbox.id) { + return Err(Status::already_exists("sandbox already exists")); + } + let record = self.provision(&sandbox, driver)?; + let snapshot = record.snapshot.clone(); + records.insert(sandbox.id.clone(), record); + drop(records); + self.publish_snapshot(snapshot); + Ok(Response::new(CreateSandboxResponse {})) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let records = self.records.lock().await; + let sandbox = records + .values() + .find(|record| { + (!request.sandbox_id.is_empty() && record.snapshot.id == request.sandbox_id) + || (!request.sandbox_name.is_empty() + && record.snapshot.name == request.sandbox_name) + }) + .map(|record| record.snapshot.clone()) + .ok_or_else(|| Status::not_found("sandbox not found"))?; + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _: Request, + ) -> Result, Status> { + Ok(Response::new(ListSandboxesResponse { + sandboxes: self.snapshots().await, + })) + } + + async fn stop_sandbox( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "stop is not implemented by the Firecracker prototype", + )) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let mut records = self.records.lock().await; + let id = if request.sandbox_id.is_empty() { + records + .values() + .find(|record| record.snapshot.name == request.sandbox_name) + .map(|record| record.snapshot.id.clone()) + .unwrap_or_default() + } else { + request.sandbox_id + }; + let Some(mut record) = records.remove(&id) else { + return Ok(Response::new(DeleteSandboxResponse { deleted: false })); + }; + let _ = record.supervisor.kill(); + let _ = record.supervisor.wait(); + let _ = record.vm.terminate(); + let run_dir = record.run_dir.clone(); + drop(record); + let _ = std::fs::remove_dir_all(run_dir); + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id: id }, + )), + }); + Ok(Response::new(DeleteSandboxResponse { deleted: true })) + } + + type WatchSandboxesStream = + Pin> + Send + 'static>>; + + async fn watch_sandboxes( + &self, + _: Request, + ) -> Result, Status> { + let initial = self.snapshots().await; + let mut events = self.events.subscribe(); + let (tx, rx) = mpsc::channel(WATCH_BUFFER); + tokio::spawn(async move { + for sandbox in initial { + if tx + .send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + })) + .await + .is_err() + { + return; + } + } + loop { + match events.recv().await { + Ok(event) => { + if tx.send(Ok(event)).await.is_err() { + return; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + Ok(Response::new(Box::pin(ReceiverStream::new(rx)))) + } +} + +fn ready_snapshot(sandbox: &DriverSandbox) -> DriverSandbox { + DriverSandbox { + id: sandbox.id.clone(), + name: sandbox.name.clone(), + namespace: sandbox.namespace.clone(), + spec: None, + status: Some(DriverSandboxStatus { + sandbox_name: sandbox.name.clone(), + instance_id: sandbox.id.clone(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![DriverCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "FirecrackerStarted".to_string(), + message: "Firecracker VM and host supervisor started".to_string(), + last_transition_time: String::new(), + }], + deleting: false, + }), + workspace: sandbox.workspace.clone(), + } +} + +fn validate_id(id: &str) -> Result<(), Status> { + if id.is_empty() + || id.len() > 128 + || matches!(id, "." | "..") + || !id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + { + return Err(Status::invalid_argument( + "sandbox id must match [A-Za-z0-9._-]{1,128}", + )); + } + Ok(()) +} + +fn random_token() -> String { + let mut token = String::with_capacity(64); + for byte in rand::random::<[u8; 32]>() { + write!(&mut token, "{byte:02x}").expect("writing to String cannot fail"); + } + token +} + +fn inject_guest_file(disk: &Path, source: &Path, destination: &str) -> Result<(), Status> { + run_debugfs(disk, &format!("rm {destination}")).ok(); + run_debugfs(disk, &format!("write {} {destination}", source.display())) +} + +fn run_debugfs(disk: &Path, operation: &str) -> Result<(), Status> { + run_checked( + "debugfs", + &[ + "-w".as_ref(), + "-R".as_ref(), + operation.as_ref(), + disk.as_os_str(), + ], + ) +} + +fn run_checked(program: &str, args: &[&std::ffi::OsStr]) -> Result<(), Status> { + let output = Command::new(program) + .args(args) + .output() + .map_err(|error| Status::internal(format!("run {program}: {error}")))?; + if output.status.success() { + Ok(()) + } else { + Err(Status::internal(format!( + "{program} failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bootstrap_token_is_32_random_bytes_in_hex() { + let token = random_token(); + assert_eq!(token.len(), 64); + assert!(token.bytes().all(|byte| byte.is_ascii_hexdigit())); + } + + #[test] + fn sandbox_ids_are_safe_state_directory_components() { + assert!(validate_id("sandbox-123._ok").is_ok()); + assert!(validate_id("../escape").is_err()); + assert!(validate_id("").is_err()); + } +} diff --git a/crates/openshell-driver-firecracker/src/guest.rs b/crates/openshell-driver-firecracker/src/guest.rs new file mode 100644 index 0000000000..fdad645bc4 --- /dev/null +++ b/crates/openshell-driver-firecracker/src/guest.rs @@ -0,0 +1,601 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Private guest mode for the Firecracker driver. +//! +//! This is transport and lifecycle glue, not another supervisor model. When +//! the host authorizes `start_agent`, it invokes the existing +//! `openshell-supervisor-process` implementation inside the VM. + +#![allow(unsafe_code)] + +use std::ffi::CString; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::mem::size_of; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::path::Path; +use std::sync::atomic::AtomicU32; +use std::sync::{Arc, Condvar, Mutex}; +use std::time::Duration; + +use openshell_core::proposals::AgentProposals; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_supervisor_process::boundary_io::BoundaryRuntimeState; +use openshell_supervisor_process::process::{ + ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity, +}; +use openshell_supervisor_process::run::{AgentSignaler, spawn_workload}; +use serde::{Deserialize, Serialize}; + +use crate::protocol::{ + AgentSpecWire, ExitStatusWire, Request, RequestEnvelope, Response, ResponseEnvelope, + SandboxPolicyWire, SignalWire, read_frame, write_frame, +}; + +pub const DEFAULT_GUEST_CONFIG_PATH: &str = "/etc/openshell/firecracker.json"; +const DEFAULT_CONTROL_PORT: u32 = 5500; +const DEFAULT_AGENT_UID: u32 = 10_001; +const DEFAULT_AGENT_GID: u32 = 10_001; +const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30); + +/// Driver-private configuration injected into the guest image at provision time. +#[derive(Clone, Serialize, Deserialize)] +pub struct GuestConfig { + pub boundary_id: String, + pub bootstrap_token: String, + #[serde(default = "default_control_port")] + pub control_port: u32, + #[serde(default = "default_agent_uid")] + pub agent_uid: u32, + #[serde(default = "default_agent_gid")] + pub agent_gid: u32, +} + +impl std::fmt::Debug for GuestConfig { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("GuestConfig") + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("control_port", &self.control_port) + .field("agent_uid", &self.agent_uid) + .field("agent_gid", &self.agent_gid) + .finish() + } +} + +const fn default_control_port() -> u32 { + DEFAULT_CONTROL_PORT +} + +const fn default_agent_uid() -> u32 { + DEFAULT_AGENT_UID +} + +const fn default_agent_gid() -> u32 { + DEFAULT_AGENT_GID +} + +pub fn run_guest(config_path: &Path) -> Result<(), String> { + if std::process::id() == 1 { + prepare_pid1_filesystems()?; + } + let bytes = std::fs::read(config_path) + .map_err(|error| format!("read guest config {}: {error}", config_path.display()))?; + let config: GuestConfig = serde_json::from_slice(&bytes) + .map_err(|error| format!("decode guest config {}: {error}", config_path.display()))?; + validate_config(&config)?; + let process_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create guest process runtime: {error}"))?; + let runtime = Arc::new(GuestRuntime::new( + config.clone(), + process_runtime.handle().clone(), + )); + serve(config.control_port, runtime) +} + +fn validate_config(config: &GuestConfig) -> Result<(), String> { + if config.boundary_id.is_empty() { + return Err("guest boundary ID must not be empty".to_string()); + } + if config.bootstrap_token.len() < 32 { + return Err("guest bootstrap token must contain at least 32 bytes".to_string()); + } + if config.control_port == 0 { + return Err("guest control port must be nonzero".to_string()); + } + if config.agent_uid == 0 || config.agent_gid == 0 { + return Err("guest agent UID and GID must be nonzero".to_string()); + } + Ok(()) +} + +fn prepare_pid1_filesystems() -> Result<(), String> { + for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] { + std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?; + } + mount_if_needed("proc", "/proc", "proc")?; + mount_if_needed("sysfs", "/sys", "sysfs")?; + mount_if_needed("devtmpfs", "/dev", "devtmpfs")?; + std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?; + mount_if_needed("devpts", "/dev/pts", "devpts")?; + Ok(()) +} + +fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> { + let source = CString::new(source).map_err(|error| error.to_string())?; + let target_c = CString::new(target).map_err(|error| error.to_string())?; + let file_system = CString::new(file_system).map_err(|error| error.to_string())?; + let result = unsafe { + libc::mount( + source.as_ptr(), + target_c.as_ptr(), + file_system.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if result == 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EBUSY) { + Ok(()) + } else { + Err(format!("mount {file_system:?} on {target}: {error}")) + } +} + +fn serve(port: u32, runtime: Arc) -> Result<(), String> { + let listener = VsockListener::bind(port) + .map_err(|error| format!("bind guest control vsock port {port}: {error}"))?; + eprintln!("Firecracker process supervisor leaf listening on vsock port {port}"); + loop { + match listener.accept() { + Ok(stream) => { + let runtime = runtime.clone(); + std::thread::spawn(move || { + if let Err(error) = serve_one(stream, &runtime) { + eprintln!("Firecracker guest control request failed: {error}"); + } + }); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(format!("accept guest control connection: {error}")), + } + } +} + +fn serve_one(mut stream: VsockStream, runtime: &GuestRuntime) -> Result<(), String> { + stream + .set_timeout(CONTROL_IO_TIMEOUT) + .map_err(|error| format!("set control timeout: {error}"))?; + let request: RequestEnvelope = + read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?; + let response = ResponseEnvelope { + request_id: request.request_id, + response: runtime.dispatch(request), + }; + write_frame(&mut stream, &response).map_err(|error| format!("write control frame: {error}")) +} + +struct GuestRuntime { + config: GuestConfig, + process_runtime: tokio::runtime::Handle, + state: Mutex, +} + +enum RuntimeState { + AwaitingAttach, + Bound, + Ready, + Running(Arc), +} + +impl GuestRuntime { + fn new(config: GuestConfig, process_runtime: tokio::runtime::Handle) -> Self { + Self { + config, + process_runtime, + state: Mutex::new(RuntimeState::AwaitingAttach), + } + } + + fn dispatch(&self, envelope: RequestEnvelope) -> Response { + if !constant_time_eq( + envelope.boundary_id.as_bytes(), + self.config.boundary_id.as_bytes(), + ) || !constant_time_eq( + envelope.bootstrap_token.as_bytes(), + self.config.bootstrap_token.as_bytes(), + ) { + return guest_error("denied", "control authentication failed"); + } + match envelope.request { + Request::Attach => self.attach(), + Request::Confirm => self.confirm(), + Request::StartAgent { + sandbox_id, + spec, + policy, + } => self.start_agent(sandbox_id, spec, *policy), + Request::Wait { process_id } => self.wait(&process_id), + Request::Signal { process_id, signal } => self.signal(&process_id, signal), + Request::Terminate { process_id } => self.terminate(&process_id), + } + } + + fn attach(&self) -> Response { + let mut state = lock(&self.state); + match *state { + RuntimeState::AwaitingAttach => { + *state = RuntimeState::Bound; + Response::Attached + } + RuntimeState::Bound => Response::Attached, + _ => guest_error("invalid", "boundary has already advanced past attach"), + } + } + + fn confirm(&self) -> Response { + let mut state = lock(&self.state); + match *state { + RuntimeState::Bound => { + *state = RuntimeState::Ready; + Response::Confirmed + } + RuntimeState::Ready => Response::Confirmed, + RuntimeState::AwaitingAttach => { + guest_error("invalid", "boundary must be attached before confirm") + } + RuntimeState::Running(_) => { + guest_error("invalid", "boundary has already started its agent") + } + } + } + + fn start_agent( + &self, + sandbox_id: String, + spec: AgentSpecWire, + policy: SandboxPolicyWire, + ) -> Response { + let mut state = lock(&self.state); + if !matches!(*state, RuntimeState::Ready) { + return guest_error("invalid", "boundary must be confirmed before start_agent"); + } + let process = match ManagedProcess::spawn( + &self.process_runtime, + sandbox_id, + spec, + policy.into(), + ResolvedProcessIdentity::new(Some(self.config.agent_uid), Some(self.config.agent_gid)), + ) { + Ok(process) => Arc::new(process), + Err(error) => return guest_error("failed", error), + }; + let process_id = process.process_id(); + *state = RuntimeState::Running(process); + Response::Started { process_id } + } + + fn wait(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.wait() { + Ok(status) => Response::Exited { status }, + Err(error) => guest_error("failed", error), + } + } + + fn signal(&self, process_id: &str, signal: SignalWire) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(signal) { + Ok(()) => Response::Signaled, + Err(error) => guest_error("terminated", error), + } + } + + fn terminate(&self, process_id: &str) -> Response { + let process = match self.running_process(process_id) { + Ok(process) => process, + Err(response) => return response, + }; + match process.signal(SignalWire::Kill) { + Ok(()) => Response::Terminated, + Err(_) if process.has_exited() => Response::Terminated, + Err(error) => guest_error("failed", error), + } + } + + fn running_process(&self, process_id: &str) -> Result, Response> { + let state = lock(&self.state); + let RuntimeState::Running(process) = &*state else { + return Err(guest_error("invalid", "agent process has not been started")); + }; + if process.process_id() != process_id { + return Err(guest_error("invalid", "unknown process ID")); + } + Ok(process.clone()) + } +} + +type ProcessExit = Result; +type SharedProcessExit = Arc<(Mutex>, Condvar)>; + +struct ManagedProcess { + pid: i32, + signaler: AgentSignaler, + exit: SharedProcessExit, +} + +impl ManagedProcess { + fn spawn( + runtime: &tokio::runtime::Handle, + sandbox_id: String, + spec: AgentSpecWire, + policy: openshell_core::policy::SandboxPolicy, + resolved_identity: ResolvedProcessIdentity, + ) -> Result { + if spec.program.is_empty() { + return Err("agent program must not be empty".to_string()); + } + let boundary_runtime = BoundaryRuntimeState::new(); + let mut spawned = runtime + .block_on(spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + Some(&sandbox_id), + None, + None, + false, + &policy, + resolved_identity, + ProcessEnforcementMode::Full, + Arc::new(AtomicU32::new(0)), + None, + ProviderCredentialState::from_child_env_snapshot( + 0, + std::collections::HashMap::new(), + ), + std::collections::HashMap::new(), + None, + AgentProposals::default(), + None, + None, + None, + Some(boundary_runtime), + )) + .map_err(|error| format!("start process supervisor leaf: {error}"))?; + let pid = i32::try_from(spawned.pid()) + .map_err(|_| "process supervisor PID does not fit i32".to_string())?; + let signaler = spawned.signaler(); + let exit = Arc::new((Mutex::new(None), Condvar::new())); + let reaper_exit = exit.clone(); + runtime.spawn(async move { + let result = spawned + .wait() + .await + .map(process_status) + .map_err(|error| format!("wait for process supervisor leaf: {error}")); + let (state, changed) = &*reaper_exit; + *lock(state) = Some(result); + changed.notify_all(); + }); + Ok(Self { + pid, + signaler, + exit, + }) + } + + fn process_id(&self) -> String { + self.pid.to_string() + } + + fn wait(&self) -> ProcessExit { + let (state, changed) = &*self.exit; + let mut exit = lock(state); + while exit.is_none() { + exit = changed + .wait(exit) + .unwrap_or_else(std::sync::PoisonError::into_inner); + } + exit.as_ref().expect("exit checked above").clone() + } + + fn signal(&self, signal: SignalWire) -> Result<(), String> { + if self.has_exited() { + return Err("agent process has already exited".to_string()); + } + let result = match signal { + SignalWire::Term => self.signaler.term(), + SignalWire::Kill => self.signaler.kill(), + SignalWire::Int => self.signaler.interrupt(), + SignalWire::Hup => self.signaler.hangup(), + }; + result.map_err(|error| format!("signal process supervisor group: {error}")) + } + + fn has_exited(&self) -> bool { + let (state, _) = &*self.exit; + lock(state).is_some() + } +} + +fn lock(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn process_status(status: ProcessStatus) -> ExitStatusWire { + status.signal().map_or_else( + || ExitStatusWire::Exited(status.code()), + ExitStatusWire::Signaled, + ) +} + +fn guest_error(kind: &str, message: impl Into) -> Response { + Response::Error { + kind: kind.to_string(), + message: message.into(), + } +} + +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let max_len = left.len().max(right.len()); + let mut difference = left.len() ^ right.len(); + for index in 0..max_len { + let left_byte = left.get(index).copied().unwrap_or_default(); + let right_byte = right.get(index).copied().unwrap_or_default(); + difference |= usize::from(left_byte ^ right_byte); + } + difference == 0 +} + +struct VsockListener { + fd: OwnedFd, +} + +impl VsockListener { + fn bind(port: u32) -> io::Result { + let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t") + })?; + let address_length = + libc::socklen_t::try_from(size_of::()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t") + })?; + let raw_fd = + unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) }; + if raw_fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) }; + let address = libc::sockaddr_vm { + svm_family: family, + svm_reserved1: 0, + svm_port: port, + svm_cid: libc::VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let result = unsafe { + libc::bind( + fd.as_raw_fd(), + (&raw const address).cast::(), + address_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 { + return Err(io::Error::last_os_error()); + } + Ok(Self { fd }) + } + + fn accept(&self) -> io::Result { + let raw_fd = unsafe { + libc::accept4( + self.fd.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + libc::SOCK_CLOEXEC, + ) + }; + if raw_fd < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(VsockStream { + file: unsafe { File::from_raw_fd(raw_fd) }, + }) + } + } +} + +struct VsockStream { + file: File, +} + +impl VsockStream { + fn set_timeout(&self, timeout: Duration) -> io::Result<()> { + let option_length = + libc::socklen_t::try_from(size_of::()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t") + })?; + let timeout = libc::timeval { + tv_sec: timeout.as_secs().try_into().unwrap_or(libc::time_t::MAX), + tv_usec: timeout.subsec_micros().into(), + }; + for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] { + let result = unsafe { + libc::setsockopt( + self.file.as_raw_fd(), + libc::SOL_SOCKET, + option, + (&raw const timeout).cast(), + option_length, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) + } +} + +impl Read for VsockStream { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.file.read(buffer) + } +} + +impl Write for VsockStream { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.file.write(buffer) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn guest_config_debug_redacts_token() { + let config = GuestConfig { + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this-never-log-this".to_string(), + control_port: 5500, + agent_uid: DEFAULT_AGENT_UID, + agent_gid: DEFAULT_AGENT_GID, + }; + let debug = format!("{config:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn constant_time_comparison_checks_length_and_content() { + assert!(constant_time_eq(b"same", b"same")); + assert!(!constant_time_eq(b"same", b"different")); + assert!(!constant_time_eq(b"same", b"sam")); + } +} diff --git a/crates/openshell-driver-firecracker/src/lib.rs b/crates/openshell-driver-firecracker/src/lib.rs new file mode 100644 index 0000000000..b3f4c7c69b --- /dev/null +++ b/crates/openshell-driver-firecracker/src/lib.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental Firecracker Isolation Backend. +//! +//! The logical supervisor and RFC 0012 lifecycle live on the host. A private +//! guest mode of the same driver binary only transports lifecycle operations +//! and invokes the existing `openshell-supervisor-process` implementation. + +pub mod backend; +pub mod compute; +pub mod guest; +mod protocol; +pub mod runtime; + +pub use backend::{BACKEND_NAME, FirecrackerHostBackend, FirecrackerTopology}; +pub use compute::{FirecrackerComputeConfig, FirecrackerComputeDriver}; +pub use guest::{DEFAULT_GUEST_CONFIG_PATH, GuestConfig, run_guest}; +pub use runtime::{FirecrackerLaunchConfig, FirecrackerVm}; diff --git a/crates/openshell-driver-firecracker/src/main.rs b/crates/openshell-driver-firecracker/src/main.rs new file mode 100644 index 0000000000..c0725fbdfc --- /dev/null +++ b/crates/openshell-driver-firecracker/src/main.rs @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::os::unix::fs::{FileTypeExt as _, PermissionsExt as _}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use clap::{Args, Parser, Subcommand}; +use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, +}; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_firecracker::{ + BACKEND_NAME, DEFAULT_GUEST_CONFIG_PATH, FirecrackerComputeConfig, FirecrackerComputeDriver, + FirecrackerHostBackend, FirecrackerLaunchConfig, FirecrackerTopology, FirecrackerVm, run_guest, +}; +use openshell_isolation::AgentSpec; +use openshell_isolation::contract::{ + BackendRegistry, BoundaryExitStatus, INTERFACE_VERSION, SandboxContext, TopologyDescriptor, +}; +use tokio::net::UnixListener; +use tokio_stream::wrappers::UnixListenerStream; + +#[derive(Debug, Parser)] +#[command(about = "Experimental OpenShell Firecracker isolation driver")] +struct Cli { + #[command(subcommand)] + command: Option, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// Boot and wait for one Firecracker VM. + Launch(LaunchArgs), + /// Drive the RFC 0012 lifecycle from the host. + Supervise(SuperviseArgs), + /// Run the private process-supervisor leaf transport inside the guest. + Guest(GuestArgs), + /// Serve the gateway compute-driver contract over a Unix socket. + ComputeDriver(ComputeDriverArgs), +} + +#[derive(Debug, Args)] +struct LaunchArgs { + #[arg( + long, + env = "OPENSHELL_FIRECRACKER_BINARY", + default_value = "firecracker" + )] + firecracker_binary: PathBuf, + #[arg(long)] + kernel_image: PathBuf, + #[arg(long)] + root_disk: PathBuf, + #[arg(long)] + run_dir: PathBuf, + #[arg(long)] + console_output: PathBuf, + #[arg( + long, + default_value = "/opt/openshell/bin/openshell-driver-firecracker" + )] + guest_init: String, + #[arg(long, default_value_t = 2)] + vcpus: u8, + #[arg(long, default_value_t = 512)] + mem_mib: u32, + #[arg(long)] + vsock_cid: u32, +} + +#[derive(Debug, Args)] +struct SuperviseArgs { + #[arg(long)] + boundary_id: String, + #[arg(long)] + vsock_uds_path: PathBuf, + #[arg(long, default_value_t = 5500)] + vsock_port: u32, + #[arg(long)] + bootstrap_token_file: PathBuf, + #[arg(long, default_value = "/sandbox")] + workdir: String, + #[arg(long, default_value_t = 300)] + timeout_seconds: u64, + #[arg(required = true, trailing_var_arg = true)] + command: Vec, +} + +#[derive(Debug, Args)] +struct GuestArgs { + #[arg(long, default_value = DEFAULT_GUEST_CONFIG_PATH)] + config: PathBuf, +} + +#[derive(Debug, Args)] +struct ComputeDriverArgs { + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: PathBuf, + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + gateway_endpoint: String, + #[arg(long, env = "OPENSHELL_FIRECRACKER_STATE_DIR")] + state_dir: PathBuf, + #[arg(long, env = "OPENSHELL_FIRECRACKER_BINARY")] + firecracker_binary: PathBuf, + #[arg(long, env = "OPENSHELL_FIRECRACKER_KERNEL_IMAGE")] + kernel_image: PathBuf, + #[arg(long, env = "OPENSHELL_FIRECRACKER_ROOT_DISK")] + root_disk: PathBuf, + #[arg(long, env = "OPENSHELL_FIRECRACKER_SUPERVISOR_BIN")] + supervisor_binary: PathBuf, + #[arg(long, default_value = "firecracker-rootfs")] + default_image: String, + #[arg(long, default_value_t = 2)] + vcpus: u8, + #[arg(long, default_value_t = 512)] + mem_mib: u32, +} + +fn main() { + if let Err(error) = run() { + eprintln!("Firecracker driver failed: {error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + let cli = Cli::parse(); + let command = match cli.command { + Some(command) => command, + None if std::process::id() == 1 => Command::Guest(GuestArgs { + config: PathBuf::from(DEFAULT_GUEST_CONFIG_PATH), + }), + None => return Err("a subcommand is required outside a Firecracker guest".to_string()), + }; + match command { + Command::Guest(args) => run_guest(&args.config), + Command::Launch(args) => launch(args), + Command::Supervise(args) => { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create host runtime: {error}"))?; + runtime.block_on(supervise(args)) + } + Command::ComputeDriver(args) => { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| format!("create compute-driver runtime: {error}"))?; + runtime.block_on(serve_compute_driver(args)) + } + } +} + +async fn serve_compute_driver(args: ComputeDriverArgs) -> Result<(), String> { + nix::sys::prctl::set_pdeathsig(nix::sys::signal::Signal::SIGTERM) + .map_err(|error| format!("arm compute-driver parent-death signal: {error}"))?; + prepare_compute_socket(&args.bind_socket)?; + let driver_binary = std::env::current_exe() + .map_err(|error| format!("resolve Firecracker driver binary: {error}"))?; + let driver = FirecrackerComputeDriver::new(FirecrackerComputeConfig { + gateway_endpoint: args.gateway_endpoint, + state_dir: args.state_dir, + firecracker_binary: resolve_executable(&args.firecracker_binary)?, + kernel_image: args.kernel_image, + root_disk: args.root_disk, + supervisor_binary: args.supervisor_binary, + driver_binary, + default_image: args.default_image, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + })?; + let listener = UnixListener::bind(&args.bind_socket) + .map_err(|error| format!("bind compute-driver socket: {error}"))?; + std::fs::set_permissions(&args.bind_socket, std::fs::Permissions::from_mode(0o600)) + .map_err(|error| format!("restrict compute-driver socket: {error}"))?; + eprintln!( + "Firecracker compute driver listening on {}", + args.bind_socket.display() + ); + let result = tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming(UnixListenerStream::new(listener)) + .await + .map_err(|error| format!("serve compute driver: {error}")); + let _ = std::fs::remove_file(&args.bind_socket); + result +} + +fn prepare_compute_socket(path: &Path) -> Result<(), String> { + let parent = path + .parent() + .ok_or_else(|| "compute-driver socket requires a parent directory".to_string())?; + std::fs::create_dir_all(parent) + .map_err(|error| format!("create compute-driver socket directory: {error}"))?; + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("restrict compute-driver socket directory: {error}"))?; + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_socket() => std::fs::remove_file(path) + .map_err(|error| format!("remove stale compute-driver socket: {error}")), + Ok(_) => Err(format!( + "refusing to replace non-socket compute-driver path {}", + path.display() + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("inspect compute-driver socket: {error}")), + } +} + +fn launch(args: LaunchArgs) -> Result<(), String> { + let config = FirecrackerLaunchConfig { + firecracker_binary: resolve_executable(&args.firecracker_binary)?, + kernel_image: args.kernel_image, + root_disk: args.root_disk, + run_dir: args.run_dir, + console_output: args.console_output, + guest_init: args.guest_init, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + vsock_cid: args.vsock_cid, + }; + let mut vm = FirecrackerVm::launch(&config)?; + let status = vm.wait()?; + if status.success() { + Ok(()) + } else { + Err(format!("Firecracker exited with status {status}")) + } +} + +async fn supervise(args: SuperviseArgs) -> Result<(), String> { + let bootstrap_token = std::fs::read_to_string(&args.bootstrap_token_file) + .map_err(|error| { + format!( + "read bootstrap token file {}: {error}", + args.bootstrap_token_file.display() + ) + })? + .trim() + .to_string(); + let topology = FirecrackerTopology { + boundary_id: args.boundary_id.clone(), + vsock_uds_path: args.vsock_uds_path, + control_port: args.vsock_port, + bootstrap_token, + }; + let descriptor = TopologyDescriptor { + version: INTERFACE_VERSION, + backend_name: BACKEND_NAME.to_string(), + payload: topology.encode().map_err(|error| error.to_string())?, + }; + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(FirecrackerHostBackend)) + .map_err(|error| error.to_string())?; + let (backend, verified) = registry + .resolve(descriptor, BACKEND_NAME) + .map_err(|error| error.to_string())?; + let (program, command_args) = args + .command + .split_first() + .ok_or_else(|| "agent command must not be empty".to_string())?; + let sandbox = SandboxContext { + sandbox_id: args.boundary_id, + policy: restrictive_host_policy(), + agent: AgentSpec { + program: program.clone(), + args: command_args.to_vec(), + workdir: Some(args.workdir), + timeout_secs: args.timeout_seconds, + interactive: false, + }, + }; + let bound = backend + .attach(verified, sandbox) + .await + .map_err(|error| error.to_string())?; + let ready = bound.confirm().await.map_err(|error| error.to_string())?; + let running = ready + .start_agent() + .await + .map_err(|error| error.to_string())?; + let process = running.agent(); + let wait = process.wait(); + tokio::pin!(wait); + let status = tokio::select! { + result = &mut wait => result.map_err(|error| error.to_string())?, + signal = tokio::signal::ctrl_c() => { + signal.map_err(|error| format!("wait for interrupt: {error}"))?; + process.terminate().await.map_err(|error| error.to_string())?; + process.wait().await.map_err(|error| error.to_string())? + } + }; + println!("agent exited: {status:?}"); + match status { + BoundaryExitStatus::Exited(0) => Ok(()), + _ => Err(format!("agent exited unsuccessfully: {status:?}")), + } +} + +fn restrictive_host_policy() -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: ["/usr", "/lib", "/proc", "/dev/urandom", "/etc", "/var/log"] + .into_iter() + .map(PathBuf::from) + .collect(), + read_write: ["/sandbox", "/tmp", "/dev/null"] + .into_iter() + .map(PathBuf::from) + .collect(), + include_workdir: true, + }, + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } +} + +fn resolve_executable(path: &Path) -> Result { + if path.components().count() > 1 || path.is_absolute() { + return path + .is_file() + .then(|| path.to_path_buf()) + .ok_or_else(|| format!("executable not found: {}", path.display())); + } + let search_path = std::env::var_os("PATH").unwrap_or_default(); + std::env::split_paths(&search_path) + .map(|directory| directory.join(path)) + .find(|candidate| candidate.is_file()) + .ok_or_else(|| format!("executable not found on PATH: {}", path.display())) +} diff --git a/crates/openshell-driver-firecracker/src/protocol.rs b/crates/openshell-driver-firecracker/src/protocol.rs new file mode 100644 index 0000000000..1f994f2899 --- /dev/null +++ b/crates/openshell-driver-firecracker/src/protocol.rs @@ -0,0 +1,361 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-private, length-delimited JSON protocol carried over virtio-vsock. + +use std::fmt; +use std::io::{self, Read, Write}; + +use openshell_core::policy::{ + FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy, + ProcessPolicy, ProxyPolicy, SandboxPolicy, +}; +use openshell_isolation::AgentSpec; +use openshell_isolation::contract::{BoundaryExitStatus, BoundarySignal}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RequestEnvelope { + pub request_id: u64, + pub boundary_id: String, + pub bootstrap_token: String, + pub request: Request, +} + +impl fmt::Debug for RequestEnvelope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RequestEnvelope") + .field("request_id", &self.request_id) + .field("boundary_id", &self.boundary_id) + .field("bootstrap_token", &"") + .field("request", &self.request) + .finish() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "operation", rename_all = "snake_case")] +pub enum Request { + Attach, + Confirm, + StartAgent { + sandbox_id: String, + spec: AgentSpecWire, + policy: Box, + }, + Wait { + process_id: String, + }, + Signal { + process_id: String, + signal: SignalWire, + }, + Terminate { + process_id: String, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResponseEnvelope { + pub request_id: u64, + pub response: Response, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum Response { + Attached, + Confirmed, + Started { process_id: String }, + Exited { status: ExitStatusWire }, + Signaled, + Terminated, + Error { kind: String, message: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AgentSpecWire { + pub program: String, + pub args: Vec, + pub workdir: Option, + pub timeout_secs: u64, + pub interactive: bool, +} + +impl From for AgentSpecWire { + fn from(spec: AgentSpec) -> Self { + Self { + program: spec.program, + args: spec.args, + workdir: spec.workdir, + timeout_secs: spec.timeout_secs, + interactive: spec.interactive, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct SandboxPolicyWire { + pub version: u32, + pub read_only: Vec, + pub read_write: Vec, + pub include_workdir: bool, + pub network: NetworkModeWire, + pub proxy_addr: Option, + pub landlock: LandlockCompatibilityWire, + pub run_as_user: Option, + pub run_as_group: Option, +} + +impl From for SandboxPolicyWire { + fn from(policy: SandboxPolicy) -> Self { + Self { + version: policy.version, + read_only: policy.filesystem.read_only, + read_write: policy.filesystem.read_write, + include_workdir: policy.filesystem.include_workdir, + network: NetworkModeWire::from(policy.network.mode), + proxy_addr: policy.network.proxy.and_then(|proxy| proxy.http_addr), + landlock: LandlockCompatibilityWire::from(policy.landlock.compatibility), + run_as_user: policy.process.run_as_user, + run_as_group: policy.process.run_as_group, + } + } +} + +impl From for SandboxPolicy { + fn from(policy: SandboxPolicyWire) -> Self { + let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy { + http_addr: policy.proxy_addr, + }); + Self { + version: policy.version, + filesystem: FilesystemPolicy { + read_only: policy.read_only, + read_write: policy.read_write, + include_workdir: policy.include_workdir, + }, + network: NetworkPolicy { + mode: policy.network.into(), + proxy, + }, + landlock: LandlockPolicy { + compatibility: policy.landlock.into(), + }, + process: ProcessPolicy { + run_as_user: policy.run_as_user, + run_as_group: policy.run_as_group, + }, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NetworkModeWire { + Block, + Proxy, + Allow, +} + +impl From for NetworkModeWire { + fn from(mode: NetworkMode) -> Self { + match mode { + NetworkMode::Block => Self::Block, + NetworkMode::Proxy => Self::Proxy, + NetworkMode::Allow => Self::Allow, + } + } +} + +impl From for NetworkMode { + fn from(mode: NetworkModeWire) -> Self { + match mode { + NetworkModeWire::Block => Self::Block, + NetworkModeWire::Proxy => Self::Proxy, + NetworkModeWire::Allow => Self::Allow, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LandlockCompatibilityWire { + BestEffort, + HardRequirement, +} + +impl From for LandlockCompatibilityWire { + fn from(compatibility: LandlockCompatibility) -> Self { + match compatibility { + LandlockCompatibility::BestEffort => Self::BestEffort, + LandlockCompatibility::HardRequirement => Self::HardRequirement, + } + } +} + +impl From for LandlockCompatibility { + fn from(compatibility: LandlockCompatibilityWire) -> Self { + match compatibility { + LandlockCompatibilityWire::BestEffort => Self::BestEffort, + LandlockCompatibilityWire::HardRequirement => Self::HardRequirement, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignalWire { + Term, + Kill, + Int, + Hup, +} + +impl From for SignalWire { + fn from(signal: BoundarySignal) -> Self { + match signal { + BoundarySignal::Term => Self::Term, + BoundarySignal::Kill => Self::Kill, + BoundarySignal::Int => Self::Int, + BoundarySignal::Hup => Self::Hup, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum ExitStatusWire { + Exited(i32), + Signaled(i32), +} + +impl From for BoundaryExitStatus { + fn from(status: ExitStatusWire) -> Self { + match status { + ExitStatusWire::Exited(code) => Self::Exited(code), + ExitStatusWire::Signaled(signal) => Self::Signaled(signal), + } + } +} + +pub fn encode_frame(message: &T) -> Result, FrameError> { + let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?; + if payload.len() > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(payload.len())); + } + let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?; + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.extend_from_slice(&length.to_be_bytes()); + frame.extend_from_slice(&payload); + Ok(frame) +} + +pub fn decode_frame(frame: &[u8]) -> Result { + let header: [u8; 4] = frame + .get(..4) + .ok_or(FrameError::Truncated)? + .try_into() + .map_err(|_| FrameError::Truncated)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let payload = frame.get(4..).ok_or(FrameError::Truncated)?; + if payload.len() != declared { + return Err(FrameError::LengthMismatch { + declared, + actual: payload.len(), + }); + } + serde_json::from_slice(payload).map_err(FrameError::Deserialize) +} + +pub fn read_frame(reader: &mut impl Read) -> Result { + let mut header = [0_u8; 4]; + reader.read_exact(&mut header)?; + let declared = u32::from_be_bytes(header) as usize; + if declared > MAX_CONTROL_FRAME_BYTES { + return Err(FrameError::TooLarge(declared)); + } + let mut frame = Vec::with_capacity(4 + declared); + frame.extend_from_slice(&header); + frame.resize(4 + declared, 0); + reader.read_exact(&mut frame[4..])?; + decode_frame(&frame) +} + +pub fn write_frame(writer: &mut impl Write, message: &T) -> Result<(), FrameError> { + let frame = encode_frame(message)?; + writer.write_all(&frame)?; + writer.flush()?; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum FrameError { + #[error("control frame is truncated")] + Truncated, + #[error("control frame is too large: {0} bytes")] + TooLarge(usize), + #[error("control frame declared {declared} bytes but contained {actual}")] + LengthMismatch { declared: usize, actual: usize }, + #[error("serialize control frame: {0}")] + Serialize(serde_json::Error), + #[error("deserialize control frame: {0}")] + Deserialize(serde_json::Error), + #[error("read or write control frame: {0}")] + Io(#[from] io::Error), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_and_redacts_token() { + let request = RequestEnvelope { + request_id: 7, + boundary_id: "sandbox-1".to_string(), + bootstrap_token: "never-log-this".to_string(), + request: Request::StartAgent { + sandbox_id: "sandbox-1".to_string(), + spec: AgentSpecWire { + program: "/bin/true".to_string(), + args: Vec::new(), + workdir: Some("/sandbox".to_string()), + timeout_secs: 5, + interactive: false, + }, + policy: Box::new(SandboxPolicyWire::from(SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + })), + }, + }; + let frame = encode_frame(&request).expect("encode request"); + let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request"); + assert_eq!(decoded, request); + let debug = format!("{request:?}"); + assert!(debug.contains("")); + assert!(!debug.contains("never-log-this")); + } + + #[test] + fn rejects_declared_oversize() { + let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32"); + let mut frame = Vec::from(oversized.to_be_bytes()); + frame.extend_from_slice(b"{}"); + assert!(matches!( + decode_frame::(&frame), + Err(FrameError::TooLarge(_)) + )); + } +} diff --git a/crates/openshell-driver-firecracker/src/runtime.rs b/crates/openshell-driver-firecracker/src/runtime.rs new file mode 100644 index 0000000000..132f4ee51f --- /dev/null +++ b/crates/openshell-driver-firecracker/src/runtime.rs @@ -0,0 +1,401 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small standalone Firecracker launcher owned by this driver crate. + +#![allow(unsafe_code)] + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::unix::net::UnixStream; +use std::os::unix::process::CommandExt as _; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::time::{Duration, Instant}; + +use nix::sys::signal::Signal; +use serde_json::{Value, json}; + +const API_START_TIMEOUT: Duration = Duration::from_secs(5); +const API_IO_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_API_RESPONSE_BYTES: usize = 1024 * 1024; + +#[derive(Debug, Clone)] +pub struct FirecrackerLaunchConfig { + pub firecracker_binary: PathBuf, + pub kernel_image: PathBuf, + pub root_disk: PathBuf, + pub run_dir: PathBuf, + pub console_output: PathBuf, + pub guest_init: String, + pub vcpus: u8, + pub mem_mib: u32, + pub vsock_cid: u32, +} + +/// A configured Firecracker child. Dropping it terminates the VMM. +pub struct FirecrackerVm { + child: Child, + api_socket: PathBuf, + vsock_socket: PathBuf, +} + +impl FirecrackerVm { + pub fn launch(config: &FirecrackerLaunchConfig) -> Result { + validate_config(config)?; + check_kvm_access()?; + std::fs::create_dir_all(&config.run_dir).map_err(|error| { + format!( + "create Firecracker run dir {}: {error}", + config.run_dir.display() + ) + })?; + let api_socket = config.run_dir.join("firecracker-api.sock"); + let vsock_socket = config.run_dir.join("firecracker-vsock.sock"); + remove_stale_socket(&api_socket)?; + remove_stale_socket(&vsock_socket)?; + + let console = File::create(&config.console_output).map_err(|error| { + format!( + "create Firecracker console log {}: {error}", + config.console_output.display() + ) + })?; + let stderr = console + .try_clone() + .map_err(|error| format!("clone Firecracker console log: {error}"))?; + let mut command = Command::new(&config.firecracker_binary); + command + .arg("--api-sock") + .arg(&api_socket) + .stdin(Stdio::null()) + .stdout(Stdio::from(console)) + .stderr(Stdio::from(stderr)); + unsafe { + command.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|error| io::Error::other(format!("pdeathsig: {error}"))) + }); + } + let mut child = command + .spawn() + .map_err(|error| format!("start Firecracker: {error}"))?; + if let Err(error) = configure(&mut child, &api_socket, &vsock_socket, config) { + terminate_child(&mut child); + return Err(error); + } + Ok(Self { + child, + api_socket, + vsock_socket, + }) + } + + pub fn vsock_uds_path(&self) -> &Path { + &self.vsock_socket + } + + pub fn wait(&mut self) -> Result { + self.child + .wait() + .map_err(|error| format!("wait for Firecracker: {error}")) + } + + pub fn terminate(&mut self) -> Result<(), String> { + terminate_child(&mut self.child); + Ok(()) + } +} + +impl Drop for FirecrackerVm { + fn drop(&mut self) { + if self.child.try_wait().ok().flatten().is_none() { + terminate_child(&mut self.child); + } + let _ = std::fs::remove_file(&self.api_socket); + let _ = std::fs::remove_file(&self.vsock_socket); + } +} + +fn validate_config(config: &FirecrackerLaunchConfig) -> Result<(), String> { + for (label, path) in [ + ("Firecracker binary", &config.firecracker_binary), + ("kernel image", &config.kernel_image), + ("root disk", &config.root_disk), + ] { + if !path.is_file() { + return Err(format!("{label} not found: {}", path.display())); + } + } + if config.vcpus == 0 { + return Err("Firecracker vCPU count must be nonzero".to_string()); + } + if config.mem_mib < 128 { + return Err("Firecracker memory must be at least 128 MiB".to_string()); + } + if config.vsock_cid < 3 { + return Err("Firecracker guest CID must be at least 3".to_string()); + } + if !config.guest_init.starts_with('/') { + return Err("Firecracker guest init path must be absolute".to_string()); + } + Ok(()) +} + +fn check_kvm_access() -> Result<(), String> { + OpenOptions::new() + .read(true) + .write(true) + .open("/dev/kvm") + .map(|_| ()) + .map_err(|error| { + format!( + "open /dev/kvm read/write: {error}; start a login session whose supplementary groups include kvm" + ) + }) +} + +fn configure( + child: &mut Child, + api_socket: &Path, + vsock_socket: &Path, + config: &FirecrackerLaunchConfig, +) -> Result<(), String> { + wait_for_api_socket(child, api_socket)?; + for request in configuration_requests(config, vsock_socket) { + put_json(api_socket, request.path, &request.body)?; + } + Ok(()) +} + +struct ApiRequest { + path: &'static str, + body: Value, +} + +fn configuration_requests( + config: &FirecrackerLaunchConfig, + vsock_socket: &Path, +) -> Vec { + vec![ + ApiRequest { + path: "/machine-config", + body: json!({ + "vcpu_count": config.vcpus, + "mem_size_mib": config.mem_mib, + "smt": false + }), + }, + ApiRequest { + path: "/boot-source", + body: json!({ + "kernel_image_path": config.kernel_image, + "boot_args": kernel_command_line(&config.guest_init) + }), + }, + ApiRequest { + path: "/drives/rootfs", + body: json!({ + "drive_id": "rootfs", + "path_on_host": config.root_disk, + "is_root_device": true, + "is_read_only": false + }), + }, + ApiRequest { + path: "/vsock", + body: json!({ + "guest_cid": config.vsock_cid, + "uds_path": vsock_socket + }), + }, + ApiRequest { + path: "/actions", + body: json!({ "action_type": "InstanceStart" }), + }, + ] +} + +fn kernel_command_line(init: &str) -> String { + format!("console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw init={init}") +} + +fn wait_for_api_socket(child: &mut Child, socket: &Path) -> Result<(), String> { + let deadline = Instant::now() + API_START_TIMEOUT; + loop { + if UnixStream::connect(socket).is_ok() { + return Ok(()); + } + if let Some(status) = child + .try_wait() + .map_err(|error| format!("check Firecracker process: {error}"))? + { + return Err(format!( + "Firecracker exited before its API socket was ready: {status}" + )); + } + if Instant::now() >= deadline { + return Err(format!( + "timed out waiting for Firecracker API socket {}", + socket.display() + )); + } + std::thread::sleep(Duration::from_millis(20)); + } +} + +fn put_json(socket: &Path, path: &str, body: &Value) -> Result<(), String> { + let body = serde_json::to_vec(body).map_err(|error| format!("encode {path}: {error}"))?; + let mut stream = UnixStream::connect(socket) + .map_err(|error| format!("connect to Firecracker API {}: {error}", socket.display()))?; + stream + .set_read_timeout(Some(API_IO_TIMEOUT)) + .map_err(|error| format!("set API read timeout: {error}"))?; + stream + .set_write_timeout(Some(API_IO_TIMEOUT)) + .map_err(|error| format!("set API write timeout: {error}"))?; + write!( + stream, + "PUT {path} HTTP/1.1\r\nHost: localhost\r\nAccept: application/json\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .and_then(|()| stream.write_all(&body)) + .map_err(|error| format!("write Firecracker API request {path}: {error}"))?; + let response = read_http_response(&mut stream, path)?; + check_http_status(path, &response) +} + +fn read_http_response(stream: &mut UnixStream, path: &str) -> Result, String> { + let mut response = Vec::with_capacity(1024); + let mut buffer = [0_u8; 4096]; + loop { + let count = stream + .read(&mut buffer) + .map_err(|error| format!("read Firecracker API response {path}: {error}"))?; + if count == 0 { + return if response.is_empty() { + Err(format!("empty Firecracker API response for {path}")) + } else { + Ok(response) + }; + } + response.extend_from_slice(&buffer[..count]); + if response.len() > MAX_API_RESPONSE_BYTES { + return Err(format!( + "Firecracker API response for {path} exceeds {MAX_API_RESPONSE_BYTES} bytes" + )); + } + let Some(header_end) = response.windows(4).position(|window| window == b"\r\n\r\n") else { + continue; + }; + let body_start = header_end + 4; + let headers = String::from_utf8_lossy(&response[..header_end]); + let content_length = headers.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }); + match content_length { + Some(length) if response.len() >= body_start + length => { + response.truncate(body_start + length); + return Ok(response); + } + Some(_) => {} + None => return Ok(response), + } + } +} + +fn check_http_status(path: &str, response: &[u8]) -> Result<(), String> { + let text = String::from_utf8_lossy(response); + let status_line = text + .lines() + .next() + .ok_or_else(|| format!("empty Firecracker API response for {path}"))?; + let status = status_line + .split_whitespace() + .nth(1) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| format!("invalid Firecracker API response for {path}: {status_line}"))?; + if (200..300).contains(&status) { + Ok(()) + } else { + let body = text.split_once("\r\n\r\n").map_or("", |(_, body)| body); + Err(format!( + "Firecracker API request {path} failed with status {status}: {}", + body.trim() + )) + } +} + +fn remove_stale_socket(path: &Path) -> Result<(), String> { + match std::fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(format!("remove stale socket {}: {error}", path.display())), + } +} + +fn terminate_child(child: &mut Child) { + let _ = child.kill(); + let _ = child.wait(); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config() -> FirecrackerLaunchConfig { + FirecrackerLaunchConfig { + firecracker_binary: PathBuf::from("/firecracker"), + kernel_image: PathBuf::from("/vmlinux"), + root_disk: PathBuf::from("/root.ext4"), + run_dir: PathBuf::from("/run/firecracker"), + console_output: PathBuf::from("/run/firecracker/console.log"), + guest_init: "/opt/openshell/bin/openshell-driver-firecracker".to_string(), + vcpus: 2, + mem_mib: 512, + vsock_cid: 4, + } + } + + #[test] + fn configures_no_network_device() { + let config = config(); + let requests = configuration_requests(&config, Path::new("/tmp/vsock.sock")); + let paths = requests + .iter() + .map(|request| request.path) + .collect::>(); + assert_eq!( + paths, + [ + "/machine-config", + "/boot-source", + "/drives/rootfs", + "/vsock", + "/actions" + ] + ); + assert!( + !requests + .iter() + .any(|request| request.path == "/network-interfaces") + ); + assert_eq!(requests[2].body["is_read_only"], false); + } + + #[test] + fn boot_source_uses_driver_as_init() { + let command_line = kernel_command_line("/opt/openshell/bin/openshell-driver-firecracker"); + assert!(command_line.contains("root=/dev/vda rw")); + assert!(command_line.contains("init=/opt/openshell/bin/openshell-driver-firecracker")); + } + + #[test] + fn accepts_only_successful_api_statuses() { + assert!(check_http_status("/actions", b"HTTP/1.1 204 No Content\r\n\r\n").is_ok()); + assert!(check_http_status("/actions", b"HTTP/1.1 400 Bad\r\n\r\nnope").is_err()); + } +} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 2c38967e06..08bbe3945b 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } openshell-isolation = { path = "../openshell-isolation" } +openshell-driver-firecracker = { path = "../openshell-driver-firecracker" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-supervisor-network = { path = "../openshell-supervisor-network" } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index ef74fa5f24..8eb8688b1a 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -362,32 +362,46 @@ pub async fn run_sandbox( let mediation_ready = Arc::new(AtomicBool::new(false)); let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); let proxy_bind_ip = Arc::new(std::sync::Mutex::new(None)); - let backend = Arc::new(inpod::InPodBackend::new(inpod::InPodConfig { - require_exclusive_pid_namespace: true, - network_enabled, - process_enabled, - entrypoint_pid: entrypoint_pid.clone(), - provider_credentials: provider_credentials.clone(), - provider_env: std::sync::Mutex::new(provider_env), - process_enforcement_mode, - resolved_process_identity, - agent_proposals: agent_proposals.clone(), - openshell_endpoint: openshell_endpoint_for_proxy.clone(), - ssh_socket_path, - #[cfg(target_os = "linux")] - bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx), - #[cfg(target_os = "linux")] - bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx), - mediation_ready: mediation_ready.clone(), - ca_file_paths: ca_file_paths.clone(), - proxy_bind_ip: proxy_bind_ip.clone(), - })); + let admitted_backend_name = descriptor.backend_name.clone(); + let backend: Arc = + match admitted_backend_name.as_str() { + inpod::IN_POD_BACKEND_NAME => { + Arc::new(inpod::InPodBackend::new(inpod::InPodConfig { + require_exclusive_pid_namespace: true, + network_enabled, + process_enabled, + entrypoint_pid: entrypoint_pid.clone(), + provider_credentials: provider_credentials.clone(), + provider_env: std::sync::Mutex::new(provider_env), + process_enforcement_mode, + resolved_process_identity, + agent_proposals: agent_proposals.clone(), + openshell_endpoint: openshell_endpoint_for_proxy.clone(), + ssh_socket_path, + #[cfg(target_os = "linux")] + bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx), + #[cfg(target_os = "linux")] + bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx), + mediation_ready: mediation_ready.clone(), + ca_file_paths: ca_file_paths.clone(), + proxy_bind_ip: proxy_bind_ip.clone(), + })) + } + openshell_driver_firecracker::BACKEND_NAME => { + Arc::new(openshell_driver_firecracker::FirecrackerHostBackend) + } + other => { + return Err(miette::miette!( + "unsupported admitted isolation backend {other:?}" + )); + } + }; let mut registry = openshell_isolation::contract::BackendRegistry::new(); registry .register(backend) .map_err(|error| miette::miette!(error.to_string()))?; let (backend, verified) = registry - .resolve(descriptor, inpod::IN_POD_BACKEND_NAME) + .resolve(descriptor, &admitted_backend_name) .map_err(|error| miette::miette!(error.to_string()))?; let context = openshell_isolation::contract::SandboxContext { sandbox_id: sandbox_id.clone().unwrap_or_default(), @@ -565,14 +579,15 @@ pub async fn run_sandbox( drop(running); drop(networking); - // This admitted topology exclusively owns its PID namespace and the + // The in-pod topology exclusively owns its PID namespace and the // supervisor is PID 1. Exiting namespace init is the kernel's - // race-free whole-boundary teardown: no workload descendant can - // escape it by changing process group or forking during cleanup. + // race-free whole-boundary teardown. Delegated backends, including + // Firecracker, own their boundary teardown and return normally. #[cfg(target_os = "linux")] - std::process::exit(result?); + if admitted_backend_name == inpod::IN_POD_BACKEND_NAME { + std::process::exit(result?); + } - #[cfg(not(target_os = "linux"))] return result; } diff --git a/e2e/firecracker/README.md b/e2e/firecracker/README.md new file mode 100644 index 0000000000..358a01af10 --- /dev/null +++ b/e2e/firecracker/README.md @@ -0,0 +1,23 @@ +# Firecracker smoke test + +This runner boots a real Firecracker VM, drives the RFC 0012 lifecycle from the +host, and verifies that the guest process leaf runs the admitted command through +the existing OpenShell process supervisor. + +It does not use `sudo`, a TAP interface, a guest NIC, nftables, or +`CAP_NET_ADMIN`. The current login session must have read/write access to +`/dev/kvm`. + +Default fixtures live in `/tmp/openshell-firecracker-e2e-fixtures`. Override +them with: + +```shell +OPENSHELL_FIRECRACKER_BINARY=/path/to/firecracker \ +OPENSHELL_FIRECRACKER_KERNEL_IMAGE=/path/to/vmlinux \ +OPENSHELL_FIRECRACKER_ROOT_DISK=/path/to/root.ext4 \ +mise run e2e:firecracker +``` + +The runner clones the root disk into a temporary directory and injects the +current `openshell-driver-firecracker` binary plus a one-time authenticated +guest configuration. It never modifies the source fixture. diff --git a/e2e/firecracker/smoke.sh b/e2e/firecracker/smoke.sh new file mode 100755 index 0000000000..1b2734fe59 --- /dev/null +++ b/e2e/firecracker/smoke.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Boot the host-supervised Firecracker backend and exercise its RFC 0012 +# lifecycle against the current process supervisor implementation in the guest. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +FIXTURE_DIR="${OPENSHELL_FIRECRACKER_FIXTURE_DIR:-/tmp/openshell-firecracker-e2e-fixtures}" +DRIVER_BIN="${OPENSHELL_FIRECRACKER_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-firecracker}" +FIRECRACKER_BIN="${OPENSHELL_FIRECRACKER_BINARY:-${FIXTURE_DIR}/release/release-v1.16.1-aarch64/firecracker-v1.16.1-aarch64}" +KERNEL_IMAGE="${OPENSHELL_FIRECRACKER_KERNEL_IMAGE:-${FIXTURE_DIR}/vmlinux-6.1.155}" +ROOT_DISK_FIXTURE="${OPENSHELL_FIRECRACKER_ROOT_DISK:-${FIXTURE_DIR}/openshell-ubuntu-24.04.ext4}" +BOOT_TIMEOUT_SECONDS="${OPENSHELL_FIRECRACKER_BOOT_TIMEOUT_SECONDS:-60}" +KEEP_STATE="${OPENSHELL_FIRECRACKER_KEEP_STATE:-0}" +CONTROL_PORT="${OPENSHELL_FIRECRACKER_CONTROL_PORT:-5500}" + +RUN_DIR="$(mktemp -d /tmp/openshell-firecracker-e2e.XXXXXX)" +ROOT_DISK="${RUN_DIR}/root.ext4" +TOKEN_FILE="${RUN_DIR}/bootstrap.token" +GUEST_CONFIG="${RUN_DIR}/firecracker.json" +CONSOLE_LOG="${RUN_DIR}/console.log" +LAUNCHER_LOG="${RUN_DIR}/launcher.log" +SUPERVISOR_LOG="${RUN_DIR}/supervisor.log" +VSOCK_SOCKET="${RUN_DIR}/firecracker-vsock.sock" +LAUNCHER_PID="" + +BOUNDARY_ID="firecracker-e2e-$$" +VSOCK_CID=$(( ($$ % 60000) + 1024 )) +SMOKE_MARKER="openshell-firecracker-e2e-ok-${BOUNDARY_ID}" + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +launcher_is_running() { + [ -n "${LAUNCHER_PID}" ] && kill -0 "${LAUNCHER_PID}" 2>/dev/null +} + +print_logs() { + local log + for log in "${LAUNCHER_LOG}" "${SUPERVISOR_LOG}" "${CONSOLE_LOG}"; do + if [ -s "${log}" ]; then + echo "=== ${log} ===" >&2 + tail -n 200 "${log}" >&2 || true + echo "=== end ${log} ===" >&2 + fi + done +} + +cleanup() { + local exit_code=$? + set +e + if launcher_is_running; then + kill -TERM "${LAUNCHER_PID}" 2>/dev/null + for _ in 1 2 3 4 5 6 7 8 9 10; do + launcher_is_running || break + sleep 0.2 + done + launcher_is_running && kill -KILL "${LAUNCHER_PID}" 2>/dev/null + fi + [ -n "${LAUNCHER_PID}" ] && wait "${LAUNCHER_PID}" 2>/dev/null + [ "${exit_code}" -eq 0 ] || print_logs + if [ "${KEEP_STATE}" = "1" ]; then + echo "Firecracker E2E state preserved at ${RUN_DIR}" >&2 + else + case "${RUN_DIR}" in + /tmp/openshell-firecracker-e2e.*) rm -rf -- "${RUN_DIR}" ;; + *) echo "Refusing to remove unexpected run directory: ${RUN_DIR}" >&2 ;; + esac + fi + exit "${exit_code}" +} +trap cleanup EXIT + +wait_for_console_marker() { + local marker="$1" + local label="$2" + local deadline=$(( SECONDS + BOOT_TIMEOUT_SECONDS )) + while [ "${SECONDS}" -lt "${deadline}" ]; do + grep -Fq -- "${marker}" "${CONSOLE_LOG}" 2>/dev/null && return 0 + launcher_is_running || fail "Firecracker exited while waiting for ${label}" + sleep 0.2 + done + fail "timed out after ${BOOT_TIMEOUT_SECONDS}s waiting for ${label}" +} + +inject_file() { + local source="$1" + local destination="$2" + debugfs -w -R "rm ${destination}" "${ROOT_DISK}" >/dev/null 2>&1 || true + debugfs -w -R "write ${source} ${destination}" "${ROOT_DISK}" >/dev/null +} + +[ "$(uname -s)" = "Linux" ] || fail "Firecracker E2E requires Linux" +[ -r /dev/kvm ] && [ -w /dev/kvm ] || \ + fail "/dev/kvm is not readable and writable; start a login session with the kvm supplementary group" +[ -x "${FIRECRACKER_BIN}" ] || fail "Firecracker binary is not executable: ${FIRECRACKER_BIN}" +[ -f "${KERNEL_IMAGE}" ] || fail "kernel image not found: ${KERNEL_IMAGE}" +[ -f "${ROOT_DISK_FIXTURE}" ] || fail "root disk fixture not found: ${ROOT_DISK_FIXTURE}" +[[ "${BOOT_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] || fail "boot timeout must be positive" +[[ "${CONTROL_PORT}" =~ ^[1-9][0-9]*$ ]] || fail "control port must be positive" + +for tool in cargo cp debugfs grep jq openssl tail; do + command -v "${tool}" >/dev/null 2>&1 || fail "required host tool not found: ${tool}" +done + +if [ -n "${RUSTC_WRAPPER:-}" ] && [ "${OPENSHELL_E2E_FIRECRACKER_ALLOW_RUSTC_WRAPPER:-0}" != "1" ]; then + unset RUSTC_WRAPPER +fi +if [ -z "${OPENSHELL_FIRECRACKER_DRIVER_BIN:-}" ]; then + echo "==> Building standalone Firecracker driver" + cargo build --package openshell-driver-firecracker +fi +[ -x "${DRIVER_BIN}" ] || fail "driver binary not found: ${DRIVER_BIN}" + +echo "==> Preparing an isolated guest disk" +cp --reflink=auto "${ROOT_DISK_FIXTURE}" "${ROOT_DISK}" +umask 077 +openssl rand -hex 32 >"${TOKEN_FILE}" +jq -n \ + --arg boundary_id "${BOUNDARY_ID}" \ + --arg bootstrap_token "$(tr -d '\n' <"${TOKEN_FILE}")" \ + --argjson control_port "${CONTROL_PORT}" \ + '{boundary_id: $boundary_id, bootstrap_token: $bootstrap_token, control_port: $control_port}' \ + >"${GUEST_CONFIG}" + +debugfs -w -R "mkdir /etc/openshell" "${ROOT_DISK}" >/dev/null 2>&1 || true +inject_file "${GUEST_CONFIG}" /etc/openshell/firecracker.json +inject_file "${DRIVER_BIN}" /opt/openshell/bin/openshell-driver-firecracker +debugfs -w -R \ + "set_inode_field /opt/openshell/bin/openshell-driver-firecracker mode 0100755" \ + "${ROOT_DISK}" >/dev/null + +echo "==> Starting Firecracker guest ${BOUNDARY_ID} (vsock only, no NIC)" +"${DRIVER_BIN}" launch \ + --firecracker-binary "${FIRECRACKER_BIN}" \ + --kernel-image "${KERNEL_IMAGE}" \ + --root-disk "${ROOT_DISK}" \ + --run-dir "${RUN_DIR}" \ + --console-output "${CONSOLE_LOG}" \ + --vsock-cid "${VSOCK_CID}" \ + >"${LAUNCHER_LOG}" 2>&1 & +LAUNCHER_PID=$! + +wait_for_console_marker \ + "Firecracker process supervisor leaf listening on vsock port ${CONTROL_PORT}" \ + "guest process leaf" +[ -S "${VSOCK_SOCKET}" ] || fail "Firecracker vsock UDS was not created: ${VSOCK_SOCKET}" + +echo "==> Driving attach, confirm, start_agent, and wait from the host" +"${DRIVER_BIN}" supervise \ + --boundary-id "${BOUNDARY_ID}" \ + --vsock-uds-path "${VSOCK_SOCKET}" \ + --vsock-port "${CONTROL_PORT}" \ + --bootstrap-token-file "${TOKEN_FILE}" \ + --workdir /sandbox \ + --timeout-seconds 30 \ + -- /bin/sh -lc "printf '%s\\n' '${SMOKE_MARKER}'" \ + >"${SUPERVISOR_LOG}" 2>&1 + +grep -Fq 'agent exited: Exited(0)' "${SUPERVISOR_LOG}" || \ + fail "host supervisor did not observe a successful agent exit" +wait_for_console_marker "${SMOKE_MARKER}" "agent output" + +echo "==> Firecracker host-supervisor E2E passed" +echo " boundary: ${BOUNDARY_ID}" +echo " lifecycle: attach -> confirm -> start_agent -> wait" +echo " process leaf: openshell-supervisor-process" +echo " transport: authenticated Firecracker virtio-vsock" +echo " network: no guest NIC" diff --git a/tasks/gateway.toml b/tasks/gateway.toml index 83cf35d8fb..65d1121834 100644 --- a/tasks/gateway.toml +++ b/tasks/gateway.toml @@ -19,3 +19,7 @@ run = "bash tasks/scripts/gateway-docker.sh" ["gateway:vm"] description = "Run a standalone gateway with the bundled VM compute driver" run = "bash tasks/scripts/gateway-vm.sh" + +["gateway:firecracker"] +description = "Run a standalone gateway with the experimental Firecracker compute driver" +run = "bash tasks/scripts/gateway-firecracker.sh" diff --git a/tasks/scripts/gateway-firecracker.sh b/tasks/scripts/gateway-firecracker.sh new file mode 100755 index 0000000000..402afa3c4d --- /dev/null +++ b/tasks/scripts/gateway-firecracker.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Start a plaintext local gateway backed by the experimental external +# Firecracker compute driver. The logical openshell-sandbox supervisor runs on +# the host; each workload runs in a no-NIC Firecracker VM. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PORT="${OPENSHELL_SERVER_PORT:-18082}" +GATEWAY_NAME="${OPENSHELL_FIRECRACKER_GATEWAY_NAME:-firecracker-dev}" +STATE_DIR="${OPENSHELL_FIRECRACKER_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-firecracker}" +FIXTURE_DIR="${OPENSHELL_FIRECRACKER_FIXTURE_DIR:-/tmp/openshell-firecracker-e2e-fixtures}" +FIRECRACKER_BIN="${OPENSHELL_FIRECRACKER_BINARY:-${FIXTURE_DIR}/release/release-v1.16.1-aarch64/firecracker-v1.16.1-aarch64}" +KERNEL_IMAGE="${OPENSHELL_FIRECRACKER_KERNEL_IMAGE:-${FIXTURE_DIR}/vmlinux-6.1.155}" +ROOT_DISK="${OPENSHELL_FIRECRACKER_ROOT_DISK:-${FIXTURE_DIR}/openshell-ubuntu-24.04.ext4}" +GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +DRIVER_BIN="${ROOT}/target/debug/openshell-driver-firecracker" +SUPERVISOR_BIN="${ROOT}/target/debug/openshell-sandbox" +LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" +STATE_LABEL="$(printf '%s' "${GATEWAY_NAME}" | tr -cs '[:alnum:]._-' '-')" +DRIVER_STATE_DIR="${OPENSHELL_FIRECRACKER_STATE_DIR:-/tmp/openshell-firecracker-${USER:-user}-${STATE_LABEL}}" +DRIVER_SOCKET="${DRIVER_STATE_DIR}/compute-driver.sock" +DRIVER_LOG="${DRIVER_STATE_DIR}/driver.log" +GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" +DRIVER_PID="" + +ensure_kvm_access() { + [ -e /dev/kvm ] || fail "/dev/kvm does not exist; enable KVM on this host" + if [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + return 0 + fi + if [ "${OPENSHELL_FIRECRACKER_KVM_REEXEC:-0}" != "1" ] \ + && command -v sg >/dev/null 2>&1 \ + && [[ " $(id -nG "$(id -un)") " == *" kvm "* ]]; then + echo "==> Entering the configured kvm group for the Firecracker gateway" + export OPENSHELL_FIRECRACKER_KVM_REEXEC=1 + export OPENSHELL_FIRECRACKER_GATEWAY_SCRIPT="${ROOT}/tasks/scripts/gateway-firecracker.sh" + exec sg kvm -c 'exec "$OPENSHELL_FIRECRACKER_GATEWAY_SCRIPT"' + fi + fail "/dev/kvm is not readable and writable; add $(id -un) to the kvm group" +} + +configure_bindgen_include() { + local gcc_include + command -v gcc >/dev/null 2>&1 || return 0 + gcc_include="$(gcc -print-file-name=include)" + [ -f "${gcc_include}/stdbool.h" ] || return 0 + case " ${BINDGEN_EXTRA_CLANG_ARGS:-} " in + *" -isystem ${gcc_include} "*) ;; + *) + export BINDGEN_EXTRA_CLANG_ARGS="${BINDGEN_EXTRA_CLANG_ARGS:+${BINDGEN_EXTRA_CLANG_ARGS} }-isystem ${gcc_include}" + ;; + esac +} + +fail() { + echo "ERROR: $*" >&2 + exit 1 +} + +port_is_in_use() { + local port=$1 + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + return $? + fi + if command -v nc >/dev/null 2>&1; then + nc -z 127.0.0.1 "${port}" >/dev/null 2>&1 + return $? + fi + (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 +} + +cleanup() { + local exit_code=$? + if [ -n "${DRIVER_PID}" ] && kill -0 "${DRIVER_PID}" 2>/dev/null; then + kill -TERM "${DRIVER_PID}" 2>/dev/null || true + wait "${DRIVER_PID}" 2>/dev/null || true + fi + exit "${exit_code}" +} +trap cleanup EXIT + +register_gateway() { + local config_home gateway_dir + config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" + gateway_dir="${config_home}/openshell/gateways/${GATEWAY_NAME}" + mkdir -p "${gateway_dir}" + chmod 700 "${gateway_dir}" 2>/dev/null || true + cat >"${gateway_dir}/metadata.json" </dev/null || true + printf '%s' "${GATEWAY_NAME}" >"${config_home}/openshell/active_gateway" +} + +[ "$(uname -s)" = "Linux" ] || fail "gateway:firecracker requires Linux" +ensure_kvm_access +configure_bindgen_include +[ -x "${FIRECRACKER_BIN}" ] || fail "Firecracker binary is not executable: ${FIRECRACKER_BIN}" +[ -f "${KERNEL_IMAGE}" ] || fail "kernel image not found: ${KERNEL_IMAGE}" +[ -f "${ROOT_DISK}" ] || fail "root disk fixture not found: ${ROOT_DISK}" +command -v debugfs >/dev/null 2>&1 || fail "debugfs is required" +port_is_in_use "${PORT}" && fail "port ${PORT} is already in use; set OPENSHELL_SERVER_PORT" + +echo "==> Building gateway, host supervisor, and Firecracker driver" +cargo build -p openshell-server -p openshell-sandbox -p openshell-driver-firecracker + +mkdir -p "${STATE_DIR}" "${DRIVER_STATE_DIR}" +chmod 700 "${DRIVER_STATE_DIR}" +TLS_DIR="${STATE_DIR}/tls" +echo "==> Generating local gateway credentials" +"${GATEWAY_BIN}" generate-certs \ + --output-dir "${TLS_DIR}" \ + --server-san 127.0.0.1 \ + --server-san localhost + +CONFIG_PATH="${STATE_DIR}/gateway.toml" +install -m 600 /dev/null "${CONFIG_PATH}" +cat >"${CONFIG_PATH}" < Starting Firecracker compute driver" +"${DRIVER_BIN}" compute-driver \ + --bind-socket "${DRIVER_SOCKET}" \ + --gateway-endpoint "${GATEWAY_ENDPOINT}" \ + --state-dir "${DRIVER_STATE_DIR}" \ + --firecracker-binary "${FIRECRACKER_BIN}" \ + --kernel-image "${KERNEL_IMAGE}" \ + --root-disk "${ROOT_DISK}" \ + --supervisor-binary "${SUPERVISOR_BIN}" \ + >"${DRIVER_LOG}" 2>&1 & +DRIVER_PID=$! + +for _ in $(seq 1 100); do + [ -S "${DRIVER_SOCKET}" ] && break + kill -0 "${DRIVER_PID}" 2>/dev/null || { + tail -n 200 "${DRIVER_LOG}" >&2 || true + fail "Firecracker compute driver exited before creating its socket" + } + sleep 0.1 +done +[ -S "${DRIVER_SOCKET}" ] || fail "timed out waiting for ${DRIVER_SOCKET}" + +register_gateway +echo "Starting standalone Firecracker gateway..." +echo " gateway: ${GATEWAY_NAME}" +echo " endpoint: ${GATEWAY_ENDPOINT}" +echo " driver socket: ${DRIVER_SOCKET}" +echo " driver log: ${DRIVER_LOG}" +echo " topology: host supervisor + no-NIC Firecracker workload VM" +echo + +exec "${GATEWAY_BIN}" \ + --config "${CONFIG_PATH}" \ + --port "${PORT}" \ + --log-level "${LOG_LEVEL}" \ + --drivers firecracker \ + --disable-tls \ + --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" diff --git a/tasks/test.toml b/tasks/test.toml index ceb1c30086..1640fb5592 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -155,6 +155,10 @@ run = "e2e/rust/e2e-kubernetes.sh" description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" +["e2e:firecracker"] +description = "Boot Firecracker and run the host-supervised RFC 0012 lifecycle smoke test" +run = "e2e/firecracker/smoke.sh" + ["e2e:docker"] description = "Run smoke e2e against a standalone gateway with the Docker compute driver" run = "e2e/rust/e2e-docker.sh"