From 1c8c93006da362cfeaad1be80470a56a5badb58e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 25 Aug 2026 12:18:32 +0200 Subject: [PATCH 1/2] test(e2e): isolate VM-specific smoke assertions Signed-off-by: Evan Lezar --- TESTING.md | 3 ++ e2e/rust/Cargo.toml | 5 ++++ e2e/rust/e2e-vm.sh | 2 +- e2e/rust/tests/smoke.rs | 42 ---------------------------- e2e/rust/tests/vm_overlay.rs | 54 ++++++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 43 deletions(-) create mode 100644 e2e/rust/tests/vm_overlay.rs diff --git a/TESTING.md b/TESTING.md index 6c0829060d..1befde66b8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -153,6 +153,9 @@ Suites: - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage. +VM overlay and TLS-key permission assertions run only in the VM suite; the +driver-neutral smoke test does not include them. + GPU device-selection tests compare OpenShell sandboxes against a plain Docker or Podman container that requests `--device nvidia.com/gpu=all`. The probe image defaults to the image used by the `gateway` stage in diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 42f989ce42..faf21fe485 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -41,6 +41,11 @@ name = "oidc_pkce" path = "tests/oidc_pkce.rs" required-features = ["e2e-oidc-pkce"] +[[test]] +name = "vm_overlay" +path = "tests/vm_overlay.rs" +required-features = ["e2e-vm"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 1960f83588..8ba001a552 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -371,7 +371,6 @@ fi # The CLI uses the raw endpoint but still resolves matching metadata so it # can find the mTLS client bundle. -export OPENSHELL_E2E_EXPECT_VM_OVERLAY=1 export OPENSHELL_E2E_DRIVER="vm" export OPENSHELL_E2E_VM_STATE_DIR="${RUN_STATE_DIR}" e2e_export_gateway_restart_metadata \ @@ -405,5 +404,6 @@ if [ -n "${E2E_TEST_OVERRIDE}" ]; then else run_e2e_test smoke run_e2e_test host_gateway_alias + run_e2e_test vm_overlay run_e2e_test vm_gateway_start fi diff --git a/e2e/rust/tests/smoke.rs b/e2e/rust/tests/smoke.rs index c27255e5ef..172afa22b8 100644 --- a/e2e/rust/tests/smoke.rs +++ b/e2e/rust/tests/smoke.rs @@ -68,10 +68,6 @@ async fn gateway_smoke() { sb.create_output, ); - if std::env::var_os("OPENSHELL_E2E_EXPECT_VM_OVERLAY").is_some() { - assert_vm_overlay_root(&sb.name).await; - } - // ── 3. Verify the sandbox appeared in the list ─────────────────── let mut list_cmd = openshell_cmd(); list_cmd @@ -99,41 +95,3 @@ async fn gateway_smoke() { // ── 4. Cleanup ─────────────────────────────────────────────────── sb.cleanup().await; } - -async fn assert_vm_overlay_root(sandbox_name: &str) { - let script = concat!( - "set -eu; ", - "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", - "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", - "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", - "if [ -e /opt/openshell/tls/tls.key ]; then ", - "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", - "fi; ", - "echo vm-overlay-ok", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .arg("sh") - .arg("-lc") - .arg(script) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = exec_cmd - .output() - .await - .expect("failed to run VM overlay assertion"); - let combined = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert!( - output.status.success() && combined.contains("vm-overlay-ok"), - "VM overlay assertion failed (status {:?}):\n{combined}", - output.status.code(), - ); -} diff --git a/e2e/rust/tests/vm_overlay.rs b/e2e/rust/tests/vm_overlay.rs new file mode 100644 index 0000000000..078c6f01de --- /dev/null +++ b/e2e/rust/tests/vm_overlay.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM-driver-specific assertions for the sandbox root filesystem. + +use std::process::Stdio; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +#[tokio::test] +async fn vm_overlay() { + let mut sandbox = SandboxGuard::create(&["--", "echo", "vm-sandbox-ready"]) + .await + .expect("sandbox create should succeed"); + + let script = concat!( + "set -eu; ", + "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", + "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", + "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", + "if [ -e /opt/openshell/tls/tls.key ]; then ", + "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", + "fi; ", + "echo vm-overlay-ok", + ); + + let mut exec_cmd = openshell_cmd(); + exec_cmd + .args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"]) + .arg("sh") + .arg("-lc") + .arg(script) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = exec_cmd + .output() + .await + .expect("failed to run VM overlay assertion"); + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + assert!( + output.status.success() && combined.contains("vm-overlay-ok"), + "VM overlay assertion failed (status {:?}):\n{combined}", + output.status.code(), + ); + + sandbox.cleanup().await; +} From a257f6ab9622edac20077302d9b29329874c7d78 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 25 Aug 2026 12:21:10 +0200 Subject: [PATCH 2/2] test(e2e): add portable CLI conformance baseline Signed-off-by: Evan Lezar --- TESTING.md | 46 +- e2e/rust/Cargo.toml | 10 +- e2e/rust/e2e-docker.sh | 2 +- e2e/rust/e2e-kubernetes.sh | 1 + e2e/rust/src/harness/conformance.rs | 952 +++++++++++++++++++ e2e/rust/src/harness/conformance/executor.rs | 52 + e2e/rust/src/harness/mod.rs | 1 + e2e/rust/tests/smoke.rs | 278 ++++-- e2e/with-docker-gateway.sh | 1 + e2e/with-podman-gateway.sh | 1 + tasks/test.toml | 8 + 11 files changed, 1266 insertions(+), 86 deletions(-) create mode 100644 e2e/rust/src/harness/conformance.rs create mode 100644 e2e/rust/src/harness/conformance/executor.rs diff --git a/TESTING.md b/TESTING.md index 1befde66b8..863599875b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -148,14 +148,16 @@ lifecycle management, output parsing, and cleanup. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. -- Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. +- CLI conformance (`--features e2e-cli-conformance`) - the portable deployment + smoke scenario plus focused tests for its reusable command runner. +- Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or + `e2e-vm`) - CLI conformance plus the common and driver-specific coverage for + the selected deployment. +- Docker suite (`--features e2e-docker`) - includes Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage. -VM overlay and TLS-key permission assertions run only in the VM suite; the -driver-neutral smoke test does not include them. - GPU device-selection tests compare OpenShell sandboxes against a plain Docker or Podman container that requests `--device nvidia.com/gpu=all`. The probe image defaults to the image used by the `gateway` stage in @@ -172,6 +174,42 @@ Run the Docker-backed Rust CLI e2e suite: mise run e2e:rust ``` +Run the minimal portable CLI conformance profile against the gateway selected +in your OpenShell CLI configuration: + +```shell +mise run e2e:cli-conformance +``` + +The gateway must already be installed, reachable, and selected before the task +starts. The task does not provision a gateway or select a compute driver. Set +`OPENSHELL_BIN` to test a prebuilt CLI; otherwise, the task builds the CLI from +the current checkout. + +The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without +depending on how the gateway was installed or which driver is configured. It +requires machine-readable gRPC status, creates a uniquely named detached +sandbox with `--from base`, verifies the sandbox is `Ready` by finding its +unique name in paginated JSON list output, executes `echo` with a run-specific +marker, deletes the sandbox, and verifies that its name no longer appears. +Driver suites enable the same profile +instead of maintaining a separate smoke implementation. Sandbox lifecycle, +label matrices, VM overlay, and TLS-key permission assertions remain regular +E2E coverage. + +Each invocation prints a ten-character run ID before creating resources. +Conformance sandboxes use names such as `ct--01`. The runner tracks the +exact name and uses it for cleanup; phase 1 does not add ownership labels. + +The runner deletes owned resources after both success and failure. If the test +process is interrupted before cleanup, locate leftovers without touching +unrelated gateway state: + +```shell +openshell sandbox list --output json +openshell sandbox delete +``` + Run the Podman-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index faf21fe485..98e4d5cd4a 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -16,7 +16,10 @@ license = "Apache-2.0" publish = false [features] -e2e = [] +# Selects the minimal portable CLI conformance profile. +e2e-cli-conformance = [] +# Selects the common E2E suite, including the CLI conformance baseline. +e2e = ["e2e-cli-conformance"] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the # alias unconditionally; the kube driver only does so when the chart's @@ -41,6 +44,11 @@ name = "oidc_pkce" path = "tests/oidc_pkce.rs" required-features = ["e2e-oidc-pkce"] +[[test]] +name = "smoke" +path = "tests/smoke.rs" +required-features = ["e2e-cli-conformance"] + [[test]] name = "vm_overlay" path = "tests/vm_overlay.rs" diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 6c28868083..d43cd58471 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}" -E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e,e2e-docker}" +E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e-docker}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && [ ! -f "${DEFAULT_WORKLOAD_MANIFEST}" ]; then diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index cf28e35728..1ed81132f3 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -26,6 +26,7 @@ E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kube # local CLI needs to be built when CI has not supplied a prebuilt one. if [ -z "${OPENSHELL_BIN:-}" ]; then cargo build -p openshell-cli + export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi test_filter=() diff --git a/e2e/rust/src/harness/conformance.rs b/e2e/rust/src/harness/conformance.rs new file mode 100644 index 0000000000..8b8498260b --- /dev/null +++ b/e2e/rust/src/harness/conformance.rs @@ -0,0 +1,952 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reusable support for portable `OpenShell` CLI conformance scenarios. + +mod executor; + +use std::collections::BTreeSet; +use std::error::Error; +use std::fmt; +use std::path::PathBuf; +use std::process::ExitStatus; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use rand::Rng; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use tokio::time::sleep; + +use self::executor::{CliExecutionError, CliExecutor, ProcessCli}; + +const CLEANUP_TIMEOUT: Duration = Duration::from_secs(120); +pub const STATUS_TIMEOUT: Duration = Duration::from_secs(30); +const GATEWAY_STATUS_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); +const GATEWAY_STATUS_INTERVAL: Duration = Duration::from_secs(2); + +/// Generate the suite-owned identifier used in resource names and diagnostics. +fn generate_run_id() -> String { + const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + let mut rng = rand::rng(); + (0..10) + .map(|_| ALPHABET[rng.random_range(0..ALPHABET.len())] as char) + .collect() +} + +/// The completed outcome of one `OpenShell` CLI process. +#[derive(Debug)] +pub struct CommandResult { + run_id: String, + scenario: String, + step: String, + expectation: String, + command: String, + status: ExitStatus, + elapsed: Duration, + stdout: String, + stderr: String, +} + +impl CommandResult { + pub fn success(&self) -> bool { + self.status.success() + } + + pub fn exit_code(&self) -> Option { + self.status.code() + } + + pub fn stdout(&self) -> &str { + &self.stdout + } + + pub fn stderr(&self) -> &str { + &self.stderr + } + + pub fn elapsed(&self) -> Duration { + self.elapsed + } + + pub fn json(&self) -> Result { + serde_json::from_str(&self.stdout).map_err(|source| RunnerError::InvalidJson { + context: self.context(), + source, + stdout: self.stdout.clone(), + stderr: self.stderr.clone(), + }) + } + + pub fn require_success(&self) -> Result<(), String> { + if self.success() { + return Ok(()); + } + Err(self.failure_diagnostic(&self.expectation)) + } + + pub fn failure_diagnostic(&self, expectation: &str) -> String { + format!( + "{}\nexpected: {expectation}\nactual: exit {} after {:.1?}\ncommand: {}\nstdout:\n{}\nstderr:\n{}", + self.context(), + exit_description(self.status), + self.elapsed, + self.command, + self.stdout, + self.stderr, + ) + } + + fn context(&self) -> String { + format!("[run {}][{}/{}]", self.run_id, self.scenario, self.step) + } +} + +/// Failures in reusable runner mechanics rather than scenario assertions. +#[derive(Debug)] +pub enum RunnerError { + BinaryUnavailable(String), + Spawn { + context: String, + command: String, + source: std::io::Error, + }, + Timeout { + context: String, + command: String, + timeout: Duration, + }, + InvalidJson { + context: String, + source: serde_json::Error, + stdout: String, + stderr: String, + }, + PollTimeout { + context: String, + timeout: Duration, + last_observation: String, + }, + ObservationFailed { + context: String, + diagnostic: String, + }, +} + +impl fmt::Display for RunnerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BinaryUnavailable(message) => write!(f, "{message}"), + Self::Spawn { + context, + command, + source, + } => write!( + f, + "{context} failed to spawn command: {source}\ncommand: {command}" + ), + Self::Timeout { + context, + command, + timeout, + } => write!( + f, + "{context} timed out after {timeout:.1?}\ncommand: {command}" + ), + Self::InvalidJson { + context, + source, + stdout, + stderr, + } => write!( + f, + "{context} returned invalid JSON: {source}\nstdout:\n{stdout}\nstderr:\n{stderr}" + ), + Self::PollTimeout { + context, + timeout, + last_observation, + } => write!( + f, + "{context} did not satisfy the observation within {timeout:.1?}\nlast observation:\n{last_observation}" + ), + Self::ObservationFailed { + context, + diagnostic, + } => write!(f, "{context} observation failed early:\n{diagnostic}"), + } + } +} + +impl Error for RunnerError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Spawn { source, .. } => Some(source), + Self::InvalidJson { source, .. } => Some(source), + _ => None, + } + } +} + +/// Result of one read-only polling observation. +pub enum Poll { + Ready(T), + Pending(String), + Failed(String), +} + +#[derive(Debug, Deserialize)] +struct StatusOutput { + gateway: Option, + server: Option, + status: String, + version: Option, + authentication: Option, +} + +#[derive(Debug, Deserialize)] +struct AuthenticationOutput { + status: String, +} + +/// Runs `OpenShell` commands for one conformance scenario and owns its cleanup. +pub struct OpenShellRunner { + cli: Arc, + run_id: String, + scenario: String, + known_sandboxes: BTreeSet, + finished: bool, +} + +/// A runner command with diagnostic context but no timeout yet. +pub struct CommandStep<'a> { + runner: &'a OpenShellRunner, + step: String, + description: Option, +} + +/// A fully configured runner command ready to execute. +pub struct OpenShellCommand<'a> { + runner: &'a OpenShellRunner, + step: String, + description: String, + timeout: Duration, +} + +impl OpenShellRunner { + pub fn new(scenario: &str) -> Result { + let binary = std::env::var_os("OPENSHELL_BIN") + .map(PathBuf::from) + .ok_or_else(|| { + RunnerError::BinaryUnavailable( + "OPENSHELL_BIN is required for CLI conformance tests".to_string(), + ) + })?; + Self::with_binary(binary, scenario) + } + + fn with_binary(binary: PathBuf, scenario: &str) -> Result { + if !binary.is_file() { + return Err(RunnerError::BinaryUnavailable(format!( + "OpenShell CLI binary not found at {}", + binary.display() + ))); + } + Ok(Self::with_cli(Arc::new(ProcessCli::new(binary)), scenario)) + } + + fn with_cli(cli: Arc, scenario: &str) -> Self { + Self { + cli, + run_id: generate_run_id(), + scenario: scenario.to_string(), + known_sandboxes: BTreeSet::new(), + finished: false, + } + } + + pub fn id(&self) -> &str { + &self.run_id + } + + pub fn scenario(&self) -> &str { + &self.scenario + } + + pub fn step(&self, step: impl Into) -> CommandStep<'_> { + CommandStep { + runner: self, + step: step.into(), + description: None, + } + } + + /// Confirm that the configured gateway is reachable before a scenario runs. + pub async fn check_gateway_status(&mut self) -> Result<(), String> { + let status = self + .poll_until( + "preflight", + STATUS_TIMEOUT, + GATEWAY_STATUS_INTERVAL, + async |runner| match runner + .step("status/json") + .description("machine-readable gateway status succeeds") + .with_timeout(GATEWAY_STATUS_ATTEMPT_TIMEOUT) + .run(&["status", "--output", "json"]) + .await + { + Ok(result) if !result.success() => Poll::Pending( + result.failure_diagnostic("gateway status command succeeds"), + ), + Ok(result) => match result.json::() { + Ok(status) if status.status == "connected" => Poll::Ready(status), + Ok(status) if status.status == "not_configured" => Poll::Failed( + "no active gateway; register and select a gateway before running conformance" + .to_string(), + ), + Ok(status) => Poll::Pending(format!( + "gateway status is {:?}; expected \"connected\"\nstderr:\n{}", + status.status, + result.stderr() + )), + Err(error) => Poll::Failed(error.to_string()), + }, + Err(error) => Poll::Pending(error.to_string()), + }, + ) + .await + .map_err(|error| error.to_string())?; + + println!( + "gateway preflight connected: gateway={}, server={}, version={}, authentication={}", + status.gateway.as_deref().unwrap_or("unknown"), + status.server.as_deref().unwrap_or("unknown"), + status.version.as_deref().unwrap_or("unknown"), + status + .authentication + .as_ref() + .map_or("unknown", |authentication| authentication.status.as_str()), + ); + Ok(()) + } + + /// Register a sandbox name for cleanup. + /// + /// Call this before running the command that may create the sandbox. The + /// scenario remains responsible for constructing and running that command. + pub fn track_sandbox(&mut self, name: &str) { + self.known_sandboxes.insert(name.to_string()); + } + + /// Stop tracking a sandbox after the scenario confirms it is absent. + pub fn forget_sandbox(&mut self, name: &str) { + self.known_sandboxes.remove(name); + } + + pub async fn poll_until( + &mut self, + step: &str, + poll_timeout: Duration, + interval: Duration, + mut observe: F, + ) -> Result + where + F: AsyncFnMut(&mut OpenShellRunner) -> Poll, + { + let started = Instant::now(); + let context = self.context(step); + + loop { + match observe(self).await { + Poll::Ready(value) => return Ok(value), + Poll::Pending(diagnostic) => { + if started.elapsed() >= poll_timeout { + return Err(RunnerError::PollTimeout { + context, + timeout: poll_timeout, + last_observation: diagnostic, + }); + } + } + Poll::Failed(diagnostic) => { + return Err(RunnerError::ObservationFailed { + context, + diagnostic, + }); + } + } + sleep(interval).await; + } + } + + pub async fn finish(mut self, scenario_result: Result<(), String>) -> Result<(), String> { + let cleanup_result = self.cleanup().await; + self.finished = true; + combine_results(scenario_result, cleanup_result) + } + + async fn run_strings( + &self, + step: &str, + expectation: &str, + args: Vec, + command_timeout: Duration, + ) -> Result { + let context = self.context(step); + let command = sanitized_command(&args); + println!("{context} running: {command}"); + + let started = Instant::now(); + let output = + self.cli + .execute(args, command_timeout) + .await + .map_err(|error| match error { + CliExecutionError::Timeout => RunnerError::Timeout { + context: context.clone(), + command: command.clone(), + timeout: command_timeout, + }, + CliExecutionError::Spawn(source) => RunnerError::Spawn { + context: context.clone(), + command: command.clone(), + source, + }, + })?; + let elapsed = started.elapsed(); + println!( + "{context} completed in {:.1?}: exit {}", + elapsed, + exit_description(output.status) + ); + + Ok(CommandResult { + run_id: self.run_id.clone(), + scenario: self.scenario.clone(), + step: step.to_string(), + expectation: expectation.to_string(), + command, + status: output.status, + elapsed, + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }) + } + + async fn cleanup(&mut self) -> Result<(), String> { + if self.known_sandboxes.is_empty() { + return Ok(()); + } + + let cleanup_started = Instant::now(); + let mut failures = Vec::new(); + for name in self.known_sandboxes.clone() { + let remaining = remaining_cleanup_time(cleanup_started); + if remaining.is_zero() { + failures.push(format!( + "{} cleanup budget expired before deleting sandbox '{name}'", + self.context("cleanup/delete") + )); + break; + } + match self + .step("cleanup/delete") + .description(format!("sandbox '{name}' is deleted or already absent")) + .with_timeout(remaining) + .run(&["sandbox", "delete", &name]) + .await + { + Ok(result) if result.success() || output_reports_not_found(&result) => {} + Ok(result) => { + failures.push(result.failure_diagnostic(&format!( + "sandbox '{name}' is deleted or already absent" + ))); + } + Err(error) => failures.push(error.to_string()), + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("\n\n")) + } + } + + fn context(&self, step: &str) -> String { + format!("[run {}][{}/{}]", self.run_id, self.scenario, step) + } +} + +impl<'a> CommandStep<'a> { + #[must_use] + pub fn description(mut self, description: impl Into) -> Self { + self.description = Some(description.into()); + self + } + + pub fn with_timeout(self, timeout: Duration) -> OpenShellCommand<'a> { + let description = self + .description + .unwrap_or_else(|| format!("step '{}' succeeds", self.step)); + OpenShellCommand { + runner: self.runner, + step: self.step, + description, + timeout, + } + } +} + +impl OpenShellCommand<'_> { + pub async fn run(&self, args: &[&str]) -> Result { + self.runner + .run_strings( + &self.step, + &self.description, + args.iter().map(|arg| (*arg).to_string()).collect(), + self.timeout, + ) + .await + } +} + +impl Drop for OpenShellRunner { + fn drop(&mut self) { + if self.finished { + return; + } + let resources = if self.known_sandboxes.is_empty() { + "none explicitly tracked".to_string() + } else { + self.known_sandboxes + .iter() + .cloned() + .collect::>() + .join(", ") + }; + eprintln!( + "[run {}][{}] WARNING: runner dropped without finish(); known sandboxes: {}", + self.run_id, self.scenario, resources + ); + } +} + +fn combine_results( + scenario_result: Result<(), String>, + cleanup_result: Result<(), String>, +) -> Result<(), String> { + match (scenario_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(functional), Ok(())) => Err(functional), + (Ok(()), Err(cleanup)) => Err(format!("cleanup failed:\n{cleanup}")), + (Err(functional), Err(cleanup)) => Err(format!( + "{functional}\n\nsecondary cleanup failure:\n{cleanup}" + )), + } +} + +fn remaining_cleanup_time(started: Instant) -> Duration { + CLEANUP_TIMEOUT.saturating_sub(started.elapsed()) +} + +fn output_reports_not_found(result: &CommandResult) -> bool { + result.stderr().to_ascii_lowercase().contains("not found") + || result.stdout().to_ascii_lowercase().contains("not found") +} + +fn exit_description(status: ExitStatus) -> String { + status.code().map_or_else( + || "terminated by signal".to_string(), + |code| code.to_string(), + ) +} + +fn sanitized_command(args: &[String]) -> String { + let mut redact_next = false; + let rendered = args.iter().map(|arg| { + let rendered = if redact_next { + redact_next = false; + "".to_string() + } else if sensitive_flag(arg) { + redact_next = !arg.contains('='); + arg.split_once('=') + .map_or_else(|| arg.clone(), |(flag, _)| format!("{flag}=")) + } else { + arg.clone() + }; + shell_escape(&rendered) + }); + std::iter::once("openshell".to_string()) + .chain(rendered) + .collect::>() + .join(" ") +} + +fn sensitive_flag(arg: &str) -> bool { + let flag = arg.split_once('=').map_or(arg, |(flag, _)| flag); + matches!( + flag, + "--credential" + | "--credentials" + | "--env" + | "--header" + | "--material" + | "--password" + | "--secret" + | "--token" + ) || flag.contains("client-secret") + || flag.contains("private-key") + || flag.ends_with("-token") +} + +fn shell_escape(arg: &str) -> String { + if arg + .chars() + .all(|character| character.is_ascii_alphanumeric() || "-_./:@=,".contains(character)) + { + return arg.to_string(); + } + format!("'{}'", arg.replace('\'', "'\\''")) +} + +#[cfg(all(test, unix))] +mod tests { + use std::collections::VecDeque; + use std::os::unix::process::ExitStatusExt; + use std::process::Output; + use std::sync::Mutex; + + use serde::Deserialize; + + use super::executor::CliExecution; + use super::*; + + struct MockCli { + state: Mutex, + } + + struct MockCliState { + responses: VecDeque, + invocations: Vec>, + } + + enum MockResponse { + Output { + exit_code: i32, + stdout: String, + stderr: String, + }, + Timeout, + } + + impl MockResponse { + fn output(exit_code: i32, stdout: &str, stderr: &str) -> Self { + Self::Output { + exit_code, + stdout: stdout.to_string(), + stderr: stderr.to_string(), + } + } + + fn success(stdout: &str) -> Self { + Self::output(0, stdout, "") + } + } + + impl MockCli { + fn new(responses: Vec) -> Self { + Self { + state: Mutex::new(MockCliState { + responses: responses.into(), + invocations: Vec::new(), + }), + } + } + + fn invocations(&self) -> Vec> { + self.state + .lock() + .expect("lock mock CLI state") + .invocations + .clone() + } + } + + impl CliExecutor for MockCli { + fn execute(&self, args: Vec, _command_timeout: Duration) -> CliExecution<'_> { + let response = { + let mut state = self.state.lock().expect("lock mock CLI state"); + state.invocations.push(args); + state + .responses + .pop_front() + .expect("mock CLI received an unexpected invocation") + }; + + Box::pin(async move { + match response { + MockResponse::Output { + exit_code, + stdout, + stderr, + } => Ok(Output { + status: ExitStatus::from_raw(exit_code << 8), + stdout: stdout.into_bytes(), + stderr: stderr.into_bytes(), + }), + MockResponse::Timeout => Err(CliExecutionError::Timeout), + } + }) + } + } + + fn test_runner(responses: Vec) -> (OpenShellRunner, Arc) { + let cli = Arc::new(MockCli::new(responses)); + let runner = OpenShellRunner::with_cli(cli.clone(), "smoke"); + (runner, cli) + } + + #[test] + fn with_binary_rejects_a_missing_file() { + let directory = tempfile::tempdir().expect("create missing CLI directory"); + let Err(error) = OpenShellRunner::with_binary(directory.path().join("missing"), "smoke") + else { + panic!("missing CLI should be rejected"); + }; + + assert!(matches!(error, RunnerError::BinaryUnavailable(_))); + } + + #[test] + fn exposes_generated_identity() { + let (mut runner, _cli) = test_runner(Vec::new()); + + assert_eq!(runner.id().len(), 10); + assert_eq!(runner.scenario(), "smoke"); + runner.finished = true; + } + + #[tokio::test] + async fn captures_streams_and_nonzero_exit_as_result() { + let (mut runner, _cli) = test_runner(vec![MockResponse::output(7, "stdout", "stderr")]); + + let result = runner + .step("capture") + .description("sandbox list succeeds") + .with_timeout(Duration::from_secs(1)) + .run(&["sandbox", "list"]) + .await + .expect("completed nonzero exit is a result"); + + assert_eq!(result.exit_code(), Some(7)); + assert_eq!(result.stdout(), "stdout"); + assert_eq!(result.stderr(), "stderr"); + assert!( + result + .require_success() + .expect_err("nonzero result should fail its expectation") + .contains("expected: sandbox list succeeds") + ); + runner.finished = true; + } + + #[tokio::test] + async fn reports_timeout_as_typed_error() { + let (mut runner, _cli) = test_runner(vec![MockResponse::Timeout]); + + let error = runner + .step("timeout") + .with_timeout(Duration::from_millis(10)) + .run(&["status"]) + .await + .expect_err("command should time out"); + + assert!(matches!(error, RunnerError::Timeout { .. })); + runner.finished = true; + } + + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct JsonFixture { + status: String, + } + + #[tokio::test] + async fn deserializes_stdout_json_without_stderr() { + let (mut runner, _cli) = test_runner(vec![MockResponse::output( + 0, + "{\"status\":\"connected\"}", + "diagnostic", + )]); + + let result = runner + .step("json") + .with_timeout(Duration::from_secs(1)) + .run(&["status"]) + .await + .expect("run fake CLI"); + + assert_eq!( + result.json::().expect("parse JSON"), + JsonFixture { + status: "connected".to_string() + } + ); + runner.finished = true; + } + + #[tokio::test] + async fn accepts_connected_gateway_status() { + let (mut runner, _cli) = + test_runner(vec![MockResponse::success("{\"status\":\"connected\"}")]); + + runner + .check_gateway_status() + .await + .expect("connected gateway should satisfy preflight"); + + runner.finished = true; + } + + #[tokio::test] + async fn polling_returns_ready_value() { + let (mut runner, _cli) = test_runner(Vec::new()); + let mut attempts = 0; + + let value = runner + .poll_until( + "poll", + Duration::from_secs(1), + Duration::from_millis(1), + async |_runner| { + attempts += 1; + if attempts == 2 { + Poll::Ready("ready") + } else { + Poll::Pending("not ready".to_string()) + } + }, + ) + .await + .expect("poll should become ready"); + + assert_eq!(value, "ready"); + runner.finished = true; + } + + #[tokio::test] + async fn polling_timeout_preserves_last_observation() { + let (mut runner, _cli) = test_runner(Vec::new()); + + let error = runner + .poll_until( + "poll", + Duration::from_millis(1), + Duration::from_millis(1), + async |_runner| Poll::<()>::Pending("still pending".to_string()), + ) + .await + .expect_err("poll should time out"); + + assert!(matches!( + error, + RunnerError::PollTimeout { + last_observation, + .. + } if last_observation == "still pending" + )); + runner.finished = true; + } + + #[tokio::test] + async fn finish_skips_cleanup_commands_without_registered_resources() { + let (runner, cli) = test_runner(Vec::new()); + + let error = runner + .finish(Err("preflight failed".to_string())) + .await + .expect_err("functional failure should be preserved"); + + assert_eq!(error, "preflight failed"); + assert!(cli.invocations().is_empty()); + } + + #[test] + fn tracks_sandbox_for_cleanup() { + let (mut runner, _cli) = test_runner(Vec::new()); + + runner.track_sandbox("ct-0123456789-01"); + + assert!(runner.known_sandboxes.contains("ct-0123456789-01")); + runner.finished = true; + } + + #[test] + fn forgets_exact_name_cleanup() { + let (mut runner, _cli) = test_runner(Vec::new()); + + runner.track_sandbox("ct-0123456789-01"); + runner.forget_sandbox("ct-0123456789-01"); + + assert!(runner.known_sandboxes.is_empty()); + runner.finished = true; + } + + #[tokio::test] + async fn finish_deletes_explicitly_tracked_resources() { + let (mut runner, cli) = test_runner(vec![MockResponse::success("")]); + runner.track_sandbox("ct-0123456789-01"); + + runner + .finish(Ok(())) + .await + .expect("owned resource cleanup should succeed"); + + assert_eq!( + cli.invocations(), + vec![vec![ + "sandbox".to_string(), + "delete".to_string(), + "ct-0123456789-01".to_string(), + ]] + ); + } + + #[test] + fn functional_failure_remains_primary_when_cleanup_also_fails() { + let error = combine_results( + Err("functional failure".to_string()), + Err("cleanup failure".to_string()), + ) + .expect_err("combined result should fail"); + + assert!(error.starts_with("functional failure")); + assert!(error.contains("secondary cleanup failure:\ncleanup failure")); + } + + #[test] + fn generated_run_id_is_portable_and_compact() { + let run_id = generate_run_id(); + assert_eq!(run_id.len(), 10); + assert!( + run_id + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + ); + } + + #[test] + fn sanitizes_sensitive_cli_arguments() { + let command = sanitized_command(&[ + "provider".to_string(), + "create".to_string(), + "--credential".to_string(), + "TOKEN=secret".to_string(), + "--client-secret=hunter2".to_string(), + ]); + + assert!(!command.contains("TOKEN=secret")); + assert!(!command.contains("hunter2")); + assert!(command.contains("")); + } +} diff --git a/e2e/rust/src/harness/conformance/executor.rs b/e2e/rust/src/harness/conformance/executor.rs new file mode 100644 index 0000000000..53c255d937 --- /dev/null +++ b/e2e/rust/src/harness/conformance/executor.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process execution boundary for the CLI conformance runner. + +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::process::{Output, Stdio}; +use std::time::Duration; + +use tokio::time::timeout; + +pub(super) type CliExecution<'a> = + Pin> + Send + 'a>>; + +pub(super) trait CliExecutor: Send + Sync { + fn execute(&self, args: Vec, command_timeout: Duration) -> CliExecution<'_>; +} + +pub(super) enum CliExecutionError { + Spawn(std::io::Error), + Timeout, +} + +pub(super) struct ProcessCli { + binary: PathBuf, +} + +impl ProcessCli { + pub(super) fn new(binary: PathBuf) -> Self { + Self { binary } + } +} + +impl CliExecutor for ProcessCli { + fn execute(&self, args: Vec, command_timeout: Duration) -> CliExecution<'_> { + Box::pin(async move { + let mut process = tokio::process::Command::new(&self.binary); + process + .args(&args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + timeout(command_timeout, process.output()) + .await + .map_err(|_| CliExecutionError::Timeout)? + .map_err(CliExecutionError::Spawn) + }) + } +} diff --git a/e2e/rust/src/harness/mod.rs b/e2e/rust/src/harness/mod.rs index f2dfd5ec9c..ee392ce459 100644 --- a/e2e/rust/src/harness/mod.rs +++ b/e2e/rust/src/harness/mod.rs @@ -5,6 +5,7 @@ pub mod binary; pub mod cli; +pub mod conformance; pub mod container; pub mod gateway; pub mod output; diff --git a/e2e/rust/tests/smoke.rs b/e2e/rust/tests/smoke.rs index 172afa22b8..7ac7d72ffe 100644 --- a/e2e/rust/tests/smoke.rs +++ b/e2e/rust/tests/smoke.rs @@ -1,97 +1,215 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#![cfg(feature = "e2e")] +#![cfg(feature = "e2e-cli-conformance")] -//! Smoke test: verify the gateway is healthy, create a sandbox, exec a -//! command inside it, and tear it down. -//! -//! This test is cluster-agnostic — it works against any running gateway -//! (Docker-based cluster or openshell-driver-vm microVM). The `e2e:vm` mise -//! task uses it to validate the VM gateway after boot. +//! Portable phase-1 CLI conformance scenario. -use std::process::Stdio; use std::time::Duration; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; +use openshell_e2e::harness::conformance::{OpenShellRunner, Poll, STATUS_TIMEOUT}; +use serde::Deserialize; -/// End-to-end smoke test: status → create → exec → list → delete. +const CREATE_TIMEOUT: Duration = Duration::from_secs(600); +const LIST_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); +const LIST_PAGE_SIZE: u32 = 1_000; +const EXEC_TIMEOUT: Duration = Duration::from_secs(120); +const DELETE_TIMEOUT: Duration = Duration::from_secs(120); +const DELETE_POLL_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, Deserialize)] +struct SandboxListEntry { + name: String, + phase: String, +} + +/// Certify status -> create -> list Ready -> exec -> delete -> list empty. #[tokio::test] async fn gateway_smoke() { - // ── 1. Gateway must be reachable ────────────────────────────────── - let mut clean_status = String::new(); - let mut status_ok = false; - for _ in 0..15 { - let mut status_cmd = openshell_cmd(); - status_cmd - .arg("status") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let status_out = status_cmd - .output() - .await - .expect("failed to run openshell status"); - - let status_text = format!( - "{}{}", - String::from_utf8_lossy(&status_out.stdout), - String::from_utf8_lossy(&status_out.stderr), - ); - clean_status = strip_ansi(&status_text); - - if status_out.status.success() && clean_status.contains("Connected") { - status_ok = true; - break; - } + let mut runner = OpenShellRunner::new("smoke") + .unwrap_or_else(|error| panic!("failed to initialize conformance runner: {error}")); + println!("CLI conformance run ID: {}", runner.id()); + let scenario_result = match runner.check_gateway_status().await { + Ok(()) => run_smoke(&mut runner).await, + Err(error) => Err(error), + }; + + runner + .finish(scenario_result) + .await + .unwrap_or_else(|error| panic!("CLI conformance smoke failed:\n{error}")); +} - tokio::time::sleep(Duration::from_secs(2)).await; +async fn run_smoke(runner: &mut OpenShellRunner) -> Result<(), String> { + let status = runner + .step("status") + .description("openshell status succeeds") + .with_timeout(STATUS_TIMEOUT) + .run(&["status"]) + .await + .map_err(|error| error.to_string())?; + status.require_success()?; + + let sandbox_name = format!("ct-{}-01", runner.id()); + runner.track_sandbox(&sandbox_name); + let create = runner + .step("create") + .description("sandbox creation succeeds") + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + &sandbox_name, + "--from", + "base", + "--detach", + ]) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + + let get = runner + .step("get-ready") + .description(format!("sandbox '{sandbox_name}' can be retrieved")) + .with_timeout(LIST_ATTEMPT_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + get.require_success()?; + + let sandbox = get + .json::() + .map_err(|error| error.to_string())?; + if sandbox.name != sandbox_name { + return Err(format!( + "sandbox get returned {:?}; expected sandbox '{sandbox_name}'", + sandbox.name + )); + } + if sandbox.phase != "Ready" { + return Err(format!( + "sandbox '{sandbox_name}' is in phase {:?}; expected Ready", + sandbox.phase + )); } - assert!( - status_ok, - "openshell status never became healthy:\n{clean_status}", - ); + check_sandbox_listed(runner, &sandbox_name).await?; + + let marker = format!("openshell-conformance-{}", runner.id()); + let exec = runner + .step("exec") + .description("sandbox exec exits successfully") + .with_timeout(EXEC_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &sandbox_name, + "--no-tty", + "--", + "echo", + &marker, + ]) + .await + .map_err(|error| error.to_string())?; + exec.require_success()?; + let expected_stdout = format!("{marker}\n"); + if exec.stdout() != expected_stdout { + return Err(exec.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}"))); + } - // ── 2. Create a sandbox and exec a command ─────────────────────── - // Default behaviour keeps the sandbox alive after the command exits, - // so we can verify it in the list before cleaning up. - let mut sb = SandboxGuard::create(&["--", "echo", "smoke-ok"]) + let delete = runner + .step("delete") + .description("sandbox deletion succeeds") + .with_timeout(DELETE_TIMEOUT) + .run(&["sandbox", "delete", &sandbox_name]) .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("smoke-ok"), - "expected 'smoke-ok' in sandbox output:\n{}", - sb.create_output, - ); - - // ── 3. Verify the sandbox appeared in the list ─────────────────── - let mut list_cmd = openshell_cmd(); - list_cmd - .args(["sandbox", "list", "--names"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let list_out = list_cmd - .output() + .map_err(|error| error.to_string())?; + delete.require_success()?; + + check_empty_list(runner, &sandbox_name).await?; + runner.forget_sandbox(&sandbox_name); + Ok(()) +} + +async fn check_sandbox_listed(runner: &OpenShellRunner, sandbox_name: &str) -> Result<(), String> { + if find_sandbox(runner, sandbox_name, "list-visible") + .await? + .is_some() + { + return Ok(()); + } + Err(format!( + "sandbox '{sandbox_name}' does not appear in sandbox list" + )) +} + +async fn check_empty_list(runner: &mut OpenShellRunner, sandbox_name: &str) -> Result<(), String> { + runner + .poll_until( + "list-empty", + DELETE_TIMEOUT, + DELETE_POLL_INTERVAL, + async |runner| match find_sandbox(runner, sandbox_name, "list-empty/query").await { + Ok(None) => Poll::Ready(()), + Ok(Some(sandbox)) => Poll::Pending(format!( + "sandbox '{sandbox_name}' remains listed in phase {:?}", + sandbox.phase + )), + Err(error) => Poll::Failed(error), + }, + ) .await - .expect("failed to run openshell sandbox list"); - - let list_text = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&list_out.stdout), - String::from_utf8_lossy(&list_out.stderr), - )); - - assert!( - list_text.contains(&sb.name), - "sandbox '{}' should appear in list output:\n{list_text}", - sb.name, - ); - - // ── 4. Cleanup ─────────────────────────────────────────────────── - sb.cleanup().await; + .map_err(|error| error.to_string()) +} + +async fn find_sandbox( + runner: &OpenShellRunner, + sandbox_name: &str, + step: &str, +) -> Result, String> { + let mut offset = 0u32; + + loop { + let limit = LIST_PAGE_SIZE.to_string(); + let page_offset = offset.to_string(); + let result = runner + .step(format!("{step}/{offset}")) + .description(format!("sandbox list page at offset {offset} succeeds")) + .with_timeout(LIST_ATTEMPT_TIMEOUT) + .run(&[ + "sandbox", + "list", + "--limit", + &limit, + "--offset", + &page_offset, + "--output", + "json", + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + + let sandboxes = result + .json::>() + .map_err(|error| error.to_string())?; + if let Some(sandbox) = sandboxes + .iter() + .find(|sandbox| sandbox.name == sandbox_name) + { + return Ok(Some(SandboxListEntry { + name: sandbox.name.clone(), + phase: sandbox.phase.clone(), + })); + } + if sandboxes.len() < LIST_PAGE_SIZE as usize { + return Ok(None); + } + + offset = offset + .checked_add(LIST_PAGE_SIZE) + .ok_or_else(|| "sandbox list pagination offset overflowed".to_string())?; + } } diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 958ea1b0eb..0a767576dd 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -437,6 +437,7 @@ ensure_sandbox_image_available() { } e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +export OPENSHELL_BIN="${CLI_BIN}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then e2e_build_external_driver \ "${ROOT}" openshell-driver-docker openshell-driver-docker DRIVER_BIN diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index dcb2d43fe0..fc3419e182 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -383,6 +383,7 @@ fi ensure_podman_api_socket e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +export OPENSHELL_BIN="${CLI_BIN}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then e2e_build_external_driver \ "${ROOT}" openshell-driver-podman openshell-driver-podman DRIVER_BIN diff --git a/tasks/test.toml b/tasks/test.toml index e3328d12bd..103f529380 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -86,6 +86,14 @@ run = [ "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", ] +["e2e:cli-conformance"] +description = "Run the portable CLI conformance baseline against the configured gateway" +run = [ + "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi", + "OPENSHELL_BIN=\"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\" cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-cli-conformance --lib harness::conformance::tests", + "OPENSHELL_BIN=\"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\" cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-cli-conformance --test smoke", +] + ["e2e:websocket-conformance"] description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" run = [