diff --git a/.github/actions/setup-e2e-cli/action.yml b/.github/actions/setup-e2e-cli/action.yml index fb894194c8..f5b2c93968 100644 --- a/.github/actions/setup-e2e-cli/action.yml +++ b/.github/actions/setup-e2e-cli/action.yml @@ -1,10 +1,14 @@ name: Setup E2E CLI -description: Download an architecture-matched prebuilt OpenShell CLI for E2E tests +description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests inputs: artifact-prefix: description: Artifact name prefix; linux- is appended automatically required: true + conformance-artifact-prefix: + description: Optional conformance CLI artifact name prefix; linux- is appended automatically + required: false + default: "" runs: using: composite @@ -27,3 +31,24 @@ runs: chmod +x "$cli" "$cli" --version echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV" + + - name: Download prebuilt conformance CLI + if: inputs.conformance-artifact-prefix != '' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ format('{0}-linux-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }} + path: .e2e/prebuilt-conformance + + - name: Configure prebuilt conformance CLI + if: inputs.conformance-artifact-prefix != '' + shell: bash + run: | + set -euo pipefail + conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance" + if [[ ! -f "$conformance" ]]; then + echo "downloaded artifact is missing $conformance" >&2 + exit 1 + fi + chmod +x "$conformance" + "$conformance" list --output json >/dev/null + echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 2b4d9d5d46..c4a5cb1333 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -109,6 +109,17 @@ jobs: component: cli platform: linux/amd64,linux/arm64 + build-conformance: + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true' + permissions: + contents: read + packages: write + uses: ./.github/workflows/docker-build.yml + with: + component: conformance + platform: linux/amd64,linux/arm64 + build-driver-vm-linux: needs: [pr_metadata] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -124,7 +135,7 @@ jobs: [{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}] e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -135,11 +146,12 @@ jobs: image-tag: ${{ github.sha }} runner: linux-arm64-cpu8 cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance gateway-artifact-prefix: rust-binary-gateway vm-driver-artifact-name: driver-vm-linux-amd64 gpu-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' permissions: actions: read @@ -149,10 +161,11 @@ jobs: with: image-tag: ${{ github.sha }} cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance gateway-artifact-prefix: rust-binary-gateway kubernetes-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' strategy: fail-fast: false @@ -181,9 +194,10 @@ jobs: agent-sandbox-version: ${{ matrix.agent_sandbox_version }} extra-helm-values: ${{ matrix.extra_helm_values }} cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance kubernetes-workspace-managed-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -195,9 +209,10 @@ jobs: job-name: Kubernetes E2E (workspace managed mode) e2e-task: e2e:kubernetes:workspace-managed cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance kubernetes-external-driver-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -209,9 +224,10 @@ jobs: job-name: Kubernetes E2E (external compute driver) e2e-task: e2e:kubernetes:external-driver cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance kubernetes-workspace-operator-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' permissions: actions: read @@ -223,9 +239,10 @@ jobs: job-name: Kubernetes E2E (workspace operator mode) e2e-task: e2e:kubernetes:workspace-operator cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance kubernetes-ha-e2e: - needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' permissions: actions: read @@ -238,9 +255,10 @@ jobs: extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml external-postgres-secret: openshell-ha-pg cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance kubernetes-credential-drivers-e2e: - needs: [pr_metadata, build-gateway, build-supervisor] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' permissions: actions: read @@ -251,10 +269,12 @@ jobs: image-tag: ${{ github.sha }} job-name: Kubernetes Credential Drivers E2E e2e-task: e2e:kubernetes:credential-drivers + cli-artifact-prefix: rust-binary-cli + conformance-artifact-prefix: rust-binary-conformance core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -263,6 +283,7 @@ jobs: BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} BUILD_CLI_RESULT: ${{ needs.build-cli.result }} + BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }} BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} @@ -276,6 +297,7 @@ jobs: "build-gateway:$BUILD_GATEWAY_RESULT" \ "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ "build-cli:$BUILD_CLI_RESULT" \ + "build-conformance:$BUILD_CONFORMANCE_RESULT" \ "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ @@ -293,7 +315,7 @@ jobs: gpu-e2e-result: name: GPU E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, gpu-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, gpu-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true' runs-on: ubuntu-latest steps: @@ -302,6 +324,7 @@ jobs: BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} BUILD_CLI_RESULT: ${{ needs.build-cli.result }} + BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }} GPU_E2E_RESULT: ${{ needs.gpu-e2e.result }} run: | set -euo pipefail @@ -310,6 +333,7 @@ jobs: "build-gateway:$BUILD_GATEWAY_RESULT" \ "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ "build-cli:$BUILD_CLI_RESULT" \ + "build-conformance:$BUILD_CONFORMANCE_RESULT" \ "gpu-e2e:$GPU_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" @@ -322,7 +346,7 @@ jobs: kubernetes-ha-e2e-result: name: Kubernetes HA E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, kubernetes-ha-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-ha-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true' runs-on: ubuntu-latest steps: @@ -331,6 +355,7 @@ jobs: BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} BUILD_CLI_RESULT: ${{ needs.build-cli.result }} + BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }} KUBERNETES_HA_E2E_RESULT: ${{ needs.kubernetes-ha-e2e.result }} run: | set -euo pipefail @@ -339,6 +364,7 @@ jobs: "build-gateway:$BUILD_GATEWAY_RESULT" \ "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ "build-cli:$BUILD_CLI_RESULT" \ + "build-conformance:$BUILD_CONFORMANCE_RESULT" \ "kubernetes-ha-e2e:$KUBERNETES_HA_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" @@ -351,7 +377,7 @@ jobs: kubernetes-credential-drivers-e2e-result: name: Kubernetes Credential Drivers E2E result - needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-credential-drivers-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true' runs-on: ubuntu-latest steps: @@ -359,6 +385,8 @@ jobs: env: BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }} BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }} + BUILD_CLI_RESULT: ${{ needs.build-cli.result }} + BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }} KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }} run: | set -euo pipefail @@ -366,6 +394,8 @@ jobs: for item in \ "build-gateway:$BUILD_GATEWAY_RESULT" \ "build-supervisor:$BUILD_SUPERVISOR_RESULT" \ + "build-cli:$BUILD_CLI_RESULT" \ + "build-conformance:$BUILD_CONFORMANCE_RESULT" \ "kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do name="${item%%:*}" result="${item#*:}" diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2e7abe2008..d83acfec87 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: component: - description: "Component to build (gateway, supervisor, cli)" + description: "Component to build (gateway, supervisor, cli, conformance)" required: true type: string timeout-minutes: @@ -104,6 +104,12 @@ jobs: features="" has_image=false ;; + conformance) + binary_component=conformance + binary_name=openshell-conformance + features="" + has_image=false + ;; *) echo "unsupported component: $component" >&2 exit 1 diff --git a/.github/workflows/e2e-gpu-test.yaml b/.github/workflows/e2e-gpu-test.yaml index be2b66e1d4..b540beb2ff 100644 --- a/.github/workflows/e2e-gpu-test.yaml +++ b/.github/workflows/e2e-gpu-test.yaml @@ -12,6 +12,11 @@ on: required: false type: string default: "" + conformance-artifact-prefix: + description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-)" + required: false + type: string + default: "" gateway-artifact-prefix: description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-)" required: false @@ -74,6 +79,7 @@ jobs: uses: ./.github/actions/setup-e2e-cli with: artifact-prefix: ${{ inputs.cli-artifact-prefix }} + conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact-prefix != '' diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 3e36570144..4b54758cfa 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -52,6 +52,11 @@ on: required: false type: string default: "" + conformance-artifact-prefix: + description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-)" + required: false + type: string + default: "" permissions: actions: read @@ -85,6 +90,7 @@ jobs: uses: ./.github/actions/setup-e2e-cli with: artifact-prefix: ${{ inputs.cli-artifact-prefix }} + conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Install mise run: | diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index f176953048..2d6fea75fa 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -22,6 +22,11 @@ on: required: false type: string default: "" + conformance-artifact-prefix: + description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-)" + required: false + type: string + default: "" gateway-artifact-prefix: description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-)" required: false @@ -57,7 +62,7 @@ jobs: cmd: "mise run --no-deps --skip-deps e2e:oidc-pkce:docker" apt_packages: "openssh-client" - suite: rust-docker - cmd: "mise run --no-deps --skip-deps e2e:rust" + cmd: "mise run --no-deps --skip-deps e2e:docker" apt_packages: "openssh-client" - suite: rust-docker-external-driver cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver" @@ -93,6 +98,7 @@ jobs: uses: ./.github/actions/setup-e2e-cli with: artifact-prefix: ${{ inputs.cli-artifact-prefix }} + conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact-prefix != '' @@ -183,6 +189,7 @@ jobs: uses: ./.github/actions/setup-e2e-cli with: artifact-prefix: ${{ inputs.cli-artifact-prefix }} + conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact-prefix != '' @@ -340,6 +347,7 @@ jobs: uses: ./.github/actions/setup-e2e-cli with: artifact-prefix: ${{ inputs.cli-artifact-prefix }} + conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }} - name: Use prebuilt OpenShell gateway if: inputs.gateway-artifact-prefix != '' diff --git a/.github/workflows/rust-native-build.yml b/.github/workflows/rust-native-build.yml index 3496f4779f..6b9a2ebcbf 100644 --- a/.github/workflows/rust-native-build.yml +++ b/.github/workflows/rust-native-build.yml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Rust Image Binary Build (openshell-gateway / openshell-sandbox / openshell-cli) +name: Rust Image Binary Build (openshell-gateway / openshell-sandbox / openshell-cli / openshell-conformance) # Build Rust binaries per Linux architecture before the Docker image build # consumes them as prebuilt artifacts. Gateway images use GNU-linked binaries @@ -18,7 +18,7 @@ on: workflow_call: inputs: component: - description: "Binary component to build (gateway, sandbox, or cli)" + description: "Binary component to build (gateway, sandbox, cli, or conformance)" required: true type: string arch: @@ -142,6 +142,11 @@ jobs: binary=openshell zig_target= ;; + conformance) + crate=openshell-conformance-cli + binary=openshell-conformance + zig_target= + ;; *) echo "unsupported component: $COMPONENT" >&2 exit 1 @@ -169,7 +174,7 @@ jobs: if [[ "$COMPONENT" == "sandbox" && "$static_libc" == "gnu" ]]; then target=x86_64-unknown-linux-gnu zig_target= - elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" ]]; then + elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" || "$COMPONENT" == "conformance" ]]; then target=x86_64-unknown-linux-musl zig_target=x86_64-linux-musl else @@ -181,7 +186,7 @@ jobs: if [[ "$COMPONENT" == "sandbox" && "$static_libc" == "gnu" ]]; then target=aarch64-unknown-linux-gnu zig_target= - elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" ]]; then + elif [[ "$COMPONENT" == "sandbox" || "$COMPONENT" == "cli" || "$COMPONENT" == "conformance" ]]; then target=aarch64-unknown-linux-musl zig_target=aarch64-linux-musl else @@ -309,9 +314,13 @@ jobs: run: | set -euo pipefail BIN="target/${{ steps.target.outputs.target }}/release/${{ steps.target.outputs.binary }}" - OUTPUT="$("$BIN" --version)" - echo "$OUTPUT" - grep -q "^${{ steps.target.outputs.binary }} " <<<"$OUTPUT" + if [[ "${{ inputs.component }}" == "conformance" ]]; then + "$BIN" list --output json >/dev/null + else + OUTPUT="$("$BIN" --version)" + echo "$OUTPUT" + grep -q "^${{ steps.target.outputs.binary }} " <<<"$OUTPUT" + fi # Record linkage so image runtime drift is visible in logs. ldd --version ldd "$BIN" || true diff --git a/Cargo.lock b/Cargo.lock index 65a50f0f39..cf0cdc7864 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3889,6 +3889,28 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-conformance" +version = "0.0.0" +dependencies = [ + "rand 0.9.4", + "serde", + "serde_json", + "tempfile", + "tokio", +] + +[[package]] +name = "openshell-conformance-cli" +version = "0.0.0" +dependencies = [ + "clap", + "openshell-conformance", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "openshell-core" version = "0.0.0" diff --git a/TESTING.md b/TESTING.md index 6c0829060d..35ab54e731 100644 --- a/TESTING.md +++ b/TESTING.md @@ -148,7 +148,12 @@ lifecycle management, output parsing, and cleanup. Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. -- Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. +- CLI conformance (`--features e2e-cli-conformance`) - the portable deployment + smoke scenario plus focused tests for its reusable command runner. +- Driver suites (`--features e2e-docker`, `e2e-podman`, `e2e-kubernetes`, or + `e2e-vm`) - CLI conformance plus the common and driver-specific coverage for + the selected deployment. +- Docker suite (`--features e2e-docker`) - includes Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway start. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. - VM suite (`--features e2e-vm`) - runs e2e tests on a VM. - Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage. @@ -166,10 +171,48 @@ key. Run the Docker-backed Rust CLI e2e suite: ```shell -mise run e2e:rust +mise run e2e:docker ``` -Run the Podman-backed Rust CLI e2e suite: +Run the minimal portable CLI conformance profile against the gateway selected +in your OpenShell CLI configuration: + +```shell +mise run e2e:cli-conformance +``` + +The gateway must already be installed, reachable, and selected before the task +starts. The task does not provision a gateway or select a compute driver. Set +`OPENSHELL_BIN` to test a prebuilt CLI; otherwise, the task builds the CLI from +the current checkout. + +The phase-1 scenario verifies the complete CLI-to-gateway-to-driver path without +depending on how the gateway was installed or which driver is configured. It +requires machine-readable gRPC status, creates a uniquely named detached +sandbox with `--from base`, verifies the sandbox is `Ready` by finding its +unique name in paginated JSON list output, executes `echo` with a run-specific +marker, deletes the sandbox, and verifies that its name no longer appears. +Driver suites enable the same profile +instead of maintaining a separate smoke implementation. Sandbox lifecycle, +label matrices, VM overlay, and TLS-key permission assertions remain regular +E2E coverage. + +Each invocation prints a ten-character run ID before creating resources. +Conformance sandboxes use names such as `ct--01`. The runner tracks the +exact name and uses it for cleanup; phase 1 does not add ownership labels. + +The runner deletes owned resources after both success and failure. If the test +process is interrupted before cleanup, locate leftovers without touching +unrelated gateway state: + +```shell +openshell sandbox list --output json +openshell sandbox delete +``` + +Gateway-backed Rust E2E tasks build the standalone conformance CLI, run its +registered scenarios against the configured gateway, then run any lane-specific +Rust tests that still apply. Run the Podman-backed Rust CLI e2e suite: ```shell mise run e2e:podman diff --git a/crates/openshell-conformance-cli/Cargo.toml b/crates/openshell-conformance-cli/Cargo.toml new file mode 100644 index 0000000000..ce9f89c17c --- /dev/null +++ b/crates/openshell-conformance-cli/Cargo.toml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-conformance-cli" +description = "Standalone OpenShell CLI conformance test runner" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-conformance" +path = "src/main.rs" + +[dependencies] +clap.workspace = true +openshell-conformance = { path = "../openshell-conformance" } +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[lints] +workspace = true diff --git a/crates/openshell-conformance-cli/src/main.rs b/crates/openshell-conformance-cli/src/main.rs new file mode 100644 index 0000000000..b024d3d1b0 --- /dev/null +++ b/crates/openshell-conformance-cli/src/main.rs @@ -0,0 +1,243 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Standalone runner for `OpenShell` CLI conformance scenarios. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Parser, Subcommand, ValueEnum}; +use openshell_conformance::{OpenShellRunner, Scenario, scenario, scenarios}; +use serde::Serialize; + +#[derive(Debug, Parser)] +#[command( + name = "openshell-conformance", + about = "Run OpenShell CLI conformance scenarios" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, Subcommand)] +enum Command { + /// List registered scenarios. + List { + #[arg(long, value_enum, default_value_t = OutputFormat::Text)] + output: OutputFormat, + }, + /// Run all registered scenarios, or named scenarios. + Run { + /// Scenario names. Omit to run every registered scenario. + scenarios: Vec, + /// Explicit path to the `OpenShell` CLI. Defaults to `openshell` on PATH. + #[arg(long)] + openshell_bin: Option, + #[arg(long, value_enum, default_value_t = OutputFormat::Text)] + output: OutputFormat, + }, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum OutputFormat { + Text, + Json, +} + +#[derive(Serialize)] +struct ScenarioDescription<'a> { + name: &'a str, + description: &'a str, +} + +#[derive(Serialize)] +struct ScenarioResult<'a> { + name: &'a str, + passed: bool, + diagnostic: Option, +} + +#[derive(Serialize)] +struct RunReport<'a> { + scenarios: Vec>, + passed: bool, +} + +#[tokio::main] +async fn main() -> ExitCode { + match execute(Cli::parse()).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => { + eprintln!("openshell-conformance: {error}"); + ExitCode::FAILURE + } + } +} + +async fn execute(cli: Cli) -> Result<(), String> { + match cli.command { + Command::List { output } => list(output), + Command::Run { + scenarios: requested, + openshell_bin, + output, + } => run(&requested, openshell_bin, output).await, + } +} + +fn list(output: OutputFormat) -> Result<(), String> { + match output { + OutputFormat::Text => { + for candidate in scenarios() { + println!("{:<16} {}", candidate.name, candidate.description); + } + } + OutputFormat::Json => { + let result = scenarios() + .iter() + .map(|candidate| ScenarioDescription { + name: candidate.name, + description: candidate.description, + }) + .collect::>(); + println!( + "{}", + serde_json::to_string_pretty(&result).map_err(|error| error.to_string())? + ); + } + } + Ok(()) +} + +async fn run( + requested: &[String], + binary: Option, + output: OutputFormat, +) -> Result<(), String> { + let selected = select_scenarios(requested)?; + let mut results = Vec::with_capacity(selected.len()); + for candidate in selected { + let runner = binary.as_ref().map_or_else( + || OpenShellRunner::new(candidate.name), + |path| OpenShellRunner::with_binary(path.clone(), candidate.name), + ); + let mut runner = match runner { + Ok(runner) => runner, + Err(error) => { + results.push(ScenarioResult { + name: candidate.name, + passed: false, + diagnostic: Some(error.to_string()), + }); + continue; + } + }; + eprintln!("CLI conformance run ID: {}", runner.id()); + let scenario_result = match runner.check_gateway_status().await { + Ok(()) => candidate.run(&mut runner).await, + Err(error) => Err(error), + }; + let outcome = runner.finish(scenario_result).await; + results.push(ScenarioResult { + name: candidate.name, + passed: outcome.is_ok(), + diagnostic: outcome.err(), + }); + } + + let passed = results.iter().all(|result| result.passed); + match output { + OutputFormat::Text => { + for result in &results { + if result.passed { + println!("PASS {}", result.name); + } else { + println!( + "FAIL {}\n{}", + result.name, + result.diagnostic.as_deref().unwrap_or("unknown failure") + ); + } + } + } + OutputFormat::Json => println!( + "{}", + serde_json::to_string_pretty(&RunReport { + scenarios: results, + passed + }) + .map_err(|error| error.to_string())? + ), + } + if passed { + Ok(()) + } else { + Err("one or more scenarios failed".to_string()) + } +} + +fn select_scenarios(requested: &[String]) -> Result, String> { + if requested.is_empty() { + return Ok(scenarios().iter().collect()); + } + requested + .iter() + .map(|name| { + scenario(name).ok_or_else(|| { + format!("unknown scenario '{name}'; run `openshell-conformance list`") + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + use super::*; + + #[test] + fn selects_all_scenarios_by_default() { + assert_eq!( + select_scenarios(&[]).expect("select all").len(), + scenarios().len() + ); + } + + #[test] + fn selects_named_scenario() { + let selected = select_scenarios(&["smoke".to_string()]).expect("select smoke"); + assert_eq!(selected[0].name, "smoke"); + } + + #[test] + fn unknown_scenario_has_actionable_diagnostic() { + let error = select_scenarios(&["missing".to_string()]).expect_err("unknown scenario"); + assert!(error.contains("openshell-conformance list")); + } + + #[test] + fn parses_binary_override_and_json_output() { + let cli = Cli::try_parse_from([ + "openshell-conformance", + "run", + "smoke", + "--openshell-bin", + "/opt/openshell", + "--output", + "json", + ]) + .expect("parse CLI"); + let Command::Run { + openshell_bin, + output, + .. + } = cli.command + else { + panic!("expected run") + }; + assert_eq!(openshell_bin, Some(PathBuf::from("/opt/openshell"))); + assert_eq!(output, OutputFormat::Json); + } +} diff --git a/crates/openshell-conformance/Cargo.toml b/crates/openshell-conformance/Cargo.toml new file mode 100644 index 0000000000..7b67258045 --- /dev/null +++ b/crates/openshell-conformance/Cargo.toml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-conformance" +description = "Reusable OpenShell CLI conformance scenarios and runner" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio.workspace = true + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/openshell-conformance/src/executor.rs b/crates/openshell-conformance/src/executor.rs new file mode 100644 index 0000000000..b45c102ee1 --- /dev/null +++ b/crates/openshell-conformance/src/executor.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Process execution boundary for the CLI conformance runner. + +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::process::{Output, Stdio}; +use std::time::Duration; + +use tokio::time::timeout; + +pub type CliExecution<'a> = + Pin> + Send + 'a>>; + +pub trait CliExecutor: Send + Sync { + fn execute(&self, args: Vec, command_timeout: Duration) -> CliExecution<'_>; +} + +pub enum CliExecutionError { + Spawn(std::io::Error), + Timeout, +} + +pub struct ProcessCli { + binary: PathBuf, +} + +impl ProcessCli { + pub fn new(binary: PathBuf) -> Self { + Self { binary } + } +} + +impl CliExecutor for ProcessCli { + fn execute(&self, args: Vec, command_timeout: Duration) -> CliExecution<'_> { + Box::pin(async move { + let mut process = tokio::process::Command::new(&self.binary); + process + .args(&args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + + timeout(command_timeout, process.output()) + .await + .map_err(|_| CliExecutionError::Timeout)? + .map_err(CliExecutionError::Spawn) + }) + } +} diff --git a/crates/openshell-conformance/src/lib.rs b/crates/openshell-conformance/src/lib.rs new file mode 100644 index 0000000000..aa6f69c387 --- /dev/null +++ b/crates/openshell-conformance/src/lib.rs @@ -0,0 +1,986 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Reusable support for portable `OpenShell` CLI conformance scenarios. + +pub mod executor; +mod scenarios; + +use std::collections::BTreeSet; +use std::error::Error; +use std::fmt; +use std::future::Future; +use std::path::PathBuf; +use std::pin::Pin; +use std::process::ExitStatus; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use rand::Rng; +use serde::Deserialize; +use serde::de::DeserializeOwned; +use tokio::time::sleep; + +use self::executor::{CliExecutionError, CliExecutor, ProcessCli}; + +pub use scenarios::SMOKE_SCENARIO; + +/// An installed conformance scenario. +#[derive(Debug)] +pub struct Scenario { + pub name: &'static str, + pub description: &'static str, + run: for<'a> fn(&'a mut OpenShellRunner) -> ScenarioFuture<'a>, +} + +pub type ScenarioFuture<'a> = Pin> + Send + 'a>>; + +impl Scenario { + pub async fn run(&self, runner: &mut OpenShellRunner) -> Result<(), String> { + (self.run)(runner).await + } +} + +const SCENARIOS: &[Scenario] = &[SMOKE_SCENARIO]; + +/// Returns every scenario compiled into this distribution. +pub fn scenarios() -> &'static [Scenario] { + SCENARIOS +} + +/// Finds a scenario by its stable command-line name. +pub fn scenario(name: &str) -> Option<&'static Scenario> { + scenarios().iter().find(|candidate| candidate.name == name) +} + +const CLEANUP_TIMEOUT: Duration = Duration::from_secs(120); +pub const STATUS_TIMEOUT: Duration = Duration::from_secs(30); +const GATEWAY_STATUS_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); +const GATEWAY_STATUS_INTERVAL: Duration = Duration::from_secs(2); + +/// Generate the suite-owned identifier used in resource names and diagnostics. +fn generate_run_id() -> String { + const ALPHABET: &[u8] = b"0123456789abcdefghijklmnopqrstuvwxyz"; + let mut rng = rand::rng(); + (0..10) + .map(|_| ALPHABET[rng.random_range(0..ALPHABET.len())] as char) + .collect() +} + +/// The completed outcome of one `OpenShell` CLI process. +#[derive(Debug)] +pub struct CommandResult { + run_id: String, + scenario: String, + step: String, + expectation: String, + command: String, + status: ExitStatus, + elapsed: Duration, + stdout: String, + stderr: String, +} + +impl CommandResult { + pub fn success(&self) -> bool { + self.status.success() + } + + pub fn exit_code(&self) -> Option { + self.status.code() + } + + pub fn stdout(&self) -> &str { + &self.stdout + } + + pub fn stderr(&self) -> &str { + &self.stderr + } + + pub fn elapsed(&self) -> Duration { + self.elapsed + } + + pub fn json(&self) -> Result { + serde_json::from_str(&self.stdout).map_err(|source| RunnerError::InvalidJson { + context: self.context(), + source, + stdout: self.stdout.clone(), + stderr: self.stderr.clone(), + }) + } + + pub fn require_success(&self) -> Result<(), String> { + if self.success() { + return Ok(()); + } + Err(self.failure_diagnostic(&self.expectation)) + } + + pub fn failure_diagnostic(&self, expectation: &str) -> String { + format!( + "{}\nexpected: {expectation}\nactual: exit {} after {:.1?}\ncommand: {}\nstdout:\n{}\nstderr:\n{}", + self.context(), + exit_description(self.status), + self.elapsed, + self.command, + self.stdout, + self.stderr, + ) + } + + fn context(&self) -> String { + format!("[run {}][{}/{}]", self.run_id, self.scenario, self.step) + } +} + +/// Failures in reusable runner mechanics rather than scenario assertions. +#[derive(Debug)] +pub enum RunnerError { + BinaryUnavailable(String), + Spawn { + context: String, + command: String, + source: std::io::Error, + }, + Timeout { + context: String, + command: String, + timeout: Duration, + }, + InvalidJson { + context: String, + source: serde_json::Error, + stdout: String, + stderr: String, + }, + PollTimeout { + context: String, + timeout: Duration, + last_observation: String, + }, + ObservationFailed { + context: String, + diagnostic: String, + }, +} + +impl fmt::Display for RunnerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::BinaryUnavailable(message) => write!(f, "{message}"), + Self::Spawn { + context, + command, + source, + } => write!( + f, + "{context} failed to spawn command: {source}\ncommand: {command}" + ), + Self::Timeout { + context, + command, + timeout, + } => write!( + f, + "{context} timed out after {timeout:.1?}\ncommand: {command}" + ), + Self::InvalidJson { + context, + source, + stdout, + stderr, + } => write!( + f, + "{context} returned invalid JSON: {source}\nstdout:\n{stdout}\nstderr:\n{stderr}" + ), + Self::PollTimeout { + context, + timeout, + last_observation, + } => write!( + f, + "{context} did not satisfy the observation within {timeout:.1?}\nlast observation:\n{last_observation}" + ), + Self::ObservationFailed { + context, + diagnostic, + } => write!(f, "{context} observation failed early:\n{diagnostic}"), + } + } +} + +impl Error for RunnerError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Spawn { source, .. } => Some(source), + Self::InvalidJson { source, .. } => Some(source), + _ => None, + } + } +} + +/// Result of one read-only polling observation. +pub enum Poll { + Ready(T), + Pending(String), + Failed(String), +} + +#[derive(Debug, Deserialize)] +struct StatusOutput { + gateway: Option, + server: Option, + status: String, + version: Option, + authentication: Option, +} + +#[derive(Debug, Deserialize)] +struct AuthenticationOutput { + status: String, +} + +/// Runs `OpenShell` commands for one conformance scenario and owns its cleanup. +pub struct OpenShellRunner { + cli: Arc, + run_id: String, + scenario: String, + known_sandboxes: BTreeSet, + finished: bool, +} + +/// A runner command with diagnostic context but no timeout yet. +pub struct CommandStep<'a> { + runner: &'a OpenShellRunner, + step: String, + description: Option, +} + +/// A fully configured runner command ready to execute. +pub struct OpenShellCommand<'a> { + runner: &'a OpenShellRunner, + step: String, + description: String, + timeout: Duration, +} + +impl OpenShellRunner { + pub fn new(scenario: &str) -> Result { + Ok(Self::with_executor( + Arc::new(ProcessCli::new(PathBuf::from("openshell"))), + scenario, + )) + } + + /// Uses an explicit `openshell` binary rather than resolving it on `PATH`. + pub fn with_binary(binary: PathBuf, scenario: &str) -> Result { + if !binary.is_file() { + return Err(RunnerError::BinaryUnavailable(format!( + "OpenShell CLI binary not found at {}", + binary.display() + ))); + } + Ok(Self::with_executor( + Arc::new(ProcessCli::new(binary)), + scenario, + )) + } + + /// Creates a runner with an injected executor. This is useful for harness tests. + pub fn with_executor(cli: Arc, scenario: &str) -> Self { + Self { + cli, + run_id: generate_run_id(), + scenario: scenario.to_string(), + known_sandboxes: BTreeSet::new(), + finished: false, + } + } + + pub fn id(&self) -> &str { + &self.run_id + } + + pub fn scenario(&self) -> &str { + &self.scenario + } + + pub fn step(&self, step: impl Into) -> CommandStep<'_> { + CommandStep { + runner: self, + step: step.into(), + description: None, + } + } + + /// Confirm that the configured gateway is reachable before a scenario runs. + pub async fn check_gateway_status(&mut self) -> Result<(), String> { + let status = self + .poll_until( + "preflight", + STATUS_TIMEOUT, + GATEWAY_STATUS_INTERVAL, + async |runner| match runner + .step("status/json") + .description("machine-readable gateway status succeeds") + .with_timeout(GATEWAY_STATUS_ATTEMPT_TIMEOUT) + .run(&["status", "--output", "json"]) + .await + { + Ok(result) if !result.success() => Poll::Pending( + result.failure_diagnostic("gateway status command succeeds"), + ), + Ok(result) => match result.json::() { + Ok(status) if status.status == "connected" => Poll::Ready(status), + Ok(status) if status.status == "not_configured" => Poll::Failed( + "no active gateway; register and select a gateway before running conformance" + .to_string(), + ), + Ok(status) => Poll::Pending(format!( + "gateway status is {:?}; expected \"connected\"\nstderr:\n{}", + status.status, + result.stderr() + )), + Err(error) => Poll::Failed(error.to_string()), + }, + Err(error) => Poll::Pending(error.to_string()), + }, + ) + .await + .map_err(|error| error.to_string())?; + + eprintln!( + "gateway preflight connected: gateway={}, server={}, version={}, authentication={}", + status.gateway.as_deref().unwrap_or("unknown"), + status.server.as_deref().unwrap_or("unknown"), + status.version.as_deref().unwrap_or("unknown"), + status + .authentication + .as_ref() + .map_or("unknown", |authentication| authentication.status.as_str()), + ); + Ok(()) + } + + /// Register a sandbox name for cleanup. + /// + /// Call this before running the command that may create the sandbox. The + /// scenario remains responsible for constructing and running that command. + pub fn track_sandbox(&mut self, name: &str) { + self.known_sandboxes.insert(name.to_string()); + } + + /// Stop tracking a sandbox after the scenario confirms it is absent. + pub fn forget_sandbox(&mut self, name: &str) { + self.known_sandboxes.remove(name); + } + + pub async fn poll_until( + &mut self, + step: &str, + poll_timeout: Duration, + interval: Duration, + mut observe: F, + ) -> Result + where + F: AsyncFnMut(&mut Self) -> Poll, + { + let started = Instant::now(); + let context = self.context(step); + + loop { + match observe(self).await { + Poll::Ready(value) => return Ok(value), + Poll::Pending(diagnostic) => { + if started.elapsed() >= poll_timeout { + return Err(RunnerError::PollTimeout { + context, + timeout: poll_timeout, + last_observation: diagnostic, + }); + } + } + Poll::Failed(diagnostic) => { + return Err(RunnerError::ObservationFailed { + context, + diagnostic, + }); + } + } + sleep(interval).await; + } + } + + pub async fn finish(mut self, scenario_result: Result<(), String>) -> Result<(), String> { + let cleanup_result = self.cleanup().await; + self.finished = true; + combine_results(scenario_result, cleanup_result) + } + + async fn run_strings( + &self, + step: &str, + expectation: &str, + args: Vec, + command_timeout: Duration, + ) -> Result { + let context = self.context(step); + let command = sanitized_command(&args); + eprintln!("{context} running: {command}"); + + let started = Instant::now(); + let output = + self.cli + .execute(args, command_timeout) + .await + .map_err(|error| match error { + CliExecutionError::Timeout => RunnerError::Timeout { + context: context.clone(), + command: command.clone(), + timeout: command_timeout, + }, + CliExecutionError::Spawn(source) => RunnerError::Spawn { + context: context.clone(), + command: command.clone(), + source, + }, + })?; + let elapsed = started.elapsed(); + eprintln!( + "{context} completed in {:.1?}: exit {}", + elapsed, + exit_description(output.status) + ); + + Ok(CommandResult { + run_id: self.run_id.clone(), + scenario: self.scenario.clone(), + step: step.to_string(), + expectation: expectation.to_string(), + command, + status: output.status, + elapsed, + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + }) + } + + async fn cleanup(&self) -> Result<(), String> { + if self.known_sandboxes.is_empty() { + return Ok(()); + } + + let cleanup_started = Instant::now(); + let mut failures = Vec::new(); + for name in self.known_sandboxes.clone() { + let remaining = remaining_cleanup_time(cleanup_started); + if remaining.is_zero() { + failures.push(format!( + "{} cleanup budget expired before deleting sandbox '{name}'", + self.context("cleanup/delete") + )); + break; + } + match self + .step("cleanup/delete") + .description(format!("sandbox '{name}' is deleted or already absent")) + .with_timeout(remaining) + .run(&["sandbox", "delete", &name]) + .await + { + Ok(result) if result.success() || output_reports_not_found(&result) => {} + Ok(result) => { + failures.push(result.failure_diagnostic(&format!( + "sandbox '{name}' is deleted or already absent" + ))); + } + Err(error) => failures.push(error.to_string()), + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("\n\n")) + } + } + + fn context(&self, step: &str) -> String { + format!("[run {}][{}/{}]", self.run_id, self.scenario, step) + } +} + +impl<'a> CommandStep<'a> { + #[must_use] + pub fn description(mut self, description: impl Into) -> Self { + self.description = Some(description.into()); + self + } + + pub fn with_timeout(self, timeout: Duration) -> OpenShellCommand<'a> { + let description = self + .description + .unwrap_or_else(|| format!("step '{}' succeeds", self.step)); + OpenShellCommand { + runner: self.runner, + step: self.step, + description, + timeout, + } + } +} + +impl OpenShellCommand<'_> { + pub async fn run(&self, args: &[&str]) -> Result { + self.runner + .run_strings( + &self.step, + &self.description, + args.iter().map(|arg| (*arg).to_string()).collect(), + self.timeout, + ) + .await + } +} + +impl Drop for OpenShellRunner { + fn drop(&mut self) { + if self.finished { + return; + } + let resources = if self.known_sandboxes.is_empty() { + "none explicitly tracked".to_string() + } else { + self.known_sandboxes + .iter() + .cloned() + .collect::>() + .join(", ") + }; + eprintln!( + "[run {}][{}] WARNING: runner dropped without finish(); known sandboxes: {}", + self.run_id, self.scenario, resources + ); + } +} + +fn combine_results( + scenario_result: Result<(), String>, + cleanup_result: Result<(), String>, +) -> Result<(), String> { + match (scenario_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(functional), Ok(())) => Err(functional), + (Ok(()), Err(cleanup)) => Err(format!("cleanup failed:\n{cleanup}")), + (Err(functional), Err(cleanup)) => Err(format!( + "{functional}\n\nsecondary cleanup failure:\n{cleanup}" + )), + } +} + +fn remaining_cleanup_time(started: Instant) -> Duration { + CLEANUP_TIMEOUT.saturating_sub(started.elapsed()) +} + +fn output_reports_not_found(result: &CommandResult) -> bool { + result.stderr().to_ascii_lowercase().contains("not found") + || result.stdout().to_ascii_lowercase().contains("not found") +} + +fn exit_description(status: ExitStatus) -> String { + status.code().map_or_else( + || "terminated by signal".to_string(), + |code| code.to_string(), + ) +} + +fn sanitized_command(args: &[String]) -> String { + let mut redact_next = false; + let rendered = args.iter().map(|arg| { + let rendered = if redact_next { + redact_next = false; + "".to_string() + } else if sensitive_flag(arg) { + redact_next = !arg.contains('='); + arg.split_once('=') + .map_or_else(|| arg.clone(), |(flag, _)| format!("{flag}=")) + } else { + arg.clone() + }; + shell_escape(&rendered) + }); + std::iter::once("openshell".to_string()) + .chain(rendered) + .collect::>() + .join(" ") +} + +fn sensitive_flag(arg: &str) -> bool { + let flag = arg.split_once('=').map_or(arg, |(flag, _)| flag); + matches!( + flag, + "--credential" + | "--credentials" + | "--env" + | "--header" + | "--material" + | "--password" + | "--secret" + | "--token" + ) || flag.contains("client-secret") + || flag.contains("private-key") + || flag.ends_with("-token") +} + +fn shell_escape(arg: &str) -> String { + if arg + .chars() + .all(|character| character.is_ascii_alphanumeric() || "-_./:@=,".contains(character)) + { + return arg.to_string(); + } + format!("'{}'", arg.replace('\'', "'\\''")) +} + +#[cfg(all(test, unix))] +mod tests { + use std::collections::VecDeque; + use std::os::unix::process::ExitStatusExt; + use std::process::Output; + use std::sync::Mutex; + + use serde::Deserialize; + + use super::executor::CliExecution; + use super::*; + + struct MockCli { + state: Mutex, + } + + struct MockCliState { + responses: VecDeque, + invocations: Vec>, + } + + enum MockResponse { + Output { + exit_code: i32, + stdout: String, + stderr: String, + }, + Timeout, + } + + impl MockResponse { + fn output(exit_code: i32, stdout: &str, stderr: &str) -> Self { + Self::Output { + exit_code, + stdout: stdout.to_string(), + stderr: stderr.to_string(), + } + } + + fn success(stdout: &str) -> Self { + Self::output(0, stdout, "") + } + } + + impl MockCli { + fn new(responses: Vec) -> Self { + Self { + state: Mutex::new(MockCliState { + responses: responses.into(), + invocations: Vec::new(), + }), + } + } + + fn invocations(&self) -> Vec> { + self.state + .lock() + .expect("lock mock CLI state") + .invocations + .clone() + } + } + + impl CliExecutor for MockCli { + fn execute(&self, args: Vec, _command_timeout: Duration) -> CliExecution<'_> { + let response = { + let mut state = self.state.lock().expect("lock mock CLI state"); + state.invocations.push(args); + state + .responses + .pop_front() + .expect("mock CLI received an unexpected invocation") + }; + + Box::pin(async move { + match response { + MockResponse::Output { + exit_code, + stdout, + stderr, + } => Ok(Output { + status: ExitStatus::from_raw(exit_code << 8), + stdout: stdout.into_bytes(), + stderr: stderr.into_bytes(), + }), + MockResponse::Timeout => Err(CliExecutionError::Timeout), + } + }) + } + } + + fn test_runner(responses: Vec) -> (OpenShellRunner, Arc) { + let cli = Arc::new(MockCli::new(responses)); + let runner = OpenShellRunner::with_executor(cli.clone(), "smoke"); + (runner, cli) + } + + #[test] + fn with_binary_rejects_a_missing_file() { + let directory = tempfile::tempdir().expect("create missing CLI directory"); + let Err(error) = OpenShellRunner::with_binary(directory.path().join("missing"), "smoke") + else { + panic!("missing CLI should be rejected"); + }; + + assert!(matches!(error, RunnerError::BinaryUnavailable(_))); + } + + #[test] + fn exposes_generated_identity() { + let (mut runner, _cli) = test_runner(Vec::new()); + + assert_eq!(runner.id().len(), 10); + assert_eq!(runner.scenario(), "smoke"); + runner.finished = true; + } + + #[tokio::test] + async fn captures_streams_and_nonzero_exit_as_result() { + let (mut runner, _cli) = test_runner(vec![MockResponse::output(7, "stdout", "stderr")]); + + let result = runner + .step("capture") + .description("sandbox list succeeds") + .with_timeout(Duration::from_secs(1)) + .run(&["sandbox", "list"]) + .await + .expect("completed nonzero exit is a result"); + + assert_eq!(result.exit_code(), Some(7)); + assert_eq!(result.stdout(), "stdout"); + assert_eq!(result.stderr(), "stderr"); + assert!( + result + .require_success() + .expect_err("nonzero result should fail its expectation") + .contains("expected: sandbox list succeeds") + ); + runner.finished = true; + } + + #[tokio::test] + async fn reports_timeout_as_typed_error() { + let (mut runner, _cli) = test_runner(vec![MockResponse::Timeout]); + + let error = runner + .step("timeout") + .with_timeout(Duration::from_millis(10)) + .run(&["status"]) + .await + .expect_err("command should time out"); + + assert!(matches!(error, RunnerError::Timeout { .. })); + runner.finished = true; + } + + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct JsonFixture { + status: String, + } + + #[tokio::test] + async fn deserializes_stdout_json_without_stderr() { + let (mut runner, _cli) = test_runner(vec![MockResponse::output( + 0, + "{\"status\":\"connected\"}", + "diagnostic", + )]); + + let result = runner + .step("json") + .with_timeout(Duration::from_secs(1)) + .run(&["status"]) + .await + .expect("run fake CLI"); + + assert_eq!( + result.json::().expect("parse JSON"), + JsonFixture { + status: "connected".to_string() + } + ); + runner.finished = true; + } + + #[tokio::test] + async fn accepts_connected_gateway_status() { + let (mut runner, _cli) = + test_runner(vec![MockResponse::success("{\"status\":\"connected\"}")]); + + runner + .check_gateway_status() + .await + .expect("connected gateway should satisfy preflight"); + + runner.finished = true; + } + + #[tokio::test] + async fn polling_returns_ready_value() { + let (mut runner, _cli) = test_runner(Vec::new()); + let mut attempts = 0; + + let value = runner + .poll_until( + "poll", + Duration::from_secs(1), + Duration::from_millis(1), + async |_runner| { + attempts += 1; + if attempts == 2 { + Poll::Ready("ready") + } else { + Poll::Pending("not ready".to_string()) + } + }, + ) + .await + .expect("poll should become ready"); + + assert_eq!(value, "ready"); + runner.finished = true; + } + + #[tokio::test] + async fn polling_timeout_preserves_last_observation() { + let (mut runner, _cli) = test_runner(Vec::new()); + + let error = runner + .poll_until( + "poll", + Duration::from_millis(1), + Duration::from_millis(1), + async |_runner| Poll::<()>::Pending("still pending".to_string()), + ) + .await + .expect_err("poll should time out"); + + assert!(matches!( + error, + RunnerError::PollTimeout { + last_observation, + .. + } if last_observation == "still pending" + )); + runner.finished = true; + } + + #[tokio::test] + async fn finish_skips_cleanup_commands_without_registered_resources() { + let (runner, cli) = test_runner(Vec::new()); + + let error = runner + .finish(Err("preflight failed".to_string())) + .await + .expect_err("functional failure should be preserved"); + + assert_eq!(error, "preflight failed"); + assert!(cli.invocations().is_empty()); + } + + #[test] + fn tracks_sandbox_for_cleanup() { + let (mut runner, _cli) = test_runner(Vec::new()); + + runner.track_sandbox("ct-0123456789-01"); + + assert!(runner.known_sandboxes.contains("ct-0123456789-01")); + runner.finished = true; + } + + #[test] + fn forgets_exact_name_cleanup() { + let (mut runner, _cli) = test_runner(Vec::new()); + + runner.track_sandbox("ct-0123456789-01"); + runner.forget_sandbox("ct-0123456789-01"); + + assert!(runner.known_sandboxes.is_empty()); + runner.finished = true; + } + + #[tokio::test] + async fn finish_deletes_explicitly_tracked_resources() { + let (mut runner, cli) = test_runner(vec![MockResponse::success("")]); + runner.track_sandbox("ct-0123456789-01"); + + runner + .finish(Ok(())) + .await + .expect("owned resource cleanup should succeed"); + + assert_eq!( + cli.invocations(), + vec![vec![ + "sandbox".to_string(), + "delete".to_string(), + "ct-0123456789-01".to_string(), + ]] + ); + } + + #[test] + fn functional_failure_remains_primary_when_cleanup_also_fails() { + let error = combine_results( + Err("functional failure".to_string()), + Err("cleanup failure".to_string()), + ) + .expect_err("combined result should fail"); + + assert!(error.starts_with("functional failure")); + assert!(error.contains("secondary cleanup failure:\ncleanup failure")); + } + + #[test] + fn generated_run_id_is_portable_and_compact() { + let run_id = generate_run_id(); + assert_eq!(run_id.len(), 10); + assert!( + run_id + .chars() + .all(|character| character.is_ascii_lowercase() || character.is_ascii_digit()) + ); + } + + #[test] + fn sanitizes_sensitive_cli_arguments() { + let command = sanitized_command(&[ + "provider".to_string(), + "create".to_string(), + "--credential".to_string(), + "TOKEN=secret".to_string(), + "--client-secret=hunter2".to_string(), + ]); + + assert!(!command.contains("TOKEN=secret")); + assert!(!command.contains("hunter2")); + assert!(command.contains("")); + } +} diff --git a/crates/openshell-conformance/src/scenarios/mod.rs b/crates/openshell-conformance/src/scenarios/mod.rs new file mode 100644 index 0000000000..bd6abdc7af --- /dev/null +++ b/crates/openshell-conformance/src/scenarios/mod.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Registered, portable conformance scenarios. + +mod smoke; + +pub use smoke::SMOKE_SCENARIO; diff --git a/crates/openshell-conformance/src/scenarios/smoke.rs b/crates/openshell-conformance/src/scenarios/smoke.rs new file mode 100644 index 0000000000..7a71cb9099 --- /dev/null +++ b/crates/openshell-conformance/src/scenarios/smoke.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Portable phase-1 CLI conformance scenario. + +use std::time::{Duration, Instant}; + +use crate::{OpenShellRunner, STATUS_TIMEOUT, Scenario, ScenarioFuture}; +use serde::Deserialize; +use tokio::time::sleep; + +const CREATE_TIMEOUT: Duration = Duration::from_secs(600); +const LIST_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(10); +const LIST_PAGE_SIZE: u32 = 1_000; +const EXEC_TIMEOUT: Duration = Duration::from_secs(120); +const DELETE_TIMEOUT: Duration = Duration::from_secs(120); +const DELETE_POLL_INTERVAL: Duration = Duration::from_secs(1); + +#[derive(Debug, Deserialize)] +struct SandboxListEntry { + name: String, + phase: String, +} + +/// Certify status -> create -> list Ready -> exec -> delete -> list empty. +pub const SMOKE_SCENARIO: Scenario = Scenario { + name: "smoke", + description: "Create, inspect, execute in, and delete a base sandbox.", + run: run_smoke, +}; + +fn run_smoke(runner: &mut OpenShellRunner) -> ScenarioFuture<'_> { + Box::pin(async move { run_smoke_inner(runner).await }) +} + +async fn run_smoke_inner(runner: &mut OpenShellRunner) -> Result<(), String> { + let status = runner + .step("status") + .description("openshell status succeeds") + .with_timeout(STATUS_TIMEOUT) + .run(&["status"]) + .await + .map_err(|error| error.to_string())?; + status.require_success()?; + + let sandbox_name = format!("ct-{}-01", runner.id()); + runner.track_sandbox(&sandbox_name); + let create = runner + .step("create") + .description("sandbox creation succeeds") + .with_timeout(CREATE_TIMEOUT) + .run(&[ + "sandbox", + "create", + "--name", + &sandbox_name, + "--from", + "base", + "--detach", + ]) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + + let get = runner + .step("get-ready") + .description(format!("sandbox '{sandbox_name}' can be retrieved")) + .with_timeout(LIST_ATTEMPT_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + get.require_success()?; + + let sandbox = get + .json::() + .map_err(|error| error.to_string())?; + if sandbox.name != sandbox_name { + return Err(format!( + "sandbox get returned {:?}; expected sandbox '{sandbox_name}'", + sandbox.name + )); + } + if sandbox.phase != "Ready" { + return Err(format!( + "sandbox '{sandbox_name}' is in phase {:?}; expected Ready", + sandbox.phase + )); + } + + check_sandbox_listed(runner, &sandbox_name).await?; + + let marker = format!("openshell-conformance-{}", runner.id()); + let exec = runner + .step("exec") + .description("sandbox exec exits successfully") + .with_timeout(EXEC_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &sandbox_name, + "--no-tty", + "--", + "echo", + &marker, + ]) + .await + .map_err(|error| error.to_string())?; + exec.require_success()?; + let expected_stdout = format!("{marker}\n"); + if exec.stdout() != expected_stdout { + return Err(exec.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}"))); + } + + let delete = runner + .step("delete") + .description("sandbox deletion succeeds") + .with_timeout(DELETE_TIMEOUT) + .run(&["sandbox", "delete", &sandbox_name]) + .await + .map_err(|error| error.to_string())?; + delete.require_success()?; + + check_empty_list(runner, &sandbox_name).await?; + runner.forget_sandbox(&sandbox_name); + Ok(()) +} + +async fn check_sandbox_listed(runner: &OpenShellRunner, sandbox_name: &str) -> Result<(), String> { + if find_sandbox(runner, sandbox_name, "list-visible") + .await? + .is_some() + { + return Ok(()); + } + Err(format!( + "sandbox '{sandbox_name}' does not appear in sandbox list" + )) +} + +async fn check_empty_list(runner: &OpenShellRunner, sandbox_name: &str) -> Result<(), String> { + let started = Instant::now(); + + loop { + match find_sandbox(runner, sandbox_name, "list-empty/query").await? { + None => return Ok(()), + Some(sandbox) if started.elapsed() >= DELETE_TIMEOUT => { + return Err(format!( + "sandbox '{sandbox_name}' remains listed in phase {:?} after {DELETE_TIMEOUT:.1?}", + sandbox.phase + )); + } + Some(_) => sleep(DELETE_POLL_INTERVAL).await, + } + } +} + +async fn find_sandbox( + runner: &OpenShellRunner, + sandbox_name: &str, + step: &str, +) -> Result, String> { + let mut offset = 0u32; + + loop { + let limit = LIST_PAGE_SIZE.to_string(); + let page_offset = offset.to_string(); + let result = runner + .step(format!("{step}/{offset}")) + .description(format!("sandbox list page at offset {offset} succeeds")) + .with_timeout(LIST_ATTEMPT_TIMEOUT) + .run(&[ + "sandbox", + "list", + "--limit", + &limit, + "--offset", + &page_offset, + "--output", + "json", + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + + let sandboxes = result + .json::>() + .map_err(|error| error.to_string())?; + if let Some(sandbox) = sandboxes + .iter() + .find(|sandbox| sandbox.name == sandbox_name) + { + return Ok(Some(SandboxListEntry { + name: sandbox.name.clone(), + phase: sandbox.phase.clone(), + })); + } + if sandboxes.len() < LIST_PAGE_SIZE as usize { + return Ok(None); + } + + offset = offset + .checked_add(LIST_PAGE_SIZE) + .ok_or_else(|| "sandbox list pagination offset overflowed".to_string())?; + } +} diff --git a/docs/index.yml b/docs/index.yml index 45db451a78..f4bd842cc8 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -27,6 +27,9 @@ navigation: title: "Kubernetes" - folder: reference title: "Reference" + contents: + - page: "OpenShell conformance CLI" + path: reference/conformance-cli.mdx - folder: security title: "Security" - folder: resources diff --git a/docs/reference/conformance-cli.mdx b/docs/reference/conformance-cli.mdx new file mode 100644 index 0000000000..fc468e312d --- /dev/null +++ b/docs/reference/conformance-cli.mdx @@ -0,0 +1,34 @@ +--- +title: OpenShell conformance CLI +--- + +`openshell-conformance` runs portable conformance scenarios against an existing OpenShell gateway. It is a standalone binary: the target system needs the OpenShell CLI and its configured gateway, but does not need Rust, Cargo, or Mise. + +Build a release binary on the system that prepares the test artifact: + +```shell +cargo build --release -p openshell-conformance-cli +``` + +Copy `target/release/openshell-conformance` to the target system and ensure `openshell` is on `PATH`. Run all installed scenarios: + +```shell +openshell-conformance run +``` + +Run one scenario or select an explicit OpenShell CLI binary: + +```shell +openshell-conformance run smoke --openshell-bin /opt/openshell/bin/openshell +``` + +Use JSON for automation: + +```shell +openshell-conformance list --output json +openshell-conformance run --output json +``` + +The initial `smoke` scenario verifies gateway status, creates a base sandbox, checks that it is ready and listed, executes a marker command, and deletes the sandbox. It cleans up any tracked sandbox when a later assertion fails. + +Scenario implementations live under `crates/openshell-conformance/src/scenarios/`. Add a scenario there and register it in the library's `SCENARIOS` list. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 44881e1682..6dca586879 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -16,6 +16,7 @@ license = "Apache-2.0" publish = false [features] +# Selects the common E2E suite. e2e = [] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the @@ -47,6 +48,11 @@ name = "provider_refresh_keycloak" path = "tests/provider_refresh_keycloak.rs" required-features = ["e2e-provider-refresh-keycloak"] +[[test]] +name = "vm_overlay" +path = "tests/vm_overlay.rs" +required-features = ["e2e-vm"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 6c28868083..ba97b85a5b 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -2,23 +2,47 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Run a Rust e2e test against a standalone gateway running the bundled Docker -# compute driver. Set OPENSHELL_GATEWAY_ENDPOINT=http://host:port to reuse an -# existing plaintext gateway instead of starting an ephemeral one. +# Run standalone CLI conformance, and optionally a focused Rust e2e test, +# against a gateway using the bundled Docker compute driver. Set +# OPENSHELL_GATEWAY_ENDPOINT=http://host:port to reuse an existing plaintext +# gateway instead of starting an ephemeral one. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}" -E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e,e2e-docker}" +E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-}" +E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES-e2e-docker}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" +RUN_WITH_GATEWAY_COMMAND="__openshell_run_docker_e2e" +# shellcheck source=e2e/support/conformance.sh +source "${ROOT}/e2e/support/conformance.sh" + +run_e2e() { + e2e_run_openshell_conformance "Docker" + + if [ -z "${E2E_FEATURES}" ]; then + return 0 + fi + + local cargo_args=( + cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" + --features "${E2E_FEATURES}" + ) + if [ -n "${E2E_TEST}" ]; then + cargo_args+=(--test "${E2E_TEST}") + fi + cargo_args+=(-- --nocapture) + "${cargo_args[@]}" +} if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && [ ! -f "${DEFAULT_WORKLOAD_MANIFEST}" ]; then echo "note: running GPU e2e without a workload manifest; workload validation will log an explicit skip. Build one with 'mise run e2e:workloads:build' or set OPENSHELL_E2E_WORKLOAD_MANIFEST." fi +if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then + run_e2e + exit 0 +fi + exec "${ROOT}/e2e/with-docker-gateway.sh" \ - cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ - --features "${E2E_FEATURES}" \ - --test "${E2E_TEST}" \ - -- --nocapture + bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index cf28e35728..5ff5e002bc 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -6,7 +6,7 @@ # via Helm. Set OPENSHELL_E2E_KUBE_CONTEXT to target an existing cluster; # otherwise an ephemeral k3d cluster is created and torn down by # with-kube-gateway.sh. Set OPENSHELL_E2E_KUBE_TEST to scope to a single -# integration test (e.g. smoke) for local debugging. +# integration test for local debugging. # # Features: the default set includes `e2e-host-gateway` so tests that rely on # the sandbox-side `host.openshell.internal` alias compile and run. The @@ -18,14 +18,18 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUN_WITH_GATEWAY_COMMAND="__openshell_run_kubernetes_e2e" +# shellcheck source=e2e/support/conformance.sh +source "${ROOT}/e2e/support/conformance.sh" -E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kubernetes}" +E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES-e2e,e2e-host-gateway,e2e-kubernetes}" # Docker and Podman build their local gateway and CLI together in the shared # gateway wrapper. Kubernetes consumes published gateway images, so only its # local CLI needs to be built when CI has not supplied a prebuilt one. if [ -z "${OPENSHELL_BIN:-}" ]; then cargo build -p openshell-cli + export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi test_filter=() @@ -33,15 +37,85 @@ if [ -n "${OPENSHELL_E2E_KUBE_TEST:-}" ]; then test_filter+=(--test "${OPENSHELL_E2E_KUBE_TEST}") fi +is_operator_workspace_mode() { + [[ ",${E2E_FEATURES}," == *",e2e-kubernetes-workspace-operator,"* ]] +} + +kubectl_active() { + kubectl --context "${OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE}" "$@" +} + +run_operator_conformance() { + local workspace="cf-${RANDOM}" + local previous_workspace="${OPENSHELL_WORKSPACE-}" + local had_previous_workspace=0 + if [ -n "${OPENSHELL_WORKSPACE+x}" ]; then + had_previous_workspace=1 + fi + + kubectl_active create namespace "${workspace}" + kubectl_active label namespace "${workspace}" "openshell.ai/e2e-operator-workspace=true" + kubectl_active create serviceaccount openshell-sandbox -n "${workspace}" + "${OPENSHELL_BIN}" workspace create --name "${workspace}" + + export OPENSHELL_WORKSPACE="${workspace}" + local deadline=$((SECONDS + 30)) + local status=1 + while true; do + if e2e_run_openshell_conformance "Kubernetes"; then + status=0 + break + fi + status=$? + if [ "${SECONDS}" -ge "${deadline}" ]; then + break + fi + sleep 2 + done + + if [ "${had_previous_workspace}" -eq 1 ]; then + export OPENSHELL_WORKSPACE="${previous_workspace}" + else + unset OPENSHELL_WORKSPACE + fi + + "${OPENSHELL_BIN}" workspace delete "${workspace}" || true + kubectl_active delete namespace "${workspace}" --ignore-not-found --wait=false || true + return "${status}" +} + +run_conformance() { + if is_operator_workspace_mode; then + run_operator_conformance + return + fi + + e2e_run_openshell_conformance "Kubernetes" +} + run_suite() { "${ROOT}/e2e/with-kube-gateway.sh" \ - cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ - --features "${E2E_FEATURES}" \ - --no-fail-fast \ - ${test_filter[@]+"${test_filter[@]}"} \ - -- --nocapture + bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" } +run_e2e() { + run_conformance + if [ -z "${E2E_FEATURES}" ]; then + return 0 + fi + + cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ + --features "${E2E_FEATURES}" \ + --no-fail-fast \ + ${test_filter[@]+"${test_filter[@]}"} \ + -- --nocapture +} + +if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then + run_e2e + exit 0 +fi + if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVERS:-0}" = "1" ] \ && [ -z "${OPENSHELL_E2E_CREDENTIAL_DRIVER:-}" ]; then OPENSHELL_E2E_CREDENTIAL_DRIVER=kubernetes-secrets run_suite @@ -50,8 +124,4 @@ if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVERS:-0}" = "1" ] \ fi exec "${ROOT}/e2e/with-kube-gateway.sh" \ - cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ - --features "${E2E_FEATURES}" \ - --no-fail-fast \ - ${test_filter[@]+"${test_filter[@]}"} \ - -- --nocapture + bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index f5f1effd18..736519458e 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -10,28 +10,43 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}" +E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES-e2e-podman}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" +RUN_WITH_GATEWAY_COMMAND="__openshell_run_podman_e2e" +# shellcheck source=e2e/support/conformance.sh +source "${ROOT}/e2e/support/conformance.sh" if [ "${E2E_TEST}" = "gpu" ] && [ -z "${OPENSHELL_E2E_WORKLOAD_MANIFEST:-}" ] && [ ! -f "${DEFAULT_WORKLOAD_MANIFEST}" ]; then echo "note: running Podman GPU e2e without a workload manifest; workload validation will log an explicit skip. Build one with 'CONTAINER_ENGINE=podman mise run e2e:workloads:build' or set OPENSHELL_E2E_WORKLOAD_MANIFEST." fi +if [ "${1:-}" = "${RUN_WITH_GATEWAY_COMMAND}" ]; then + e2e_run_openshell_conformance "Podman" + if [ -z "${E2E_FEATURES}" ]; then + exit 0 + fi + + TEST_ARGS=( + cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" + --features "${E2E_FEATURES}" + ) + if [ -n "${E2E_TEST}" ]; then + TEST_ARGS+=(--test "${E2E_TEST}") + fi + TEST_ARGS+=(-- --nocapture) + "${TEST_ARGS[@]}" + exit 0 +fi + # An empty selector runs the full Podman suite, including provider_token_exchange. if [ -z "${E2E_TEST}" ] || [ "${E2E_TEST}" = "provider_token_exchange" ]; then export OPENSHELL_E2E_SPIFFE_FIXTURE="${OPENSHELL_E2E_SPIFFE_FIXTURE:-1}" fi -cargo build -p openshell-cli - -TEST_ARGS=( - cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" - --features "${E2E_FEATURES}" -) -if [ -n "${E2E_TEST}" ]; then - TEST_ARGS+=(--test "${E2E_TEST}") +if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ] && [ -z "${OPENSHELL_BIN:-}" ]; then + cargo build -p openshell-cli + export OPENSHELL_BIN="${ROOT}/target/debug/openshell" fi -TEST_ARGS+=(-- --nocapture) exec "${ROOT}/e2e/with-podman-gateway.sh" \ - "${TEST_ARGS[@]}" + bash "${BASH_SOURCE[0]}" "${RUN_WITH_GATEWAY_COMMAND}" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 1960f83588..a157a73d62 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -46,13 +46,15 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "${ROOT}/e2e/support/gateway-common.sh" +# shellcheck source=e2e/support/conformance.sh +source "${ROOT}/e2e/support/conformance.sh" COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${OPENSHELL_GATEWAY_BIN:-${ROOT}/target/debug/openshell-gateway}" DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}" CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" +E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES-e2e-vm}" SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. @@ -371,7 +373,6 @@ fi # The CLI uses the raw endpoint but still resolves matching metadata so it # can find the mTLS client bundle. -export OPENSHELL_E2E_EXPECT_VM_OVERLAY=1 export OPENSHELL_E2E_DRIVER="vm" export OPENSHELL_E2E_VM_STATE_DIR="${RUN_STATE_DIR}" e2e_export_gateway_restart_metadata \ @@ -386,6 +387,8 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" +e2e_run_openshell_conformance "VM" + run_e2e_test() { local test_target="$1" shift @@ -403,7 +406,7 @@ run_e2e_test() { if [ -n "${E2E_TEST_OVERRIDE}" ]; then run_e2e_test "${E2E_TEST_OVERRIDE}" else - run_e2e_test smoke run_e2e_test host_gateway_alias + run_e2e_test vm_overlay run_e2e_test vm_gateway_start fi diff --git a/e2e/rust/tests/smoke.rs b/e2e/rust/tests/smoke.rs deleted file mode 100644 index c27255e5ef..0000000000 --- a/e2e/rust/tests/smoke.rs +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! Smoke test: verify the gateway is healthy, create a sandbox, exec a -//! command inside it, and tear it down. -//! -//! This test is cluster-agnostic — it works against any running gateway -//! (Docker-based cluster or openshell-driver-vm microVM). The `e2e:vm` mise -//! task uses it to validate the VM gateway after boot. - -use std::process::Stdio; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::strip_ansi; -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// End-to-end smoke test: status → create → exec → list → delete. -#[tokio::test] -async fn gateway_smoke() { - // ── 1. Gateway must be reachable ────────────────────────────────── - let mut clean_status = String::new(); - let mut status_ok = false; - for _ in 0..15 { - let mut status_cmd = openshell_cmd(); - status_cmd - .arg("status") - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let status_out = status_cmd - .output() - .await - .expect("failed to run openshell status"); - - let status_text = format!( - "{}{}", - String::from_utf8_lossy(&status_out.stdout), - String::from_utf8_lossy(&status_out.stderr), - ); - clean_status = strip_ansi(&status_text); - - if status_out.status.success() && clean_status.contains("Connected") { - status_ok = true; - break; - } - - tokio::time::sleep(Duration::from_secs(2)).await; - } - - assert!( - status_ok, - "openshell status never became healthy:\n{clean_status}", - ); - - // ── 2. Create a sandbox and exec a command ─────────────────────── - // Default behaviour keeps the sandbox alive after the command exits, - // so we can verify it in the list before cleaning up. - let mut sb = SandboxGuard::create(&["--", "echo", "smoke-ok"]) - .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("smoke-ok"), - "expected 'smoke-ok' in sandbox output:\n{}", - sb.create_output, - ); - - if std::env::var_os("OPENSHELL_E2E_EXPECT_VM_OVERLAY").is_some() { - assert_vm_overlay_root(&sb.name).await; - } - - // ── 3. Verify the sandbox appeared in the list ─────────────────── - let mut list_cmd = openshell_cmd(); - list_cmd - .args(["sandbox", "list", "--names"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let list_out = list_cmd - .output() - .await - .expect("failed to run openshell sandbox list"); - - let list_text = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&list_out.stdout), - String::from_utf8_lossy(&list_out.stderr), - )); - - assert!( - list_text.contains(&sb.name), - "sandbox '{}' should appear in list output:\n{list_text}", - sb.name, - ); - - // ── 4. Cleanup ─────────────────────────────────────────────────── - sb.cleanup().await; -} - -async fn assert_vm_overlay_root(sandbox_name: &str) { - let script = concat!( - "set -eu; ", - "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", - "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", - "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", - "if [ -e /opt/openshell/tls/tls.key ]; then ", - "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", - "fi; ", - "echo vm-overlay-ok", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .arg("sh") - .arg("-lc") - .arg(script) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = exec_cmd - .output() - .await - .expect("failed to run VM overlay assertion"); - let combined = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert!( - output.status.success() && combined.contains("vm-overlay-ok"), - "VM overlay assertion failed (status {:?}):\n{combined}", - output.status.code(), - ); -} diff --git a/e2e/rust/tests/vm_overlay.rs b/e2e/rust/tests/vm_overlay.rs new file mode 100644 index 0000000000..078c6f01de --- /dev/null +++ b/e2e/rust/tests/vm_overlay.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM-driver-specific assertions for the sandbox root filesystem. + +use std::process::Stdio; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +#[tokio::test] +async fn vm_overlay() { + let mut sandbox = SandboxGuard::create(&["--", "echo", "vm-sandbox-ready"]) + .await + .expect("sandbox create should succeed"); + + let script = concat!( + "set -eu; ", + "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", + "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", + "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", + "if [ -e /opt/openshell/tls/tls.key ]; then ", + "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", + "fi; ", + "echo vm-overlay-ok", + ); + + let mut exec_cmd = openshell_cmd(); + exec_cmd + .args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"]) + .arg("sh") + .arg("-lc") + .arg(script) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = exec_cmd + .output() + .await + .expect("failed to run VM overlay assertion"); + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + assert!( + output.status.success() && combined.contains("vm-overlay-ok"), + "VM overlay assertion failed (status {:?}):\n{combined}", + output.status.code(), + ); + + sandbox.cleanup().await; +} diff --git a/e2e/support/conformance.sh b/e2e/support/conformance.sh new file mode 100644 index 0000000000..dd2cbcc596 --- /dev/null +++ b/e2e/support/conformance.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared helpers for running standalone conformance suites against an already +# configured e2e gateway. + +e2e_run_openshell_conformance() { + local gateway_label=${1:-OpenShell} + + if [ -z "${OPENSHELL_BIN:-}" ]; then + echo "ERROR: OPENSHELL_BIN must point to the openshell CLI under test" >&2 + return 2 + fi + + if [ -z "${OPENSHELL_CONFORMANCE_BIN:-}" ]; then + echo "ERROR: OPENSHELL_CONFORMANCE_BIN must point to the openshell-conformance CLI under test" >&2 + return 2 + fi + + if [ ! -x "${OPENSHELL_CONFORMANCE_BIN}" ]; then + echo "ERROR: openshell conformance binary is not executable: ${OPENSHELL_CONFORMANCE_BIN}" >&2 + return 2 + fi + + echo "==> Running standalone CLI conformance against the ${gateway_label} gateway" + "${OPENSHELL_CONFORMANCE_BIN}" run \ + --openshell-bin "${OPENSHELL_BIN}" \ + --output json +} diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 958ea1b0eb..0a767576dd 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -437,6 +437,7 @@ ensure_sandbox_image_available() { } e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +export OPENSHELL_BIN="${CLI_BIN}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then e2e_build_external_driver \ "${ROOT}" openshell-driver-docker openshell-driver-docker DRIVER_BIN diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index dcb2d43fe0..fc3419e182 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -383,6 +383,7 @@ fi ensure_podman_api_socket e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +export OPENSHELL_BIN="${CLI_BIN}" if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then e2e_build_external_driver \ "${ROOT}" openshell-driver-podman openshell-driver-podman DRIVER_BIN diff --git a/tasks/test.toml b/tasks/test.toml index dfb632dda0..c88219f2ab 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -47,7 +47,7 @@ hide = true [e2e] description = "Run all end-to-end tests (Rust + Python + MCP)" -depends = ["e2e:rust", "e2e:python", "e2e:mcp"] +depends = ["e2e:docker", "e2e:python", "e2e:mcp"] ["e2e:test"] description = "Build the current checkout and run a named host or Nix test-guest E2E suite" @@ -80,12 +80,19 @@ env = { UV_NO_SYNC = "1" } run = "uv run pytest python/" hide = true -["e2e:rust"] -description = "Run Rust CLI e2e tests against a Docker-backed gateway" +["e2e:cli-conformance"] +description = "Build and run the standalone CLI conformance suite against the configured gateway" +depends = ["e2e:conformance:build"] run = [ - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", + "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi", + "\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" run --openshell-bin \"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\"", ] +["e2e:conformance:build"] +description = "Build the standalone CLI conformance binary" +run = "if [ -z \"${OPENSHELL_CONFORMANCE_BIN:-}\" ]; then cargo build -p openshell-conformance-cli; fi" +hide = true + ["e2e:websocket-conformance"] description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" run = [ @@ -108,7 +115,8 @@ run = "e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py *_tes ["e2e:podman"] description = "Run Rust CLI e2e tests against a Podman-backed gateway" -run = "e2e/rust/e2e-podman.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" ["e2e:oidc-pkce"] description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Podman gateway" @@ -138,85 +146,101 @@ run = [ ["e2e:podman:rootless"] description = "Run Rust CLI e2e tests against a rootless Podman-backed gateway" -run = "e2e/rust/e2e-podman-rootless.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman-rootless.sh" ["e2e:podman:gpu"] description = "Run GPU e2e against a standalone gateway with the Podman compute driver" env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" } -run = "e2e/rust/e2e-podman.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" ["e2e:kubernetes"] description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST= to scope to one test)" -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:v1alpha1"] description = "Run Kubernetes e2e against Agent Sandbox v1alpha1" env = { AGENT_SANDBOX_VERSION = "v0.4.6" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:agent-sandbox-versions"] description = "Run Kubernetes e2e against Agent Sandbox v1beta1 and v1alpha1" +depends = ["e2e:conformance:build"] run = [ - "e2e/rust/e2e-kubernetes.sh", - "AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", + "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh", + "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] ["e2e:kubernetes:sidecar"] description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:credential-drivers"] description = "Run Kubernetes e2e for provider credential storage backed by Kubernetes Secrets and Vault" env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-managed"] description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:kubernetes:workspace-operator"] description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)" env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" } -run = "e2e/rust/e2e-kubernetes.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" -run = "e2e/rust/e2e-vm.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" ["e2e:gateway:no-compute-drivers"] description = "Build and launch-check openshell-gateway without compiled compute drivers" run = "bash e2e/no-compute-driver-gateway.sh" ["e2e:docker:external-driver"] -description = "Run Docker smoke E2E with a driver-free gateway and external Docker driver binary" -env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } -run = "e2e/rust/e2e-docker.sh" +description = "Run Docker conformance with a driver-free gateway and external Docker driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "" } +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" ["e2e:podman:external-driver"] -description = "Run Podman E2E with a driver-free gateway and external Podman driver binary" -env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_TEST = "smoke" } -run = "e2e/rust/e2e-podman.sh" +description = "Run Podman conformance with a driver-free gateway and external Podman driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "" } +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh" ["e2e:vm:external-driver"] description = "Run VM E2E with a driver-free gateway and external VM driver binary" env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } -run = "e2e/rust/e2e-vm.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh" ["e2e:kubernetes:external-driver"] -description = "Run Kubernetes smoke E2E with a driver-free gateway and external Kubernetes driver sidecar" -env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBE_TEST = "smoke" } -run = "e2e/rust/e2e-kubernetes.sh" +description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver sidecar" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" } +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh" ["e2e:docker"] -description = "Run smoke e2e against a standalone gateway with the Docker compute driver" -run = "e2e/rust/e2e-docker.sh" +description = "Run Docker conformance and Rust e2e tests against a standalone gateway" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" ["e2e:mechanistic-smoke"] description = "Run mechanistic L4 smoke against a Docker-backed gateway" @@ -235,7 +259,8 @@ run = [ ["e2e:docker:gpu"] description = "Run GPU e2e against a standalone gateway with the Docker compute driver" env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "gpu", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-docker-gpu" } -run = "e2e/rust/e2e-docker.sh" +depends = ["e2e:conformance:build"] +run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh" ["e2e:openshift"] description = "Run OpenShift database-backend integration scenarios against a live cluster (requires oc CLI authenticated to an OpenShift cluster)"