test(e2e): run conformance in gateway lanes

.github/actions/setup-e2e-cli/action.yml

@@ -1,10 +1,14 @@
 name: Setup E2E CLI
-description: Download an architecture-matched prebuilt OpenShell CLI for E2E tests
+description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests
 
 inputs:
   artifact-prefix:
     description: Artifact name prefix; linux-<arch> is appended automatically
     required: true
+  conformance-artifact-prefix:
+    description: Optional conformance CLI artifact name prefix; linux-<arch> is appended automatically
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -27,3 +31,24 @@ runs:
         chmod +x "$cli"
         "$cli" --version
         echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV"
+
+    - name: Download prebuilt conformance CLI
+      if: inputs.conformance-artifact-prefix != ''
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ${{ format('{0}-linux-{1}', inputs.conformance-artifact-prefix, runner.arch == 'X64' && 'amd64' || 'arm64') }}
+        path: .e2e/prebuilt-conformance
+
+    - name: Configure prebuilt conformance CLI
+      if: inputs.conformance-artifact-prefix != ''
+      shell: bash
+      run: |
+        set -euo pipefail
+        conformance="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance"
+        if [[ ! -f "$conformance" ]]; then
+          echo "downloaded artifact is missing $conformance" >&2
+          exit 1
+        fi
+        chmod +x "$conformance"
+        "$conformance" list --output json >/dev/null
+        echo "OPENSHELL_CONFORMANCE_BIN=$conformance" >> "$GITHUB_ENV"

.github/workflows/branch-e2e.yml

@@ -109,6 +109,17 @@ jobs:
       component: cli
       platform: linux/amd64,linux/arm64
 
+  build-conformance:
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true'
+    permissions:
+      contents: read
+      packages: write
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      component: conformance
+      platform: linux/amd64,linux/arm64
+
   build-driver-vm-linux:
     needs: [pr_metadata]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
@@ -124,7 +135,7 @@ jobs:
         [{"arch":"amd64","runner":"linux-amd64-cpu8","target":"x86_64-unknown-linux-gnu","zig_target":"x86_64-unknown-linux-gnu.2.28","platform":"linux-x86_64","guest_arch":"x86_64"}]
 
   e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -135,11 +146,12 @@ jobs:
       image-tag: ${{ github.sha }}
       runner: linux-arm64-cpu8
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
       gateway-artifact-prefix: rust-binary-gateway
       vm-driver-artifact-name: driver-vm-linux-amd64
 
   gpu-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true'
     permissions:
       actions: read
@@ -149,10 +161,11 @@ jobs:
     with:
       image-tag: ${{ github.sha }}
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
       gateway-artifact-prefix: rust-binary-gateway
 
   kubernetes-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     strategy:
       fail-fast: false
@@ -181,9 +194,10 @@ jobs:
       agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
       extra-helm-values: ${{ matrix.extra_helm_values }}
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   kubernetes-workspace-managed-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -195,9 +209,10 @@ jobs:
       job-name: Kubernetes E2E (workspace managed mode)
       e2e-task: e2e:kubernetes:workspace-managed
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   kubernetes-external-driver-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -209,9 +224,10 @@ jobs:
       job-name: Kubernetes E2E (external compute driver)
       e2e-task: e2e:kubernetes:external-driver
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   kubernetes-workspace-operator-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
@@ -223,9 +239,10 @@ jobs:
       job-name: Kubernetes E2E (workspace operator mode)
       e2e-task: e2e:kubernetes:workspace-operator
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   kubernetes-ha-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
     permissions:
       actions: read
@@ -238,9 +255,10 @@ jobs:
       extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml
       external-postgres-secret: openshell-ha-pg
       cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   kubernetes-credential-drivers-e2e:
-    needs: [pr_metadata, build-gateway, build-supervisor]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
     permissions:
       actions: read
@@ -251,10 +269,12 @@ jobs:
       image-tag: ${{ github.sha }}
       job-name: Kubernetes Credential Drivers E2E
       e2e-task: e2e:kubernetes:credential-drivers
+      cli-artifact-prefix: rust-binary-cli
+      conformance-artifact-prefix: rust-binary-conformance
 
   core-e2e-result:
     name: Core E2E result
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e]
     if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     runs-on: ubuntu-latest
     steps:
@@ -263,6 +283,7 @@ jobs:
           BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
           BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
           BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
+          BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
           BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }}
           E2E_RESULT: ${{ needs.e2e.result }}
           KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }}
@@ -276,6 +297,7 @@ jobs:
             "build-gateway:$BUILD_GATEWAY_RESULT" \
             "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
             "build-cli:$BUILD_CLI_RESULT" \
+            "build-conformance:$BUILD_CONFORMANCE_RESULT" \
             "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \
             "e2e:$E2E_RESULT" \
             "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \
@@ -293,7 +315,7 @@ jobs:
 
   gpu-e2e-result:
     name: GPU E2E result
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, gpu-e2e]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, gpu-e2e]
     if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_gpu_e2e == 'true'
     runs-on: ubuntu-latest
     steps:
@@ -302,6 +324,7 @@ jobs:
           BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
           BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
           BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
+          BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
           GPU_E2E_RESULT: ${{ needs.gpu-e2e.result }}
         run: |
           set -euo pipefail
@@ -310,6 +333,7 @@ jobs:
             "build-gateway:$BUILD_GATEWAY_RESULT" \
             "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
             "build-cli:$BUILD_CLI_RESULT" \
+            "build-conformance:$BUILD_CONFORMANCE_RESULT" \
             "gpu-e2e:$GPU_E2E_RESULT"; do
             name="${item%%:*}"
             result="${item#*:}"
@@ -322,7 +346,7 @@ jobs:
 
   kubernetes-ha-e2e-result:
     name: Kubernetes HA E2E result
-    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, kubernetes-ha-e2e]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-ha-e2e]
     if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
     runs-on: ubuntu-latest
     steps:
@@ -331,6 +355,7 @@ jobs:
           BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
           BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
           BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
+          BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
           KUBERNETES_HA_E2E_RESULT: ${{ needs.kubernetes-ha-e2e.result }}
         run: |
           set -euo pipefail
@@ -339,6 +364,7 @@ jobs:
             "build-gateway:$BUILD_GATEWAY_RESULT" \
             "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
             "build-cli:$BUILD_CLI_RESULT" \
+            "build-conformance:$BUILD_CONFORMANCE_RESULT" \
             "kubernetes-ha-e2e:$KUBERNETES_HA_E2E_RESULT"; do
             name="${item%%:*}"
             result="${item#*:}"
@@ -351,21 +377,25 @@ jobs:
 
   kubernetes-credential-drivers-e2e-result:
     name: Kubernetes Credential Drivers E2E result
-    needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e]
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-conformance, kubernetes-credential-drivers-e2e]
     if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Verify Kubernetes credential drivers E2E jobs
         env:
           BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
           BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
+          BUILD_CLI_RESULT: ${{ needs.build-cli.result }}
+          BUILD_CONFORMANCE_RESULT: ${{ needs.build-conformance.result }}
           KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }}
         run: |
           set -euo pipefail
           failed=0
           for item in \
             "build-gateway:$BUILD_GATEWAY_RESULT" \
             "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
+            "build-cli:$BUILD_CLI_RESULT" \
+            "build-conformance:$BUILD_CONFORMANCE_RESULT" \
             "kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do
             name="${item%%:*}"
             result="${item#*:}"

.github/workflows/docker-build.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     inputs:
       component:
-        description: "Component to build (gateway, supervisor, cli)"
+        description: "Component to build (gateway, supervisor, cli, conformance)"
         required: true
         type: string
       timeout-minutes:
@@ -104,6 +104,12 @@ jobs:
               features=""
               has_image=false
               ;;
+            conformance)
+              binary_component=conformance
+              binary_name=openshell-conformance
+              features=""
+              has_image=false
+              ;;
             *)
               echo "unsupported component: $component" >&2
               exit 1

.github/workflows/e2e-gpu-test.yaml

@@ -12,6 +12,11 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
+        required: false
+        type: string
+        default: ""
       gateway-artifact-prefix:
         description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-<arch>)"
         required: false
@@ -74,6 +79,7 @@ jobs:
         uses: ./.github/actions/setup-e2e-cli
         with:
           artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact-prefix != ''

.github/workflows/e2e-kubernetes-test.yml

@@ -52,6 +52,11 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
+        required: false
+        type: string
+        default: ""
 
 permissions:
   actions: read
@@ -85,6 +90,7 @@ jobs:
         uses: ./.github/actions/setup-e2e-cli
         with:
           artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Install mise
         run: |

.github/workflows/e2e-test.yml

@@ -22,6 +22,11 @@ on:
         required: false
         type: string
         default: ""
+      conformance-artifact-prefix:
+        description: "Optional prebuilt conformance CLI artifact prefix (artifact suffix is linux-<arch>)"
+        required: false
+        type: string
+        default: ""
       gateway-artifact-prefix:
         description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-<arch>)"
         required: false
@@ -57,7 +62,7 @@ jobs:
             cmd: "mise run --no-deps --skip-deps e2e:oidc-pkce:docker"
             apt_packages: "openssh-client"
           - suite: rust-docker
-            cmd: "mise run --no-deps --skip-deps e2e:rust"
+            cmd: "mise run --no-deps --skip-deps e2e:docker"
             apt_packages: "openssh-client"
           - suite: rust-docker-external-driver
             cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver"
@@ -93,6 +98,7 @@ jobs:
         uses: ./.github/actions/setup-e2e-cli
         with:
           artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact-prefix != ''
@@ -183,6 +189,7 @@ jobs:
         uses: ./.github/actions/setup-e2e-cli
         with:
           artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact-prefix != ''
@@ -340,6 +347,7 @@ jobs:
         uses: ./.github/actions/setup-e2e-cli
         with:
           artifact-prefix: ${{ inputs.cli-artifact-prefix }}
+          conformance-artifact-prefix: ${{ inputs.conformance-artifact-prefix }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact-prefix != ''