diff --git a/.github/scripts/smoke-install.sh b/.github/scripts/smoke-install.sh index af6ca11..a01a90d 100755 --- a/.github/scripts/smoke-install.sh +++ b/.github/scripts/smoke-install.sh @@ -1,21 +1,70 @@ #!/usr/bin/env bash # Install this repo's skill with the Vercel `skills` CLI into a throwaway HOME, once per -# agent named on the command line, and run the skill's own self-check from each installed -# copy. Usage: .github/scripts/smoke-install.sh claude-code codex cursor gemini-cli ... +# agent, and run the skill's own self-check from each installed copy. With no arguments, +# the agent list is read from the CLI itself, so agents it gains are covered automatically. +# Usage: .github/scripts/smoke-install.sh [agent...] set -euo pipefail repo=$(cd "$(dirname "$0")/../.." && pwd) -[ $# -gt 0 ] || { echo "usage: $0 [agent...]" >&2; exit 2; } -# Each agent gets its own throwaway home: all agents except claude-code install into the -# shared ~/.agents/skills/, so a single home would make the one-installed-copy assert below -# pass for the first agent and then see the previous agent's copy for the rest. -for agent in "$@"; do +# Agents the CLI lists but cannot install globally: "|" each. They +# are reported at the end and excluded from the pass count rather than silently skipped. +exceptions=( + "eve|the CLI reports Eve does not support global skill installation" + "promptscript|the CLI reports PromptScript does not support global skill installation" +) + +strip_ansi() { sed -E $'s/\033\\[[0-9;?]*[a-zA-Z]//g'; } + +reason_for() { + local e + for e in ${exceptions[@]+"${exceptions[@]}"}; do + [ "${e%%|*}" = "$1" ] && { echo "${e#*|}"; return; } + done + return 0 +} + +# Ask the CLI for its own agent list: an invalid -a makes it print "Valid agents: a, b, ...". +# Output is joined into one line first, so a wrapped list is parsed the same as an unwrapped one. +list_agents() { + local home out + home=$(mktemp -d) + out=$(HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --skill interface-matrix -g \ + -a __invalid__ -y --copy 2>&1 | strip_ansi | tr '\n' ' ' || true) + rm -rf "$home" + printf '%s' "${out##*Valid agents:}" \ + | grep -oE '[A-Za-z0-9._-]+(, *[A-Za-z0-9._-]+)+' \ + | head -1 | tr ',' '\n' | sed -E 's/^ +//; s/ +$//' +} + +agents=() +if [ $# -gt 0 ]; then + agents=("$@") +else + while IFS= read -r a; do [ -n "$a" ] && agents+=("$a"); done < <(list_agents) + [ "${#agents[@]}" -gt 0 ] || { echo "could not parse the skills CLI agent list" >&2; exit 1; } +fi + +echo "== skills add --list" +home=$(mktemp -d); HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --list; rm -rf "$home" + +total=${#agents[@]} +passed=0 +skipped=() + +# Each agent gets its own throwaway home: most agents install into the shared +# ~/.agents/skills/, so a single home would make the one-installed-copy assert below pass +# for the first agent and then see the previous agent's copy for the rest. +for agent in "${agents[@]}"; do + reason=$(reason_for "$agent") + if [ -n "$reason" ]; then + echo "== $agent: SKIPPED — $reason" + skipped+=("$agent: $reason") + continue + fi home=$(mktemp -d) - echo "== skills add --list ($agent)" - HOME="$home" npx --yes skills add "$repo" --list echo "== skills add -a $agent" - HOME="$home" npx --yes skills add "$repo" --skill interface-matrix -g -a "$agent" -y --copy + HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --skill interface-matrix -g -a "$agent" -y --copy found=$(find "$home" -path "*/interface-matrix/SKILL.md" -print) count=$(printf '%s' "$found" | grep -c . || true) @@ -27,4 +76,8 @@ for agent in "$@"; do done ( cd "$dir" && python3 scripts/test_interface_matrix.py && python3 -O scripts/test_interface_matrix.py ) rm -rf "$home" + passed=$((passed + 1)) done + +for s in ${skipped[@]+"${skipped[@]}"}; do echo "exception: $s"; done +echo "$passed of $total agents installed and tested" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c79086c..f2147d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,8 +51,8 @@ jobs: - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" - - name: skills CLI install smoke test (six agents) - run: .github/scripts/smoke-install.sh claude-code codex cursor gemini-cli github-copilot opencode + - name: skills CLI install smoke test (all agents) + run: .github/scripts/smoke-install.sh commit-emails: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bfbb3f..493168f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- CI installs and self-tests the skill for every agent the `skills` CLI supports, with the + agent list read from the CLI at run time. - Copyright holder in `LICENSE` and the README is now `macblackstuff`. - Code of conduct (Contributor Covenant 2.1); reports go to `conduct@macblackstuff.com`. - README rewritten: what it does, who it is for, a worked example, the diff --git a/README.md b/README.md index 121398a..75cba77 100644 --- a/README.md +++ b/README.md @@ -125,12 +125,12 @@ python3 scripts/test_interface_matrix.py # Ran 83 tests ... OK ### Harnesses tested -CI installs the skill with the `skills` CLI on every push and pull request, once per agent with `--copy`, and -runs the 83 tests from each installed copy. Six agents are covered: `claude-code`, `codex`, -`cursor`, `gemini-cli`, `github-copilot` and `opencode`. Five of the six share one user-level skills -directory (the `skills` CLI decides the target; see its documentation for each agent's path), so -on a real machine a single installed copy can serve all five. CI still installs and tests each -agent separately, in its own throwaway home, so a change to any one agent's target is caught. +CI installs the skill with the [`skills` CLI`](https://github.com/vercel-labs/skills) on every push +and pull request, once per agent in its own throwaway home, and runs the 83 tests from each +installed copy. Every agent the CLI supports is covered — 79 at the time of writing (`skills` +1.7.0), of which 77 are installed and tested; the list is read from the CLI at run time. Two +agents are excluded with reasons recorded in `.github/scripts/smoke-install.sh`: `eve` and +`promptscript` — the CLI reports that neither supports global skill installation. The skill itself is harness-neutral: it is a `SKILL.md` plus standard-library Python, with no agent-specific commands.