From d1301b952d4834da60d96ef18b2139a811cd6b30 Mon Sep 17 00:00:00 2001 From: macblackstuff <148771651+macblackstuff@users.noreply.github.com> Date: Sun, 27 Sep 2026 01:57:01 +0200 Subject: [PATCH 1/2] ci: install and self-test the skill for every agent the skills CLI supports The smoke script now reads the agent list from the skills CLI at run time, so newly supported agents are covered without a code change. Agents the CLI cannot install globally must be listed with a reason; eve and promptscript are. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/smoke-install.sh | 71 ++++++++++++++++++++++++---- .github/workflows/tests.yml | 4 +- CHANGELOG.md | 2 + README.md | 79 +++++++++++++++++++++++++++++--- 4 files changed, 139 insertions(+), 17 deletions(-) diff --git a/.github/scripts/smoke-install.sh b/.github/scripts/smoke-install.sh index af6ca11..af48ccc 100755 --- a/.github/scripts/smoke-install.sh +++ b/.github/scripts/smoke-install.sh @@ -1,19 +1,68 @@ #!/usr/bin/env bash # Install this repo's skill with the Vercel `skills` CLI into a throwaway HOME, once per -# agent named on the command line, and run the skill's own self-check from each installed -# copy. Usage: .github/scripts/smoke-install.sh claude-code codex cursor gemini-cli ... +# agent, and run the skill's own self-check from each installed copy. With no arguments, +# the agent list is read from the CLI itself, so agents it gains are covered automatically. +# Usage: .github/scripts/smoke-install.sh [agent...] set -euo pipefail repo=$(cd "$(dirname "$0")/../.." && pwd) -[ $# -gt 0 ] || { echo "usage: $0 [agent...]" >&2; exit 2; } -# Each agent gets its own throwaway home: all agents except claude-code install into the -# shared ~/.agents/skills/, so a single home would make the one-installed-copy assert below -# pass for the first agent and then see the previous agent's copy for the rest. -for agent in "$@"; do +# Agents the CLI lists but cannot install globally: "|" each. They +# are reported at the end and excluded from the pass count rather than silently skipped. +exceptions=( + "eve|the CLI reports Eve does not support global skill installation" + "promptscript|the CLI reports PromptScript does not support global skill installation" +) + +strip_ansi() { sed -E $'s/\033\\[[0-9;?]*[a-zA-Z]//g'; } + +reason_for() { + local e + for e in ${exceptions[@]+"${exceptions[@]}"}; do + [ "${e%%|*}" = "$1" ] && { echo "${e#*|}"; return; } + done + return 0 +} + +# Ask the CLI for its own agent list: an invalid -a makes it print "Valid agents: a, b, ...". +# Output is joined into one line first, so a wrapped list is parsed the same as an unwrapped one. +list_agents() { + local home out + home=$(mktemp -d) + out=$(HOME="$home" npx --yes skills add "$repo" --skill interface-matrix -g \ + -a __invalid__ -y --copy 2>&1 | strip_ansi | tr '\n' ' ' || true) + rm -rf "$home" + printf '%s' "${out##*Valid agents:}" \ + | grep -oE '[A-Za-z0-9._-]+(, *[A-Za-z0-9._-]+)+' \ + | head -1 | tr ',' '\n' | sed -E 's/^ +//; s/ +$//' +} + +agents=() +if [ $# -gt 0 ]; then + agents=("$@") +else + while IFS= read -r a; do [ -n "$a" ] && agents+=("$a"); done < <(list_agents) + [ "${#agents[@]}" -gt 0 ] || { echo "could not parse the skills CLI agent list" >&2; exit 1; } +fi + +echo "== skills add --list" +home=$(mktemp -d); HOME="$home" npx --yes skills add "$repo" --list; rm -rf "$home" + +total=${#agents[@]} +passed=0 +skipped=() + +# Each agent gets its own throwaway home: most agents install into the shared +# ~/.agents/skills/, so a single home would make the one-installed-copy assert below pass +# for the first agent and then see the previous agent's copy for the rest. +for agent in "${agents[@]}"; do + reason=$(reason_for "$agent") + if [ -n "$reason" ]; then + echo "== $agent: SKIPPED — $reason" + skipped+=("$agent: $reason") + continue + fi home=$(mktemp -d) - echo "== skills add --list ($agent)" - HOME="$home" npx --yes skills add "$repo" --list echo "== skills add -a $agent" HOME="$home" npx --yes skills add "$repo" --skill interface-matrix -g -a "$agent" -y --copy @@ -27,4 +76,8 @@ for agent in "$@"; do done ( cd "$dir" && python3 scripts/test_interface_matrix.py && python3 -O scripts/test_interface_matrix.py ) rm -rf "$home" + passed=$((passed + 1)) done + +for s in ${skipped[@]+"${skipped[@]}"}; do echo "exception: $s"; done +echo "$passed of $total agents installed and tested" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c79086c..f2147d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -51,8 +51,8 @@ jobs: - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" - - name: skills CLI install smoke test (six agents) - run: .github/scripts/smoke-install.sh claude-code codex cursor gemini-cli github-copilot opencode + - name: skills CLI install smoke test (all agents) + run: .github/scripts/smoke-install.sh commit-emails: runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 654e3f0..92762c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- CI installs and self-tests the skill for every agent the `skills` CLI supports, with the + agent list read from the CLI at run time. - Copyright holder in `LICENSE` and the README is now `macblackstuff`. - Code of conduct (Contributor Covenant 2.1); reports go to `conduct@macblackstuff.com`. diff --git a/README.md b/README.md index af04550..1a3dde6 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,79 @@ every command below. ## Harnesses tested -CI installs the skill with the `skills` CLI on every push and pull request, once per agent, and -runs the 83 tests from each installed copy. Six agents are covered: `claude-code`, `codex`, -`cursor`, `gemini-cli`, `github-copilot` and `opencode`. Five of the six share one user-level skills -directory (the `skills` CLI decides the target; see its documentation for each agent's path), so -on a real machine a single installed copy can serve all five. CI still installs and tests each -agent separately, in its own throwaway home, so a change to any one agent's target is caught. +CI installs the skill with the [`skills` CLI](https://github.com/vercel-labs/skills) on every push +and pull request, once per agent in its own throwaway home, and runs the 83 tests from each +installed copy. Every agent the CLI supports is covered — 79 at the time of writing (`skills` +1.7.0), of which 77 are installed and tested. The list is read from the CLI at run time, so agents +it gains later are covered automatically. Two agents are excluded, each with a reason recorded in +`.github/scripts/smoke-install.sh`: `eve` and `promptscript` — the CLI reports that neither +supports global skill installation. + +Many agents share a global skills directory, so on a real machine one installed copy serves all of +them; CI still installs and tests each agent separately, so a change to any one agent's target is +caught. Paths below are from a real run, relative to `~`: + +| Global install path | Agents | +|---|---| +| `~/.agents/skills` | `amp`, `antigravity`, `antigravity-cli`, `cline`, `codex`, `cursor`, `deepagents`, `dexto`, `droid`, `firebender`, `gemini-cli`, `github-copilot`, `kilo`, `kimi-code-cli`, `loaf`, `opencode`, `replit`, `sarvam-code`, `universal`, `warp`, `zed` | +| `~/.zencoder/skills` | `zencoder`, `zenflow` | +| `~/.adal/skills` | `adal` | +| `~/.aider-desk/skills` | `aider-desk` | +| `~/.astrbot/data/skills` | `astrbot` | +| `~/.augment/skills` | `augment` | +| `~/.autohand/skills` | `autohand-code` | +| `~/.bob/skills` | `bob` | +| `~/.claude/skills` | `claude-code` | +| `~/.codeartsdoer/skills` | `codearts-agent` | +| `~/.codebuddy/skills` | `codebuddy` | +| `~/.codeium/windsurf/skills` | `windsurf` | +| `~/.codemaker/skills` | `codemaker` | +| `~/.codestudio/skills` | `codestudio` | +| `~/.commandcode/skills` | `command-code` | +| `~/.config/crush/skills` | `crush` | +| `~/.config/devin/skills` | `devin` | +| `~/.config/goose/skills` | `goose` | +| `~/.config/kimchi/harness/skills` | `kimchi` | +| `~/.continue/skills` | `continue` | +| `~/.forge/skills` | `forgecode` | +| `~/.fx/skills` | `fx` | +| `~/.grok/skills` | `grok` | +| `~/.hermes/skills` | `hermes-agent` | +| `~/.iflow/skills` | `iflow-cli` | +| `~/.inferencesh/skills` | `inference-sh` | +| `~/.jazz/skills` | `jazz` | +| `~/.junie/skills` | `junie` | +| `~/.kiro/skills` | `kiro-cli` | +| `~/.kode/skills` | `kode` | +| `~/.lingma/skills` | `lingma` | +| `~/.mcpjam/skills` | `mcpjam` | +| `~/.minimax/skills` | `minimax-code` | +| `~/.moxby/skills` | `moxby` | +| `~/.mux/skills` | `mux` | +| `~/.neovate/skills` | `neovate` | +| `~/.ona/skills` | `ona` | +| `~/.openclaw/skills` | `openclaw` | +| `~/.openhands/skills` | `openhands` | +| `~/.pi/agent/skills` | `pi` | +| `~/.pochi/skills` | `pochi` | +| `~/.posit/assistant/skills` | `posit-assistant` | +| `~/.qoder-cn/skills` | `qoder-cn` | +| `~/.qoder/skills` | `qoder` | +| `~/.qwen/skills` | `qwen-code` | +| `~/.reasonix/skills` | `reasonix` | +| `~/.roo/skills` | `roo` | +| `~/.rovodev/skills` | `rovodev` | +| `~/.snowflake/cortex/skills` | `cortex` | +| `~/.tabnine/agent/skills` | `tabnine-cli` | +| `~/.terramind/skills` | `terramind` | +| `~/.tinycloud/skills` | `tinycloud` | +| `~/.trae-cn/skills` | `trae-cn` | +| `~/.trae/skills` | `trae` | +| `~/.vibe/skills` | `mistral-vibe` | +| `~/.zcode/skills` | `zcode` | + +Tested: the skill installs for each agent and its own test suite passes from the installed copy. +Not tested: each agent's own runtime behaviour when it loads the skill. The skill itself is harness-neutral: it is a `SKILL.md` plus standard-library Python, with no agent-specific commands. From c8c169ecdca2ea7d2213238ba117005132e75412 Mon Sep 17 00:00:00 2001 From: macblackstuff <148771651+macblackstuff@users.noreply.github.com> Date: Sun, 27 Sep 2026 02:03:22 +0200 Subject: [PATCH 2/2] ci: keep XDG dirs inside the throwaway home Agents that resolve ~/.config through XDG_CONFIG_HOME (crush, devin, goose, kimchi) otherwise install into the runner's real home and the assert sees none. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/smoke-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/smoke-install.sh b/.github/scripts/smoke-install.sh index af48ccc..a01a90d 100755 --- a/.github/scripts/smoke-install.sh +++ b/.github/scripts/smoke-install.sh @@ -29,7 +29,7 @@ reason_for() { list_agents() { local home out home=$(mktemp -d) - out=$(HOME="$home" npx --yes skills add "$repo" --skill interface-matrix -g \ + out=$(HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --skill interface-matrix -g \ -a __invalid__ -y --copy 2>&1 | strip_ansi | tr '\n' ' ' || true) rm -rf "$home" printf '%s' "${out##*Valid agents:}" \ @@ -46,7 +46,7 @@ else fi echo "== skills add --list" -home=$(mktemp -d); HOME="$home" npx --yes skills add "$repo" --list; rm -rf "$home" +home=$(mktemp -d); HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --list; rm -rf "$home" total=${#agents[@]} passed=0 @@ -64,7 +64,7 @@ for agent in "${agents[@]}"; do fi home=$(mktemp -d) echo "== skills add -a $agent" - HOME="$home" npx --yes skills add "$repo" --skill interface-matrix -g -a "$agent" -y --copy + HOME="$home" XDG_CONFIG_HOME="$home/.config" XDG_DATA_HOME="$home/.local/share" npx --yes skills add "$repo" --skill interface-matrix -g -a "$agent" -y --copy found=$(find "$home" -path "*/interface-matrix/SKILL.md" -print) count=$(printf '%s' "$found" | grep -c . || true)