diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml index 932c550..2c43dee 100644 --- a/.github/workflows/nightly-release.yml +++ b/.github/workflows/nightly-release.yml @@ -66,7 +66,8 @@ concurrency: jobs: nightly: runs-on: ubuntu-latest - # Rehearsal (~1h) + release-fidelity integration (~2h) + live release (~1.5h). + # Rehearsal (~1h) + release-fidelity integration (~2h) + live release (~1.5h) + # — far past the App token's 60-minute life, so the driver re-mints it. timeout-minutes: 350 steps: - name: Checkout PyAutoBrain @@ -113,6 +114,12 @@ jobs: id: driver env: GH_TOKEN: ${{ steps.app_token.outputs.token }} + # The App token lives 60 min; the driver re-mints it before long waits + # (bin/gh_app_token.py, the same permission set as the mint step above). + # The key is used for that re-mint only. + APP_CLIENT_ID: ${{ vars.APP_CLIENT_ID }} + APP_PRIVATE_KEY: ${{ secrets.APP_PRIVATE_KEY }} + APP_OWNER: PyAutoLabs # the same installation the mint step's `owner:` names PYAUTO_RELEASE_WEBHOOK_URL: ${{ secrets.PYAUTO_RELEASE_WEBHOOK_URL }} NIGHTLY_RELEASES: ${{ vars.NIGHTLY_RELEASES }} # Scheduled runs have no inputs: default 'false' — ARMED 2026-07-09 diff --git a/agents/conductors/release/_nightly_lib.sh b/agents/conductors/release/_nightly_lib.sh new file mode 100644 index 0000000..5543ded --- /dev/null +++ b/agents/conductors/release/_nightly_lib.sh @@ -0,0 +1,75 @@ +# agents/conductors/release/_nightly_lib.sh — token refresh + run polling for +# the nightly driver. Sourced by nightly.sh after _common.sh; the tests source +# it on its own. +# +# A GitHub App installation token lives 60 minutes and the nightly job runs for +# hours. refresh_gh_token re-mints GH_TOKEN once it is 45 minutes old, and +# await_run polls a run (refreshing as it goes) instead of one long blocking +# `gh run watch` that dies with "HTTP 401: Bad credentials" when the token +# expires underneath it. +# +# Without APP_CLIENT_ID + APP_PRIVATE_KEY (a local run on a personal gh login) +# refresh_gh_token does nothing. + +_NIGHTLY_LIB_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +# The workflow mints the first token seconds before the driver starts. +_GH_TOKEN_MINTED_AT="$(date +%s)" +_GH_TOKEN_MAX_AGE=2700 # 45 min: refresh well inside the 60-min life + +# refresh_gh_token [--force] — re-mint GH_TOKEN when it is old (or always with +# --force). Returns 1, with the reason on stderr, if a mint fails. +refresh_gh_token() { + [[ -z "${APP_CLIENT_ID:-}" || -z "${APP_PRIVATE_KEY:-}" ]] && return 0 + local now token + now="$(date +%s)" + if [[ "${1:-}" != "--force" ]] && (( now - _GH_TOKEN_MINTED_AT <= _GH_TOKEN_MAX_AGE )); then + return 0 + fi + if [[ -n "${NIGHTLY_MINT_CMD:-}" ]]; then + token="$($NIGHTLY_MINT_CMD)" || token="" + else + token="$(python3 "$_NIGHTLY_LIB_DIR/../../../bin/gh_app_token.py")" || token="" + fi + if [[ -z "$token" ]]; then + echo "nightly: could not mint a fresh GitHub App token (see above)" >&2 + return 1 + fi + export GH_TOKEN="$token" + _GH_TOKEN_MINTED_AT="$now" + printf '== nightly: GitHub token refreshed\n' +} + +# await_run — poll until the run completes. +# Returns 0 success, 2 completed with any other conclusion, 4 could not +# observe the run (5 failed reads in a row, even after re-minting the token). +await_run() { + local repo="$1" run_id="$2" interval="${NIGHTLY_POLL_SECONDS:-60}" + local fails=0 out status conclusion last="" err + err="$(mktemp "${TMPDIR:-/tmp}/nightly-await.XXXXXX")" + while :; do + refresh_gh_token || true + if out="$(gh run view "$run_id" -R "$repo" --json status,conclusion 2>"$err")" \ + && out="$(printf '%s' "$out" | python3 -c 'import json, sys +r = json.load(sys.stdin) +print(r["status"], r.get("conclusion") or "")' 2>>"$err")"; then + fails=0 + read -r status conclusion <<< "$out" + if [[ "$status" != "$last" ]]; then + printf '== nightly: run %s %s\n' "$run_id" "$status" + last="$status" + fi + if [[ "$status" == "completed" ]]; then + rm -f "$err" + [[ "$conclusion" == "success" ]] && return 0 + printf '== nightly: run %s concluded %s\n' "$run_id" "${conclusion:-unknown}" + return 2 + fi + else + fails=$((fails + 1)) + echo "nightly: could not read run $run_id ($fails/5): $(head -c 200 "$err")" >&2 + (( fails >= 5 )) && { rm -f "$err"; return 4; } + refresh_gh_token --force || true + fi + sleep "$interval" + done +} diff --git a/agents/conductors/release/nightly.sh b/agents/conductors/release/nightly.sh index 18abd1b..509bb65 100644 --- a/agents/conductors/release/nightly.sh +++ b/agents/conductors/release/nightly.sh @@ -45,6 +45,8 @@ # DRY_RUN "true" (default) → step 6 logs instead of shipping # GITHUB_EVENT_NAME "schedule" engages the kill switch; anything else # (manual dispatch, local) proceeds +# APP_CLIENT_ID, APP_PRIVATE_KEY App credentials to re-mint GH_TOKEN (CI); +# unset locally, where no refresh happens # # Usage: # nightly.sh [--dry-run] [--no-dispatch] @@ -55,8 +57,16 @@ # NIGHTLY_WINDOW_START (env) — override the activity-window start (ISO 8601) # for local verification; the persisted/24h anchor is used when unset. # +# Token lifetime: in CI GH_TOKEN is an App installation token, which lives 60 +# minutes, and this job runs for hours. _nightly_lib.sh re-mints it (from +# APP_CLIENT_ID + APP_PRIVATE_KEY) once it is 45 minutes old, before every long +# wait and every gh call after one, and run watches poll instead of blocking. +# A watch that still loses its credentials did not judge the run, so it is a +# driver error (exit 1), never a stage failure. +# # Exit: 0 on a reported outcome (shipped / skipped / dry-run) — 2 blocked at a -# gate (paged) — 3 not GREEN / preflight red (paged) — 1 driver error (paged). +# gate (paged) — 3 not GREEN / preflight red (paged) — 1 driver error (paged), +# including a run watch that lost its GitHub credentials. # # These codes are a CONTRACT, not just a status: .github/workflows/ # nightly-release.yml maps 0/2/3 to a green run (a blocked night is the gate @@ -73,6 +83,7 @@ HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" . "$HERE/../../../bin/_gh.sh" require_gh nightly || exit $? source "$HERE/../../_common.sh" +source "$HERE/_nightly_lib.sh" BRAIN_REPO="PyAutoLabs/PyAutoHands" # where release-blocker labels are curated first ANCHOR_REPO="PyAutoLabs/PyAutoBrain" @@ -142,6 +153,7 @@ advance_anchor() { return 0 fi local err + refresh_gh_token if ! err="$(gh api -X PATCH "repos/$ANCHOR_REPO/actions/variables/$ANCHOR_VAR" \ -f name="$ANCHOR_VAR" -f value="$WINDOW_END" 2>&1 >/dev/null)"; then echo "nightly: could not write $ANCHOR_REPO variable $ANCHOR_VAR — does the token have the Variables (write) permission? ${err:0:200}" >&2 @@ -310,11 +322,15 @@ print($2) } # dispatch_and_await [ ] -# Dispatches, resolves the run id, watches to completion (fails loudly), and -# optionally downloads an artifact. Echoes the run URL on success. +# Dispatches, resolves the run id, polls it to completion (await_run, which +# keeps the token fresh), and optionally downloads an artifact. Returns 0 ok, +# 1 dispatch failed, 2 run red, 3 artifact missing, 4 lost the watch (the run +# was never judged — callers treat it as a driver error). dispatch_and_await() { local repo="$1" wf="$2" inputs="$3" artifact="${4:-}" dest="${5:-}" local before run_id rc + # The step 6 dispatch comes hours in: never dispatch on a stale token. + refresh_gh_token before="$(date -u +%Y-%m-%dT%H:%M:%SZ)" local fargs=() while IFS=$'\t' read -r k v; do @@ -338,20 +354,23 @@ for k, v in (json.load(sys.stdin) or {}).items(): return 1 fi echo "https://github.com/$repo/actions/runs/$run_id" >&2 - gh run watch "$run_id" -R "$repo" --interval 60 --exit-status >/dev/null - rc=$? LAST_RUN_URL="https://github.com/$repo/actions/runs/$run_id" + await_run "$repo" "$run_id" + rc=$? + [[ $rc -eq 4 ]] && return 4 if [[ $rc -ne 0 ]]; then # The report artifacts upload `if: always()`, so a FAILED run still carries # the evidence of which scripts failed — fetch it so the page can name # them. Best-effort: paging must never be blocked by a missing artifact. if [[ -n "$artifact" ]]; then + refresh_gh_token gh run download "$run_id" -R "$repo" -n "$artifact" -D "$dest" 2>/dev/null \ || echo "nightly: failed run's artifact '$artifact' not downloadable" >&2 fi return 2 fi if [[ -n "$artifact" ]]; then + refresh_gh_token gh run download "$run_id" -R "$repo" -n "$artifact" -D "$dest" || return 3 fi return 0 @@ -376,13 +395,19 @@ s2_artifact="$(plan_field "$phase_a" "[s for s in plan['stage2_plan']['steps'] i head_repos="$(plan_field "$phase_a" "chr(10).join([s for s in plan['stage2_plan']['steps'] if s['step']=='capture-heads'][0]['repos'])")" log "step 4a — Stage 2 rehearsal: dispatch $s2_repo/$s2_wf" -if ! dispatch_and_await "$s2_repo" "$s2_wf" "$s2_inputs" "$s2_artifact" "$ART_DIR"; then +dispatch_and_await "$s2_repo" "$s2_wf" "$s2_inputs" "$s2_artifact" "$ART_DIR" +rc=$? +if [[ $rc -eq 4 ]]; then + page "driver lost its GitHub credentials while watching <${LAST_RUN_URL:-$RUN_URL}|the run> — the run itself was not judged (token refresh failed?)" + exit 1 +elif [[ $rc -ne 0 ]]; then page "Stage 2 (TestPyPI rehearsal) failed — <${LAST_RUN_URL:-$RUN_URL}|run>" exit 2 fi # capture-heads: {bare repo name: main HEAD sha} — Stage 2's commit_shas.json # is the single source of truth downstream (validate.sh header). +refresh_gh_token python3 - "$ART_DIR/commit_shas.json" < — the run itself was not judged (token refresh failed?)" + exit 1 +elif [[ $rc -ne 0 ]]; then # Name the failing scripts straight from the downloaded stage report so the # page is actionable without opening the run (no report → plain page). # Lives in its own file, not a heredoc, so it carries a regression test: @@ -429,6 +459,7 @@ HEART_DIR="$(cd "$(dirname "$(readlink -f "$(command -v "$heart" || echo "$heart export HEART_STATE_DIR="${HEART_STATE_DIR:-$WORK/heart-state}" mkdir -p "$HEART_STATE_DIR" +refresh_gh_token bash "$HEART_DIR/heart/checks/ci_status.sh" || echo "nightly: ci_status check failed (verdict will show unknowns)" >&2 bash "$HEART_DIR/heart/checks/open_prs.sh" || echo "nightly: open_prs check failed" >&2 "$heart" validate --ingest "$ART_DIR" --commit-shas "$ART_DIR/commit_shas.json" || { @@ -463,8 +494,15 @@ if [[ "$DRY_RUN" == "true" ]]; then fi log "step 6 — dispatching the LIVE release ($version)" -if ! dispatch_and_await "$BUILD_REPO" "$RELEASE_WORKFLOW" \ - '{"rehearsal": "false", "minor_version": "1"}'; then +dispatch_and_await "$BUILD_REPO" "$RELEASE_WORKFLOW" \ + '{"rehearsal": "false", "minor_version": "1"}' +rc=$? +if [[ $rc -eq 4 ]]; then + # Unjudged, not failed: the release may well have published, so not page() + # ("No release was made"). The anchor stays put for a human to settle. + notify "🚨" "*nightly driver error* — driver lost its GitHub credentials while watching <${LAST_RUN_URL:-$RUN_URL}|the run> — the run itself was not judged (token refresh failed?). Check whether $version published before the next night. <$RUN_URL|nightly run>" + exit 1 +elif [[ $rc -ne 0 ]]; then # A red release run is not always a release that did not happen: release.yml # publishes (tag + PyPI) from its `release` job, which does not wait on the # post-publish `run_smoke_tests` leg. On 2026-09-11 (run 34577342347) that leg @@ -473,6 +511,7 @@ if ! dispatch_and_await "$BUILD_REPO" "$RELEASE_WORKFLOW" \ # the anchor behind, so the shipped window would have been re-judged the # next night. The tag is the evidence: present means shipped, and the night # is reported as shipped with a red smoke leg for a human to look at. + refresh_gh_token if released_today; then notify "⚠️" "*nightly release shipped with a red smoke leg* — $version published, but the release run went red after publishing (the post-release smoke leg is the usual cause) — <${LAST_RUN_URL:-$RUN_URL}|release run>. $summary (window $anchor → $WINDOW_END). <$RUN_URL|nightly run>" advance_anchor diff --git a/bin/README.md b/bin/README.md index 47cd3cb..798a6b8 100644 --- a/bin/README.md +++ b/bin/README.md @@ -10,6 +10,9 @@ Executable tooling for the PyAuto organism. `state.json` (fallback: local `pyauto-heart readiness --json`), read-only, never ticks; exit 0 green · 1 yellow/stale · 2 red · 3 grey. Run first by `start_dev` step 0a and `route`. +- **`gh_app_token.py`** — mints a fresh pyautolabs-bot App installation token + (stdlib + `openssl`) so the nightly release driver can outlive the 60-minute + token the workflow starts with. Prints only the token. The installer and guard used to live in `admin_jammy/skills/`. They moved here because they are organism-wide infrastructure, not admin_jammy's own tooling — diff --git a/bin/gh_app_token.py b/bin/gh_app_token.py new file mode 100755 index 0000000..af5366e --- /dev/null +++ b/bin/gh_app_token.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""bin/gh_app_token.py — mint a fresh pyautolabs-bot App installation token. + +WHY. An App installation token lives 60 minutes, but the nightly release job +runs for hours. The workflow mints one token at the start; the nightly driver +calls this helper to mint a new one before the old one runs out, so a long +watch on a passing run never ends in "HTTP 401: Bad credentials". + +WHAT IT DOES. Signs a short-lived JWT with the App's private key (via the +`openssl` command, so there are no Python dependencies), finds the App's +installation on the owner, and asks for an installation token with exactly +the permission set the workflow's mint step uses. Prints the token, nothing +else. + +Environment: + APP_CLIENT_ID the App's client id (the JWT issuer) + APP_PRIVATE_KEY the App's private key, PEM text + APP_OWNER the installation owner (the workflow sets it beside the mint step's `owner:`) + +Exit 0 with the token on stdout; exit 1 with a one-line reason on stderr. The +key, the JWT and the token never appear in an error message. +""" + +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request + +API = "https://api.github.com" + +# Must match the `permission-*` inputs of the `app_token` step in +# .github/workflows/nightly-release.yml (a test pins this). `actions_variables` +# is the installation's name for the App's "Variables" permission. +INSTALLATION_PERMISSIONS = { + "actions": "write", + "contents": "read", + "issues": "read", + "actions_variables": "write", +} + + +class MintError(Exception): + """A failure whose message is safe to print (no secrets in it).""" + + +def _b64url(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii") + + +def _sign(message: bytes, private_key: str) -> bytes: + """RS256-sign `message` with `openssl dgst`; the key file is 0600 and always removed.""" + fd, key_path = tempfile.mkstemp(prefix="gh-app-key-") + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w") as f: + f.write(private_key) + proc = subprocess.run( + ["openssl", "dgst", "-sha256", "-sign", key_path], + input=message, + capture_output=True, + check=False, + ) + except OSError as exc: + raise MintError(f"could not run openssl ({exc.strerror})") from None + finally: + try: + os.unlink(key_path) + except OSError: + pass + if proc.returncode != 0: + raise MintError( + "openssl could not sign the JWT (is APP_PRIVATE_KEY a valid PEM key?)" + ) + return proc.stdout + + +def make_jwt(client_id: str, private_key: str, now: int | None = None) -> str: + """A 10-minute App JWT; `iat` is back-dated 60 s for clock drift.""" + now = int(time.time()) if now is None else int(now) + header = {"alg": "RS256", "typ": "JWT"} + claims = {"iat": now - 60, "exp": now + 540, "iss": client_id} + signing_input = ( + _b64url(json.dumps(header, separators=(",", ":")).encode()) + + "." + + _b64url(json.dumps(claims, separators=(",", ":")).encode()) + ) + signature = _sign(signing_input.encode("ascii"), private_key) + return signing_input + "." + _b64url(signature) + + +def _request(method: str, url: str, headers: dict, body: dict | None = None): + """One GitHub API call; returns the decoded JSON. All HTTP goes through here.""" + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.load(resp) + except urllib.error.HTTPError as exc: + raise MintError(f"{method} {url} returned HTTP {exc.code}") from None + except (urllib.error.URLError, OSError, ValueError) as exc: + raise MintError(f"{method} {url} failed ({type(exc).__name__})") from None + + +def mint( + client_id: str, private_key: str, owner: str, request=_request +) -> str: + """Mint an installation token for `owner` with INSTALLATION_PERMISSIONS.""" + headers = { + "Authorization": f"Bearer {make_jwt(client_id, private_key)}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + installations = request("GET", f"{API}/app/installations", headers, None) + match = [ + i + for i in installations or [] + if str((i.get("account") or {}).get("login", "")).lower() == owner.lower() + ] + if not match: + raise MintError(f"the App has no installation on '{owner}'") + result = request( + "POST", + f"{API}/app/installations/{match[0]['id']}/access_tokens", + headers, + {"permissions": INSTALLATION_PERMISSIONS}, + ) + token = (result or {}).get("token") + if not token: + raise MintError("GitHub returned no token") + return token + + +def main() -> int: + client_id = os.environ.get("APP_CLIENT_ID", "") + private_key = os.environ.get("APP_PRIVATE_KEY", "") + owner = os.environ.get("APP_OWNER", "") + if not client_id or not private_key or not owner: + print( + "gh_app_token: APP_CLIENT_ID, APP_PRIVATE_KEY and APP_OWNER must all be set", + file=sys.stderr, + ) + return 1 + try: + token = mint(client_id, private_key, owner) + except MintError as exc: + print(f"gh_app_token: {exc}", file=sys.stderr) + return 1 + except Exception as exc: # noqa: BLE001 — never dump a traceback that could carry secrets + print(f"gh_app_token: unexpected {type(exc).__name__}", file=sys.stderr) + return 1 + print(token) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_gh_app_token.py b/tests/test_gh_app_token.py new file mode 100644 index 0000000..245b989 --- /dev/null +++ b/tests/test_gh_app_token.py @@ -0,0 +1,124 @@ +"""tests/test_gh_app_token.py — the nightly driver's App token re-mint helper. + +The workflow mints one 60-minute token; bin/gh_app_token.py mints the next ones +mid-run. It must sign a valid RS256 JWT and ask for exactly the permission set +the workflow's mint step declares — never more. +""" + +from __future__ import annotations + +import base64 +import importlib.util +import json +import shutil +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location( + "gh_app_token", ROOT / "bin" / "gh_app_token.py" +) +gh_app_token = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gh_app_token) + +needs_openssl = pytest.mark.skipif( + shutil.which("openssl") is None, reason="openssl not installed" +) + + +def _b64decode(part: str) -> bytes: + return base64.urlsafe_b64decode(part + "=" * (-len(part) % 4)) + + +@pytest.fixture +def rsa_key(tmp_path): + key = tmp_path / "key.pem" + subprocess.run( + ["openssl", "genrsa", "-out", str(key), "2048"], check=True, capture_output=True + ) + return key + + +@needs_openssl +def test_make_jwt_is_a_verifiable_rs256_jwt(rsa_key, tmp_path): + jwt = gh_app_token.make_jwt("Iv1.client", rsa_key.read_text(), now=1_000_000) + parts = jwt.split(".") + assert len(parts) == 3 + assert all("=" not in p for p in parts) + assert json.loads(_b64decode(parts[0])) == {"alg": "RS256", "typ": "JWT"} + claims = json.loads(_b64decode(parts[1])) + assert claims["iss"] == "Iv1.client" + assert claims["iat"] == 1_000_000 - 60 + assert claims["exp"] - claims["iat"] == 600 + + pub = tmp_path / "pub.pem" + subprocess.run( + ["openssl", "rsa", "-in", str(rsa_key), "-pubout", "-out", str(pub)], + check=True, + capture_output=True, + ) + sig = tmp_path / "sig.bin" + sig.write_bytes(_b64decode(parts[2])) + verify = subprocess.run( + ["openssl", "dgst", "-sha256", "-verify", str(pub), "-signature", str(sig)], + input=f"{parts[0]}.{parts[1]}".encode(), + capture_output=True, + check=False, + ) + assert verify.returncode == 0, verify.stderr + + +@needs_openssl +def test_mint_picks_the_owner_installation_and_asks_for_exact_permissions(rsa_key): + calls = [] + + def fake_request(method, url, headers, body): + calls.append((method, url, headers, body)) + if method == "GET": + return [ + {"id": 1, "account": {"login": "someone-else"}}, + {"id": 42, "account": {"login": "example-org"}}, + ] + return {"token": "ghs_fresh"} + + token = gh_app_token.mint( + "Iv1.client", rsa_key.read_text(), "example-org", request=fake_request + ) + assert token == "ghs_fresh" + assert calls[0][:2] == ("GET", "https://api.github.com/app/installations") + method, url, headers, body = calls[1] + assert (method, url) == ( + "POST", + "https://api.github.com/app/installations/42/access_tokens", + ) + assert headers["Authorization"].startswith("Bearer ") + assert body == {"permissions": gh_app_token.INSTALLATION_PERMISSIONS} + + +@needs_openssl +def test_mint_fails_cleanly_without_an_owner_installation(rsa_key): + with pytest.raises(gh_app_token.MintError, match="no installation"): + gh_app_token.mint( + "Iv1.client", + rsa_key.read_text(), + "example-org", + request=lambda *a: [{"id": 1, "account": {"login": "x"}}], + ) + + +def test_permissions_match_the_workflow_mint_step(): + workflow = yaml.safe_load( + (ROOT / ".github" / "workflows" / "nightly-release.yml").read_text() + ) + step = next( + s for s in workflow["jobs"]["nightly"]["steps"] if s.get("id") == "app_token" + ) + declared = { + key[len("permission-") :].replace("-", "_"): value + for key, value in step["with"].items() + if key.startswith("permission-") + } + assert declared == gh_app_token.INSTALLATION_PERMISSIONS diff --git a/tests/test_nightly_await.py b/tests/test_nightly_await.py new file mode 100644 index 0000000..5d9770a --- /dev/null +++ b/tests/test_nightly_await.py @@ -0,0 +1,157 @@ +"""tests/test_nightly_await.py — the nightly driver keeps its token alive. + +An App token lives 60 minutes; a Stage 3 watch once outlived it, `gh` returned +"HTTP 401: Bad credentials", and a passing run was paged as a failure (#577). +await_run must ride out transient read failures, report a red run as red, and +report "could not observe" (4) — never a verdict — when reads keep failing. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + +LIB = ( + Path(__file__).resolve().parents[1] + / "agents" + / "conductors" + / "release" + / "_nightly_lib.sh" +) + + +def _lean_path(tmp_path: Path) -> str: + """Only the tools the lib calls, plus a fake `gh` (so no real gh is reachable).""" + lean = tmp_path / "bin" + lean.mkdir(exist_ok=True) + for tool in ( + "bash", + "python3", + "date", + "sleep", + "mktemp", + "head", + "rm", + "cat", + "dirname", + "readlink", + ): + found = shutil.which(tool) + if found and not (lean / tool).exists(): + (lean / tool).symlink_to(found) + return str(lean) + + +def _fake_gh(tmp_path: Path, responses: list[str]) -> None: + """A `gh` that answers call N with responses[N] (the last repeats). + + "FAIL" prints a 401 and exits 1; anything else is printed as the JSON body. + """ + script_dir = tmp_path / "gh-script" + script_dir.mkdir() + for i, r in enumerate(responses): + (script_dir / str(i)).write_text(r) + gh = tmp_path / "bin" / "gh" + gh.write_text(f"""#!/usr/bin/env bash +d="{script_dir}" +n=$(cat "$d/count" 2>/dev/null || echo 0) +echo $((n + 1)) > "$d/count" +i=$n; [[ -f "$d/$i" ]] || i={len(responses) - 1} +r=$(cat "$d/$i") +if [[ "$r" == FAIL ]]; then echo "failed to get run: HTTP 401: Bad credentials" >&2; exit 1; fi +echo "$r" +""") + gh.chmod(0o755) + + +def _run(tmp_path: Path, body: str, env_extra: dict | None = None): + env = { + "PATH": _lean_path(tmp_path), + "NIGHTLY_POLL_SECONDS": "0", + "TMPDIR": str(tmp_path), + "GH_TOKEN": "oldtoken", + } + env.update(env_extra or {}) + return subprocess.run( + ["bash", "-c", f'set -uo pipefail; source "{LIB}"; {body}'], + capture_output=True, + text=True, + env=env, + timeout=60, + check=False, + ) + + +def _calls(tmp_path: Path) -> int: + return int((tmp_path / "gh-script" / "count").read_text()) + + +def test_transient_401s_then_success_returns_0(tmp_path): + _lean_path(tmp_path) + _fake_gh( + tmp_path, ["FAIL", "FAIL", '{"status":"completed","conclusion":"success"}'] + ) + out = _run(tmp_path, 'await_run o/r 7; echo "rc=$?"') + assert "rc=0" in out.stdout, out.stderr + assert _calls(tmp_path) == 3 + + +def test_permanent_failure_returns_4_after_five_polls(tmp_path): + _lean_path(tmp_path) + _fake_gh(tmp_path, ["FAIL"]) + out = _run(tmp_path, 'await_run o/r 7; echo "rc=$?"') + assert "rc=4" in out.stdout, out.stderr + assert _calls(tmp_path) == 5 + assert "Bad credentials" in out.stderr + + +def test_red_run_returns_2(tmp_path): + _lean_path(tmp_path) + _fake_gh( + tmp_path, + [ + '{"status":"in_progress","conclusion":null}', + '{"status":"completed","conclusion":"failure"}', + ], + ) + out = _run(tmp_path, 'await_run o/r 7; echo "rc=$?"') + assert "rc=2" in out.stdout, out.stderr + + +def test_refresh_is_a_no_op_without_app_credentials(tmp_path): + out = _run( + tmp_path, + 'refresh_gh_token --force; echo "rc=$? tok=$GH_TOKEN"', + {"NIGHTLY_MINT_CMD": "false"}, + ) + assert "rc=0 tok=oldtoken" in out.stdout + assert "refreshed" not in out.stdout + + +def test_forced_refresh_exports_the_new_token(tmp_path): + mint = tmp_path / "mint.sh" + mint.write_text("#!/usr/bin/env bash\necho newtoken\n") + mint.chmod(0o755) + creds = { + "APP_CLIENT_ID": "id", + "APP_PRIVATE_KEY": "key", + "NIGHTLY_MINT_CMD": str(mint), + } + out = _run(tmp_path, 'refresh_gh_token --force; echo "rc=$? tok=$GH_TOKEN"', creds) + assert "rc=0 tok=newtoken" in out.stdout, out.stderr + assert "GitHub token refreshed" in out.stdout + # A fresh token is not re-minted until it is old. + out = _run(tmp_path, 'refresh_gh_token; echo "tok=$GH_TOKEN"', creds) + assert "tok=oldtoken" in out.stdout + + +def test_failed_mint_returns_1_and_keeps_the_old_token(tmp_path): + creds = { + "APP_CLIENT_ID": "id", + "APP_PRIVATE_KEY": "key", + "NIGHTLY_MINT_CMD": str(shutil.which("false")), + } + out = _run(tmp_path, 'refresh_gh_token --force; echo "rc=$? tok=$GH_TOKEN"', creds) + assert "rc=1 tok=oldtoken" in out.stdout + assert "could not mint" in out.stderr