diff --git a/.claude/hooks/end-at-deliverable.sh b/.claude/hooks/end-at-deliverable.sh new file mode 100755 index 0000000..1e52f75 --- /dev/null +++ b/.claude/hooks/end-at-deliverable.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# GENERATED — canonical source: PyAutoMind/policy/end_at_deliverable_hook.sh +# Installed into every checked-out repo as .claude/hooks/end-at-deliverable.sh by +# `python3 PyAutoMind/scripts/repos_sync.py --write`, and drift-checked by +# `--check`. Edit the canonical file, never a copy. +# +# --------------------------------------------------------------------------- +# Sessions end at their deliverable (PyAutoMind/policy/end_at_deliverable.md). +# +# A PreToolUse guard on the tools that outlive the turn. It exists because the +# prose rule was already written and was still broken twice: five batch members +# armed hourly check-ins on 2026-08-31 (fixed for batch members only), and on +# 2026-09-02/03 a mobile `/prm` re-armed a 60-minute `send_later` hourly from +# 02:39 to 12:11 UTC with no task active, leaving twenty fired one-shots and a +# drained usage window. A rule a session can talk itself past is not a rule, so +# this one is enforced by the harness. +# +# Registered under `hooks.PreToolUse` with the matcher +# ^(send_later|subscribe_pr_activity|ScheduleWakeup|CronCreate|RemoteTrigger|mcp__.*(send_later|subscribe_pr_activity).*)$ +# +# Allowed through: +# * `RemoteTrigger` with action list / get / list_runs / get_run_log — reading +# what already exists never outlives the turn; +# * anything at all when PYAUTO_ALLOW_TIMERS=1 — the human-authorised escape +# for a routine the human actually asked for. +# +# Everything else exits 2 (the harness treats stderr as the reason and blocks +# the call). An unreadable payload also exits 2: this fails closed, because the +# failure it guards against is silent and costs a night of usage. +set -uo pipefail + +# Human-authorised routine: the one way past this guard, and it has to be set +# deliberately in the environment. +[ "${PYAUTO_ALLOW_TIMERS:-}" = "1" ] && exit 0 + +payload="$(cat)" + +# python3 rather than jq: python3 is present everywhere this hook is installed +# (it is what the SessionStart hook guarantees), jq is not. The payload rides in +# the environment rather than on stdin so the heredoc keeps its own stdin. +PYAUTO_HOOK_PAYLOAD="$payload" python3 <<'PY' +import json +import os +import sys + +# Read-only RemoteTrigger actions: they inspect what exists and schedule nothing. +READ_ONLY_REMOTE_TRIGGER = {"list", "get", "list_runs", "get_run_log"} + +REASON = ( + "policy end_at_deliverable: sessions end at their deliverable " + "— {tool} would outlive the turn.\n" + "Report and stop; the human re-runs /prm. Set PYAUTO_ALLOW_TIMERS=1 only " + "for a routine the human asked for.\n" +) + + +def block(tool): + sys.stderr.write(REASON.format(tool=tool)) + raise SystemExit(2) + + +try: + event = json.loads(os.environ.get("PYAUTO_HOOK_PAYLOAD", "")) + if not isinstance(event, dict): + raise ValueError("PreToolUse payload is not a JSON object") + tool = event.get("tool_name") + if not isinstance(tool, str) or not tool: + raise ValueError("PreToolUse payload carries no tool_name") + tool_input = event.get("tool_input") + if not isinstance(tool_input, dict): + tool_input = {} +except Exception: + # Fail closed. A payload this hook cannot read is a call it cannot clear. + block("an unreadable tool call") + +if "RemoteTrigger" in tool: + action = tool_input.get("action") + if isinstance(action, str) and action in READ_ONLY_REMOTE_TRIGGER: + raise SystemExit(0) + +block(tool) +PY diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100755 index 0000000..591c2d7 --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,755 @@ +#!/usr/bin/env bash +# GENERATED — canonical source: PyAutoMind/policy/session_start_hook.sh +# Installed into every checked-out repo as .claude/hooks/session-start.sh by +# `python3 PyAutoMind/scripts/repos_sync.py --write`, and drift-checked by +# `--check`. Edit the canonical file, never a copy. +# +# --------------------------------------------------------------------------- +# Python 3.12 is the default in Claude Code web/mobile sessions. +# +# The remote container ships /usr/local/bin/python{,3} -> /usr/bin/python3.11, +# a pip whose shebang is #!/usr/bin/python3, and a uv-managed tool set (pytest, +# ruff, black, mypy, pyright, flake8, poetry) every one of which was built on +# 3.11 — one minor version below the floor the organism set for itself in the +# Python 3.12 floor campaign, and below every CI leg it runs. The image's own +# `use-python 3.12` does not fix it: it moves the update-alternatives links +# under /usr/bin, which /usr/local/bin/python{,3} then shadow. +# +# This hook makes a session 3.12 on three surfaces: +# +# 1. a 3.12 virtualenv first on PATH — python, python3, pip, pytest; +# 2. the /usr/local/bin/python{,3} symlinks repointed at 3.12, so anything +# resolving PATH without this session's env (a subprocess with a scrubbed +# environment, a `#!/usr/bin/env python3` script) also gets 3.12; +# 3. the uv-managed tools rebuilt on 3.12 — mypy and flake8 read the +# interpreter's version, so on 3.11 they judged code against 3.11 rules — +# and then repaired, because (2) is what breaks each tool env's own +# `bin/python`, and a rebuild cannot fix a link it is handed. +# +# What it deliberately does NOT touch: the update-alternatives links under +# /usr/bin. Scripts with a literal `#!/usr/bin/python3` shebang follow those, +# and some of the image's own tools (conan) are installed for 3.11 only — a +# flip there breaks them for no gain the three surfaces above don't already +# give. +# +# Remote-only (a local checkout keeps whatever the developer's shell provides), +# idempotent (everything is skipped once it already reads 3.12, so the second +# repo's copy in the same session costs ~0.2s), and non-blocking: every step +# degrades to a logged warning rather than failing the session start. +# +# Per-repo dependencies: a repo that needs more than pytest + PyYAML declares it +# in .claude/session-python.txt — one pip argument per line (`-e .`, a package +# spec, `-r requirements.txt`; `#` comments ignored). Installed additively into +# the shared venv, so the file stays out of this generated hook and the hook +# stays byte-identical in every repo. +set -euo pipefail + +[ "${CLAUDE_CODE_REMOTE:-}" = "true" ] || exit 0 + +VENV="${PYAUTO_SESSION_VENV:-$HOME/.pyauto/session-py312}" +# pytest-xdist is a base dep, not a nicety: a remote container has 4 cores and +# the Brain suite takes 96s on one of them and 28s on four. A session that has +# to `pip install` before it can run tests fast will simply not run them fast. +BASE_DEPS=(pytest PyYAML pytest-xdist) + +# Which checkout is this copy of the hook installed in? +# +# NOT $CLAUDE_PROJECT_DIR. That is the session's project directory, which equals +# the repo only when the session holds exactly ONE repo. A session scoped to +# several organs clones them side by side under the project directory +# (/home/user/PyAutoMind, /home/user/PyAutoBrain, ...), and then +# $CLAUDE_PROJECT_DIR is their parent — so reading session-python.txt from it +# found nothing, silently, in exactly the sessions that hold the most repos. +# +# Derive it from where this script actually is, in both the installed and the +# canonical location. +HOOK_SELF="$(readlink -f "$0")" +case "$(dirname "$HOOK_SELF")" in + */.claude/hooks) REPO_DIR="$(cd "$(dirname "$HOOK_SELF")/../.." && pwd)" ;; + */policy) REPO_DIR="$(cd "$(dirname "$HOOK_SELF")/.." && pwd)" ;; + *) REPO_DIR="${CLAUDE_PROJECT_DIR:-$PWD}" ;; +esac +EXTRAS_FILE="$REPO_DIR/.claude/session-python.txt" + +# stderr, not stdout: a SessionStart hook's stdout is fed to the agent as +# session context. +log() { printf '[session-start] %s\n' "$*" >&2; } + +# And where is the WORKSPACE ROOT — the directory holding the checkouts? +# +# NOT `dirname "$REPO_DIR"`, which is what this was. That is the root only in a +# FLAT workspace. Group the checkouts into family directories and the parent of +# a repo is a family directory, which really exists, really is writable and +# really is not a repo — so every guard below passes and all three consumers go +# wrong in silence: $PYAUTO_ROOT is exported to the whole session as the family +# directory, a second `.claude/` root is written INTO it, and the unshallow +# sweep and the generated fan-out reach that one family instead of every +# checkout. +# +# The rule is the one `PyAutoBrain/bin/_pyauto_root.sh` already ships, in the +# same order, so the session and the Brain's resolvers cannot disagree: +# +# 1. an explicit $PYAUTO_ROOT — the operator's word, taken verbatim; +# 2. the nearest ANCESTOR holding a .pyauto-root marker — the only rule that +# survives the workspace growing subdirectories, because "does this +# directory hold a checkout?" is answered yes by a family directory the +# moment the checkouts move into one; +# 3. the parent of this checkout when it holds a sibling organ; +# 4. the parent regardless, reported as unverified. +# +# The resolver itself is used where it is reachable, so there is one +# implementation to correct. But it CANNOT be a dependency: this hook is +# installed into every repo and runs in containers that hold a single checkout, +# no PyAutoBrain and no marker anywhere above them — the session that most +# needs the hook is the one least likely to have a workspace around it. So the +# same rule runs inline wherever the resolver is absent, steps 3 and 4 never +# require a marker, and the resolver is read in a subshell that reports only +# its two answers: a resolver that is broken, or from a version this copy of +# the hook has never seen, can then never fail a session start. +PYAUTO_ROOT_MARKER=".pyauto-root" + +# The nearest ancestor of $1 holding the marker. A checkout is never its own +# workspace root, so the walk starts one level up. +marked_root() { + local d + d="$(dirname "$1")" + while [ "$d" != "/" ]; do + if [ -f "$d/$PYAUTO_ROOT_MARKER" ]; then + printf '%s' "$d" + return 0 + fi + d="$(dirname "$d")" + done + if [ -f "/$PYAUTO_ROOT_MARKER" ]; then + printf '%s' "/" + return 0 + fi + return 1 +} + +# A directory counts as a workspace root if it holds an organ checkout. Any +# organ: a remote session may hold exactly one, and it is still a root. +holds_an_organ() { + [ -d "$1/PyAutoBrain" ] || [ -d "$1/PyAutoMind" ] || [ -d "$1/PyAutoCortex" ] \ + || [ -d "$1/PyAutoMemory" ] || [ -d "$1/PyAutoEyes" ] || [ -d "$1/PyAutoHeart" ] \ + || [ -d "$1/PyAutoHands" ] || [ -d "$1/PyAutoPulse" ] || [ -d "$1/PyAutoInsight" ] || [ -d "$1/PyAutoNerves" ] \ + || [ -d "$1/PyAutoGut" ] \ + || [ -d "$1/organs/PyAutoBrain" ] || [ -d "$1/organs/PyAutoMind" ] \ + || [ -d "$1/organs/PyAutoCortex" ] || [ -d "$1/organs/PyAutoMemory" ] \ + || [ -d "$1/organs/PyAutoEyes" ] || [ -d "$1/organs/PyAutoHeart" ] \ + || [ -d "$1/organs/PyAutoHands" ] || [ -d "$1/organs/PyAutoPulse" ] || [ -d "$1/organs/PyAutoInsight" ] \ + || [ -d "$1/organs/PyAutoNerves" ] || [ -d "$1/organs/PyAutoGut" ] +} + +# Assigns WORKSPACE_ROOT and WORKSPACE_ROOT_REASON rather than printing: one +# decision produces two answers, and a command substitution would drop the +# reason — which is the whole lesson of a defect whose every consequence was +# silent. +resolve_workspace_root() { + local helper answer + # 1. The operator's word, never second-guessed. The marker only decides + # how the reason reads, so a wrong value from a caller stays visible. + if [ -n "${PYAUTO_ROOT:-}" ]; then + WORKSPACE_ROOT="$PYAUTO_ROOT" + if [ -f "$WORKSPACE_ROOT/$PYAUTO_ROOT_MARKER" ]; then + WORKSPACE_ROOT_REASON="PYAUTO_ROOT" + else + WORKSPACE_ROOT_REASON="PYAUTO_ROOT (unverified - no $PYAUTO_ROOT_MARKER marker)" + fi + return 0 + fi + # The shared resolver, if this checkout can see one: either this checkout + # IS the Brain, or the Brain is beside it. + # + # Its answer is ACCEPTED ONLY IF IT CONTAINS THIS CHECKOUT, because the + # resolver anchors on the BRAIN checkout, not on this one. In a worktree + # bundle the bundle's PyAutoBrain is a symlink into the canonical + # workspace, so the resolver follows it and answers with the canonical + # root — and this session would then export somebody else's workspace to + # the whole session, write a `.claude/` root into it and unshallow its + # repos. Measured on this very bundle: the resolver said + # the canonical workspace for a hook running in a task bundle beside it. + # `scripts/session_bootstrap.sh` guards the same trap the same way; the + # test is "is it an ancestor", not "is the checkout directly under it", so + # a checkout inside a family directory still passes. + local repo_real root_real + repo_real="$(readlink -f "$REPO_DIR" 2>/dev/null || printf '%s' "$REPO_DIR")" + for helper in "$REPO_DIR/bin/_pyauto_root.sh" \ + "$(dirname "$REPO_DIR")/PyAutoBrain/bin/_pyauto_root.sh" \ + "$(dirname "$REPO_DIR")/organs/PyAutoBrain/bin/_pyauto_root.sh" \ + "$(dirname "$(dirname "$REPO_DIR")")/organs/PyAutoBrain/bin/_pyauto_root.sh"; do + [ -r "$helper" ] || continue + # `|` separates the two answers; no reason the resolver states contains + # one. The subshell drops this script's `set -euo pipefail` so that a + # resolver which fails, or which one day needs something this container + # does not have, degrades to the inline rule instead of to a dead + # session. + answer="$( ( + set +euo pipefail + # shellcheck source=/dev/null + . "$helper" >/dev/null 2>&1 + printf '%s|%s' "${PYAUTO_ROOT:-}" "${PYAUTO_ROOT_REASON:-}" + ) 2>/dev/null )" || answer="" + [ -n "${answer%%|*}" ] && [ -d "${answer%%|*}" ] || continue + root_real="$(readlink -f "${answer%%|*}" 2>/dev/null || printf '%s' "${answer%%|*}")" + case "$repo_real" in + "${root_real%/}"/*) ;; + *) continue ;; + esac + WORKSPACE_ROOT="${answer%%|*}" + WORKSPACE_ROOT_REASON="${answer#*|}" + return 0 + done + # 2. A marked ancestor — the root naming itself, which stays right however + # deep under it this checkout sits. + if answer="$(marked_root "$REPO_DIR")"; then + WORKSPACE_ROOT="$answer" + WORKSPACE_ROOT_REASON="$PYAUTO_ROOT_MARKER marker" + return 0 + fi + # 3. Beside this checkout — nothing above us says anything, so what sits + # beside us is all there is: a single-repo remote session, a CI matrix, + # a spawned template. 4. Failing that, the parent anyway: the best guess + # available, and a real path a diagnostic can name. + local parent + parent="$(dirname "$REPO_DIR")" + [ "$(basename "$parent")" = organs ] && parent="$(dirname "$parent")" + WORKSPACE_ROOT="$parent" + if holds_an_organ "$WORKSPACE_ROOT"; then + WORKSPACE_ROOT_REASON="beside this checkout" + else + WORKSPACE_ROOT_REASON="unverified (no sibling organ beside this checkout)" + fi +} +resolve_workspace_root + +is_py312() { + [ -x "$1" ] && "$1" -c 'import sys; raise SystemExit(sys.version_info[:2] != (3, 12))' >/dev/null 2>&1 +} + +find_base_python() { + local candidate + for candidate in /usr/bin/python3.12 /usr/local/bin/python3.12 \ + "$(command -v python3.12 2>/dev/null || true)"; do + if [ -n "$candidate" ] && is_py312 "$candidate"; then + printf '%s\n' "$candidate" + return 0 + fi + done + # No system 3.12 (a future base image could drop it) — uv can fetch one. + if command -v uv >/dev/null 2>&1; then + log "no system python3.12; asking uv to install one" + uv python install 3.12 >&2 || return 1 + candidate="$(uv python find 3.12 2>/dev/null || true)" + if [ -n "$candidate" ] && is_py312 "$candidate"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + return 1 +} + +venv_ready() { + is_py312 "$VENV/bin/python" \ + && [ -x "$VENV/bin/pip" ] \ + && "$VENV/bin/python" -c 'import pytest, yaml, xdist' >/dev/null 2>&1 +} + +# 1. The interpreter a session types: python, python3, pip, pytest. +ensure_venv() { + local base_python + if venv_ready; then + log "reusing $VENV ($("$VENV/bin/python" -V 2>&1))" + return 0 + fi + if ! base_python="$(find_base_python)"; then + log "WARNING: no Python 3.12 in this container; PATH left unchanged" + return 1 + fi + log "building $VENV on $base_python" + rm -rf "$VENV" + mkdir -p "$(dirname "$VENV")" + # --system-site-packages makes the venv a strict SUPERSET of the base + # interpreter. That is what lets `point_system_default` make this venv the + # session's `python3` without silently removing anything the image + # installed: an isolated venv would swap one set of missing modules for + # another, and the session would find out through a ModuleNotFoundError + # that reads like broken code. + if command -v uv >/dev/null 2>&1; then + # --seed puts pip inside the venv too, so `pip install` targets 3.12 + # rather than falling through to the container's 3.11 /usr/bin/pip. + uv venv --seed --system-site-packages --python "$base_python" "$VENV" >&2 + uv pip install --python "$VENV/bin/python" --quiet "${BASE_DEPS[@]}" >&2 + else + "$base_python" -m venv --system-site-packages "$VENV" >&2 + "$VENV/bin/python" -m pip install --quiet --upgrade pip >&2 + "$VENV/bin/python" -m pip install --quiet "${BASE_DEPS[@]}" >&2 + fi + venv_ready || { log "WARNING: could not build a 3.12 venv at $VENV"; return 1; } +} + +# This repo's own dependencies, if it declares any. Additive and marked, so a +# session holding several repos installs each repo's set exactly once. +ensure_repo_extras() { + [ -r "$EXTRAS_FILE" ] || return 0 + local marker args=() + marker="$VENV/.extras-$(cksum <"$EXTRAS_FILE" | tr -d ' /')" + [ -e "$marker" ] && return 0 + while IFS= read -r line; do + line="${line%%#*}" + line="$(printf '%s' "$line" | tr -d '\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -n "$line" ] && args+=("$line") + done <"$EXTRAS_FILE" + [ ${#args[@]} -gt 0 ] || return 0 + log "installing this repo's declared deps: ${args[*]}" + if (cd "$(dirname "$(dirname "$EXTRAS_FILE")")" && "$VENV/bin/python" -m pip install --quiet "${args[@]}" >&2); then + : >"$marker" + else + log "WARNING: $EXTRAS_FILE install failed; continuing without it" + fi +} + +# 2. What PATH means without this session's env file. +# +# This leg is the one that matters most, because the env file is exactly what a +# multi-repo session does NOT get: Claude Code registers project hooks from the +# session's project directory, that directory is the repos' parent, and no hook +# runs there (see leg 5). So every Bash call in such a session resolves whatever +# the image put on PATH, and these two names are the fallback that has to be +# right on its own. +# +# It used to point them at the BASE interpreter — `point_system_default +# "$(readlink -f "$VENV/bin/python")"` resolved the venv's python straight +# through to /usr/bin/python3.12. That satisfied the version question and lost +# everything else: `python3 -m pytest` answered "No module named pytest", and +# the session's own venv, sitting one directory away with pytest in it, was +# unreachable from any shell. +# +# Two details make the fix work: +# +# * A wrapper script, not a symlink. Python resolves a symlinked executable +# before looking for `pyvenv.cfg`, so a symlink to $VENV/bin/python lands on +# the base interpreter's prefix and the venv is lost again — silently, and +# in the same shape as the bug being fixed. `exec`ing it keeps the venv. +# * --system-site-packages on the venv (see ensure_venv), so pointing the +# system default here adds the session's packages without removing the +# image's. +# Bounded, because this predicate is what runs a candidate interpreter — and +# the one failure mode worth surviving is an interpreter that never returns. +venv_backed() { + [ -x "$1" ] || return 1 + [ "$(timeout 20 "$1" -c 'import sys; print(sys.prefix)' 2>/dev/null)" = "$VENV" ] +} + +# Does following $1's symlinks pass through $2? A venv's python legitimately +# RESOLVES to the same base interpreter a system default points at, so comparing +# endpoints refuses safe rewrites. The question that matters is narrower: is the +# path being rewritten a link in the target's own chain — because then the +# wrapper execs itself. +links_through() { + local node="$1" needle="$2" hops=0 + needle="$(cd "$(dirname "$needle")" 2>/dev/null && printf '%s/%s' "$(pwd -P)" "$(basename "$needle")")" + while [ -L "$node" ] && [ "$hops" -lt 40 ]; do + node="$(cd "$(dirname "$node")" && cd "$(dirname "$(readlink "$node")")" 2>/dev/null && printf '%s/%s' "$(pwd -P)" "$(basename "$(readlink "$node")")")" + [ "$node" = "$needle" ] && return 0 + hops=$((hops + 1)) + done + return 1 +} + +# `rm -f` first, and it is not tidiness. /usr/local/bin/python3 is a SYMLINK to +# the real interpreter, and a redirect opens the link's TARGET: `cat >` there +# overwrites /usr/bin/python3.12 itself with this wrapper. The venv's own +# python symlinks to that same file, so the wrapper then execs itself — the +# container loses its interpreter and every `python3` spins at 100% CPU. Writing +# a fresh file at the path replaces the link instead of following it. +# +# `links_through` is the second half of the same guard: a target that reaches +# the destination through its own symlink chain builds the identical loop by the +# other route, so refuse rather than write it. +write_venv_shim() { + local dest="$1" target="$2" + [ -w "$(dirname "$dest")" ] || { log "WARNING: cannot rewrite $dest (not writable)"; return 1; } + [ -x "$target" ] || { log "WARNING: $target is not executable; leaving $dest alone"; return 1; } + if links_through "$target" "$dest"; then + log "WARNING: refusing to point $dest at $target — it links back through $dest" + return 1 + fi + rm -f "$dest" + cat >"$dest" < $VENV/bin/python (venv, with pytest and yaml)" \ + || log "WARNING: $bin/python3 is still $(timeout 20 "$bin/python3" -V 2>&1)" +} + +# 2b. The `pytest` that PATH actually finds. +# +# $HOME/.local/bin precedes /usr/local/bin, and it holds uv's tool shims — so +# fixing python3 alone still leaves bare `pytest` resolving to uv's ISOLATED +# pytest, which by design cannot see PyYAML or a repo's own extras. In this +# workspace that made `pytest` exit on four collection ImportErrors that read +# like broken source, in a session where the suite was in fact green. +# +# A session should have exactly one pytest, and it should be the venv's. +# Confined to uv's own shim directory: a distro-packaged pytest is not ours to +# overwrite. Runs after retool_uv_tools, which rewrites these same shims. +point_pytest_at_venv() { + local shim_dir="${HOME}/.local/bin" name path + [ -x "$VENV/bin/pytest" ] || return 0 + for name in pytest py.test; do + path="$(command -v "$name" 2>/dev/null)" || continue + [ -n "$path" ] || continue + case "$path" in + "$VENV"/*) continue ;; + "$shim_dir"/*) ;; + *) log "WARNING: $name resolves to $path, outside uv's shim dir — left alone"; continue ;; + esac + write_venv_shim "$path" "$VENV/bin/$name" \ + && log "$path -> $VENV/bin/$name (so \`$name\` sees this repo's deps)" + done +} + +# 2c. Every OTHER console script the venv owns. +# +# `pytest` and `python3` survive without the venv on PATH because 2 and 2b give +# each one its own shim. Nothing else does — and PATH is exactly what a +# multi-repo session never gets, because the env file that exports it is written +# by Claude Code around a hook it never registers (see leg 5). So a dependency +# declared in `.claude/session-python.txt` that ships a CONSOLE SCRIPT rather +# than an importable module was installed into the venv and then invisible. +# +# Measured, not hypothetical: PyAutoHands shells out to `ipynb-py-convert`, and +# with the package installed its suite still failed five tests with +# `FileNotFoundError: ipynb-py-convert` — the binary sitting in $VENV/bin, +# unreachable, while `--check` called the session healthy. A missing binary and +# an unreachable one are the same symptom, and neither says "environment". +# +# The claim policy is 2b's, because the reasoning is 2b's: a name we can reach +# only through uv's shim dir or not at all is ours to provide; a name the image +# owns elsewhere is not ours to overwrite. +point_venv_scripts_at_venv() { + local shim_dir="${HOME}/.local/bin" entry name path + [ -d "$VENV/bin" ] || return 0 + mkdir -p "$shim_dir" 2>/dev/null || return 0 + for entry in "$VENV"/bin/*; do + [ -f "$entry" ] && [ -x "$entry" ] || continue + name="$(basename "$entry")" + case "$name" in + # python/pip are legs 2 and the venv's own plumbing; pytest is 2b. + # The activate family is meant to be sourced, never executed. + python*|pip*|pytest|py.test|activate*|deactivate*|*.bat|*.ps1|*.csh|*.fish|*.nu) continue ;; + esac + path="$(command -v "$name" 2>/dev/null)" || path="" + case "$path" in + "$VENV"/*) continue ;; + "") ;; + "$shim_dir"/*) ;; + *) log "WARNING: $name resolves to $path, outside uv's shim dir — left alone"; continue ;; + esac + write_venv_shim "$shim_dir/$name" "$entry" \ + && log "$shim_dir/$name -> $entry (venv console script)" + done +} + +# 3. The uv-managed tools — rebuilt on 3.12 only where they are not already. +# +# Pinned to the version already installed: this hook changes the INTERPRETER a +# tool runs on, and nothing else. Unpinned, `uv tool install --force` fetches +# the latest release, which quietly moved mypy across a major version (1.19 -> +# 2.3) the first time this ran — a lint-result change nobody asked for, riding +# in on a Python upgrade. Falls back to unpinned only if the pin cannot be +# resolved for 3.12. +retool_uv_tools() { + command -v uv >/dev/null 2>&1 || return 0 + local tools_dir tool name version spec + tools_dir="$(uv tool dir 2>/dev/null || echo "$HOME/.local/share/uv/tools")" + [ -d "$tools_dir" ] || return 0 + for tool in "$tools_dir"/*/; do + name="$(basename "$tool")" + is_py312 "$tool/bin/python" && continue + version="$(uv tool list 2>/dev/null | awk -v n="$name" '$1 == n {print substr($2, 2); exit}')" + spec="$name" + [ -n "$version" ] && spec="$name==$version" + if uv tool install --python 3.12 --force "$spec" >/dev/null 2>&1 \ + || uv tool install --python 3.12 --force "$name" >/dev/null 2>&1; then + log "rebuilt ${spec} on 3.12" + else + log "WARNING: could not rebuild $name on 3.12; it stays on $("$tool/bin/python" -V 2>&1)" + fi + done +} + +# 3b. The link uv's rebuild cannot fix from inside. +# +# uv creates each tool env with `bin/python` as a SYMLINK to whatever `python3` +# was at install time — here `/usr/local/bin/python3`, which leg 2 has already +# replaced with a wrapper that `exec`s the session venv. Every tool env's python +# then resolves its prefix to the VENV: `sys.prefix` is the venv, the tool's own +# site-packages never reaches `sys.path`, and the console script dies with +# `ModuleNotFoundError: No module named 'flake8'` — with flake8 sitting +# installed two directories away. +# +# TWO paths reach that state, which is why this runs after leg 3 rather than +# only on the envs leg 3 rebuilt: leg 3 rebuilds a 3.11 tool and hands the new +# env the hijacked path, and leg 2 separately breaks every PRE-EXISTING tool env +# that already pointed there and that leg 3 skips (`is_py312 … && continue`). +# Running here covers both, because leg 2 runs before leg 3. +# +# Measured 2026-08-27, post-bootstrap: mypy, flake8, black, poetry and pyright +# all dead this way; `ruff` survived (a native binary) and `pytest` survived +# (leg 2b points its shim straight at the venv, which has pytest). The +# bootstrap's `--check` called every one of them "3.12 OK", because the +# interpreter they reach IS 3.12 — it is simply the wrong one. A session then +# lints clean by not linting at all, and CI is the thing that finds out. +# +# The fix is one link: a venv's `bin/python` must resolve to a BASE interpreter, +# never to a path this hook hijacks. +# +# This lived in `scripts/session_bootstrap.sh` for one pass, which is the wrong +# home. The bootstrap is what a MULTI-repo session runs; a SINGLE-repo session +# registers this hook and never calls the bootstrap, so it got leg 2's breakage +# and none of the repair. +repair_uv_tools() { + local tools_dir base tool link prefix + base="$("$VENV/bin/python" -c 'import sys, os; print(os.path.join(sys.base_prefix, "bin", "python3.12"))' 2>/dev/null)" || base="" + [ -x "$base" ] || base="$(command -v python3.12 2>/dev/null)" || base="" + [ -x "$base" ] || return 0 + tools_dir="${PYAUTO_UV_TOOLS_DIR:-$(uv tool dir 2>/dev/null || echo "$HOME/.local/share/uv/tools")}" + [ -d "$tools_dir" ] || return 0 + for tool in "$tools_dir"/*/; do + link="${tool}bin/python" + [ -L "$link" ] || continue + # Ask the interpreter where it thinks it lives, rather than tracing the + # link: `/usr/local/bin/python3` is a WRAPPER SCRIPT (a symlink there + # would lose the venv — leg 2's whole note), so `readlink -f` stops at + # the wrapper and reports nothing about the venv behind it. sys.prefix + # is the outcome; anything else is the mechanism. + prefix="$("$link" -c 'import sys; print(sys.prefix)' 2>/dev/null)" || continue + [ -n "$prefix" ] || continue + [ "$prefix" = "${tool%/}" ] && continue # resolves to its own env: correct + # Non-fatal like every other leg: an unwritable tools dir is a warning, + # never a failed session start. + ln -sfn "$base" "$link" || { + log "WARNING: could not repoint $(basename "${tool%/}") at $base" + continue + } + prefix="$("$link" -c 'import sys; print(sys.prefix)' 2>/dev/null)" || prefix="" + if [ "$prefix" = "${tool%/}" ]; then + log "repointed $(basename "${tool%/}") at $base (it resolved into $VENV, not its own env)" + else + log "WARNING: $(basename "${tool%/}") still resolves to ${prefix:-nothing} — it will not run" + fi + done +} + +# 4. Honest git history. +# +# A remote session clones shallow. `git merge-base --is-ancestor` then LIES +# across the graft boundary: it reports "not an ancestor" for a commit whose +# ancestry is simply not in the clone, and the ship/close-out skills act on that +# answer. A completion record already logged this the hard way +# (complete/2026/08/status-sh-repos-missing-source.md, "environment note"). +# +# These repos are small (single-digit MB of .git), so unshallowing costs a +# couple of seconds once per container and removes a whole class of wrong +# answer. Bounded and non-fatal: a slow or blocked network leaves a shallow +# clone and a warning, never a failed session start. +unshallow_checkout() { + local repo + repo="${1%/}/" # the callers glob, but do not rely on the trailing slash + [ -e "${repo}.git/shallow" ] || return 0 + log "unshallowing $(basename "${repo%/}") (shallow clones make ancestry checks lie)" + if timeout 120 git -C "$repo" fetch --unshallow --quiet 2>/dev/null \ + || timeout 120 git -C "$repo" fetch --depth=2147483647 --quiet 2>/dev/null; then + log " $(basename "${repo%/}"): full history ($(git -C "$repo" rev-list --count HEAD 2>/dev/null) commits)" + else + log " WARNING: $(basename "${repo%/}") is still shallow — run 'git fetch --unshallow' before trusting any ancestry check" + fi +} + +# Two levels, not one. A workspace is `/` or, once its checkouts +# are grouped, `//` — and a family directory has no `.git`, +# so a one-level walk steps straight over every checkout inside it and leaves +# them shallow while reporting nothing. A directory that IS a checkout is never +# descended into (a repo's own subdirectories are not siblings), and the walk +# stops at two levels because that is the shape of the workspace: an unbounded +# walk over every checked-out repo is not something a session start can afford. +ensure_full_clone() { + local entry child + for entry in "$WORKSPACE_ROOT"/*/; do + [ -d "$entry" ] || continue + if [ -e "${entry}.git" ]; then + unshallow_checkout "$entry" + continue + fi + for child in "$entry"*/; do + [ -d "$child" ] && [ -e "${child}.git" ] || continue + unshallow_checkout "$child" + done + done +} + +# 5. Make the NEXT session in this container start correctly. +# +# Claude Code registers project hooks from the session's project directory. In a +# one-repo session that IS the repo, and `/.claude/settings.json` is found. +# In a session holding several organs the project directory is their parent, +# which is not a repo and has no `.claude/` — so none of the per-repo hooks are +# registered and none of this script runs. That is why a multi-repo session came +# up on the container's Python 3.11: not a broken hook, an unreachable one. +# +# A repo cannot ship a file into its own parent, so install it at run time. The +# settings we write fan out to every sibling repo's own hook, so the layout +# stays "each repo owns its hook" and this file stays generated-from-one-source. +# It takes effect on the next session start in this container. +install_workspace_settings() { + # Target the WORKSPACE ROOT, derived from this checkout — not + # $CLAUDE_PROJECT_DIR, and not only when the two differ. + # + # The early return this replaces ("only in the multi-repo layout") made the + # whole leg unreachable. Writing the fan-out requires the hook to be + # running; the hook runs only in a session whose project dir is a repo — + # that is, a SINGLE-repo session — and the early return then skipped it as + # having nothing to add. So the one session type that could seed the + # container never did, and the multi-repo session that needed it never ran + # the hook to find out. Observed directly: a container with two single-repo + # sessions behind it still had no workspace-root settings, and the next + # session — three organs, project dir /home/user — fired no hook, ran on + # PATH's pytest, and left both clones shallow for three minutes until an + # unrelated verb happened to knock on session_bootstrap.sh. + # + # A single-repo session has the same sibling layout one directory up, so it + # can seed the root for free. Skip only if the root is itself a repo (then + # it owns its own hook) or is not writable. + local root="$WORKSPACE_ROOT" + [ -n "$root" ] && [ -d "$root" ] || return 0 + [ -d "$root/.git" ] && return 0 + + local settings="$root/.claude/settings.json" + local fanout="$root/.claude/hooks/session-start.sh" + [ -w "$root" ] || { log "WARNING: $root not writable ($WORKSPACE_ROOT_REASON); multi-repo sessions will keep skipping the hook"; return 0; } + + mkdir -p "$(dirname "$fanout")" + rm -f "$fanout" # never write through a symlink; see write_venv_shim + cat >"$fanout" <<'FANOUT' +#!/usr/bin/env bash +# GENERATED at session start by a PyAuto repo's own session-start hook. +# Runs every sibling repo's hook, because the workspace root is not a repo and +# therefore has no hook of its own. Each repo's hook is idempotent, so the +# second and later ones cost ~0.2s. +set -u +ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../.." && pwd)" + +# Run one checkout's hook. Non-zero only when there is no hook to run, which is +# how a family directory is told apart from a checkout. +run_repo_hook() { + hook="${1%/}/.claude/hooks/session-start.sh" + [ -x "$hook" ] || return 1 + CLAUDE_PROJECT_DIR="${1%/}" "$hook" || \ + printf '[session-start] WARNING: %s failed\n' "$hook" >&2 + return 0 +} + +# Two levels: the checkouts directly under the root, and those one level down +# inside a family directory once the workspace groups them. Anything that +# already carries a hook is run, never walked into. +for entry in "$ROOT"/*/; do + [ -d "$entry" ] || continue + run_repo_hook "$entry" && continue + for child in "$entry"*/; do + [ -d "$child" ] || continue + run_repo_hook "$child" || true + done +done +FANOUT + chmod 0755 "$fanout" + + if [ ! -f "$settings" ]; then + cat >"$settings" <<'SETTINGS' +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ] + } +} +SETTINGS + log "installed $settings ($WORKSPACE_ROOT_REASON) — multi-repo sessions in this container now run the hook" + fi +} + +# Git history and hook reachability have nothing to do with the interpreter, so +# they run first and unconditionally: a container where the 3.12 venv cannot be +# built still wants honest ancestry and a hook that fires next session. +# PYAUTO_SESSION_DEFINE_ONLY=1 defines every function and performs no action — +# the seam this hook's own tests use to drive one leg at a time against a +# temporary directory, instead of against the container they run in. Sourcing +# returns; executing exits. +if [ "${PYAUTO_SESSION_DEFINE_ONLY:-}" = "1" ]; then + return 0 2>/dev/null || exit 0 +fi + +# Run leg 3b on its own, without a session start. The door +# `scripts/session_bootstrap.sh` knocks on after it has run every repo's hook — +# a subprocess rather than a source, so this script's `set -euo pipefail` never +# leaks into a caller that is contractually "a bootstrap, never a gate". +if [ "${1:-}" = "--repair-uv-tools" ]; then + repair_uv_tools + exit 0 +fi + +ensure_full_clone +install_workspace_settings + +# PYAUTO_SESSION_SKIP_PYTHON=1 stops here — the seam the hook's own tests use to +# exercise the legs above without building an interpreter. +if [ "${PYAUTO_SESSION_SKIP_PYTHON:-}" = "1" ]; then + log "PYAUTO_SESSION_SKIP_PYTHON=1 — leaving the interpreter alone" + exit 0 +fi + +if ensure_venv; then + ensure_repo_extras + point_system_default + retool_uv_tools + repair_uv_tools + point_pytest_at_venv + point_venv_scripts_at_venv + # Every repo in the session registers this hook, so the second copy must not + # prepend the venv a second time. + if [ -n "${CLAUDE_ENV_FILE:-}" ] && ! grep -qs 'PYAUTO_SESSION_PY312=' "$CLAUDE_ENV_FILE"; then + { + echo "export PYAUTO_SESSION_PY312=\"$VENV\"" + echo "export VIRTUAL_ENV=\"$VENV\"" + echo "export PATH=\"$VENV/bin:\$PATH\"" + # The workspace root, so the Brain's shell/python resolvers agree + # with the session instead of each re-deriving it — and how it was + # reached, because an unverified root that a consumer inherits is + # exactly the answer that used to be wrong without saying so. + echo "export PYAUTO_ROOT=\"$WORKSPACE_ROOT\"" + echo "export PYAUTO_ROOT_REASON=\"$WORKSPACE_ROOT_REASON\"" + } >>"$CLAUDE_ENV_FILE" + fi + log "default python is now $("$VENV/bin/python" -V 2>&1) ($VENV/bin)" +fi diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..5578c7d --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,25 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "^(send_later|subscribe_pr_activity|ScheduleWakeup|CronCreate|RemoteTrigger|mcp__.*(send_later|subscribe_pr_activity).*)$", + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/end-at-deliverable.sh" + } + ] + } + ] + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..040cce5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,21 @@ +name: DNA checks +on: [push, pull_request, workflow_dispatch] +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python: ['3.12', '3.13'] + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoBrain + path: .brain + - uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python }}' + - run: python -m pip install -e . + - run: python -m unittest discover -s tests -v + env: + PYAUTO_BRAIN: ${{ github.workspace }}/.brain diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..749965e --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,39 @@ +name: DNA dashboard +on: + push: + branches: [main] + workflow_dispatch: +permissions: + contents: read + pages: write + id-token: write +concurrency: + group: pages + cancel-in-progress: true +jobs: + render: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: PyAutoLabs/PyAutoBrain + path: .brain + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -e . + - run: python -m dna board --brain .brain --output _site + - run: python .brain/board/_state.py _site/state.json + - uses: actions/upload-pages-artifact@v3 + with: + path: _site + deploy: + needs: render + runs-on: ubuntu-latest + environment: + name: github-pages + url: '${{ steps.deployment.outputs.page_url }}' + steps: + - uses: actions/deploy-pages@v4 + id: deployment diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..95fa43e --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.pyc +*.egg-info/ +.pytest_cache/ +.venv/ +private/ +_site/ +.scratch/ +inventories/*.json +!inventories/public.json + +.brain/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..5485076 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,87 @@ +# PyAutoDNA + +Follow workspace AGENTS.md and approved start-dev worktree flow. DNA owns stack +identities, observations, support rationale and upgrade/adoption records. Brain +coordinates, Heart validates, Hands releases, Nerves guards runtime compatibility. +No automatic upgrades, compute submission, schedules or package publication. +Never publish private records. Public feeds are allowlisted projections; retain +unknown/missing/stale evidence. Render with explicit Brain checkout and shared +board/_theme.py and validate state.json with board/_state.py. Test with +`python -m unittest discover -s tests -v` after sourcing task activate.sh. + + + +**You are one organ of the PyAuto organism** — an agentic ecosystem for +human-led, natural-language software development. The organs below are +peer repositories; this repo is one of them, not a part of another. +Canonical boundaries live in `PyAutoBrain/ORGANISM.md`; the full body map +(every repo, not just organs) is `PyAutoMind/repos.yaml`. + +| Organ | Repo | Role | +|-------|------|------| +| **Brain** | PyAutoBrain | Reasoning/orchestration layer; how work is decomposed and routed; the specialist agents. | +| **Mind** | PyAutoMind | Intent, goals, priorities, workflow state; every task starts as a markdown prompt here. | +| **Cortex** | PyAutoCortex | The Cortex — what is true in the science: the body map (`projects.yaml`) and one ledger per science project (runs, results, learnings, where to pick up); the science mirror of the Mind. | +| **Memory** | PyAutoMemory | Long-term scientific/software/project knowledge (see science pointer below). | +| **Eyes** | PyAutoEyes | The Eyes — cross-project visualization dashboard over the `_visualization` repos: their registry, the `gallery/viz_manifest.yaml` read contract and the Pages board linking their PNGs. Renders and copies nothing, never judges figures (the Brain's Eyes conductor does) and never edits plot code. | +| **Ears** | PyAutoEars | The Ears — community listening: read-only public conversation collection, the versioned community snapshot contract, coverage receipts and the dashboard. GitHub stays authoritative; Brain's Community conductor judges and drafts replies; never posts, labels, files issues or exports transcripts. | +| **Heart** | PyAutoHeart | Health/readiness — the authoritative "is it safe to release?" verdict. | +| **Hands** | PyAutoHands | Packaging, tagging, notebook generation, PyPI release execution. | +| **Pulse** | PyAutoPulse | The Pulse — cross-project profiling dashboard over the `_profiling` repos: campaign intent, instance registry, the versioned `profiling-summary` read contract, ingest receipts and the Pages board. Validates the contract only; never judges, scores or issues verdicts (the Brain's profiling conductor judges). | +| **Insight** | PyAutoInsight | Inference campaign intent, the cross-project inference instance registry, the versioned `inference-summary` read contract, ingest receipts and the evidence dashboard. Projects execute, Cortex records the science, Mind owns task state; never infers scientific acceptance or submits compute. | +| **DNA** | PyAutoDNA | Software-stack specifications, environment inventory receipts, support rationale, upgrade campaigns and adoption history. Brain coordinates, Heart validates readiness, Hands releases, Nerves enforces runtime compatibility; collection never upgrades environments. | +| **Nerves** | PyAutoNerves | The Nerves — the configuration/serialization layer connecting workspace conventions to libraries (layered config, version handshake, test_mode), delivered as the `autonerves` package. | +| **Gut** | PyAutoGut | Lifecycle of condemned self-material (stale branches, stashes, dead code/tests): held as recoverable git refs through a transit window, voided on a sweep. The storage mirror of Memory. | + +Call chain (always this order): **Brain → Heart (gate) → Build (execute)**. Brain agents are **conductors** (front-door; a human drives them; they decide *and* act) or **faculties** (read-only opinions the conductors consult; they judge and stop). New capability grows as a faculty, not a new organ, unless it owns state or effects no existing organ can. + +Generated from `PyAutoMind/repos.yaml` + `PyAutoBrain/ORGANISM.md`; edit there, then run `python3 PyAutoMind/scripts/repos_sync.py --write`. + + + +## Never rewrite history + +Never rewrite pushed history on any repo with a remote — no `git init` over a +tracked repo, no force-push to `main`, no fresh-start "Initial commit", no +`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a +clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. + + + +## Sessions end at their deliverable + +A session ends when it reports its deliverable — never arm anything that +outlives the turn to wait for CI, a review or a merge: no `send_later`, no +`subscribe_pr_activity`, no `CronCreate`, no `ScheduleWakeup`, no `/loop`, no +`RemoteTrigger` create/update/run. Judge once, report, stop; the human re-runs +`/prm` (or the batch review) when it is green. Measured: five batch members +armed hourly check-ins on 2026-08-31, and a mobile `/prm` re-armed a 60-minute +`send_later` hourly all night on 2026-09-03 with no task active, draining usage. + + + +## Where to file + +Questions, help with code or an analysis, ideas, bug reports and results from a +user or collaborator — or an agent acting for one — go to + in the matching category +(Help & Questions, Ideas & Proposals, Bugs & Errors, Show and tell; +Announcements is maintainers-only), never to this repo's Issues. An agent never +runs `gh issue create` for such a report: it drafts the title, category and +body and hands them to the human (sessions cannot create Discussions). Only the +development flow — Mind prompt → `/start_dev` → `/create_issue` → one issue per +task → PR — opens issues here. Why: `PyAutoMind/policy/community_surface.md`. + + + +## Shared standards + +Before changing a shared interface, consult the applicable +[organism standard](https://github.com/PyAutoLabs/PyAutoBrain/blob/main/docs/standards.md) +on demand, identify affected consumers, and validate their adoption. Change +generated guidance at its canonical source and regenerate. + +For board changes, follow the applicable sizing, navigation and orchestration +standards and reuse Brain’s shared components. Keep domain data, prompt meaning +and approval boundaries with the board’s owner. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..ca44103 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +# PyAutoDNA + +Specify. Compare. Evolve. + +DNA records software stacks and the environments actually running them. It keeps +upstream availability, package declarations, tested configurations, recommendations +and observations separate. Brain coordinates changes; Heart supplies validation; +Hands owns releases; Nerves owns runtime compatibility. DNA never installs packages +or submits compute during collection, rendering or campaign actions. + +Requires Python 3.12+; `python -m pip install -e .`. From this repository: + +```bash +python -m dna collect local --repo ../PyAutoBrain +python -m dna board --brain ../PyAutoBrain --output _site +python -m dna show +``` + +Collection prints a private receipt and stores it in ignored `private/records/`. +Do not upload its console output: it includes paths and distribution origins. +`--probe-backend` explicitly initializes JAX to identify the actual backend; without +it backend remains unknown. Collect on RAL or CI using their actual interpreter, +then transfer the receipt privately and `python -m dna import receipt.json`. +No remote observation is inferred from a declaration or local interpreter. + +See [operations](docs/operations.md), [contracts](docs/contracts.md) and the +[JAX incident](decisions/jax-2026-10-04.md). Checked-in environments are unobserved +until a receipt exists. Pages refresh alone cannot observe remote stacks. + +The dashboard follows Brain's shared presentation standards and publishes the +standard `state.json`, `board.json` and `badge.json` feeds. Its source includes +reviewed observations of local development and the RAL shared environment; CPU, +GPU, CI and release validation remain unknown until their own receipts arrive. +`support.yaml` records upstream Python/JAX support horizons with source links and +a policy review date. These dates do not certify PyAuto compatibility. diff --git a/campaigns/README.md b/campaigns/README.md new file mode 100644 index 0000000..7428818 --- /dev/null +++ b/campaigns/README.md @@ -0,0 +1,5 @@ +# Campaign history + +No campaign is preapproved. CLI records start in ignored private/records; reviewed +public projections expose target/rollback digest identities and explicit events. +Campaign creation, promotion, adoption and rollback are distinct commands. diff --git a/decisions/jax-2026-10-04.md b/decisions/jax-2026-10-04.md new file mode 100644 index 0000000..2f2eaf9 --- /dev/null +++ b/decisions/jax-2026-10-04.md @@ -0,0 +1,20 @@ +# JAX compatibility incident — October 4, 2026 + +Source: PyAutoMind complete/2026/10/jax-lapack-compatibility-repair.md. +Related source: complete/2026/10/restore-dashboard-green.md. + +The repair retained JAX >=0.7,<0.12 and excluded 0.10.* and 0.11.0. +The native LAPACK/Eigen deadlock was reproduced on 0.10.2. Other exclusions +were source inferred; they are not separately reproduced failures. The older +0.9.2 implementation lacks the affected path; OSS >=0.11.1 disables it. +Neither fact certifies every older version or every backend/Python pair. +Historical FFT workarounds remain distinct evidence. The autonerves guard +>2026.10.4.1 prevents fallback to permissive compatibility logic. + +These are historical findings, not duplicated current requirement authority. +Run DNA audit against live pyproject.toml files and the actual installed +interpreter. Source version stamps are not wheel equality gates: record commit, +dirty state, import provenance and distribution metadata independently. + +Status: historical decision; no current validation or promotion recorded here. +Review date: 2026-11-04. Owner: PyAutoBrain; validation authority: PyAutoHeart. diff --git a/dna/__init__.py b/dna/__init__.py new file mode 100644 index 0000000..eaace91 --- /dev/null +++ b/dna/__init__.py @@ -0,0 +1 @@ +"""PyAutoDNA: evidence, never implicit environment mutation.""" diff --git a/dna/__main__.py b/dna/__main__.py new file mode 100644 index 0000000..13be635 --- /dev/null +++ b/dna/__main__.py @@ -0,0 +1,2 @@ +from .cli import main +raise SystemExit(main()) diff --git a/dna/board.py b/dna/board.py new file mode 100644 index 0000000..a934eab --- /dev/null +++ b/dna/board.py @@ -0,0 +1,276 @@ +"""Reviewed public projections and the shared organism dashboard.""" +import html +import importlib.util +import json +from datetime import datetime, timezone +from pathlib import Path +import yaml +from packaging.requirements import Requirement +from .campaigns import Store +from .inventory import public_inventory, PUBLIC_PACKAGES, PUBLIC_REPOSITORIES +from .schema import identifier, now, read, timestamp + +URL = 'https://pyautolabs.github.io/PyAutoDNA/' +REPO = 'https://github.com/PyAutoLabs/PyAutoDNA' + + +def module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + obj = importlib.util.module_from_spec(spec) + spec.loader.exec_module(obj) + return obj + + +def registry(root): + obj = yaml.safe_load((root / 'environments.yaml').read_text()) + if obj.get('schema_version') != 1 or not isinstance(obj.get('environments'), list): + raise ValueError('invalid environment registry') + seen = set() + for env in obj['environments']: + identifier(env['id']) + if env['id'] in seen: + raise ValueError('duplicate environment') + seen.add(env['id']) + if env.get('backend') not in {'unknown', 'cpu', 'cuda'}: + raise ValueError('invalid environment backend') + if not isinstance(env.get('label'), str): + raise ValueError('environment label required') + return obj['environments'] + + +def public_records(records): + result = [] + for obj in records: + if obj['kind'] == 'inventory': + continue + if obj['kind'] == 'availability' and obj.get('name') not in PUBLIC_PACKAGES: + continue + allowed = ('digest', 'kind', 'created', 'name', 'python', 'packages', 'backend', + 'target', 'rollback', 'environments', 'campaign', 'stack', 'environment', + 'inventory', 'result', 'authority', 'review_date', 'validations', 'latest', + 'compatible_stable') + clean = {k: obj[k] for k in allowed if k in obj} + if 'packages' in clean: + clean['packages'] = {k: v for k, v in clean['packages'].items() if k in PUBLIC_PACKAGES} + if obj['kind'] == 'audit': + clean['rows'] = [] + for row in obj.get('rows', []): + if row.get('repository') not in PUBLIC_REPOSITORIES | {'explicit-ci-override'}: + continue + if row.get('package') not in PUBLIC_PACKAGES | {'declarations'} and row.get('status') != 'ci_override_review': + continue + safe = {k: row[k] for k in ('repository', 'package', 'observed', 'status', 'source', 'line') if k in row} + if 'declared' in row: + if row.get('package') == 'python': + safe['declared'] = row['declared'] + else: + req = Requirement(row['declared']) + safe['declared'] = req.name + str(req.specifier) + if req.url or row.get('direct_source') is True: + safe['direct_source'] = True + clean['rows'].append(safe) + result.append(clean) + return result + + +def snapshot(root, max_age_days=7): + if max_age_days < 1: + raise ValueError('max age must be positive') + envs = registry(root) + records = Store(root / 'private').records() + records += [read(p) for folder in ('stacks', 'decisions', 'campaigns') + for p in sorted((root / folder).glob('*.json'))] + projection_path = root / 'inventories/public.json' + projection = json.loads(projection_path.read_text()) if projection_path.exists() else {} + if projection and projection.get('schema_version') != 1: + raise ValueError('unsupported public projection schema') + current = {} + for obj in records: + if obj['kind'] == 'inventory': + inv = public_inventory(obj) + if inv['environment'] not in current or timestamp(inv['created']) > timestamp(current[inv['environment']]['created']): + current[inv['environment']] = inv + for old in projection.get('environments', []): + inv = old.get('observed') + if not inv: + continue + if inv['environment'] != old['id']: + raise ValueError('public inventory identity mismatch') + timestamp(inv['created']) + # Re-apply the allowlist even to committed public data. + clean = public_inventory(inv) + if old['id'] not in current or timestamp(clean['created']) > timestamp(current[old['id']]['created']): + current[old['id']] = clean + clock = datetime.now(timezone.utc) + rows = [] + for env in envs: + inv = current.get(env['id']) + state = 'unknown' if inv is None else 'stale' if (clock - timestamp(inv['created'])).total_seconds() > max_age_days * 86400 else 'observed' + rows.append({'id': env['id'], 'label': env['label'], 'role': env.get('role', 'unknown'), + 'declared_python': env.get('python', 'unknown'), 'declared_backend': env['backend'], + 'status': state, 'observed': inv}) + published = public_records(projection.get('records', [])) + public_records(records) + published = list({r['digest']: r for r in published}.values()) + observed_times = [r['observed']['created'] for r in rows if r['observed']] + refreshed = min(observed_times, key=timestamp) if rows and len(observed_times) == len(rows) else None + # Retain observation time across re-renders. No receipt means unknown coverage, + # even though the shared feed requires a timestamp for the generated artifact. + updated = min(observed_times, key=timestamp) if observed_times else datetime.fromtimestamp( + (root / 'environments.yaml').stat().st_mtime, timezone.utc).isoformat() + return {'schema_version': 1, 'updated': updated, 'refreshed_at': refreshed, + 'environments': rows, 'records': published} + + +def esc(value): + return html.escape(str(value), quote=True) + + +def table(headers, rows, label): + return ('
' + '' + ''.join('' for h in headers) + + '' + ''.join('' + ''.join('' for v in row) + '' for row in rows) + + '
' + esc(h) + '
' + esc(v) + '
') + + +def latest(records, kind, key): + result = {} + for obj in records: + if obj['kind'] == kind: + name = key(obj) + if name not in result or timestamp(obj['created']) > timestamp(result[name]['created']): + result[name] = obj + return result + + +def evidence_age(obj, max_age_days): + age = (datetime.now(timezone.utc) - timestamp(obj['created'])).total_seconds() + return 'Stale' if age > max_age_days * 86400 else 'Current receipt' + + +def render(root, brain, output, max_age_days=7): + theme = module(brain / 'board/_theme.py', 'dna_theme') + state_contract = module(brain / 'board/_state.py', 'dna_state') + board = snapshot(root, max_age_days) + envs, records = board['environments'], board['records'] + items = [{'id': e['id'], 'severity': 'yellow' if e['status'] == 'stale' else 'info', + 'state': 'stale' if e['status'] == 'stale' else 'unknown' if e['status'] == 'unknown' else 'active', + 'text': f"{e['label']}: {e['status']}", 'url': URL + '#environments'} for e in envs] + current_inventories = {e['observed']['digest']: e for e in envs if e['observed']} + mismatches = [] + for audit in latest(records, 'audit', lambda r: r['inventory']).values(): + env = current_inventories.get(audit['inventory']) + if env and evidence_age(audit, max_age_days) != 'Stale': + for row in audit.get('rows', []): + if row.get('status') == 'incompatible': + mismatches.append((env, row)) + for index, (env, row) in enumerate(mismatches): + items.append({'id': 'dependency-' + str(index), 'severity': 'yellow', 'state': 'action_required', + 'text': f"{env['label']}: {row.get('package')} {row.get('observed')} is outside {row['repository']} declarations", + 'url': URL + '#compatibility'}) + status = 'yellow' if mismatches else 'stale' if any(e['status'] == 'stale' for e in envs) else 'grey' + count = sum(e['status'] == 'observed' for e in envs) + headline = f'{count} / {len(envs)} environments observed' + if mismatches: + headline += f'; {len(mismatches)} declaration mismatches' + state = dict(schema_version=1, organ='dna', repo='PyAutoLabs/PyAutoDNA', status=status, + headline=headline, updated=board['updated'], pages_url=URL, items=items) + errors = state_contract.validate_state(state) + if errors: + raise ValueError('; '.join(errors)) + sections = ['environments', 'compatibility', 'decisions', 'campaigns'] + body = theme.hero('dna', 'Dashboard', navigation=[{'href': '#' + s, 'label': s.title()} for s in sections]) + body += theme.orchestration_panel('dna', 'DNA', '', + 'Review PyAutoDNA software stacks and compatibility. Read AGENTS.md and inspect current inventories, ' + 'dependency declarations, support decisions and upgrade campaigns. Explain local, RAL and CI drift. ' + 'Propose scoped upgrades through Brain and verify linked Heart evidence before promotion. Preserve ' + 'running science environments. Collection or refresh never installs packages or submits compute.', + organ='dna', refreshed_at=board['refreshed_at'], refresh_url=REPO + '/actions/workflows/pages.yml', + work_links=[{'label': 'DNA', 'href': REPO}, {'label': 'Brain', 'href': 'https://github.com/PyAutoLabs/PyAutoBrain'}]) + body += '

' + esc(headline) + '. Observed versions describe an installation; support requires matching validation evidence.

' + if mismatches: + body += '

' + esc('; '.join(f"{env['label']}: {row.get('package')} {row.get('observed')} is outside declared compatibility" for env, row in mismatches)) + '.

' + body += '

Environments

' + body += table(['Environment', 'Profile', 'Observed Python / backend', 'Last observation'], [ + [e['label'], e['declared_python'] + ' / ' + e['declared_backend'], + e['observed']['python'] + ' / ' + e['observed']['runtime']['backend'] if e['observed'] else 'Unknown', + e['status'] + ' · ' + e['observed']['created'] if e['observed'] else 'Not collected'] for e in envs], 'Environment observations; scroll for more columns') + observed = [e for e in envs if e['observed']] + if observed: + body += '

Installed packages

Compare the latest recorded inventory for each environment. Swipe or scroll the table to see all columns.

' + packages = sorted(set().union(*(set(e['observed']['packages']) for e in observed))) + body += table(['Package'] + [e['label'] for e in envs], [ + [name] + [(e['observed']['packages'].get(name, {}).get('version', 'Not installed') if e['observed'] else 'Unknown') for e in envs] + for name in packages], 'Installed package versions across environments; scroll horizontally') + for e in observed: + inv = e['observed'] + body += '
' + esc(e['label']) + ' · source identity' + body += '

Inventory ' + esc(inv['digest']) + '

' + body += table(['Repository', 'Commit', 'Working tree'], [ + [r['name'], r['sha'], 'Unknown' if r['dirty'] is None else 'Modified' if r['dirty'] else 'Clean'] for r in inv['repositories']], e['label'] + ' source commits') if inv['repositories'] else '

No source checkout recorded.

' + body += '

Source commits identify editable checkouts. Package version metadata may retain an older release stamp.

' + body += '

Compatibility

' + availability = latest(records, 'availability', lambda r: (r['name'], r['python'])) + support_path = root / 'support.yaml' + if support_path.exists(): + from .policy import support_windows + support = yaml.safe_load(support_path.read_text()) + windows = support_windows(support) + body += '

Python support horizons

' + body += table(['Python', 'CPython end of support', 'JAX guarantees support through', 'Policy status'], [[r['version'], r['upstream_end_month'], r['jax_guaranteed_through_month'], r['status']] for r in windows], 'Upstream Python support windows') + body += '

These upstream windows do not certify PyAuto compatibility. Review due ' + esc(support['review_by']) + '. Sources: CPython and JAX.

' + body += '

Upstream releases

' + if availability: + body += table(['Package', 'Latest upstream', 'Stable for Python', 'Python', 'Checked', 'Freshness'], [ + [r['name'], r.get('latest') or 'Unknown', r.get('compatible_stable') or 'None found', r['python'], r['created'], evidence_age(r, max_age_days)] + for r in availability.values()], 'Upstream availability') + body += '

Python-compatible metadata does not prove platform, GPU or full-stack compatibility. Newer releases are candidates for validation.

' + else: + body += '

No upstream release check recorded.

' + audits = latest(records, 'audit', lambda r: r['inventory']) + body += '

Declared compatibility

' + if audits: + for obj in audits.values(): + env = next((e['label'] for e in envs if e['observed'] and e['observed']['digest'] == obj['inventory']), 'Historical inventory') + rows = obj.get('rows', []) + body += '
' + esc(env) + ' · dependency audit · ' + esc(obj['created']) + ' · ' + esc(evidence_age(obj, max_age_days)) + '' + body += table(['Repository', 'Package / workflow', 'Requirement', 'Installed', 'Result'], [ + [r['repository'], r.get('package', r.get('source', 'Unknown')), 'Direct source (private)' if r.get('direct_source') else r.get('declared', 'Review installation steps'), r.get('observed') or 'Unknown', r['status'].replace('_', ' ')] for r in rows], env + ' dependency audit') + body += '
' + else: + body += '

No dependency audit recorded. Package requirements remain authoritative in each repository.

' + body += '

Validation and promotion history

' + validations = latest(records, 'validation', lambda r: (r['stack'], r['environment'])) + if validations: + body += table(['Stack', 'Environment', 'Latest supplied Heart result', 'Evidence date', 'Freshness'], [[r['stack'][:12], r['environment'], r['result'], r['created'], evidence_age(r, max_age_days)] for r in validations.values()], 'Supplied validation receipts') + else: + body += '

No matching Heart validation receipt recorded. No supported stack is inferred from an observed installation.

' + body += '

Supplied validation receipts are records, not a live Heart readiness verdict. Stale evidence must be refreshed before adoption.

' + stacks = {r['digest']: r for r in records if r['kind'] == 'stack'} + promotions = latest(records, 'promotion', lambda r: r['campaign']) + if promotions: + body += table(['Promoted stack', 'Promotion recorded', 'Campaign'], [[stacks.get(r['stack'], {}).get('name', r['stack'][:12]), r['created'], r['campaign'][:12]] for r in promotions.values()], 'Recorded promotions') + body += '

Promotion history is retained; adoption rechecks current evidence and expiry.

' + else: + body += '

No recommended stack has been promoted.

' + body += '

Decisions

Reasons for version restrictions and when to review them.

' + for path in sorted((root / 'decisions').glob('*.md')): + body += '
' + esc(path.stem.replace('-', ' ')) + '
' + esc(path.read_text()) + '
' + body += '

Campaigns

' + campaigns = [r for r in records if r['kind'] == 'campaign'] + if not campaigns: + body += '

No upgrade campaign recorded. Start a check-in to compare available releases and choose a candidate stack.

' + for campaign in campaigns: + body += '
' + esc(campaign['name']) + '' + body += table(['Target stack', 'Rollback stack', 'Environments', 'Review date'], [[stacks.get(campaign['target'], {}).get('name', campaign['target'][:12]), stacks.get(campaign['rollback'], {}).get('name', campaign['rollback'][:12]), ', '.join(campaign['environments']), campaign['review_date']]], 'Upgrade campaign') + events = [r for r in records if r.get('campaign') == campaign['digest'] and r['kind'] in {'promotion', 'adoption', 'rollback'}] + body += table(['Action', 'Environment', 'Recorded'], [[r['kind'], r.get('environment', 'Campaign'), r['created']] for r in sorted(events, key=lambda r: timestamp(r['created']))], 'Campaign history') if events else '

Candidate awaiting validation.

' + body += '
' + body += '
' + theme.boards_footer(theme.board_links('https://pyautolabs.github.io', 'dna'), 'dna') + css = '.dna-table{overflow-x:auto;margin:1rem 0}.dna-table table{width:100%;border-collapse:collapse}.dna-table th,.dna-table td{padding:.65rem .8rem;text-align:left;vertical-align:top;border-bottom:1px solid var(--border,#d0d7de)}.dna-table th{white-space:nowrap}.dna-table td{min-width:7rem}.dna-prose{white-space:pre-wrap;font:inherit;line-height:1.6}code{overflow-wrap:anywhere}' + page = 'PyAutoDNA — Software stacks and compatibility
' + body + '
' + page = theme.section_layout(page) + output.mkdir(parents=True, exist_ok=True) + (output / 'index.html').write_text(page) + for name, obj in (('board', board), ('state', state), ('badge', {'schemaVersion': 1, 'label': 'DNA', 'message': headline, 'color': 'lightgrey' if status == 'grey' else 'yellow'})): + (output / (name + '.json')).write_text(json.dumps(obj, indent=2) + '\n') + return {'output': str(output), 'status': status, 'headline': headline} diff --git a/dna/campaigns.py b/dna/campaigns.py new file mode 100644 index 0000000..5b10894 --- /dev/null +++ b/dna/campaigns.py @@ -0,0 +1,96 @@ +"""Append-only, explicit evidence gates. Recording adoption never installs packages.""" +from datetime import datetime, timezone +from pathlib import Path +from .policy import compare +from .schema import identifier, read, record, write, timestamp + +class Store: + def __init__(self, root): + self.root = Path(root) + + def records(self, kind=None): + return [read(p) for p in sorted((self.root / "records").glob("*.json")) if kind is None or read(p)["kind"] == kind] + + def get(self, digest): + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("expected a record digest") + return read(self.root / "records" / (digest + ".json")) + + def add(self, obj): + if obj.get("kind") in {"campaign", "promotion", "adoption", "rollback"}: + raise ValueError("transition records must be created through explicit gated commands") + return self._persist(obj) + + def _persist(self, obj): + path = self.root / "records" / (obj["digest"] + ".json") + if path.exists(): + if read(path) != obj: + raise ValueError("immutable record conflict") + return obj + write(path, obj) + return obj + + def require(self, digest, kind): + obj = self.get(digest) + if obj["kind"] != kind: + raise ValueError(f"expected {kind}") + return obj + + def create(self, name, target, rollback, environments, owner, review_date): + self.require(target, "stack"); self.require(rollback, "stack") + registry_path = self.root.parent / "environments.yaml" + if not registry_path.exists(): + raise ValueError("campaign requires an environment registry") + import yaml + registered = {e["id"] for e in yaml.safe_load(registry_path.read_text())["environments"]} + if not set(environments).issubset(registered): + raise ValueError("campaign environment is not registered") + return self._persist(record("campaign", name=name, target=target, rollback=rollback, + environments=environments, owner=owner, review_date=review_date)) + + def _gate(self, obj, validations): + if datetime.strptime(obj["review_date"], "%Y-%m-%d").date() < datetime.now(timezone.utc).date(): + raise ValueError("campaign review date expired") + target = self.require(obj["target"], "stack") + if target["backend"] == "unknown": + raise ValueError("promotion requires an explicit backend") + checks = [self.require(d, "validation") for d in validations] + all_checks = self.records("validation") + for env in obj["environments"]: + matching = [c for c in all_checks if c["stack"] == obj["target"] and c["environment"] == env] + latest = max(matching, key=lambda c: timestamp(c["created"])) if matching else None + if latest is None or latest["digest"] not in validations or latest["result"] != "pass": + raise ValueError(f"latest matching Heart evidence must pass: {env}") + if (datetime.now(timezone.utc) - timestamp(latest["created"])).total_seconds() > 7 * 86400: + raise ValueError("stale Heart validation") + inv = self.require(latest["inventory"], "inventory") + if inv["environment"] != env or any(r["status"] != "match" for r in compare(inv, target)): + raise ValueError("validation inventory does not match stack") + + def promote(self, campaign, validations): + obj = self.require(campaign, "campaign") + self._gate(obj, validations) + return self._persist(record("promotion", campaign=campaign, stack=obj["target"], validations=validations)) + + def adopt(self, campaign, environment, inventory, rollback=False, reason=None): + obj = self.require(campaign, "campaign") + if environment not in obj["environments"]: + raise ValueError("environment outside campaign") + if not rollback: + promotions = [p for p in self.records("promotion") if p["campaign"] == campaign] + if not promotions: + raise ValueError("campaign is not promoted") + latest_promotion = max(promotions, key=lambda p: timestamp(p["created"])) + self._gate(obj, latest_promotion["validations"]) + stack = obj["rollback"] if rollback else obj["target"] + inv = self.require(inventory, "inventory") + if (datetime.now(timezone.utc) - timestamp(inv["created"])).total_seconds() > 7 * 86400: + raise ValueError("stale adoption inventory") + if inv["environment"] != environment or any(r["status"] != "match" for r in compare(inv, self.require(stack, "stack"))): + raise ValueError("inventory does not match environment and stack, including runtime") + if rollback and not reason: + raise ValueError("rollback requires a reason") + kwargs = dict(campaign=campaign, environment=environment, stack=stack, inventory=inventory) + if rollback: + kwargs["reason"] = reason + return self._persist(record("rollback" if rollback else "adoption", **kwargs)) diff --git a/dna/cli.py b/dna/cli.py new file mode 100644 index 0000000..33579b4 --- /dev/null +++ b/dna/cli.py @@ -0,0 +1,79 @@ +"""Explicit commands; no collectors run during render, promotion or adoption.""" +import argparse +import json +from pathlib import Path +from . import schema +from .campaigns import Store +from .inventory import collect +from .policy import audit, command_plan, compare, diff_inventories + +def main(argv=None): + parser = argparse.ArgumentParser(prog="pyauto-dna") + parser.add_argument("--root", default=".", help="DNA data repository") + sub = parser.add_subparsers(dest="command", required=True) + p = sub.add_parser("upstream"); p.add_argument("package"); p.add_argument("--python", required=True) + p = sub.add_parser("export-public"); p.add_argument("--output", default="inventories/public.json") + p = sub.add_parser("collect"); p.add_argument("environment"); p.add_argument("--repo", action="append", default=[]); p.add_argument("--probe-backend", action="store_true") + p = sub.add_parser("import"); p.add_argument("file") + p = sub.add_parser("stack"); p.add_argument("name"); p.add_argument("--python", required=True); p.add_argument("--backend", choices=["cpu", "cuda", "unknown"], required=True); p.add_argument("--package", action="append", default=[]) + p = sub.add_parser("diff"); p.add_argument("left"); p.add_argument("right") + p = sub.add_parser("compare"); p.add_argument("inventory"); p.add_argument("stack") + p = sub.add_parser("audit"); p.add_argument("inventory"); p.add_argument("--repo", action="append", required=True); p.add_argument("--extra", action="append", default=[]); p.add_argument("--override", action="append", default=[]) + p = sub.add_parser("plan"); p.add_argument("stack") + p = sub.add_parser("campaign"); p.add_argument("name"); p.add_argument("--target", required=True); p.add_argument("--rollback", required=True); p.add_argument("--environment", action="append", required=True); p.add_argument("--owner", required=True); p.add_argument("--review-date", required=True) + p = sub.add_parser("promote"); p.add_argument("campaign"); p.add_argument("--validation", action="append", required=True) + for command in ("adopt", "rollback"): + p = sub.add_parser(command); p.add_argument("campaign"); p.add_argument("environment"); p.add_argument("inventory") + if command == "rollback": p.add_argument("--reason", required=True) + p = sub.add_parser("show"); p.add_argument("digest", nargs="?") + p = sub.add_parser("board"); p.add_argument("--brain", required=True); p.add_argument("--output", default="_site"); p.add_argument("--max-age-days", type=int, default=7) + args = parser.parse_args(argv) + store = Store(Path(args.root) / "private") + try: + cmd = args.command + if cmd == "upstream": + from .upstream import available + result = store.add(available(args.package, args.python)) + elif cmd == "export-public": + from .board import snapshot + result = snapshot(Path(args.root)) + path = Path(args.output); path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2) + "\n") + elif cmd == "collect": + from .board import registry + if args.environment not in {e["id"] for e in registry(Path(args.root))}: + raise ValueError("environment not in registry") + result = store.add(collect(args.environment, args.repo, args.probe_backend)) + elif cmd == "import": result = store.add(schema.read(args.file)) + elif cmd == "stack": + packages = {} + for text in args.package: + name, sep, version = text.partition("==") + if not sep: raise ValueError("stack packages require exact name==version") + from packaging.utils import canonicalize_name + name = canonicalize_name(name) + if name in packages: raise ValueError("duplicate package") + packages[name] = version + result = store.add(schema.record("stack", name=args.name, python=args.python, backend=args.backend, packages=packages)) + elif cmd == "diff": result = diff_inventories(store.require(args.left, "inventory"), store.require(args.right, "inventory")) + elif cmd == "compare": result = compare(store.require(args.inventory, "inventory"), store.require(args.stack, "stack")) + elif cmd == "audit": + from .policy import declarations + result = store.add(schema.record("audit", inventory=args.inventory, + rows=audit(store.require(args.inventory, "inventory"), args.repo, args.extra, args.override), + declarations=[dict(repository=Path(repo).name, **declarations(repo)) for repo in args.repo])) + elif cmd == "plan": result = command_plan(store.require(args.stack, "stack")) + elif cmd == "campaign": result = store.create(args.name, args.target, args.rollback, args.environment, args.owner, args.review_date) + elif cmd == "promote": result = store.promote(args.campaign, args.validation) + elif cmd in {"adopt", "rollback"}: result = store.adopt(args.campaign, args.environment, args.inventory, cmd == "rollback", getattr(args, "reason", None)) + elif cmd == "show": result = store.get(args.digest) if args.digest else store.records() + elif cmd == "board": + from .board import render + result = render(Path(args.root), Path(args.brain), Path(args.output), args.max_age_days) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + except (ValueError, OSError, KeyError) as error: + parser.exit(2, f"pyauto-dna: {error}\n") + +if __name__ == "__main__": + main() diff --git a/dna/inventory.py b/dna/inventory.py new file mode 100644 index 0000000..058211b --- /dev/null +++ b/dna/inventory.py @@ -0,0 +1,67 @@ +"""Read-only collection in the actual interpreter; private receipts are not feeds.""" +import importlib.metadata as metadata +import importlib.util +import json +import os +import platform +import subprocess +import sys +from pathlib import Path +from .schema import record + +PUBLIC_REPOSITORIES = {"PyAutoBrain", "PyAutoMind", "PyAutoDNA", "PyAutoNerves", "PyAutoFit", "PyAutoArray", "PyAutoGalaxy", "PyAutoLens", "PyAutoCTI", "PyAutoReduce", "PyAutoHeart", "PyAutoHands"} + +PUBLIC_PACKAGES = {"anesthetic", "astropy", "matplotlib", "scikit-learn", "dynesty", "nautilus-sampler", "emcee", "zeus-mcmc", "corner", "optax", "blackjax", "numpyro", "tensorflow-probability", "tfp-nightly", "nufftax", "jax-cuda12-plugin", "jax-cuda12-pjrt", "jax-cuda13-plugin", "jax-cuda13-pjrt", "autofit", "autoarray", "autogalaxy", "autolens", "autocti", "autonerves", "jax", "jaxlib", "numpy", "scipy", "numba", "python", "pyautodna", "packaging", "pyyaml"} + +FLAGS = ("JAX_ENABLE_X64", "JAX_PLATFORMS", "JAX_PLATFORM_NAME", "XLA_FLAGS", "CUDA_VISIBLE_DEVICES", "PYAUTO_TEST_MODE", "PYAUTO_DISABLE_JAX", "PYAUTO_TEST") + +def command(args, cwd=None): + try: + return subprocess.run(args, cwd=cwd, capture_output=True, text=True, timeout=10, check=True).stdout.strip() + except (OSError, subprocess.SubprocessError): + return None + +def collect(environment, repositories=(), probe_backend=False): + packages = {} + for dist in metadata.distributions(): + name = dist.metadata.get("Name") + if not name: + continue + from packaging.utils import canonicalize_name + name = canonicalize_name(name) + origin = dist.read_text("direct_url.json") + packages[name] = {"version": dist.version, "location": str(dist.locate_file("")), + "origin": json.loads(origin) if origin else None} + repos = [] + for path in repositories: + path = Path(path).resolve() + sha = command(["git", "rev-parse", "HEAD"], path) + if not sha: + raise ValueError(f"not a git checkout: {path}") + repos.append({"name": path.name, "sha": sha, "dirty": (lambda status: None if status is None else bool(status))(command(["git", "status", "--porcelain"], path)), + "path": str(path), "branch": command(["git", "branch", "--show-current"], path)}) + imports = {} + for name in ("autofit", "autoarray", "autogalaxy", "autolens", "autocti", "autonerves", "jax"): + try: + spec = importlib.util.find_spec(name) + imports[name] = spec.origin if spec else None + except (ImportError, ValueError): + imports[name] = None + runtime = {"flags": {key: os.environ[key] for key in FLAGS if key in os.environ}, + "backend": "unknown", "devices": [], "gpu": command(["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"])} + if probe_backend: + try: + import jax + runtime.update(backend={"gpu": "cuda"}.get(jax.default_backend(), jax.default_backend()), devices=[str(d) for d in jax.devices()], x64=bool(jax.config.x64_enabled)) + except Exception as error: + runtime["probe_error"] = type(error).__name__ + return record("inventory", environment=environment, python=platform.python_version(), + executable=sys.executable, platform={"system": platform.system(), "machine": platform.machine(), "markers": __import__("packaging.markers", fromlist=["default_environment"]).default_environment()}, + packages=packages, repositories=repos, imports=imports, runtime=runtime) + +def public_inventory(obj): + """Construct from allowed fields; never redact a raw object in place.""" + return {"digest": obj["digest"], "environment": obj["environment"], "created": obj["created"], + "python": obj["python"], "packages": {n: {"version": p["version"]} for n, p in obj["packages"].items() if n in PUBLIC_PACKAGES}, + "repositories": [{k: r[k] for k in ("name", "sha", "dirty")} for r in obj["repositories"] if r["name"] in PUBLIC_REPOSITORIES], + "runtime": {"backend": obj["runtime"].get("backend", "unknown"), "x64": obj["runtime"].get("x64")}} diff --git a/dna/policy.py b/dna/policy.py new file mode 100644 index 0000000..a646b88 --- /dev/null +++ b/dna/policy.py @@ -0,0 +1,149 @@ +"""Compare resolved identities and read authoritative project declarations on demand.""" +import ast +import re +import tomllib +from pathlib import Path +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from .schema import validate + +def compare(inventory, stack): + validate(inventory); validate(stack) + rows = [] + for name, expected in stack["packages"].items(): + observed = inventory["packages"].get(canonicalize_name(name), {}).get("version") + rows.append({"package": name, "expected": expected, "observed": observed, + "status": "unknown" if observed is None else "match" if observed == expected else "drift"}) + rows.append({"package": "python", "expected": stack["python"], "observed": inventory["python"], + "status": "match" if stack["python"] == inventory["python"] else "drift"}) + backend = inventory["runtime"].get("backend", "unknown") + rows.append({"package": "backend", "expected": stack["backend"], "observed": backend, + "status": "unknown" if "unknown" in (backend, stack["backend"]) else "match" if backend == stack["backend"] else "drift"}) + for wanted in stack.get("repositories", []): + actual = next((r for r in inventory["repositories"] if r["name"] == wanted["name"]), None) + state = "unknown" if actual is None or actual.get("dirty") is None else "match" if actual["sha"] == wanted["sha"] and actual["dirty"] == wanted.get("dirty", False) else "drift" + rows.append({"package": "git:" + wanted["name"], "status": state}) + for flag, expected in stack.get("runtime_flags", {}).items(): + actual = inventory["runtime"].get("flags", {}).get(flag) + rows.append({"package": "runtime:" + flag, "expected": expected, "observed": actual, + "status": "unknown" if actual is None else "match" if actual == expected else "drift"}) + return rows + +def declarations(repo): + repo = Path(repo) + path = repo / "pyproject.toml" + if not path.exists(): + return {"source": "pyproject.toml", "status": "unknown", "requirements": [], "python": None} + data = tomllib.loads(path.read_text()) + project = data.get("project", {}) + return {"source": "pyproject.toml", "status": "declared", "requirements": project.get("dependencies", []), + "optional": project.get("optional-dependencies", {}), "dynamic": project.get("dynamic", []), + "python": project.get("requires-python"), "sha": __import__("hashlib").sha256(path.read_bytes()).hexdigest()} + +def audit(inventory, repos, extras=(), overrides=()): + validate(inventory) + rows = [] + env = {"python_version": ".".join(inventory["python"].split(".")[:2]), "python_full_version": inventory["python"], + "sys_platform": {"Linux": "linux", "Windows": "win32", "Darwin": "darwin"}.get(inventory.get("platform", {}).get("system"), "unknown"), + "platform_system": inventory.get("platform", {}).get("system", "unknown"), + "platform_machine": inventory.get("platform", {}).get("machine", "unknown"), "extra": ""} + env.update(inventory.get("platform", {}).get("markers", {})) + for repo in repos: + declared = declarations(repo) + name = Path(repo).name + if declared["status"] == "unknown" or "dependencies" in declared.get("dynamic", []): + rows.append({"repository": name, "package": "declarations", "status": "unknown", "reason": "missing or dynamic declarations require build metadata"}) + if declared.get("python"): + rows.append({"repository": name, "package": "python", "declared": declared["python"], + "observed": inventory["python"], "status": "compatible" if inventory["python"] in SpecifierSet(declared["python"]) else "incompatible"}) + requirements = list(declared["requirements"]) + for extra in extras: + requirements.extend(declared.get("optional", {}).get(extra, [])) + for text in requirements: + req = Requirement(text) + if req.marker: + marker_variables = set(re.findall(r"\b(?:os_name|sys_platform|platform_machine|platform_python_implementation|platform_release|platform_system|platform_version|python_version|python_full_version|implementation_name|implementation_version|extra)\b", str(req.marker))) + if any(env.get(variable, "unknown") == "unknown" for variable in marker_variables): + rows.append({"repository": name, "package": req.name, "declared": str(req), "status": "unknown", "reason": "target marker facts not collected"}) + continue + if not any(req.marker.evaluate(dict(env, extra=x)) for x in ("", *extras)): + continue + observed = inventory["packages"].get(canonicalize_name(req.name), {}).get("version") + state = "unknown" if observed is None or req.url else "compatible" if observed in req.specifier else "incompatible" + rows.append({"repository": name, "package": req.name, "declared": str(req), "observed": observed, "status": state}) + for workflow in sorted((Path(repo) / ".github/workflows").glob("*.y*ml")): + for line_no, line in enumerate(workflow.read_text().splitlines(), 1): + if re.search(r'(pip|uv).*install|python-version|jax[^a-z].*[<>=]', line): + rows.append({"repository": name, "source": workflow.name, "line": line_no, + "status": "ci_override_review", "reason": "CI installation or interpreter override; inspect authoritative workflow"}) + for override in overrides: + req = Requirement(override) + observed = inventory["packages"].get(canonicalize_name(req.name), {}).get("version") + marker_unknown = False + if req.marker: + variables = set(re.findall(r"\b(?:os_name|sys_platform|platform_machine|platform_python_implementation|platform_release|platform_system|platform_version|python_version|python_full_version|implementation_name|implementation_version|extra)\b", str(req.marker))) + marker_unknown = any(env.get(v, "unknown") == "unknown" for v in variables) + if not marker_unknown and not any(req.marker.evaluate(dict(env, extra=x)) for x in ("", *extras)): + continue + rows.append({"repository": "explicit-ci-override", "package": req.name, "declared": str(req), "observed": observed, + "status": "unknown" if observed is None or req.url or marker_unknown else "compatible" if observed in req.specifier else "incompatible", + "reason": "override does not change package support declarations"}) + return rows + +def command_plan(stack): + validate(stack) + return {"python": stack["python"], "backend": stack["backend"], "requires_approval": True, + "commands": [["python", "-m", "pip", "install", "--dry-run", "--report", "resolver-report.json", + *[f"{name}=={version}" for name, version in sorted(stack["packages"].items())]]], + "runtime_constraints": stack.get("runtime_flags", {}), "repositories": stack.get("repositories", []), + "backend_note": "CUDA requires a separately reviewed CUDA/JAX plugin and driver resolution; package pins alone do not select a GPU backend.", + "next": "Use the exact declared Python interpreter. Review resolution, create isolated environment, collect inventory, obtain Heart evidence; DNA executes none of these commands."} + + +def diff_inventories(left, right): + """Compare two observed environments without inventing a recommended stack.""" + validate(left) + validate(right) + if left['kind'] != 'inventory' or right['kind'] != 'inventory': + raise ValueError('diff requires two inventory receipts') + rows = [] + def add(component, a, b, unknown=False): + rows.append({'component': component, 'left': a, 'right': b, + 'status': 'unknown' if unknown else 'same' if a == b else 'different'}) + add('python', left['python'], right['python']) + for name in sorted(set(left['packages']) | set(right['packages'])): + add('package:' + name, left['packages'].get(name, {}).get('version'), right['packages'].get(name, {}).get('version')) + for key in ('backend', 'x64'): + a, b = left['runtime'].get(key), right['runtime'].get(key) + add('runtime:' + key, a, b, a in (None, 'unknown') or b in (None, 'unknown')) + add('platform', left.get('platform'), right.get('platform'), not left.get('platform') or not right.get('platform')) + flags = set(left['runtime'].get('flags', {})) | set(right['runtime'].get('flags', {})) + for flag in sorted(flags): + add('flag:' + flag, left['runtime'].get('flags', {}).get(flag), right['runtime'].get('flags', {}).get(flag)) + names = {r['name'] for r in left['repositories'] + right['repositories']} + for name in sorted(names): + a = next((r for r in left['repositories'] if r['name'] == name), None) + b = next((r for r in right['repositories'] if r['name'] == name), None) + add('source:' + name, {k: a.get(k) for k in ('sha', 'dirty')} if a else None, + {k: b.get(k) for k in ('sha', 'dirty')} if b else None, + not a or not b or a.get('dirty') is not False or b.get('dirty') is not False) + return {'left': left['environment'], 'right': right['environment'], 'rows': rows} + + +def support_windows(policy, today=None): + """Curated upstream lifecycle dates, separate from PyAuto validation.""" + from datetime import date + today = today or date.today() + if policy.get('schema_version') != 1: + raise ValueError('unsupported support policy schema') + review = date.fromisoformat(policy['review_by']) + result = [] + for row in policy['python']: + end = row['upstream_end_month'] + date.fromisoformat(end + '-01') + date.fromisoformat(row['jax_guaranteed_through_month'] + '-01') + month = today.strftime('%Y-%m') + state = 'Upstream end of life' if month > end else 'Ends this month' if month == end else 'Policy review due' if today > review else 'Within upstream window' + result.append(dict(row, status=state)) + return result diff --git a/dna/schema.py b/dna/schema.py new file mode 100644 index 0000000..0e1f93d --- /dev/null +++ b/dna/schema.py @@ -0,0 +1,113 @@ +"""Strict identities and immutable, content-addressed evidence records.""" +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path + +KINDS = {"inventory", "stack", "validation", "decision", "campaign", "adoption", "promotion", "rollback", "availability", "audit"} + +def now(): + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + +def identifier(value): + if not isinstance(value, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9_.-]{0,100}", value): + raise ValueError("identity must be a simple name, never a path") + return value + +def timestamp(value): + if not isinstance(value, str): + raise ValueError("timestamp must be UTC ISO-8601") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.utcoffset() is None or parsed.utcoffset().total_seconds() != 0: + raise ValueError("timestamp must be UTC") + if parsed > datetime.now(timezone.utc): + raise ValueError("future evidence timestamp") + return parsed + +def digest(data): + clean = {k: v for k, v in data.items() if k != "digest"} + return hashlib.sha256(json.dumps(clean, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + +def record(kind, **data): + obj = dict(schema_version=1, kind=kind, created=now(), **data) + obj["digest"] = digest(obj) + validate(obj) + return obj + +def validate(obj): + if not isinstance(obj, dict) or obj.get("schema_version") != 1 or obj.get("kind") not in KINDS: + raise ValueError("unsupported record schema/kind") + timestamp(obj.get("created")) + if obj.get("digest") != digest(obj): + raise ValueError("record digest mismatch") + kind = obj["kind"] + required = {"audit": ["inventory", "rows", "declarations"], + "availability": ["name", "python", "source", "interpretation"], + "inventory": ["environment", "python", "packages", "repositories", "runtime"], + "stack": ["name", "python", "packages", "backend"], + "validation": ["stack", "environment", "authority", "result", "evidence", "checks", "inventory"], + "decision": ["name", "rationale", "evidence", "review_date"], + "campaign": ["name", "target", "rollback", "environments", "owner", "review_date"], + "promotion": ["campaign", "stack", "validations"], + "adoption": ["campaign", "environment", "stack", "inventory"], + "rollback": ["campaign", "environment", "stack", "inventory", "reason"]}[kind] + for key in required: + if key not in obj or obj[key] is None: + raise ValueError(f"{kind}: missing {key}") + for key in ("environment", "name", "owner"): + if key in obj: + identifier(obj[key]) + if kind in {"inventory", "stack"}: + if not isinstance(obj["packages"], dict) or not isinstance(obj["python"], str): + raise ValueError("packages must be a mapping; python must be a string") + if kind == "inventory": + for name, package in obj["packages"].items(): + identifier(name) + if not isinstance(package, dict) or not isinstance(package.get("version"), str): + raise ValueError("package requires a version") + if not isinstance(obj["repositories"], list) or not isinstance(obj["runtime"], dict): + raise ValueError("invalid inventory repositories/runtime") + if kind == "stack": + if not obj["packages"]: + raise ValueError("stack must pin at least one package") + for repo in obj.get("repositories", []): + identifier(repo["name"]) + if not re.fullmatch(r"[0-9a-f]{40,64}", repo.get("sha", "")) or not isinstance(repo.get("dirty", False), bool): + raise ValueError("invalid stack git identity") + if not isinstance(obj.get("runtime_flags", {}), dict): + raise ValueError("invalid runtime flags") + from packaging.version import Version + Version(obj["python"]) + for name, version in obj["packages"].items(): + identifier(name) + Version(version) + if obj["backend"] not in {"cpu", "cuda", "unknown"}: + raise ValueError("invalid backend") + if kind == "validation": + if obj["authority"] != "PyAutoHeart" or obj["result"] not in {"pass", "fail", "unknown"}: + raise ValueError("validation must name Heart and a valid result") + if not isinstance(obj["checks"], list) or not obj["checks"] or not obj["evidence"]: + raise ValueError("validation requires checks and evidence") + if "review_date" in obj: + datetime.strptime(obj["review_date"], "%Y-%m-%d") + if kind == "campaign": + if not isinstance(obj["environments"], list) or not obj["environments"]: + raise ValueError("campaign requires environments") + for env in obj["environments"]: + identifier(env) + if len(set(obj["environments"])) != len(obj["environments"]): + raise ValueError("duplicate campaign environment") + return obj + +def read(path): + return validate(json.loads(Path(path).read_text())) + +def write(path, obj): + validate(obj) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("x") as handle: + json.dump(obj, handle, indent=2, sort_keys=True) + handle.write("\n") + return path diff --git a/dna/upstream.py b/dna/upstream.py new file mode 100644 index 0000000..fbe0546 --- /dev/null +++ b/dna/upstream.py @@ -0,0 +1,29 @@ +"""Explicit PyPI discovery; availability is not support certification.""" +import json +from urllib.request import urlopen +from packaging.version import Version, InvalidVersion +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from .schema import record + +def available(package, python, fetch=None): + package = canonicalize_name(package) + if fetch is None: + with urlopen(f'https://pypi.org/pypi/{package}/json', timeout=20) as response: + data = json.load(response) + else: + data = fetch(package) + versions, compatible = [], [] + for text, files in data.get('releases', {}).items(): + try: version = Version(text) + except InvalidVersion: continue + if not files: continue + versions.append(version) + if version.is_prerelease or version.is_devrelease: continue + if any(not f.get('yanked', False) and python in SpecifierSet(f.get('requires_python') or '') for f in files): + compatible.append(version) + return record('availability', name=package, python=python, + source=f'https://pypi.org/pypi/{package}/json', + latest=str(max(versions)) if versions else None, + compatible_stable=str(max(compatible)) if compatible else None, + interpretation='Python-compatible non-yanked release files; platform/backend/resolver compatibility remains untested') diff --git a/docs/contracts.md b/docs/contracts.md new file mode 100644 index 0000000..23e50e5 --- /dev/null +++ b/docs/contracts.md @@ -0,0 +1,46 @@ +# Record and feed contracts + +Records use schema_version 1, kind, created (UTC ISO8601), and digest. Digests +hash canonical sorted JSON without the digest field. `dna.schema.validate` +is the normative validator. Stacks pin Python/package versions and backend; +inventories keep distribution versions/origins/import paths and separate git +SHA/dirty identities. Public projections exclude all locations, hosts, raw +runtime flags, origins, owners, free-form private reasons and evidence URLs. +Only reviewed decision Markdown is public prose. + +A Heart validation record has `stack` (digest), `environment` (registry ID), +`authority: PyAutoHeart`, `result: pass|fail|unknown`, `checks` (nonempty list), +and `evidence` (the private Heart run/artifact reference). Produce it with +`dna.schema.record('validation', ...)`, persist with `dna.schema.write`, and +import through the CLI. DNA checks structure and matching identity; it cannot +cryptographically authenticate the author of an imported Heart receipt. Human +review remains required. No record implies universal compatibility certification. + +`board.json` is DNA's v1 public snapshot. `state.json` uses Brain's shared v1 +cockpit validator. `badge.json` is a Shields endpoint. Updated timestamps record +render time; each observation separately retains collection time. Unknown is +never green; observed means collected, not supported. Evidence older than the +configured seven-day default is stale. Source declarations remain authoritative +and are read only when audit is explicitly invoked. + +Validation requires `inventory` digest pointing to the tested observation. +Promotion selects the latest known exact-target Heart receipt per environment; +newer failures/unknown evidence block older passes. Validation and adoption +observations expire after seven days. Unknown backend blocks promotion. +Imported validation is supplied evidence, not an authenticated release verdict. +Public package/repository names use explicit library/organ allowlists in +`dna.inventory`; add reviewed identities there before publishing other projects. + +Imports accept evidence records, never campaign/promotion/adoption/rollback +transitions. Those must pass the explicit command gates. Adoption reruns the +complete promotion gate, including current review date, latest matching evidence, +expiry and tested inventory identity. Target environment marker facts are captured +with packaging's default_environment in that interpreter; missing facts are unknown, +never replaced by the audit host's facts. + +`audit` records store the inventory digest, dated compatibility rows and read +pyproject declaration hashes, Python bounds, requirements/extras and dynamic flags. +These are snapshots of authoritative declarations at audit time, not new support +policy. CI findings identify source filenames/line numbers for human review; +DNA does not execute workflow code. Public compatibility rows should retain only +approved package/repository identities and declarative constraints. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..337d16c --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,62 @@ +# Operations + +All mutable actions append immutable records. A SHA-256 digest identifies a +complete record, including its timestamp. Existing files cannot be overwritten. +Use digest outputs in the following commands (substitute actual 64-digit values): + +```bash +python -m dna stack candidate --python 3.12.9 --backend cpu --package jax==0.11.1 +python -m dna compare INVENTORY_DIGEST STACK_DIGEST +python -m dna diff LOCAL_INVENTORY_DIGEST RAL_INVENTORY_DIGEST +python -m dna audit INVENTORY_DIGEST --repo ../PyAutoArray --extra jax --override 'jax==0.11.1' +python -m dna plan STACK_DIGEST +python -m dna campaign jax-update --target TARGET_DIGEST --rollback BASELINE_DIGEST --environment local --owner human --review-date 2026-11-04 +python -m dna promote CAMPAIGN_DIGEST --validation HEART_VALIDATION_DIGEST +python -m dna adopt CAMPAIGN_DIGEST local INVENTORY_DIGEST +python -m dna rollback CAMPAIGN_DIGEST local BASELINE_INVENTORY_DIGEST --reason 'Regression after adoption' +``` + +Example versions demonstrate syntax, not recommendations. `plan` emits argument +arrays for a resolver dry run; it executes nothing. The human/Brain creates and +reviews an isolated candidate environment. Heart validation is imported as a +signed-off record (see contracts), matching the exact stack and environment. +Promotion requires successful evidence for every campaign environment and an +unexpired review date. Adoption requires promotion and a matching fresh inventory +identity, Python, package pins and observed backend. Rollback records an actual +matching baseline observation and reason; it does not perform the rollback. +Retain baseline records and environments until the campaign closes. + +Audit reads current pyproject declarations, selected extras and environment +markers. Dynamic dependencies and direct URLs stay unknown; build metadata must +resolve them. CI workflow installation/interpreter lines are review findings, +not support policy. Explicit overrides are checked separately. No source +`__version__` is compared to a wheel as an equality gate. + +Suggested cadence: collect after any environment change, review drift weekly, +review support at releases and after dependency incidents. This is advice; no +schedule is enabled. Scientific runs retain their original inventory digest. +RAL observations must be collected on RAL under the workspace HPC policy; DNA +never chooses a partition or runs SSH. CI can collect its actual interpreter as +a private artifact; a Pages build interpreter is not a library validation run. + +Pages publishing defaults to checked-in declarations and public decisions only. +Private observations need an explicitly reviewed allowlisted board projection +before upload. Never commit private receipts, resolver logs or machine paths. + +Explicit upstream discovery and reviewed public publishing: + +```bash +python -m dna upstream jax --python 3.12.9 +python -m dna export-public --output inventories/public.json +python -m dna board --brain ../PyAutoBrain +``` + +Upstream receipts show absolute latest and latest stable non-yanked release with +Python-compatible files separately. Wheel platform, backend and full dependency +resolution remain untested. Review the allowlisted public JSON before committing; +Pages loads it and ages its original collection timestamps. Collection alone does +not publish. Exported records omit private evidence prose and owner identities. +For git/runtime constraints, import a stack record with `repositories` entries +(name, sha, dirty) and `runtime_flags` mapping; compare/adoption enforce them. + +The initial RAL receipt is a shared-virtualenv observation taken on the login node. It does not certify CPU or GPU compute-node behavior. Raw receipts remain private; the public board includes only approved package names and source identities. diff --git a/environments.yaml b/environments.yaml new file mode 100644 index 0000000..f6182b5 --- /dev/null +++ b/environments.yaml @@ -0,0 +1,9 @@ +schema_version: 1 +environments: + - {id: local, label: Local development, role: development, python: unknown, backend: unknown} + - {id: ral-login, label: RAL shared environment (login observation), role: inventory, python: unknown, backend: unknown} + - {id: ral-cpu, label: RAL CPU, role: science, python: unknown, backend: cpu} + - {id: ral-gpu, label: RAL GPU, role: science, python: unknown, backend: cuda} + - {id: ci-py312, label: CI Python 3.12, role: validation, python: '3.12', backend: cpu} + - {id: ci-py313, label: CI Python 3.13, role: validation, python: '3.13', backend: cpu} + - {id: release, label: Release build, role: release, python: unknown, backend: unknown} diff --git a/inventories/public.json b/inventories/public.json new file mode 100644 index 0000000..2f5fba8 --- /dev/null +++ b/inventories/public.json @@ -0,0 +1,815 @@ +{ + "schema_version": 1, + "updated": "2026-10-08T12:19:34.064025Z", + "refreshed_at": null, + "environments": [ + { + "id": "local", + "label": "Local development", + "role": "development", + "declared_python": "unknown", + "declared_backend": "unknown", + "status": "observed", + "observed": { + "digest": "7c168b4b5ae04f67582b325d07496b2e143e1dba840c3f406de2222f4d5c5556", + "environment": "local", + "created": "2026-10-08T12:19:34.064025Z", + "python": "3.12.10", + "packages": { + "anesthetic": { + "version": "2.14.6" + }, + "astropy": { + "version": "8.0.1" + }, + "autoarray": { + "version": "9999.0.0.dev0" + }, + "autocti": { + "version": "9999.0.0.dev0" + }, + "autofit": { + "version": "9999.0.0.dev0" + }, + "autogalaxy": { + "version": "9999.0.0.dev0" + }, + "autolens": { + "version": "9999.0.0.dev0" + }, + "autonerves": { + "version": "9999.0.0.dev0" + }, + "blackjax": { + "version": "1.6.2" + }, + "corner": { + "version": "2.2.2" + }, + "dynesty": { + "version": "2.1.5" + }, + "emcee": { + "version": "3.1.6" + }, + "jax": { + "version": "0.10.2" + }, + "jaxlib": { + "version": "0.10.2" + }, + "matplotlib": { + "version": "3.10.9" + }, + "nautilus-sampler": { + "version": "1.0.5" + }, + "nufftax": { + "version": "0.6.1" + }, + "numba": { + "version": "0.62.1" + }, + "numpy": { + "version": "2.2.6" + }, + "numpyro": { + "version": "0.21.0" + }, + "optax": { + "version": "0.2.5" + }, + "packaging": { + "version": "26.2" + }, + "pyyaml": { + "version": "6.0.2" + }, + "scikit-learn": { + "version": "1.5.1" + }, + "scipy": { + "version": "1.17.1" + }, + "tensorflow-probability": { + "version": "0.25.0" + }, + "tfp-nightly": { + "version": "0.26.0.dev20260413" + }, + "zeus-mcmc": { + "version": "2.5.4" + } + }, + "repositories": [ + { + "name": "PyAutoNerves", + "sha": "c5ade6050cb919e46586e04d851867d6d252a11b", + "dirty": false + }, + { + "name": "PyAutoFit", + "sha": "cf1acd7272479bf7e2cb83b33bbb7172a3ff6ca5", + "dirty": false + }, + { + "name": "PyAutoArray", + "sha": "ccddfba6744d612092ef3723361689b09bf65648", + "dirty": false + }, + { + "name": "PyAutoGalaxy", + "sha": "c32b33049ae085242f2dbf818496410d2dd6f061", + "dirty": false + }, + { + "name": "PyAutoLens", + "sha": "7a97f601d01ce18cf748db935e846368a1eacf3a", + "dirty": false + } + ], + "runtime": { + "backend": "cpu", + "x64": true + } + } + }, + { + "id": "ral-login", + "label": "RAL shared environment (login observation)", + "role": "inventory", + "declared_python": "unknown", + "declared_backend": "unknown", + "status": "observed", + "observed": { + "digest": "34d4742d11ec2a0f42c69946c3eda885a8cb24720a6f2bf50ac5f70d14898f29", + "environment": "ral-login", + "created": "2026-10-08T12:22:20.522488Z", + "python": "3.12.4", + "packages": { + "anesthetic": { + "version": "2.8.14" + }, + "astropy": { + "version": "6.1.2" + }, + "blackjax": { + "version": "1.6.2" + }, + "corner": { + "version": "2.2.2" + }, + "dynesty": { + "version": "2.1.5" + }, + "emcee": { + "version": "3.1.6" + }, + "jax": { + "version": "0.10.2" + }, + "jax-cuda12-pjrt": { + "version": "0.10.2" + }, + "jax-cuda12-plugin": { + "version": "0.10.2" + }, + "jaxlib": { + "version": "0.10.2" + }, + "matplotlib": { + "version": "3.9.4" + }, + "nautilus-sampler": { + "version": "1.0.5" + }, + "nufftax": { + "version": "0.6.1" + }, + "numba": { + "version": "0.65.1" + }, + "numpy": { + "version": "2.2.6" + }, + "optax": { + "version": "0.2.8" + }, + "packaging": { + "version": "26.2" + }, + "pyyaml": { + "version": "6.0.3" + }, + "scikit-learn": { + "version": "1.5.1" + }, + "scipy": { + "version": "1.17.1" + }, + "tensorflow-probability": { + "version": "0.25.0" + }, + "tfp-nightly": { + "version": "0.26.0.dev20260713" + } + }, + "repositories": [ + { + "name": "PyAutoNerves", + "sha": "bf104102312852537159b7fad2f998ab6677b4e4", + "dirty": null + }, + { + "name": "PyAutoFit", + "sha": "404b3e5f7760d26f4bb41c7b1535ae5be3d113a0", + "dirty": null + }, + { + "name": "PyAutoArray", + "sha": "9428eca24a02b528edde68550dbcf82e609a652a", + "dirty": null + }, + { + "name": "PyAutoGalaxy", + "sha": "c9609825660f5ee8d6b3254b5ee90971216dd272", + "dirty": null + }, + { + "name": "PyAutoLens", + "sha": "21b520be4d0dd103aaaaba800e42c5092fedeb29", + "dirty": null + } + ], + "runtime": { + "backend": "unknown", + "x64": null + } + } + }, + { + "id": "ral-cpu", + "label": "RAL CPU", + "role": "science", + "declared_python": "unknown", + "declared_backend": "cpu", + "status": "unknown", + "observed": null + }, + { + "id": "ral-gpu", + "label": "RAL GPU", + "role": "science", + "declared_python": "unknown", + "declared_backend": "cuda", + "status": "unknown", + "observed": null + }, + { + "id": "ci-py312", + "label": "CI Python 3.12", + "role": "validation", + "declared_python": "3.12", + "declared_backend": "cpu", + "status": "unknown", + "observed": null + }, + { + "id": "ci-py313", + "label": "CI Python 3.13", + "role": "validation", + "declared_python": "3.13", + "declared_backend": "cpu", + "status": "unknown", + "observed": null + }, + { + "id": "release", + "label": "Release build", + "role": "release", + "declared_python": "unknown", + "declared_backend": "unknown", + "status": "unknown", + "observed": null + } + ], + "records": [ + { + "digest": "2b5a2875c033aa4352e807a78a48221d9e3c3e242c419e1cc78c88728c8ab944", + "kind": "audit", + "created": "2026-10-08T12:25:28.725675Z", + "inventory": "34d4742d11ec2a0f42c69946c3eda885a8cb24720a6f2bf50ac5f70d14898f29", + "rows": [ + { + "repository": "PyAutoNerves", + "package": "python", + "observed": "3.12.4", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoNerves", + "package": "numpy", + "observed": "2.2.6", + "status": "compatible", + "declared": "numpy<3.0.0,>=1.24.0" + }, + { + "repository": "PyAutoNerves", + "package": "jax", + "observed": "0.10.2", + "status": "incompatible", + "declared": "jax!=0.10.*,!=0.11.0,<0.12.0,>=0.7.0" + }, + { + "repository": "PyAutoNerves", + "package": "jaxlib", + "observed": "0.10.2", + "status": "incompatible", + "declared": "jaxlib!=0.10.*,!=0.11.0,<0.12.0,>=0.7.0" + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "colab-gate.yml", + "line": 50 + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "nerves_board.yml", + "line": 58 + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "nerves_board.yml", + "line": 61 + }, + { + "repository": "PyAutoFit", + "package": "python", + "observed": "3.12.4", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoFit", + "package": "autonerves", + "observed": null, + "status": "unknown", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoFit", + "package": "optax", + "observed": "0.2.8", + "status": "compatible", + "declared": "optax>=0.2.5" + }, + { + "repository": "PyAutoFit", + "package": "anesthetic", + "observed": "2.8.14", + "status": "incompatible", + "declared": "anesthetic>=2.9.0" + }, + { + "repository": "PyAutoFit", + "package": "corner", + "observed": "2.2.2", + "status": "compatible", + "declared": "corner==2.2.2" + }, + { + "repository": "PyAutoFit", + "package": "dynesty", + "observed": "2.1.5", + "status": "compatible", + "declared": "dynesty==2.1.5" + }, + { + "repository": "PyAutoFit", + "package": "emcee", + "observed": "3.1.6", + "status": "compatible", + "declared": "emcee>=3.1.6" + }, + { + "repository": "PyAutoFit", + "package": "matplotlib", + "observed": "3.9.4", + "status": "compatible", + "declared": "matplotlib" + }, + { + "repository": "PyAutoFit", + "package": "scipy", + "observed": "1.17.1", + "status": "compatible", + "declared": "scipy<=1.17.1" + }, + { + "repository": "PyAutoArray", + "package": "python", + "observed": "3.12.4", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoArray", + "package": "autonerves", + "observed": null, + "status": "unknown", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoArray", + "package": "astropy", + "observed": "6.1.2", + "status": "compatible", + "declared": "astropy>=5.0" + }, + { + "repository": "PyAutoArray", + "package": "matplotlib", + "observed": "3.9.4", + "status": "compatible", + "declared": "matplotlib>=3.7.0" + }, + { + "repository": "PyAutoArray", + "package": "scipy", + "observed": "1.17.1", + "status": "compatible", + "declared": "scipy<=1.17.1" + }, + { + "repository": "PyAutoArray", + "package": "scikit-learn", + "observed": "1.5.1", + "status": "compatible", + "declared": "scikit-learn<=1.8.0" + }, + { + "repository": "PyAutoGalaxy", + "package": "python", + "observed": "3.12.4", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoGalaxy", + "package": "autonerves", + "observed": null, + "status": "unknown", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "autofit", + "observed": null, + "status": "unknown", + "declared": "autofit>=2026.8.22.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "autoarray", + "observed": null, + "status": "unknown", + "declared": "autoarray>=2026.8.22.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "astropy", + "observed": "6.1.2", + "status": "compatible", + "declared": "astropy>=5.0" + }, + { + "repository": "PyAutoGalaxy", + "package": "nautilus-sampler", + "observed": "1.0.5", + "status": "compatible", + "declared": "nautilus-sampler==1.0.5" + }, + { + "repository": "PyAutoLens", + "package": "python", + "observed": "3.12.4", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoLens", + "package": "autonerves", + "observed": null, + "status": "unknown", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoLens", + "package": "autogalaxy", + "observed": null, + "status": "unknown", + "declared": "autogalaxy>=2026.8.22.1" + }, + { + "repository": "PyAutoLens", + "package": "nautilus-sampler", + "observed": "1.0.5", + "status": "compatible", + "declared": "nautilus-sampler==1.0.5" + } + ] + }, + { + "digest": "50dc22639b8b66f68fe8c9d4517f79a89caf305995fe2bac787b3706382cd5c8", + "kind": "availability", + "created": "2026-10-08T12:12:15.609787Z", + "name": "jax", + "python": "3.12.9", + "latest": "0.11.2", + "compatible_stable": "0.11.2" + }, + { + "digest": "e49732f2c3e9878897b450967f5b3cbc364b830bb62503f8615055e4c39fb425", + "kind": "audit", + "created": "2026-10-08T12:20:37.346721Z", + "inventory": "7c168b4b5ae04f67582b325d07496b2e143e1dba840c3f406de2222f4d5c5556", + "rows": [ + { + "repository": "PyAutoNerves", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoNerves", + "package": "numpy", + "observed": "2.2.6", + "status": "compatible", + "declared": "numpy<3.0.0,>=1.24.0" + }, + { + "repository": "PyAutoNerves", + "package": "jax", + "observed": "0.10.2", + "status": "incompatible", + "declared": "jax!=0.10.*,!=0.11.0,<0.12.0,>=0.7.0" + }, + { + "repository": "PyAutoNerves", + "package": "jaxlib", + "observed": "0.10.2", + "status": "incompatible", + "declared": "jaxlib!=0.10.*,!=0.11.0,<0.12.0,>=0.7.0" + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "colab-gate.yml", + "line": 50 + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "nerves_board.yml", + "line": 58 + }, + { + "repository": "PyAutoNerves", + "status": "ci_override_review", + "source": "nerves_board.yml", + "line": 61 + }, + { + "repository": "PyAutoFit", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoFit", + "package": "autonerves", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoFit", + "package": "optax", + "observed": "0.2.5", + "status": "compatible", + "declared": "optax>=0.2.5" + }, + { + "repository": "PyAutoFit", + "package": "anesthetic", + "observed": "2.14.6", + "status": "compatible", + "declared": "anesthetic>=2.9.0" + }, + { + "repository": "PyAutoFit", + "package": "corner", + "observed": "2.2.2", + "status": "compatible", + "declared": "corner==2.2.2" + }, + { + "repository": "PyAutoFit", + "package": "dynesty", + "observed": "2.1.5", + "status": "compatible", + "declared": "dynesty==2.1.5" + }, + { + "repository": "PyAutoFit", + "package": "emcee", + "observed": "3.1.6", + "status": "compatible", + "declared": "emcee>=3.1.6" + }, + { + "repository": "PyAutoFit", + "package": "matplotlib", + "observed": "3.10.9", + "status": "compatible", + "declared": "matplotlib" + }, + { + "repository": "PyAutoFit", + "package": "scipy", + "observed": "1.17.1", + "status": "compatible", + "declared": "scipy<=1.17.1" + }, + { + "repository": "PyAutoArray", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoArray", + "package": "autonerves", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoArray", + "package": "astropy", + "observed": "8.0.1", + "status": "compatible", + "declared": "astropy>=5.0" + }, + { + "repository": "PyAutoArray", + "package": "matplotlib", + "observed": "3.10.9", + "status": "compatible", + "declared": "matplotlib>=3.7.0" + }, + { + "repository": "PyAutoArray", + "package": "scipy", + "observed": "1.17.1", + "status": "compatible", + "declared": "scipy<=1.17.1" + }, + { + "repository": "PyAutoArray", + "package": "scikit-learn", + "observed": "1.5.1", + "status": "compatible", + "declared": "scikit-learn<=1.8.0" + }, + { + "repository": "PyAutoGalaxy", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoGalaxy", + "package": "autonerves", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "autofit", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autofit>=2026.8.22.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "autoarray", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autoarray>=2026.8.22.1" + }, + { + "repository": "PyAutoGalaxy", + "package": "astropy", + "observed": "8.0.1", + "status": "compatible", + "declared": "astropy>=5.0" + }, + { + "repository": "PyAutoGalaxy", + "package": "nautilus-sampler", + "observed": "1.0.5", + "status": "compatible", + "declared": "nautilus-sampler==1.0.5" + }, + { + "repository": "PyAutoLens", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoLens", + "package": "autonerves", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoLens", + "package": "autogalaxy", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autogalaxy>=2026.8.22.1" + }, + { + "repository": "PyAutoLens", + "package": "nautilus-sampler", + "observed": "1.0.5", + "status": "compatible", + "declared": "nautilus-sampler==1.0.5" + } + ] + }, + { + "digest": "e6b8bbb16000f6b363ae38b1c904a07cc97d17493687e13a49ae990dd2eff551", + "kind": "audit", + "created": "2026-10-08T12:15:00.420206Z", + "inventory": "99c305d87654b5585bd94d77481b1b369626cfddb3eaa5653bc47fe4a4a73d65", + "rows": [ + { + "repository": "PyAutoArray", + "package": "python", + "observed": "3.12.10", + "status": "compatible", + "declared": ">=3.12" + }, + { + "repository": "PyAutoArray", + "package": "autonerves", + "observed": "9999.0.0.dev0", + "status": "compatible", + "declared": "autonerves>2026.10.4.1" + }, + { + "repository": "PyAutoArray", + "package": "astropy", + "observed": "8.0.1", + "status": "compatible", + "declared": "astropy>=5.0" + }, + { + "repository": "PyAutoArray", + "package": "matplotlib", + "observed": "3.10.9", + "status": "compatible", + "declared": "matplotlib>=3.7.0" + }, + { + "repository": "PyAutoArray", + "package": "scipy", + "observed": "1.17.1", + "status": "compatible", + "declared": "scipy<=1.17.1" + }, + { + "repository": "PyAutoArray", + "package": "scikit-learn", + "observed": "1.5.1", + "status": "compatible", + "declared": "scikit-learn<=1.8.0" + } + ] + } + ] +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..643b930 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" +[project] +name = "pyautodna" +version = "0.1.0" +description = "Observed software stacks, compatibility evidence and upgrade campaigns" +requires-python = ">=3.12" +dependencies = ["packaging>=23", "PyYAML>=6"] +[project.scripts] +pyauto-dna = "dna.cli:main" +[tool.setuptools.packages.find] +include = ["dna*"] diff --git a/schemas/record.schema.json b/schemas/record.schema.json new file mode 100644 index 0000000..d124941 --- /dev/null +++ b/schemas/record.schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "DNA immutable evidence envelope v1", + "type": "object", + "required": ["schema_version", "kind", "created", "digest"], + "properties": { + "schema_version": {"const": 1}, + "kind": {"enum": ["inventory", "stack", "validation", "decision", "campaign", "adoption", "promotion", "rollback", "availability", "audit"]}, + "created": {"type": "string", "format": "date-time"}, + "digest": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "$comment": "dna.schema.validate is normative for kind fields, digest verification and UTC/time constraints. This schema describes the interoperable envelope." +} diff --git a/stacks/README.md b/stacks/README.md new file mode 100644 index 0000000..dc18bff --- /dev/null +++ b/stacks/README.md @@ -0,0 +1,6 @@ +# Stack specifications + +Reviewed immutable public stack records may live here. No seeded recommendation +or certification exists. The CLI initially stores candidate records privately; +public export projects reviewed allowed fields. Exact Python, package pins and +backend are required. Optional git/runtime constraints remain part of the digest. diff --git a/support.yaml b/support.yaml new file mode 100644 index 0000000..761555d --- /dev/null +++ b/support.yaml @@ -0,0 +1,9 @@ +schema_version: 1 +reviewed: '2026-10-08' +review_by: '2027-01-08' +sources: + python: https://devguide.python.org/versions/ + jax: https://docs.jax.dev/en/latest/deprecation.html +python: + - {version: '3.12', upstream_end_month: '2028-10', jax_guaranteed_through_month: '2027-07'} + - {version: '3.13', upstream_end_month: '2029-10', jax_guaranteed_through_month: '2028-07'} diff --git a/tests/test_board.py b/tests/test_board.py new file mode 100644 index 0000000..16b4402 --- /dev/null +++ b/tests/test_board.py @@ -0,0 +1,74 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from dna.board import public_records, render, snapshot +from dna.campaigns import Store +from dna.schema import record + + +class Board(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / 'environments.yaml').write_text('schema_version: 1\nenvironments:\n - {id: local, label: Local, backend: unknown}\n - {id: remote, label: Remote, backend: cpu}\n') + self.inv = record('inventory', environment='local', python='3.12.10', packages={'jax': {'version': '0.11.2'}}, repositories=[], runtime={'backend': 'cpu'}) + Store(self.root/'private').add(self.inv) + + def test_support_expiry_and_policy_review_are_separate(self): + from datetime import date + from dna.policy import support_windows + policy = {'schema_version': 1, 'review_by': '2027-01-01', 'python': [{'version': '3.12', 'upstream_end_month': '2028-10', 'jax_guaranteed_through_month': '2027-07'}]} + self.assertEqual(support_windows(policy, date(2026, 10, 8))[0]['status'], 'Within upstream window') + self.assertEqual(support_windows(policy, date(2027, 2, 1))[0]['status'], 'Policy review due') + self.assertEqual(support_windows(policy, date(2028, 11, 1))[0]['status'], 'Upstream end of life') + + def test_inventory_diff_keeps_unknown_runtime_unknown(self): + from dna.policy import diff_inventories + other = record('inventory', environment='remote', python='3.13.1', packages={}, repositories=[], runtime={'backend': 'unknown'}) + rows = diff_inventories(self.inv, other)['rows'] + self.assertEqual(rows[0]['status'], 'different') + backend = next(r for r in rows if r['component'] == 'runtime:backend') + self.assertEqual(backend['status'], 'unknown') + self.assertIsNone(next(r for r in rows if r['component'] == 'package:jax')['right']) + + def test_partial_coverage_never_has_whole_board_refresh(self): + a = snapshot(self.root) + b = snapshot(self.root) + self.assertIsNone(a['refreshed_at']) + self.assertEqual(a['updated'], self.inv['created']) + self.assertEqual(a['updated'], b['updated']) + self.assertEqual(a['environments'][1]['status'], 'unknown') + + def test_public_audit_omits_direct_urls_and_private_projects(self): + obj = record('audit', inventory=self.inv['digest'], declarations=[], rows=[ + {'repository': 'PyAutoArray', 'package': 'jax', 'declared': 'jax @ https://private.invalid/secret.whl', 'status': 'unknown'}, + {'repository': 'PrivateScience', 'package': 'secret', 'observed': '1', 'status': 'compatible'}, + ]) + clean = json.dumps(public_records([obj])) + self.assertNotIn('private.invalid', clean) + self.assertNotIn('PrivateScience', clean) + self.assertNotIn('secret', clean) + self.assertIn('direct_source', clean) + self.assertEqual(public_records(public_records([obj])), public_records([obj])) + + def test_standard_collapsed_sections_and_visible_package_versions(self): + brain = Path(os.environ.get('PYAUTO_BRAIN', Path(__file__).resolve().parents[2] / 'PyAutoBrain')) + if not brain.is_dir(): + brain = Path(__file__).resolve().parents[1] / 'PyAutoBrain' + render(self.root, brain, self.root/'site') + page = (self.root/'site/index.html').read_text() + self.assertIn('0.11.2', page) + self.assertIn('Last updated unavailable', page) + self.assertEqual(page.count('class="board-section"'), 4) + self.assertNotIn('
=0.12", "numpy; python_version < \'3.0\'"]\n') + rows=audit(self.inv,[repo]) + self.assertEqual([r['status'] for r in rows], ['incompatible','incompatible']) + def test_override_marker_and_direct_url(self): + self.assertEqual(audit(self.inv,[],overrides=["jax==999; python_version < '3'"]),[]) + self.assertEqual(audit(self.inv,[],overrides=['jax @ https://example.com/jax.whl'])[0]['status'],'unknown') + def test_unknown_target_marker(self): + repo=self.root/'marker';repo.mkdir() + (repo/'pyproject.toml').write_text('[project]\ndependencies=["jax; implementation_name == \'cpython\'"]\n') + self.assertEqual(audit(self.inv,[repo])[0]['status'],'unknown') + def test_gates_and_newer_failure(self): + campaign=self.campaign() + with self.assertRaises(ValueError): self.store.adopt(campaign['digest'],'local',self.inv['digest']) + evidence=self.validation() + self.store.promote(campaign['digest'],[evidence['digest']]) + self.assertEqual(self.store.adopt(campaign['digest'],'local',self.inv['digest'])['kind'],'adoption') + self.validation('fail') + with self.assertRaises(ValueError): self.store.promote(campaign['digest'],[evidence['digest']]) + with self.assertRaises(ValueError): self.store.adopt(campaign['digest'],'local',self.inv['digest']) + with self.assertRaises(ValueError): self.store.adopt(campaign['digest'],'local',self.inv['digest'],True) + def test_expired_evidence_and_unknown_backend(self): + campaign=self.campaign(); evidence=self.validation() + evidence['created']=(datetime.now(timezone.utc)-timedelta(days=8)).isoformat() + evidence['digest']=digest(evidence);self.store.add(evidence) + # Remove newer pass so the stale record is the latest available evidence. + for path in (self.root/'private/records').glob('*.json'): + obj=json.loads(path.read_text()) + if obj['kind']=='validation' and obj['digest']!=evidence['digest']:path.unlink() + with self.assertRaises(ValueError):self.store.promote(campaign['digest'],[evidence['digest']]) + unknown=self.store.add(record('stack',name='unknown',python='3.12.9',backend='unknown',packages={'jax':'0.11.1'})) + c=self.store.create('unknown',unknown['digest'],self.stack['digest'],['local'],'human','2099-01-01') + with self.assertRaises(ValueError):self.store.promote(c['digest'],[]) + def test_validation_provenance(self): + campaign=self.campaign() + wrong=self.store.add(record('inventory',environment='local',python='3.13.0',packages={'jax':{'version':'0.11.1'}},repositories=[],runtime={'backend':'cpu'})) + ev=self.store.add(record('validation',stack=self.stack['digest'],environment='local',authority='PyAutoHeart',result='pass',checks=['x'],evidence='x',inventory=wrong['digest'])) + with self.assertRaises(ValueError): self.store.promote(campaign['digest'],[ev['digest']]) + def test_projection_and_unknown(self): + public=public_inventory(self.inv) + text=json.dumps(public) + self.assertNotIn('/private',text);self.assertNotIn('XLA_FLAGS',text);self.assertNotIn('secret-project',text) + snap=snapshot(self.root);self.assertEqual(snap['environments'][0]['status'],'observed') + for p in (self.root/'private/records').glob('*.json'): p.unlink() + self.assertEqual(snapshot(self.root)['environments'][0]['status'],'unknown') + def test_upstream_yanked_python_prerelease(self): + data={'releases':{'1.0':[{'requires_python':'>=3.12'}],'2.0':[{'yanked':True}],'3.0':[{'requires_python':'>=3.13'}],'4.0rc1':[{}]}} + obj=available('jax','3.12.9',lambda _:data) + self.assertEqual(obj['latest'],'4.0rc1');self.assertEqual(obj['compatible_stable'],'1.0') + def test_projection_roundtrip_and_private_injection(self): + public=snapshot(self.root);public['environments'][0]['observed']['location']='/private/injected' + (self.root/'inventories').mkdir();(self.root/'inventories/public.json').write_text(json.dumps(public)) + for p in (self.root/'private/records').glob('*.json'): p.unlink() + self.assertNotIn('/private',json.dumps(snapshot(self.root))) + +if __name__=='__main__': unittest.main()