ci: fail fast on bad nodes and cluster outages

.github/scripts/ci-outage.sh

@@ -0,0 +1,118 @@
+#!/bin/bash
+# Per-cluster circuit breaker for environment-wide outages.
+#
+# Some failures are not the node's fault and not the code's fault: pypi.org
+# unreachable from a login node, a module tree mid-upgrade, a full project
+# filesystem. Requeuing elsewhere cannot help, and every job that starts pays
+# the same discovery cost -- on 2026-08-28, 17 Frontier jobs each spent ~33
+# minutes learning that PyPI was down.
+#
+# The first job to notice records a marker on the shared filesystem (every
+# self-hosted runner for a cluster shares $HOME); later jobs check it and exit
+# immediately instead of submitting a SLURM job that is going to fail.
+#
+# The breaker is deliberately self-healing. A marker expires after
+# MFC_CI_OUTAGE_TTL_SECONDS, and a marker that cannot be parsed is ignored, so
+# neither a stale file nor a truncated write can wedge CI. Only jobs that
+# actually observe the outage re-mark it, so once the outage clears the breaker
+# closes on its own.
+#
+# Usage:
+#   ci-outage.sh mark  <cluster> <reason>   record an outage
+#   ci-outage.sh check <cluster>            exit 0 = clear, 1 = outage active
+#   ci-outage.sh clear <cluster>            reset the breaker
+#
+# Env:
+#   MFC_CI_STATE_DIR             where markers live (default ~/.mfc-ci-state)
+#   MFC_CI_OUTAGE_TTL_SECONDS    marker lifetime in seconds (default 1200)
+
+set -uo pipefail
+
+STATE_DIR="${MFC_CI_STATE_DIR:-$HOME/.mfc-ci-state}"
+TTL="${MFC_CI_OUTAGE_TTL_SECONDS:-1200}"
+
+# A non-numeric TTL would make the age comparison below emit "integer expression
+# expected" and exit with a code the caller reads as neither clear nor tripped.
+# Fall back to the default rather than letting a typo gate CI.
+case "$TTL" in
+    ''|*[!0-9]*)
+        echo "Ignoring non-numeric MFC_CI_OUTAGE_TTL_SECONDS='$TTL'; using 1200." >&2
+        TTL=1200
+        ;;
+esac
+
+EXIT_CLEAR=0
+EXIT_TRIPPED=1
+EXIT_USAGE=2
+
+usage() {
+    echo "Usage: $0 {mark <cluster> <reason>|check <cluster>|clear <cluster>}" >&2
+}
+
+# Keep the marker name filesystem-safe regardless of what the caller passes.
+marker_for() {
+    local cluster
+    cluster=$(printf '%s' "$1" | tr -c 'A-Za-z0-9_.-' '_')
+    printf '%s/outage-%s' "$STATE_DIR" "$cluster"
+}
+
+cmd="${1:-}"
+cluster="${2:-}"
+
+if [ -z "$cmd" ] || [ -z "$cluster" ]; then
+    usage
+    exit $EXIT_USAGE
+fi
+
+marker=$(marker_for "$cluster")
+
+case "$cmd" in
+    mark)
+        reason="${3:-unspecified}"
+        mkdir -p "$STATE_DIR" || exit $EXIT_USAGE
+        # Write to a temporary file and rename so a concurrent `check` never
+        # observes a half-written marker.
+        tmp="${marker}.$$.tmp"
+        {
+            date +%s
+            printf '%s\n' "$reason"
+        } > "$tmp" && mv -f "$tmp" "$marker"
+        echo "Recorded $cluster outage: $reason"
+        echo "  marker: $marker (expires after ${TTL}s)"
+        ;;
+
+    check)
+        [ -f "$marker" ] || exit $EXIT_CLEAR
+
+        stamp=$(head -n1 "$marker" 2>/dev/null)
+        reason=$(tail -n +2 "$marker" 2>/dev/null)
+
+        # A marker we cannot parse is treated as absent: an unreadable breaker
+        # must never be an un-clearable one.
+        case "$stamp" in
+            ''|*[!0-9]*)
+                echo "Ignoring unparseable outage marker $marker"
+                exit $EXIT_CLEAR
+                ;;
+        esac
+
+        age=$(( $(date +%s) - stamp ))
+        if [ "$age" -ge "$TTL" ] || [ "$age" -lt 0 ]; then
+            exit $EXIT_CLEAR
+        fi
+
+        echo "::warning::Skipping: known $cluster outage recorded ${age}s ago: ${reason:-unspecified}"
+        echo "Clear it early by deleting $marker"
+        exit $EXIT_TRIPPED
+        ;;
+
+    clear)
+        rm -f "$marker"
+        echo "Cleared any $cluster outage marker ($marker)"
+        ;;
+
+    *)
+        usage
+        exit $EXIT_USAGE
+        ;;
+esac

.github/scripts/classify-build-failure.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Decide whether a failed build was a cluster-wide dependency outage.
+#
+# MFC bootstraps its Python toolchain into build/venv on the first ./mfc.sh call
+# of a job, pulling from pypi.org. On Phoenix clean_build has just moved build/
+# aside, so that happens every time; on Frontier it happens in the login-node
+# "Fetch Dependencies" step. When the index is unreachable the build fails for a
+# reason no other node improves on, so it is worth recording once and skipping
+# the rest of the matrix rather than having each job spend ~33 minutes
+# rediscovering it (17 Frontier jobs did exactly that on 2026-08-28).
+#
+# Usage: classify-build-failure.sh <logfile> <cluster>
+#
+# Exit codes:
+#   78  cluster-wide dependency outage; it has been recorded
+#   0   ordinary build failure, caller should keep its own exit code
+
+set -uo pipefail
+
+log="${1:-}"
+cluster="${2:-}"
+
+if [ -z "$log" ] || [ -z "$cluster" ]; then
+    echo "Usage: $0 <logfile> <cluster>" >&2
+    exit 0
+fi
+
+# No log means nothing to classify. Never claim an outage on absent evidence.
+[ -f "$log" ] || exit 0
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# The URL may or may not be wrapped (uv quotes it in backticks, plain pip does
+# not), so do not require a character between the colon and the scheme.
+if grep -qE "Failed to fetch:[^h]*https?://pypi|uv install failed|\(venv\) Installation failed" "$log"; then
+    bash "$SCRIPT_DIR/ci-outage.sh" mark "$cluster" \
+        "PyPI/uv dependency install failed during build"
+    exit 78
+fi
+
+exit 0

.github/scripts/monitor_slurm_job.sh

@@ -99,6 +99,11 @@ if ! [[ "$SLURM_MAX_QUEUE_SECONDS" =~ ^[0-9]+$ ]]; then
   echo "ERROR: SLURM_MAX_QUEUE_SECONDS must be a non-negative integer (seconds), got '$SLURM_MAX_QUEUE_SECONDS'" >&2
   exit 1
 fi
+# How long to wait between status polls and between output-stabilization
+# checks. Overridable so tests can exercise this script without sleeping
+# through it; CI leaves it at the default.
+: "${MFC_MONITOR_POLL_SECONDS:=5}"
+
 queue_start=$(date +%s)
 
 abort_queue_starvation() {
@@ -142,7 +147,7 @@ while [ ! -f "$output_file" ]; do
       ;;
     PENDING|CONFIGURING)
       unknown_count=0
-      sleep 5
+      sleep "$MFC_MONITOR_POLL_SECONDS"
       ;;
     RUNNING|COMPLETING)
       unknown_count=0
@@ -155,7 +160,7 @@ while [ ! -f "$output_file" ]; do
       if [ $((unknown_count % 12)) -eq 1 ]; then
         echo "Warning: Could not query job $job_id state (SLURM may be temporarily unavailable)..."
       fi
-      sleep 5
+      sleep "$MFC_MONITOR_POLL_SECONDS"
       ;;
     *)
       # Terminal state — job finished without creating output
@@ -164,7 +169,7 @@ while [ ! -f "$output_file" ]; do
         exit 1
       fi
       # Unrecognized state, keep waiting
-      sleep 5
+      sleep "$MFC_MONITOR_POLL_SECONDS"
       ;;
   esac
 done
@@ -205,7 +210,7 @@ while true; do
     last_heartbeat=$current_time
   fi
 
-  sleep 5
+  sleep "$MFC_MONITOR_POLL_SECONDS"
 done
 
 # Give tail a moment to flush the final lines, then stop streaming.
@@ -229,7 +234,7 @@ if [ -f "$output_file" ]; then
     if [ $same_count -ge 2 ]; then
       break
     fi
-    sleep 5
+    sleep "$MFC_MONITOR_POLL_SECONDS"
   done
 fi
 
@@ -262,6 +267,23 @@ if [ -z "$exit_code" ]; then
   exit 1
 fi
 
+# Infrastructure verdicts from the in-allocation preflight come back as the
+# job's own exit code. Relay them verbatim: flattening them to 1 would leave the
+# submit wrapper unable to tell "this node is unusable" (exclude it and try
+# again) from "the tests failed" (report it).
+case "$exit_code" in
+  77:*)
+    echo "Job $job_id failed preflight: the node is unusable — signaling caller to exclude it and resubmit."
+    monitor_success=1
+    exit 77
+    ;;
+  78:*)
+    echo "Job $job_id skipped: a cluster-wide outage is already recorded."
+    monitor_success=1
+    exit 78
+    ;;
+esac
+
 # Check if job succeeded
 if [ "$exit_code" != "0:0" ]; then
   echo "ERROR: Job $job_id failed with exit code $exit_code"

.github/scripts/node-exclude.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Bookkeeping for the sbatch --exclude list used when a node fails preflight.
+#
+# The in-allocation preflight prints "MFC_FAULT_NODE=<name>" into the job's
+# output file when it finds the node unusable (dead GPU, SIGILL from a binary
+# built for another microarchitecture). The submit wrapper reads that back, adds
+# the node to --exclude and resubmits, so the retry lands elsewhere.
+#
+# Bad nodes are concentrated rather than scattered -- over 2026-08-18..31 one
+# Phoenix node accounted for 25 of 29 ECC failures -- which is why excluding the
+# offender is worth doing and why it was previously a hand-edited constant.
+#
+# Usage:
+#   node-exclude.sh node-from <output-file>   print the faulted node, if any
+#   node-exclude.sh merge <csv> <node>        print <csv> with <node> added once
+
+set -uo pipefail
+
+usage() {
+    echo "Usage: $0 {node-from <output-file>|merge <csv> <node>}" >&2
+}
+
+case "${1:-}" in
+    node-from)
+        file="${2:-}"
+        [ -f "$file" ] || exit 0
+        # Last marker wins: one output path is reused across resubmits, so an
+        # earlier attempt's marker can still be sitting above the current one.
+        sed -n 's/.*MFC_FAULT_NODE=\([A-Za-z0-9._-]\{1,\}\).*/\1/p' "$file" | tail -n1
+        ;;
+
+    merge)
+        csv="${2-}"
+        node="${3-}"
+        if [ -z "$node" ]; then
+            printf '%s\n' "$csv"
+            exit 0
+        fi
+        if [ -z "$csv" ]; then
+            printf '%s\n' "$node"
+            exit 0
+        fi
+        # Wrapping both sides in commas compares whole fields, so a shorter name
+        # that happens to be a prefix of the new one is not mistaken for a match.
+        case ",$csv," in
+            *",$node,"*) printf '%s\n' "$csv" ;;
+            *)           printf '%s,%s\n' "$csv" "$node" ;;
+        esac
+        ;;
+
+    *)
+        usage
+        exit 2
+        ;;
+esac

.github/scripts/prebuild-case-optimization.sh

@@ -90,6 +90,15 @@ if [ -n "$shard" ] && [ "$shard_count" -gt 1 ]; then
     fi
 fi
 
+# Deliberately no node probe here. This pre-build is submitted as a *cpu*
+# allocation (see test.yml: it is --dry-run, so it only builds), while the
+# binaries it produces are GPU builds. syscheck built with --gpu therefore
+# asserts omp_get_num_devices() > 0 and exits non-zero on a node that has no
+# GPU by design -- which a probe would report as a bad node. It did: three
+# healthy Phoenix nodes were condemned and two excluded before the wrapper gave
+# up. The GPU allocation that actually runs these cases is probed instead, in
+# run_case_optimization.sh.
+
 idx=0
 for case in "${benchmarks[@]}"; do
     idx=$((idx + 1))