kvlonge · Nov 5, 2026 · Nov 5, 2026
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -28,6 +28,7 @@ jobs:
       format_performance: ${{ steps.scope.outputs.format_performance }}
       rules_performance: ${{ steps.scope.outputs.rules_performance }}
       wheels: ${{ steps.scope.outputs.wheels }}
+      recovery: ${{ steps.scope.outputs.recovery }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -52,11 +53,12 @@ jobs:
               echo "wheels=$7"
               echo "compiler_differential=$8"
               echo "clickhouse=$9"
+              echo "recovery=${10}"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -68,7 +70,7 @@ jobs:
           if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \
              ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then
             echo "Could not diff ${BASE_SHA:-<none>}...${HEAD_SHA:-<none>}; running every suite" >&2
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -114,7 +116,7 @@ jobs:
                 && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then
             echo "Release Please pull request changes only release metadata; skipping the suites" \
               | tee -a "${GITHUB_STEP_SUMMARY}"
-            write_scope false false false false false false false false false
+            write_scope false false false false false false false false false false
             exit 0
           fi
 
@@ -126,6 +128,7 @@ jobs:
           rules_performance=false
           wheels=false
           compiler_differential=false
+          recovery=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -146,6 +149,11 @@ jobs:
                 wheels=true
                 ;;
             esac
+            case "${path}" in
+              crates/sqlbuild-orchestration/*|crates/sqlbuild-process-ownership/*|crates/sqlbuild-cli/*|Cargo.toml|Cargo.lock|.github/workflows/verify.yml)
+                recovery=true
+                ;;
+            esac
             case "${path}" in
               scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|.github/workflows/verify.yml|Makefile)
                 compiler_differential=true
@@ -207,7 +215,7 @@ jobs:
           done < "${changed_paths}"
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -232,7 +240,8 @@ jobs:
             "${checks}" \
             "${wheels}" \
             "${compiler_differential}" \
-            "${clickhouse}"
+            "${clickhouse}" \
+            "${recovery}"
 
   lockfile:
     name: Lockfile consistency
@@ -549,6 +558,107 @@ jobs:
           uv run --no-sync sqb playground --template python_nodes "${nodes}"
           uv run --no-sync sqb --project-dir "${nodes}" --no-color compile
 
+  scheduler_recovery:
+    name: Scheduler recovery (${{ matrix.name }})
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.recovery == 'true'
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: macOS
+            runner: macos-15
+          - name: Windows
+            runner: windows-2025
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: verify-recovery-${{ matrix.name }}
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - name: Start a native PostgreSQL server
+        run: |
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+            brew install --quiet postgresql@16
+            bin="$(brew --prefix postgresql@16)/bin"
+          else
+            bin="$(cygpath -u "${PGBIN}")"
+          fi
+          data="${RUNNER_TEMP}/postgres-data"
+          "${bin}/initdb" -D "${data}" -U postgres --auth=trust -E UTF8 > /dev/null
+          "${bin}/pg_ctl" -D "${data}" -o "-p 5433 -c listen_addresses=127.0.0.1" \
+            -l "${RUNNER_TEMP}/postgres.log" -w start
+      - name: Worker claim ownership, recovery and stop
+        run: cargo test --locked -p sqlbuild-process-ownership
+      - name: Worker supervision and person credential storage
+        run: cargo test --locked -p sqlbuild-cli --lib -- workers person_login
+      - name: Control store, scheduler and recovery contracts on SQLite and PostgreSQL
+        env:
+          SQLBUILD_TEST_CONTROL_POSTGRES_URL: postgresql://postgres@127.0.0.1:5433/postgres
+        run: cargo test --locked -p sqlbuild-orchestration --lib
+      - name: "Temporary diagnostics: commit latency and serial test timings"
+        if: always()
+        env:
+          SQLBUILD_TEST_CONTROL_POSTGRES_URL: postgresql://postgres@127.0.0.1:5433/postgres
+        run: |
+          nproc 2>/dev/null || sysctl -n hw.ncpu || true
+          python3 -c "import sys; print(sys.version)" 2>/dev/null || python -c "import sys; print(sys.version)"
+          py="$(command -v python3 || command -v python)"
+          "${py}" - "${RUNNER_TEMP}" <<'PYEOF'
+          import os, sqlite3, sys, time
+          for sync in ("NORMAL", "FULL"):
+              path = os.path.join(sys.argv[1], f"latency-{sync}.sqlite3")
+              connection = sqlite3.connect(path, isolation_level=None)
+              connection.execute(f"PRAGMA journal_mode=WAL")
+              connection.execute(f"PRAGMA synchronous={sync}")
+              connection.execute("CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)")
+              samples = []
+              for index in range(1000):
+                  started = time.perf_counter()
+                  connection.execute("BEGIN IMMEDIATE")
+                  connection.execute("INSERT INTO t (v) VALUES (?)", (str(index),))
+                  connection.execute("COMMIT")
+                  samples.append(time.perf_counter() - started)
+              samples.sort()
+              print(f"sqlite WAL synchronous={sync}: 1000 commits total={sum(samples):.2f}s "
+                    f"p50={samples[500]*1000:.2f}ms p99={samples[990]*1000:.2f}ms max={samples[-1]*1000:.2f}ms")
+          PYEOF
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+            bin="$(brew --prefix postgresql@16)/bin"
+          else
+            bin="$(cygpath -u "${PGBIN}")"
+          fi
+          echo "CREATE TABLE IF NOT EXISTS t (id bigserial PRIMARY KEY, v text);" \
+            | "${bin}/psql" -q -h 127.0.0.1 -p 5433 -U postgres postgres
+          echo "INSERT INTO t (v) VALUES ('x');" > "${RUNNER_TEMP}/insert.sql"
+          "${bin}/pgbench" -n -h 127.0.0.1 -p 5433 -U postgres -c 1 -t 1000 \
+            -f "${RUNNER_TEMP}/insert.sql" postgres | grep -E "latency|tps"
+          for name in \
+            platform::tests::test_webhook::given_a_location_outage_when_webhook_delivery_retries_then_all_health_facts_survive_restart_and_lost_acknowledgements \
+            runs::tests::test_group_admission::given_a_large_healthy_project_with_blocked_groups_when_claiming_then_unrelated_work_has_a_bounded_write_lock \
+            scheduler::tests::contracts::ticks::test_retention::given_expired_covered_ticks_when_pruned_and_cursor_reset_then_run_linked_and_latest_ticks_survive_without_resurrection \
+            sensors::tests::test_nodes::given_any_run_rebuilds_when_consuming_then_only_genuine_scoped_successes_queue_once_per_finish \
+            sensors::tests::test_runs::given_terminal_jobs_and_groups_when_selectors_retry_then_outcomes_scope_and_run_keys_are_preserved \
+            sensors::tests::test_transitions::given_independent_check_failures_when_nondecisive_results_arrive_then_only_executed_success_recovers
+          do
+            started=$(date +%s)
+            if cargo test --locked -q -p sqlbuild-orchestration --lib -- --exact "${name}" --test-threads=1 --nocapture \
+                > "${RUNNER_TEMP}/serial.log" 2>&1; then status=pass; else status=FAIL; fi
+            echo "serial ${status} $(( $(date +%s) - started ))s ${name}"
+            grep -E "admissions in|panicked|Fenced|Elapsed" "${RUNNER_TEMP}/serial.log" | head -5 || true
+          done
+      - name: PostgreSQL server log
+        if: failure()
+        run: cat "${RUNNER_TEMP}/postgres.log" || true
+
   format_performance:
     name: Formatter performance guards
     needs: [changes, lockfile]
@@ -1080,6 +1190,7 @@ jobs:
         compiler_differential,
         compiler_differential_python_versions,
         native_discovery_windows,
+        scheduler_recovery,
         format_performance,
         performance,
         cold_compile_performance,
@@ -1105,6 +1216,7 @@ jobs:
           COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
           COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }}
           NATIVE_DISCOVERY_WINDOWS_RESULT: ${{ needs.native_discovery_windows.result }}
+          SCHEDULER_RECOVERY_RESULT: ${{ needs.scheduler_recovery.result }}
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
@@ -1137,6 +1249,7 @@ jobs:
             "${COMPILER_DIFFERENTIAL_RESULT}" \
             "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \
             "${NATIVE_DISCOVERY_WINDOWS_RESULT}" \
+            "${SCHEDULER_RECOVERY_RESULT}" \
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -3,6 +3,7 @@ members = [
     "crates/sqlbuild-build-features",
     "crates/sqlbuild-core",
     "crates/sqlbuild-cursor",
+    "crates/sqlbuild-process-ownership",
     "crates/sqlbuild-state",
     "crates/sqlbuild-pysource",
     "crates/sqlbuild-cache",
@@ -33,6 +34,7 @@ order = [
     "sqlbuild-build-features",
     "sqlbuild-core",
     "sqlbuild-cursor",
+    "sqlbuild-process-ownership",
     "sqlbuild-state",
     "sqlbuild-pysource",
     "sqlbuild-cache",
@@ -108,6 +110,7 @@ thiserror = "2"
 # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it.
 ctrlc = "3"
 command-fds = "0.3"
+libc = "0.2"
 quote = "1"
 rayon = "1"
 # Static reading of project Python source; the parser fensu uses.
@@ -117,7 +120,9 @@ ruff_text_size = "=0.0.4"
 # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`.
 # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output.
 rustc-hash = "2"
-# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code in the workspace.
+# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code. The only crate
+# allowed unsafe code is sqlbuild-process-ownership, for the platform calls that prove which
+# process owns a worker claim; every other crate forbids it.
 signal-hook = { version = "0.3", default-features = false }
 # Features signal-hook's registry enables, matched in the sqb build through sqlbuild-build-features.
 errno = "0.3"
@@ -160,6 +165,7 @@ sqlbuild-refactor = { path = "crates/sqlbuild-refactor" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }
 sqlbuild-model-config = { path = "crates/sqlbuild-model-config" }
 sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" }
+sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" }
 sqlbuild-project = { path = "crates/sqlbuild-project" }
 sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
 sqlbuild-recall = { path = "crates/sqlbuild-recall" }

diff --git a/crates/README.md b/crates/README.md
@@ -9,6 +9,7 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-build-features` | No code. Enables, in the `sqb` build, the dependency features the extension build resolves, so Cargo compiles the crates both builds share (polyglot-sql, `sqlbuild-analysis`, `sqlbuild-rules` and the rest) once. `make check-build-features` (part of `make check-ci`) fails when the two builds resolve a shared crate differently and names the features to add. |
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. |
 | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. |
+| `sqlbuild-process-ownership` | Proof that a live process owns a scheduler worker claim, and the only way to stop one: an inherited claim lock on Linux and macOS, checked through `/proc` and a pidfd or through libproc with a start-time recheck, and a private named Job object on Windows. It is the only crate allowed unsafe code, for those platform calls. |
 | `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. It also owns execution history: lifecycle event batch digests and conflicts, run projection, event and run cursors, filter and page validation, and the SQLite and PostgreSQL statements the native history backends run. |
 | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. |
 | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. |
@@ -36,7 +37,7 @@ sqlbuild-python
   -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
   -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
   -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
-sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features
+sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features, sqlbuild-process-ownership
 ```
 
 `sqlbuild-scopes` reuses the macro scanner in `sqlbuild-render`; render itself depends only on

diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml
@@ -39,8 +39,8 @@ uuid.workspace = true
 [target.'cfg(unix)'.dependencies]
 rustix.workspace = true
 
-[target.'cfg(target_os = "linux")'.dependencies]
-command-fds.workspace = true
+[target.'cfg(any(target_os = "linux", target_os = "macos", windows))'.dependencies]
+sqlbuild-process-ownership.workspace = true
 
 [target.'cfg(windows)'.dependencies]
 ctrlc.workspace = true

diff --git a/crates/sqlbuild-cli/src/workers/_helpers/child.rs b/crates/sqlbuild-cli/src/workers/_helpers/child.rs
@@ -2,6 +2,11 @@
 
 use sqlbuild_orchestration::store::errors::ControlError;
 
+#[cfg(any(target_os = "linux", target_os = "macos", windows))]
+use sqlbuild_process_ownership::ownership::main::{
+    stop_worker::stop_worker, worker_exited::worker_exited,
+};
+
 use crate::workers::_helpers::supervisor::exit_code;
 use crate::workers::main::terminate_child::terminate_child;
 use crate::workers::models::{WorkerChild, WorkerExit};
@@ -23,8 +28,8 @@ impl WorkerChild {
             Self::Local(child) => Ok(child
                 .try_wait()?
                 .map(|exit| WorkerExit::Code(exit_code(exit)))),
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.try_wait(),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(worker_exited(owner)?.then_some(WorkerExit::Absent)),
             Self::Docker(child) => child.try_wait().await,
         }
     }
@@ -44,8 +49,8 @@ impl WorkerChild {
     pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> {
         match self {
             Self::Local(child) => terminate_child(child),
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(stop_worker(owner, false)?),
             Self::Docker(child) => child.signal("SIGTERM").await,
         }
     }
@@ -56,8 +61,8 @@ impl WorkerChild {
                 child.start_kill()?;
                 Ok(())
             }
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(stop_worker(owner, true)?),
             Self::Docker(child) => child.signal("SIGKILL").await,
         }
     }