feat: recover local workers on macOS and Windows

.github/workflows/verify.yml

@@ -28,6 +28,7 @@ jobs:
       format_performance: ${{ steps.scope.outputs.format_performance }}
       rules_performance: ${{ steps.scope.outputs.rules_performance }}
       wheels: ${{ steps.scope.outputs.wheels }}
+      recovery: ${{ steps.scope.outputs.recovery }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -52,11 +53,12 @@ jobs:
               echo "wheels=$7"
               echo "compiler_differential=$8"
               echo "clickhouse=$9"
+              echo "recovery=${10}"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -68,7 +70,7 @@ jobs:
           if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \
              ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then
             echo "Could not diff ${BASE_SHA:-<none>}...${HEAD_SHA:-<none>}; running every suite" >&2
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -114,7 +116,7 @@ jobs:
                 && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then
             echo "Release Please pull request changes only release metadata; skipping the suites" \
               | tee -a "${GITHUB_STEP_SUMMARY}"
-            write_scope false false false false false false false false false
+            write_scope false false false false false false false false false false
             exit 0
           fi
 
@@ -126,6 +128,7 @@ jobs:
           rules_performance=false
           wheels=false
           compiler_differential=false
+          recovery=false
           broad=false
           changed=false
           # A change that only touches the website, root-level Markdown, the README demo GIFs or the
@@ -146,6 +149,11 @@ jobs:
                 wheels=true
                 ;;
             esac
+            case "${path}" in
+              crates/sqlbuild-orchestration/*|crates/sqlbuild-process-ownership/*|crates/sqlbuild-cli/*|Cargo.toml|Cargo.lock|.github/workflows/verify.yml)
+                recovery=true
+                ;;
+            esac
             case "${path}" in
               scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|tests/goldens/*|tests/e2e/fixtures/*|website/examples/*|.github/workflows/verify.yml|Makefile)
                 compiler_differential=true
@@ -207,7 +215,7 @@ jobs:
           done < "${changed_paths}"
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true true true true
+            write_scope true true true true true true true true true true
             exit 0
           fi
 
@@ -232,7 +240,8 @@ jobs:
             "${checks}" \
             "${wheels}" \
             "${compiler_differential}" \
-            "${clickhouse}"
+            "${clickhouse}" \
+            "${recovery}"
 
   lockfile:
     name: Lockfile consistency
@@ -555,6 +564,64 @@ jobs:
           uv run --no-sync sqb playground --template python_nodes "${nodes}"
           uv run --no-sync sqb --project-dir "${nodes}" --no-color compile
 
+  scheduler_recovery:
+    name: Scheduler recovery (${{ matrix.name }})
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.recovery == 'true'
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 75
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: macOS
+            runner: macos-15
+            test_threads: 3
+          - name: Windows
+            runner: windows-2025
+            test_threads: 2
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: verify-recovery-${{ matrix.name }}
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      # Store transactions on the Windows runner are about 20 times slower than on
+      # Linux, which fences tests that hold one 30-second scheduler lease across
+      # their setup. Real-time scanning of each SQLite open and PostgreSQL write
+      # is the suspected cost; the tests cover the stores, not the scanner.
+      - name: Stop Defender scanning the test stores
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: Set-MpPreference -DisableRealtimeMonitoring $true
+      - name: Start a native PostgreSQL server
+        run: |
+          if [[ "${RUNNER_OS}" == "macOS" ]]; then
+            brew install --quiet postgresql@16
+            bin="$(brew --prefix postgresql@16)/bin"
+          else
+            bin="$(cygpath -u "${PGBIN}")"
+          fi
+          data="${RUNNER_TEMP}/postgres-data"
+          "${bin}/initdb" -D "${data}" -U postgres --auth=trust -E UTF8 > /dev/null
+          "${bin}/pg_ctl" -D "${data}" -o "-p 5433 -c listen_addresses=127.0.0.1" \
+            -l "${RUNNER_TEMP}/postgres.log" -w start
+      - name: Worker claim ownership, recovery and stop
+        run: cargo test --locked -p sqlbuild-process-ownership
+      - name: Control store, scheduler and recovery contracts on SQLite and PostgreSQL
+        env:
+          SQLBUILD_TEST_CONTROL_POSTGRES_URL: postgresql://postgres@127.0.0.1:5433/postgres
+        run: cargo test --locked -p sqlbuild-orchestration --lib -- --test-threads=${{ matrix.test_threads }}
+      - name: PostgreSQL server log
+        if: failure()
+        run: cat "${RUNNER_TEMP}/postgres.log" || true
+
   format_performance:
     name: Formatter performance guards
     needs: [changes, lockfile]
@@ -1086,6 +1153,7 @@ jobs:
         compiler_differential,
         compiler_differential_python_versions,
         native_discovery_windows,
+        scheduler_recovery,
         format_performance,
         performance,
         cold_compile_performance,
@@ -1111,6 +1179,7 @@ jobs:
           COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }}
           COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT: ${{ needs.compiler_differential_python_versions.result }}
           NATIVE_DISCOVERY_WINDOWS_RESULT: ${{ needs.native_discovery_windows.result }}
+          SCHEDULER_RECOVERY_RESULT: ${{ needs.scheduler_recovery.result }}
           FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }}
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
@@ -1143,6 +1212,7 @@ jobs:
             "${COMPILER_DIFFERENTIAL_RESULT}" \
             "${COMPILER_DIFFERENTIAL_PYTHON_VERSIONS_RESULT}" \
             "${NATIVE_DISCOVERY_WINDOWS_RESULT}" \
+            "${SCHEDULER_RECOVERY_RESULT}" \
             "${FORMAT_PERFORMANCE_RESULT}" \
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \

Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "crates/sqlbuild-build-features",
     "crates/sqlbuild-core",
     "crates/sqlbuild-cursor",
+    "crates/sqlbuild-process-ownership",
     "crates/sqlbuild-state",
     "crates/sqlbuild-pysource",
     "crates/sqlbuild-cache",
@@ -33,6 +34,7 @@ order = [
     "sqlbuild-build-features",
     "sqlbuild-core",
     "sqlbuild-cursor",
+    "sqlbuild-process-ownership",
     "sqlbuild-state",
     "sqlbuild-pysource",
     "sqlbuild-cache",
@@ -110,6 +112,7 @@ thiserror = "2"
 # Windows only: `sqb` ignores Ctrl+C while the Python CLI it started handles it.
 ctrlc = "3"
 command-fds = "0.3"
+libc = "0.2"
 quote = "1"
 rayon = "1"
 # Static reading of project Python source; the parser fensu uses.
@@ -119,7 +122,9 @@ ruff_text_size = "=0.0.4"
 # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`.
 # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output.
 rustc-hash = "2"
-# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code in the workspace.
+# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code. The only crate
+# allowed unsafe code is sqlbuild-process-ownership, for the platform calls that prove which
+# process owns a worker claim; every other crate forbids it.
 signal-hook = { version = "0.3", default-features = false }
 # Features signal-hook's registry enables, matched in the sqb build through sqlbuild-build-features.
 errno = "0.3"
@@ -162,6 +167,7 @@ sqlbuild-refactor = { path = "crates/sqlbuild-refactor" }
 sqlbuild-rules = { path = "crates/sqlbuild-rules" }
 sqlbuild-model-config = { path = "crates/sqlbuild-model-config" }
 sqlbuild-orchestration = { path = "crates/sqlbuild-orchestration" }
+sqlbuild-process-ownership = { path = "crates/sqlbuild-process-ownership" }
 sqlbuild-project = { path = "crates/sqlbuild-project" }
 sqlbuild-pysource = { path = "crates/sqlbuild-pysource" }
 sqlbuild-recall = { path = "crates/sqlbuild-recall" }

crates/README.md

@@ -9,6 +9,7 @@ that can be tested and benchmarked with `cargo test` alone.
 | `sqlbuild-build-features` | No code. Enables, in the `sqb` build, the dependency features the extension build resolves, so Cargo compiles the crates both builds share (polyglot-sql, `sqlbuild-analysis`, `sqlbuild-rules` and the rest) once. `make check-build-features` (part of `make check-ci`) fails when the two builds resolve a shared crate differently and names the features to add. |
 | `sqlbuild-core` | Constants shared by several crates, the panic boundary that turns an unwinding native failure into a named compiler error, passive identity, diagnostic and SQL value types, text positions, a JSON emitter whose output matches Python's `json.dumps` and `orjson` byte for byte, and run-cost allocation with exact Python `decimal` default-context arithmetic and Snowflake query-history SQL. |
 | `sqlbuild-cursor` | Microbatch cursor algebra: typed cursor values, grain flooring and stepping, half-open interval clamping, merging and batch splitting, timestamp text normalization and calendar durations, with Python `datetime` results and errors. Zones other than fixed offsets are resolved through a caller-supplied lookup. |
+| `sqlbuild-process-ownership` | Proof that a live process owns a scheduler worker claim, and the only way to stop one: an inherited claim lock on Linux and macOS, checked through `/proc` and a pidfd or through libproc with a start-time recheck, and a private named Job object on Windows. It is the only crate allowed unsafe code, for those platform calls. |
 | `sqlbuild-state` | Append-only warehouse state rows: content-addressed migration event IDs, typed event SQL literals and statements, and source freshness data-version normalization, hashing, equivalence, age policy and latest-state read SQL, byte-compatible with the rows Python wrote. It also owns execution history: lifecycle event batch digests and conflicts, run projection, event and run cursors, filter and page validation, and the SQLite and PostgreSQL statements the native history backends run. |
 | `sqlbuild-pysource` | Static reading of project Python source with the ruff parser: macro module structure, literal SQL arguments, definition locations and string-literal edits. It never imports or runs user code; CPython stays the authority on whether a file is valid Python. |
 | `sqlbuild-cache` | The shared native store every native cache uses (one atomically replaced, checksummed file per cache kind, keyed by content digests) and the one fingerprinting scheme for cache keys: the content digest and the project file fingerprint. |
@@ -36,7 +37,7 @@ sqlbuild-python
   -> sqlbuild-project -> sqlbuild-refactor -> sqlbuild-rules -> sqlbuild-analysis -> sqlbuild-attachments
   -> sqlbuild-model-config -> sqlbuild-scopes -> sqlbuild-render -> sqlbuild-discovery -> sqlbuild-host
   -> sqlbuild-config -> sqlbuild-sqltext -> sqlbuild-cache -> sqlbuild-pysource -> sqlbuild-state -> sqlbuild-cursor -> sqlbuild-core
-sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features
+sqlbuild-cli -> sqlbuild-cache, sqlbuild-core, sqlbuild-build-features, sqlbuild-process-ownership
 ```
 
 `sqlbuild-scopes` reuses the macro scanner in `sqlbuild-render`; render itself depends only on

crates/sqlbuild-cli/Cargo.toml

@@ -39,8 +39,8 @@ uuid.workspace = true
 [target.'cfg(unix)'.dependencies]
 rustix.workspace = true
 
-[target.'cfg(target_os = "linux")'.dependencies]
-command-fds.workspace = true
+[target.'cfg(any(target_os = "linux", target_os = "macos", windows))'.dependencies]
+sqlbuild-process-ownership.workspace = true
 
 [target.'cfg(windows)'.dependencies]
 ctrlc.workspace = true

crates/sqlbuild-cli/src/workers/_helpers/child.rs

@@ -2,6 +2,11 @@
 
 use sqlbuild_orchestration::store::errors::ControlError;
 
+#[cfg(any(target_os = "linux", target_os = "macos", windows))]
+use sqlbuild_process_ownership::ownership::main::{
+    stop_worker::stop_worker, worker_exited::worker_exited,
+};
+
 use crate::workers::_helpers::supervisor::exit_code;
 use crate::workers::main::terminate_child::terminate_child;
 use crate::workers::models::{WorkerChild, WorkerExit};
@@ -23,8 +28,8 @@ impl WorkerChild {
             Self::Local(child) => Ok(child
                 .try_wait()?
                 .map(|exit| WorkerExit::Code(exit_code(exit)))),
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.try_wait(),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(worker_exited(owner)?.then_some(WorkerExit::Absent)),
             Self::Docker(child) => child.try_wait().await,
         }
     }
@@ -44,8 +49,8 @@ impl WorkerChild {
     pub(in crate::workers) async fn terminate(&mut self) -> Result<(), ControlError> {
         match self {
             Self::Local(child) => terminate_child(child),
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::TERM),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(stop_worker(owner, false)?),
             Self::Docker(child) => child.signal("SIGTERM").await,
         }
     }
@@ -56,8 +61,8 @@ impl WorkerChild {
                 child.start_kill()?;
                 Ok(())
             }
-            #[cfg(target_os = "linux")]
-            Self::RecoveredLocal(child) => child.signal(rustix::process::Signal::KILL),
+            #[cfg(any(target_os = "linux", target_os = "macos", windows))]
+            Self::RecoveredLocal(owner) => Ok(stop_worker(owner, true)?),
             Self::Docker(child) => child.signal("SIGKILL").await,
         }
     }