From 1a0cef4ecd12b996d211b449eef916672aba17ff Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 15:17:00 -0700 Subject: [PATCH 01/10] Mint LLP 0041: central-config-driven client actions implementation design Covers the two accepted-but-uncovered decisions LLP 0036 (the action reconciler seam) and LLP 0037 (backfill-on-join, its first instance) with one neutral-minted implementation design. Specifies: the daemon-side action reconciler component and where it fires in the existing lifecycle (the confirmPoll edge + an after-activation already-confirmed pass), an onConfirmed hook on the apply engine, the run-once completion marker (config-control/client-actions.json), the per-plugin backfill config + window_days->--since resolution, subprocess execution of `hyp backfill`, failure-surfaced-not-fatal status, and a clientActions status section. Breaks the work into six independently- mergeable task seams with a matching test strategy, and carries forward the decisions' open questions. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...41-central-config-client-actions.design.md | 426 ++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 llp/0041-central-config-client-actions.design.md diff --git a/llp/0041-central-config-client-actions.design.md b/llp/0041-central-config-client-actions.design.md new file mode 100644 index 0000000..bbebe0c --- /dev/null +++ b/llp/0041-central-config-client-actions.design.md @@ -0,0 +1,426 @@ +# LLP 0041: Central-config-driven client actions — implementation design + +**Type:** design +**Status:** Active +**Systems:** Config, Daemon, Onboarding, Sources +**Generated-by:** neutral +**Related:** LLP 0036, LLP 0037 + +> [LLP 0036](./0036-central-config-driven-client-actions.decision.md) accepted a +> single seam — a daemon-side, idempotent **action reconciler** that performs a +> machine-side effect *because the central config asked*, records it, surfaces +> (never escalates) failure, isolates heavy work, gates on consent, and undoes +> reversible effects on leave. [LLP 0037](./0037-backfill-on-join.decision.md) +> accepted its first instance — **backfill on join** (run-once `hyp backfill` +> when a joined machine confirms a central config with a backfill-capable source +> enabled). Both decisions hold the *rationale*; neither has code. This document +> is the *implementation* design: where the reconciler lives, when it fires in +> the existing daemon lifecycle, the marker shape and location, the subprocess +> launch, the status surface, and the independently-mergeable task seams. + +The decisions are the authority for *why*; this design is constrained by them +and must not relitigate them. Where it makes a fresh choice (e.g. the +confirmation hook mechanism, the marker filename), that choice is called out. + +Coverage anchors (these resolve the two uncovered decisions): + +`@ref LLP 0036 — central-config-driven client action seam` +`@ref LLP 0037 — backfill-on-join instance` + +## What the code already gives us + +The seam plugs into machinery that already exists; nothing here invents a new +config section or a new lifecycle phase. + +- **The confirmation edge.** `confirmPoll()` in + [`src/core/config/apply.js`](../src/core/config/apply.js) clears the post-apply + probation marker on the first successful authenticated poll. It is called by + the central plugin's pull loop on a 200 or 304 + ([`hypaware-core/plugins-workspace/central/src/config_client.js`](../hypaware-core/plugins-workspace/central/src/config_client.js), + the `pull()` 304 and 200 branches). This is exactly the "config confirmed, + probation cleared" trigger point LLP 0036 §When-the-reconciler-runs and + [LLP 0025 §Post-apply probation](./0025-remote-config-join-flow.spec.md#post-apply-probation) + name — and the plugin reports it through the narrow facade, never touching + probation state itself. +- **Kernel-managed state.** Apply bookkeeping lives in one atomically-written + file, `config-control/state.json` (`CONTROL_DIRNAME`/`STATE_BASENAME` in + `apply.js`), under `` = `/hypaware`. The action marker + belongs here too ([LLP 0004 state directories](./0004-activation-and-paths.spec.md#state-directories)), + *not* in a plugin state dir — the reconciler is kernel surface. +- **The daemon is the only host with `configControl`.** `runDaemon` in + [`src/core/daemon/runtime.js`](../src/core/daemon/runtime.js) constructs the + engine and threads it into `bootKernel`; plain CLI boots leave + `ctx.configControl` undefined (`ConfigControlFacade` in + [`collectivus-plugin-kernel-types.d.ts`](../collectivus-plugin-kernel-types.d.ts)). + So a reconciler attached to the daemon is daemon-only by construction — + `hyp status` performs no machine effects. +- **Backfill providers are already enumerable and config-filtered.** + `ctx.backfills.list()` (`createBackfillRegistry` in + [`src/core/registry/backfills.js`](../src/core/registry/backfills.js)) yields + `{ name, plugin, datasets, run }` for every registered provider; the claude + adapter registers one at activation + ([`hypaware-core/plugins-workspace/claude/src/index.js`](../hypaware-core/plugins-workspace/claude/src/index.js), + `ctx.backfills.register(...)`), codex likewise. `selectProviders` in + [`src/core/commands/backfill.js`](../src/core/commands/backfill.js) already + computes "providers whose owning plugin is enabled in the active config" — the + reconciler reuses that exact predicate. +- **A subprocess precedent exists.** `runSmoke` in + [`src/core/cli/core_commands.js`](../src/core/cli/core_commands.js) spawns + `process.execPath bin/hypaware.js …` resolved relative to `import.meta.url`. + The reconciler launches `hyp backfill` the same way. +- **A status-section precedent exists.** `collectHypAwareStatus` in + [`src/core/daemon/status.js`](../src/core/daemon/status.js) already reads + apply state via `readConfigControlStatus({ stateRoot })` into a `remoteConfig` + section without constructing the engine. `clientActions` mirrors it. + +## Part 1 — The action seam (LLP 0036) + +### Where actions are declared (schema) + +No generic `actions[]` schema — confirmed per-instance in +[LLP 0036 §Where actions are declared](./0036-central-config-driven-client-actions.decision.md#where-actions-are-declared) +and [§Open questions](./0036-central-config-driven-client-actions.decision.md#open-questions). +Each instance rides config surface LLP 0031 already governs: + +- **Backfill** rides each source plugin's own `config.backfill` + (`plugins[]` entry) — see Part 2. +- **Attach** (future) rides the client entries the config already names (#126). + +Because both live inside `plugins[]`, the central-vs-local locking +([LLP 0031 §Merge model](./0031-layered-config.decision.md#merge-model)) falls +out with **no new merge rule**: a central-named plugin entry — and the +`backfill`/attach policy inside it — is authoritative; a colliding local entry +is dropped at the boot merge. The seam is the *reconciler*, not a config +section. On a non-joined host there is no central layer, so the reconciler is a +no-op and these stay manual local commands. + +### The reconciler component + +Add a new kernel module +**`src/core/config/action_reconciler.js`** exporting +`createActionReconciler(opts)`. It is constructed by the daemon (like +`createConfigControl`) and is the generic run-once / reconcile-on-config +machinery. It knows nothing about Claude vs Codex. + +```js +createActionReconciler({ + stateRoot, // marker location (config-control/) + now, // injectable clock (test seam) + handlers, // ordered list of ActionHandler (v1: [backfillHandler]) + log, +}) +// → { reconcile({ config, backfills }): Promise, +// readStatus(): ClientActionStatus } +``` + +An **`ActionHandler`** is the registration of detect / perform / (optional) +reverse that LLP 0036 §Options-3 names. v1 ships one: + +```ts +interface ActionHandler { + kind: 'backfill' // marker namespace + status section key + // Enumerate the (requestKey, params) units this handler wants reconciled, + // given the effective config + kernel registries. Pure — no effects. + desired(ctx): DesiredAction[] // [{ requestKey, params }] + // Run-once: has this requestKey already completed? (marker lookup) + // Reconciled/reversible handlers (attach, future) also implement reverse(). + perform(action, ctx): Promise // the effect; subprocess or in-proc + reverse?(requestKey, ctx): Promise +} +``` + +`reconcile()` is **level-triggered**: for each handler it diffs `desired()` +against the persisted marker and acts only on the gap (LLP 0036 — a missed run +is recovered on the next pass). It is safe to call repeatedly; a `done` marker +short-circuits. + +### When the reconciler runs (lifecycle integration) + +Two fire points in `runDaemon`, both **after a config is confirmed, never +mid-apply** (LLP 0036): + +1. **After activation, once, if already confirmed.** Right after the daemon + attaches apply deps and arms the watchdog + (`configControl.attachApplyDeps(...)` / `armProbationWatchdog()` in + `runtime.js`), if the central layer exists and **no probation marker is + active** (the running config already cleared probation on a prior boot), run + one reconcile pass. This is the "at boot / after activation" trigger and + recovers any pass missed while a previous probation was outstanding. +2. **On the confirmation edge.** When `confirmPoll()` *transitions* probation + from active→cleared, fire a reconcile pass. This covers the fresh-join case: + join → apply → restart → probation → first 200/304 clears it → reconcile. + +Mechanism for (2): add an **`onConfirmed` callback** to +`CreateConfigControlOptions` (in +[`src/core/config/types.d.ts`](../src/core/config/types.d.ts)), invoked from +`confirmPoll()` *only when a marker was actually cleared* (the edge, not every +poll). The daemon wires `onConfirmed` to schedule a reconcile pass. This keeps +`apply.js` ignorant of the reconciler (it just emits an edge event), and keeps +the plugin ignorant of both (it still only calls `confirmPoll()`). Chosen over +having the daemon poll `configControl.status()` on every tick: the edge is +precise and avoids a spawn-check every 300 s. + +A small **concurrency guard** in the daemon ensures only one reconcile pass runs +at a time (a confirm edge during an in-flight pass sets a "re-run when done" +flag) and that a pass never runs inside `runTick` — it is its own async task off +the tick loop. + +### Idempotency and completion state + +Marker file: **`config-control/client-actions.json`** (mode 0600), atomically +written via tmp+rename exactly like `state.json`. Shape, namespaced per handler +kind and keyed by request key: + +```jsonc +{ + "backfill": { + "@hypaware/claude": { + "status": "done", // run-once terminal state + "at": "2026-06-25T…Z", + "rows": 1234, + "request_key": "@hypaware/claude" + }, + "@hypaware/codex": { + "status": "failed", // not terminal — retried next pass + "reason": "transcript dir missing", + "last_attempt": "2026-06-25T…Z", + "attempts": 2 + } + } +} +``` + +Two flavours, both off one file (LLP 0036 §Idempotency): + +- **Run-once** (backfill): a `done` entry means *skip forever* — the action is + never auto-run again even though `hyp backfill` is independently idempotent + (`part_id` dedupe). The marker is what makes every subsequent boot *cheap* + (no history re-scan). +- **Reconciled / reversible** (attach, future): the entry records the *current + applied state*; `reverse()` runs on leave/detach when the config no longer + names the effect. The file shape already accommodates this (a second + top-level namespace, e.g. `"attach": {...}`). + +**Request key** (LLP 0036 §request-key open question): v1 backfill keys on the +owning **plugin name** — a per-(machine, provider) boolean. The marker file is +this machine's, so "machine" is implicit. A widened `window_days` does **not** +re-trigger in v1 (strict run-once; manual `hyp backfill` is the re-run path). +The key is structured (an object, not a bare bool) so a later refinement can add +a high-water input without a format break — see [Open questions](#open-questions). + +### Failure is surfaced, not fatal + +A failed action (`hyp backfill` non-zero, transcript dir missing, file not +writable) **does not** roll back the central config and **does not** flip +`overall` to `degraded` in `collectHypAwareStatus` (the gateway is functioning +on a valid config). The marker is **not advanced to `done` on failure** — a +`failed` entry is written (reason + attempt count) and the next reconcile pass +retries it. This mirrors `apply.js`'s structured-but-non-degrading rollback +surface and LLP 0031's dropped-local-entry treatment: loud (its own status line ++ a structured `client_action.*` log) but not an outage signal. + +### Execution isolation + +The handler declares whether its effect is heavy. Backfill is **subprocess** +(unbounded import; the "encoder/large import can't run inline" hazard — see the +parquet-in-daemon memory note); a future attach edit is **in-process** (bounded +file write). `perform()` for the subprocess handler spawns asynchronously and +the reconcile task awaits the child **off the tick loop**, so a multi-minute +import can never wedge `runTick` or grow daemon heap. + +### Consent gating + +Per LLP 0036 §Consent, gating is per-instance, not one global gate: + +- **Backfill — default-on, no per-machine local opt-out.** The reconciler runs + it whenever an enabled backfill provider's plugin entry has + `backfill.on_join` truthy (default true). Suppression is an operator + *scoping* decision (`backfill.on_join: false` in the locked central plugin + entry), not a local override — it rides `plugins[]` locking. +- **Attach — open.** Whether `join` implies consent to user-file edits, or + requires explicit acknowledgement, is deferred to the attach instance + (carried forward in [Open questions](#open-questions)). The handler interface + has a `consent` hook slot so the attach handler can demand acknowledgement + without changing the reconciler. + +### Undo on leave (reversible handlers) + +For reversible handlers, `reverse(requestKey)` runs when `desired()` no longer +names a request key that the marker records as applied — i.e. the central config +stopped naming the effect, or the machine left the fleet (`hyp leave`/detach, +when that lands). Backfill is **not** reversible (imported data stays), so its +handler omits `reverse()` and the reconciler never un-imports. This path is +designed-for but exercised first by the attach follow-up, not v1. + +## Part 2 — Backfill-on-join instance (LLP 0037) + +### Per-plugin capability + config + +"Backfill-capable" is **not a new manifest flag** — a plugin is backfill-capable +iff it registered a `BackfillContribution` (its presence in +`ctx.backfills.list()`). The reconciler's backfill handler enumerates those and +intersects with enabled plugins via the existing `selectProviders` predicate. + +Policy lives in the **owning plugin's own `config` block** (LLP 0037): + +```jsonc +{ "name": "@hypaware/claude", "config": { + "proxy": "@hypaware/ai-gateway", + "backfill": { "on_join": true, "window_days": 30 } +} } +``` + +`backfill.on_join` (default **true**) and `backfill.window_days` are validated by +the **owning plugin's config-section validator** (LLP 0005), the same +`config_sections` / `ConfigRegistry` path `runPerPluginSectionValidators` drives +in [`src/core/config/validate.js`](../src/core/config/validate.js). Task: extend +the claude and codex plugins' config-section schema to accept the `backfill` +sub-object. Core validates nothing new — there is no top-level `backfill` +section. + +### Run-once flow (backfill handler) + +`backfillHandler.desired({ config, backfills })`: + +1. `selectProviders({ requested: [], available: backfills.list(), activePlugins: + config.plugins })` → enabled providers. +2. For each, read the owning plugin entry's `config.backfill`. Skip when + `on_join === false`. Default `on_join` true when absent. +3. Emit `{ requestKey: provider.plugin, params: { plugin, windowDays } }`. + +`backfillHandler.perform(action)`: + +1. Resolve `--since`: if `windowDays` set, `--since (now − windowDays·days)`; + if absent, **omit `--since`** — `hyp backfill` already falls back to + `query.cache.retention.default_days` via `resolveRetentionDays` + ([LLP 0013](./0013-local-query-cache.decision.md)), so the effective span is + naturally bounded by retention. (Equivalent to LLP 0037's "fall back to + retention default_days".) +2. Spawn `process.execPath bin/hypaware.js backfill [--since ] + --json` (the `runSmoke` spawn pattern), inheriting the daemon's `env` + (notably `HYP_HOME`) so the child writes the same cache. +3. On exit 0: parse the `--json` payload (`providers[].rows_written`) → write a + `done` marker with row count. On non-zero / spawn error: write a `failed` + marker (reason + bump `attempts`); the next pass retries. + +### How imported rows reach the server + +Backfill **never talks to the server** (LLP 0037 §Context). `hyp backfill` +lands rows in the local cache tables (`writeRows`/`flushDataset` in +`backfill.js`); the **central forward sink** already drains the cache to the +server on its tick. So a subprocess import reaches the server on its own — no +new wiring. (The forward sink re-reads the whole table, #122 — tracked +separately, does not block this.) + +### Why subprocess, why post-probation + +Both are LLP 0037 restatements, realized here: post-probation because the run +fires only on the `confirmPoll` edge / the boot already-confirmed check (never +against a config that might roll back); subprocess because a months-deep import +is unbounded work that must not wedge the tick loop. + +## Module / seam breakdown (independently-mergeable tasks) + +Ordered so each lands behind the previous but merges on its own. Each names the +files/functions to add or change. + +1. **Reconciler core + marker store** — new + `src/core/config/action_reconciler.js` (`createActionReconciler`, the + `ActionHandler` interface, level-triggered `reconcile()`), marker read/write + helpers (atomic tmp+rename into `config-control/client-actions.json`), and + `readClientActionStatus({ stateRoot })`. Types in + `src/core/config/types.d.ts` (`ClientActionStatus`, `ActionMarker`, + `ActionHandler`). **Unit-testable with an injected handler + clock; no + daemon, no HTTP, no real spawn.** +2. **Confirmation edge hook** — add `onConfirmed` to `CreateConfigControlOptions` + and invoke it from `confirmPoll()` in `src/core/config/apply.js` *only on the + active→cleared transition*. Tiny, isolated; existing apply tests unaffected. +3. **Backfill action handler** — `backfillHandler` (in the reconciler module or + `src/core/config/action_backfill.js`): `desired()` over + `selectProviders` + per-plugin `config.backfill`; `perform()` resolves + `window_days`→`--since` and spawns `hyp backfill --json`. Spawn + helper mirrors `runSmoke` (resolve `bin/hypaware.js` off `import.meta.url`). + **Testable with the spawn injected (assert argv + marker writes).** +4. **Daemon wiring** — in `src/core/daemon/runtime.js`: construct the reconciler + with `[backfillHandler]`, wire `configControl`'s `onConfirmed` to schedule a + pass, run the after-activation already-confirmed pass, and add the + single-flight guard + off-tick async task. Pass `boot.runtime.backfills` and + `boot.config` (effective) into `reconcile()`. +5. **Per-plugin `backfill` config validation** — extend `@hypaware/claude` and + `@hypaware/codex` config-section schemas/validators (manifest + + `ConfigRegistry` registration) to accept `{ on_join, window_days }` + (LLP 0005). Plugin-local; no core schema change. +6. **Status surface** — `src/core/daemon/status.js`: add a `clientActions` + section to `HypAwareStatusReport` (read via `readClientActionStatus`), + per-provider `done` (with when + rows) / `failed` (reason + last attempt) / + `pending` / `n/a` (`on_join:false` or non-joined). Wire into the text/JSON + renderers. **Must not** add to `overall === 'degraded'`. Types in + `src/core/daemon/types.d.ts`. + +## Test strategy + +- **Idempotency / run-once** (task 1, 3): drive `reconcile()` twice against a + fake handler whose `perform` counts calls; assert `perform` runs once, the + second pass is a no-op (`done` marker short-circuits), and a missed pass (no + marker yet) runs on the next call. +- **Confirmation edge** (task 2, 4): with a stubbed `configControl`, assert + `onConfirmed` fires exactly on the probation active→cleared transition and not + on a no-probation poll; assert the daemon schedules exactly one pass per edge. +- **Failure surfacing** (task 3, 6): a `perform` that returns failure writes a + `failed` marker (not `done`), the next pass retries, `attempts` increments, + and `collectHypAwareStatus` reports `failed` without flipping `overall`. +- **window_days resolution** (task 3): `window_days: 30` → `--since` = now−30d; + absent → no `--since` (retention fallback). Assert the spawned argv. +- **Opt-out** (task 3, 5): `backfill.on_join: false` → `desired()` emits nothing + → no spawn, status `n/a`. A central-locked `on_join` cannot be flipped by a + local entry (merge-drop test, reusing the LLP 0031 merge harness). +- **Execution isolation** (task 4): assert the reconcile task runs off the tick + loop (a long fake `perform` does not delay `runTick`). +- **Leave/undo** (designed-for; exercised by the attach follow-up): a reversible + fake handler whose `desired()` drops a previously-applied key triggers + `reverse()` once; backfill's handler has no `reverse()` and never un-imports. +- **End-to-end (hermetic smoke)**: extend the existing fixture-backed backfill + smokes — a seeded join that confirms a config with `@hypaware/claude` enabled + runs `hyp backfill claude` once, lands rows in the cache, writes the `done` + marker, and does not re-run on a second confirmed poll. + +## Risks / open questions + +Carried forward from the decisions; settle as noted. + +- **Attach consent gate** (LLP 0036) — does `join` imply consent to + user-file edits, or require explicit acknowledgement? Open; settle with the + attach handler + onboarding ([LLP 0011](./0011-setup-and-onboarding.decision.md)). + The handler `consent` slot exists so this lands without reworking the + reconciler. +- **Auto re-trigger of run-once actions** (LLP 0036 / 0037) — v1 is strict + run-once (boolean-ish per-(machine, provider) marker). A widened `window_days` + needs a manual `hyp backfill`. The structured marker leaves room for a + high-water-window key that auto-re-imports the new slice (`part_id` dedupe + absorbs overlap); generalise only if a second run-once action wants it. +- **Subprocess resource bounds** (LLP 0037) — should the spawned backfill get a + niceness / memory ceiling so a huge first import doesn't starve live capture? + Likely yes; size it when task 4's plumbing lands (the `perform` spawn is the + single place to add it). +- **Marker reset on cache recreate** (LLP 0037) — a breaking schema change + ([LLP 0030](./0030-session-id-partition-key.decision.md)) recreates the cache + and should re-import. Should the `done` marker reset? Probably; track with the + schema-evolution work ([LLP 0029](./0029-additive-cache-schema-evolution.decision.md)). +- **Ordering relative to first ingest** (LLP 0036) — does attach (start live + routing) need to order deterministically against backfill (import history) on + a fresh join, or is the cache→forward path order-insensitive? Likely the + latter; confirm when the attach instance is designed. +- **Partial-provider failure** (LLP 0037) — the per-provider marker isolates + this (one provider `done`, another `failed`); the status surface (task 6) must + read cleanly in that mixed state — covered by a status test above. + +## References + +- [LLP 0036](./0036-central-config-driven-client-actions.decision.md) — the action seam (the decision this designs) +- [LLP 0037](./0037-backfill-on-join.decision.md) — backfill on join (the first instance this designs) +- [LLP 0011](./0011-setup-and-onboarding.decision.md) — setup and onboarding (the interactive backfill finale this reaches parity with) +- [LLP 0017](./0017-daemon-runtime.decision.md) — daemon runtime / staged restart +- [LLP 0025](./0025-remote-config-join-flow.spec.md) — join flow, apply, probation (the confirmation trigger) +- [LLP 0031](./0031-layered-config.decision.md) — layered config / merge model (plugin-entry locking) +- [LLP 0005](./0005-plugin-manifest.spec.md) — plugin manifest / config_sections (per-plugin `backfill` validation) +- [`src/core/config/apply.js`](../src/core/config/apply.js), [`src/core/daemon/runtime.js`](../src/core/daemon/runtime.js), [`src/core/commands/backfill.js`](../src/core/commands/backfill.js), [`src/core/registry/backfills.js`](../src/core/registry/backfills.js), [`src/core/daemon/status.js`](../src/core/daemon/status.js) — the code this design builds on From d8f502536ef62c8174f9959c6c4dd7beb34c1f4f Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 17:50:30 -0700 Subject: [PATCH 02/10] Mint LLP 0043: central-config-driven client actions plan Refine the LLP 0041 implementation design into six independently-mergeable tasks with explicit code dependencies, so neutral can schedule a parallel first wave (reconciler core, confirmation-edge hook, per-plugin backfill config validation) ahead of the handler, status surface, and daemon wiring. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...0043-central-config-client-actions.plan.md | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 llp/0043-central-config-client-actions.plan.md diff --git a/llp/0043-central-config-client-actions.plan.md b/llp/0043-central-config-client-actions.plan.md new file mode 100644 index 0000000..e2194f3 --- /dev/null +++ b/llp/0043-central-config-client-actions.plan.md @@ -0,0 +1,122 @@ +# LLP 0043: Central-config-driven client actions — plan + +**Type:** plan +**Status:** Active +**Related:** LLP 0041 +**Generated-by:** neutral +**Systems:** Config, Daemon, Onboarding, Sources + +> [LLP 0041](./0041-central-config-client-actions.design.md) is the implementation +> design for the daemon-side **action reconciler** (LLP 0036) and its first +> instance, **backfill on join** (LLP 0037). It already named the files and +> functions to add or change and grouped them into six independently-mergeable +> seams. This plan turns those seams into a task graph with real code +> dependencies, so the first wave can parallelize and each task merges on its own +> without leaving the tree broken. + +## How this refines the design + +The design's "Module / seam breakdown" lists six modules. This plan keeps that +exact decomposition — it is already minimal and each seam is independently +testable — and only makes the dependency edges explicit so neutral can schedule +them. + +The shape of the graph: + +- **Foundational / independent (deps `[]`).** Three tasks have no in-repo + dependency on each other and form the first wave: + - **T1 — reconciler core + marker store.** The new + `src/core/config/action_reconciler.js` (`createActionReconciler`, the + `ActionHandler` interface, level-triggered `reconcile()`), the atomic + marker read/write into `config-control/client-actions.json`, + `readClientActionStatus({ stateRoot })`, and the new types + (`ClientActionStatus`, `ActionMarker`, `ActionHandler`) in + `src/core/config/types.d.ts`. Unit-testable with an injected fake handler + and clock — no daemon, no HTTP, no real spawn. This is the contract every + other action task binds to, so it is the spine of the graph. + - **T2 — confirmation-edge hook.** Add `onConfirmed` to + `CreateConfigControlOptions` (`src/core/config/types.d.ts`) and invoke it + from `confirmPoll()` in `src/core/config/apply.js` *only on the probation + active→cleared transition*. Self-contained edit to the apply engine; existing + apply tests are unaffected because no caller wires the hook yet. Independent + of T1 — it touches a different surface and ships a no-op edge event until the + daemon (T4) consumes it. + - **T5 — per-plugin `backfill` config validation.** Extend `@hypaware/claude` + and `@hypaware/codex` to accept `{ on_join, window_days }` inside their own + plugin `config` block — a `config_sections` manifest entry plus the section + validator the kernel `ConfigRegistry` drives via + `runPerPluginSectionValidators` (LLP 0005). Plugin-local; no core schema + change; no dependency on the reconciler. It only needs to land before the + end-to-end smoke proves an opt-out, not before the handler compiles. + +- **Handler (deps `[T1]`).** **T3 — backfill action handler.** `backfillHandler` + (`desired()` over `selectProviders` + per-plugin `config.backfill`; `perform()` + resolving `window_days`→`--since` and spawning `hyp backfill --json` + via the `runSmoke` spawn pattern). It implements the `ActionHandler` interface + defined in T1, so it depends on T1 and nothing else (the spawn is injected for + tests; it does not need the daemon or the plugin schema to compile). + +- **Status (deps `[T1]`).** **T6 — status surface.** Add a `clientActions` + section to `HypAwareStatusReport` in `src/core/daemon/status.js`, read via + `readClientActionStatus`, rendered per-provider (`done` / `failed` / `pending` + / `n/a`) and explicitly excluded from `overall === 'degraded'`; types in + `src/core/daemon/types.d.ts`. It consumes the marker store / status reader from + T1 and is otherwise independent of the handler and the daemon wiring (it reads + the marker file, it never runs a pass). + +- **Integration (deps `[T1, T2, T3]`).** **T4 — daemon wiring.** In + `src/core/daemon/runtime.js`: construct the reconciler with `[backfillHandler]`, + wire `configControl`'s `onConfirmed` to schedule a pass, run the + after-activation already-confirmed pass, and add the single-flight guard + + off-tick async task; pass `boot.runtime.backfills` and `boot.config` into + `reconcile()`. This is the only task that needs all three of the reconciler + core (T1), the confirmation edge (T2), and the handler it runs (T3). It does + **not** depend on T5 (validation) or T6 (status) — an unvalidated `backfill` + block or a missing status line does not stop the daemon from running a pass, + and the end-to-end smoke that exercises the full join→backfill flow rides on + T4 with T5 already merged in the first wave. + +This yields a 3-wide first wave (T1, T2, T5), a 2-wide second wave (T3, T6 once +T1 lands), and the single integration task (T4) last. Each task leaves the tree +green: a merged-but-unwired reconciler (T1), a fired-but-unconsumed edge (T2), a +defined-but-uninstantiated handler (T3), accepted-but-unused config keys (T5), +and a status section that reads `pending`/`n/a` until a pass runs (T6) are all +inert until T4 connects them. + +## Test ownership per task + +- **T1:** run-once idempotency (drive `reconcile()` twice against a counting fake + handler; second pass is a `done` short-circuit; a missed pass runs next call); + atomic marker read/write round-trip. +- **T2:** `onConfirmed` fires exactly on the active→cleared transition, not on a + no-probation poll (stubbed state). +- **T3:** `desired()` opt-out (`on_join:false` → no action), `window_days`→ + `--since` resolution and the retention fallback (assert spawned argv), and a + `failed` `perform` writing a `failed` marker that retries with bumped + `attempts` (spawn injected). +- **T4:** the daemon schedules exactly one pass per edge; the pass runs off the + tick loop (a long fake `perform` does not delay `runTick`); the boot + already-confirmed pass fires when no probation marker is active. +- **T5:** the claude/codex section validator accepts `{ on_join, window_days }` + and rejects malformed values; the central-locked `on_join` cannot be flipped by + a colliding local entry (reuse the LLP 0031 merge-drop harness). +- **T6:** mixed `done`/`failed`/`pending`/`n/a` reads cleanly and a `failed` + backfill does not flip `overall` to `degraded`. +- **End-to-end (hermetic smoke, lands with T4):** a seeded join that confirms a + config with `@hypaware/claude` enabled runs `hyp backfill claude` once, lands + rows, writes the `done` marker, and does not re-run on a second confirmed poll. + +## Tasks +- id: T1 branch: task/central-config-client-actions/T1 deps: [] -- Reconciler core + marker store: createActionReconciler, ActionHandler interface, level-triggered reconcile(), atomic config-control/client-actions.json read/write, readClientActionStatus, and types in src/core/config/types.d.ts. Unit-testable with an injected handler + clock. +- id: T2 branch: task/central-config-client-actions/T2 deps: [] -- Confirmation-edge hook: add onConfirmed to CreateConfigControlOptions and invoke it from confirmPoll() in src/core/config/apply.js only on the probation active->cleared transition. +- id: T5 branch: task/central-config-client-actions/T5 deps: [] -- Per-plugin backfill config validation: extend @hypaware/claude and @hypaware/codex config_sections (manifest + ConfigRegistry section validator) to accept { on_join, window_days }. Plugin-local; no core schema change. +- id: T3 branch: task/central-config-client-actions/T3 deps: [T1] -- Backfill action handler: backfillHandler.desired() over selectProviders + per-plugin config.backfill, and perform() resolving window_days->--since and spawning hyp backfill --json via the runSmoke spawn pattern. Implements the T1 ActionHandler interface; spawn injected for tests. +- id: T6 branch: task/central-config-client-actions/T6 deps: [T1] -- Status surface: add a clientActions section to HypAwareStatusReport in src/core/daemon/status.js (read via readClientActionStatus), per-provider done/failed/pending/n-a, wired into text+JSON renderers and excluded from overall=degraded; types in src/core/daemon/types.d.ts. +- id: T4 branch: task/central-config-client-actions/T4 deps: [T1, T2, T3] -- Daemon wiring in src/core/daemon/runtime.js: construct the reconciler with [backfillHandler], wire onConfirmed to schedule a pass, run the after-activation already-confirmed pass, add the single-flight guard + off-tick async task, and pass boot.runtime.backfills + boot.config into reconcile(). + +## References + +- [LLP 0041](./0041-central-config-client-actions.design.md) — the implementation design this plan schedules +- [LLP 0036](./0036-central-config-driven-client-actions.decision.md) — the action seam +- [LLP 0037](./0037-backfill-on-join.decision.md) — backfill on join (the first instance) +- [`src/core/config/apply.js`](../src/core/config/apply.js), [`src/core/daemon/runtime.js`](../src/core/daemon/runtime.js), [`src/core/daemon/status.js`](../src/core/daemon/status.js), [`src/core/commands/backfill.js`](../src/core/commands/backfill.js), [`src/core/registry/backfills.js`](../src/core/registry/backfills.js) — the code these tasks build on From 457e0c991eb4c0cc306ba5e2e56960efe2c8f599 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 20:40:17 -0700 Subject: [PATCH 03/10] T2: confirmation-edge onConfirmed hook in config apply engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an optional `onConfirmed` callback to CreateConfigControlOptions and invoke it from confirmPoll() exactly once on the probation active→cleared transition (the early no-probation return guards every other poll). This emits a precise confirmation edge the daemon can wire to schedule an action-reconciler pass without polling configControl.status() each tick, while keeping apply.js ignorant of the reconciler. Ships a no-op edge until the daemon (T4) consumes it; existing apply tests are unaffected. Adds a test asserting the hook fires on the active→cleared edge and not on a no-probation poll. Annotates the edge with @ref LLP 0041. Task-Id: T2 --- src/core/config/apply.js | 10 +++++++++- src/core/config/types.d.ts | 9 +++++++++ test/core/config-apply.test.js | 25 ++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/core/config/apply.js b/src/core/config/apply.js index 4d94ec2..21287fb 100644 --- a/src/core/config/apply.js +++ b/src/core/config/apply.js @@ -154,7 +154,7 @@ export function resetCentralLayerToSeed(stateRoot) { * @ref LLP 0025#apply-engine-is-kernel-surface [implements] — the engine is kernel-owned; plugins only see the narrow facade */ export function createConfigControl(opts) { - const { stateRoot, requestRestart } = opts + const { stateRoot, requestRestart, onConfirmed } = opts const now = opts.now ?? Date.now const log = getLogger('config-control') const controlDir = path.join(stateRoot, CONTROL_DIRNAME) @@ -426,6 +426,13 @@ export function createConfigControl(opts) { return { action: /** @type {const} */ ('none') } } + // Confirmation edge: clear the post-apply probation marker on the first + // authenticated poll, then fire `onConfirmed` *only* when a marker was + // actually cleared — the active→cleared transition, not every poll. The + // early `!state.probation` return makes this exactly the edge, so the + // daemon can schedule one reconcile pass per confirmation without polling + // status each tick. apply.js stays ignorant of the reconciler. + // @ref LLP 0041#when-the-reconciler-runs-lifecycle-integration [implements] — onConfirmed fires once on the probation active→cleared edge so the daemon schedules a reconcile pass without per-tick status polling function confirmPoll() { const state = readState() if (!state.probation) return @@ -439,6 +446,7 @@ export function createConfigControl(opts) { config_etag: etag, status: 'ok', }) + if (onConfirmed) onConfirmed(etag) } /** diff --git a/src/core/config/types.d.ts b/src/core/config/types.d.ts index 849a225..1c80400 100644 --- a/src/core/config/types.d.ts +++ b/src/core/config/types.d.ts @@ -264,6 +264,15 @@ export interface CreateConfigControlOptions { stateRoot: string /** Staged restart hook; the daemon exits with the restart code. */ requestRestart(reason: string): void + /** + * Confirmation-edge hook, fired by `confirmPoll()` exactly once on the + * probation active→cleared transition (never on a no-probation poll). + * `etag` is the revision whose probation just cleared. The daemon wires + * this to schedule an action-reconciler pass; `apply.js` stays ignorant + * of the reconciler and only emits the edge event (LLP 0041). Optional — + * a plain CLI boot leaves it unset and the edge is a no-op. + */ + onConfirmed?: (etag: string) => void now?: () => number } diff --git a/test/core/config-apply.test.js b/test/core/config-apply.test.js index 47fc67d..c8ac522 100644 --- a/test/core/config-apply.test.js +++ b/test/core/config-apply.test.js @@ -97,12 +97,15 @@ function makeDeps(opts = {}) { function makeControl({ stateRoot, now }) { /** @type {string[]} */ const restarts = [] + /** @type {string[]} */ + const confirmedEdges = [] const control = createConfigControl({ stateRoot, requestRestart: (reason) => { restarts.push(reason) }, + onConfirmed: (etag) => { confirmedEdges.push(etag) }, ...(now ? { now } : {}), }) - return { control, restarts } + return { control, restarts, confirmedEdges } } test('stage applies a document: slot persisted, pointer flipped, etag staged, probation armed, restart requested', async () => { @@ -311,6 +314,26 @@ test('confirmPoll clears probation', async () => { control.confirmPoll() }) +test('onConfirmed fires exactly on the probation active→cleared edge, not on a no-probation poll', async () => { + const { stateRoot } = await makeFixture() + + // A poll before anything is under probation is not an edge. + const idle = makeControl({ stateRoot }) + idle.control.attachApplyDeps(makeDeps()) + idle.control.confirmPoll() + assert.deepEqual(idle.confirmedEdges, []) + + // Apply puts a revision under probation; the first confirming poll is the + // active→cleared edge and fires the hook once with the cleared etag. + await idle.control.stage(REMOTE_CONFIG, 'etag-1') + idle.control.confirmPoll() + assert.deepEqual(idle.confirmedEdges, ['etag-1']) + + // Further polls with no active probation do not re-fire the edge. + idle.control.confirmPoll() + assert.deepEqual(idle.confirmedEdges, ['etag-1']) +}) + test('chained applies alternate slots and roll back one revision', async () => { const { stateRoot } = await makeFixture() From aa7cb752f2b613f7be03b58efdfdcdbc031bb505 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 20:46:28 -0700 Subject: [PATCH 04/10] T1: action reconciler core + client-actions marker store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the generic, daemon-constructed client-action reconciler (LLP 0036 / LLP 0041) and its marker store — the spine every other action task binds to. - src/core/config/action_reconciler.js: createActionReconciler({ stateRoot, handlers, now, log }) -> { reconcile, readStatus }. reconcile() is level-triggered: per handler, diff desired() against the persisted markers and act only on the gap; a done marker short-circuits (run-once), a failed marker retries with bumped attempts, and reversible handlers undo a key the config no longer names. Atomic tmp+rename read/write (mode 0600) of config-control/client-actions.json, beside the apply engine's state.json. Standalone readClientActionStatus({ stateRoot }) for the status surface. - src/core/config/types.d.ts: ActionHandler, ActionMarker, ActionMarkerStore, DesiredAction, ActionOutcome, ActionContext, ReconcileInput/Report, ClientActionStatus, ActionReconciler, CreateActionReconcilerOptions. - test/core/action-reconciler.test.js: run-once idempotency + done short-circuit, missed-pass recovery, atomic marker round-trip (mode/newline), failed-then-done retry with attempts, thrown-perform normalization, a desired() throw not wedging other handlers, and the reverse path. Unit-testable with an injected handler + clock; no daemon, HTTP, or real spawn. Inert until the daemon wires it (T4). Task-Id: T1 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/core/config/action_reconciler.js | 303 ++++++++++++++++++++++++ src/core/config/types.d.ts | 186 +++++++++++++++ test/core/action-reconciler.test.js | 341 +++++++++++++++++++++++++++ 3 files changed, 830 insertions(+) create mode 100644 src/core/config/action_reconciler.js create mode 100644 test/core/action-reconciler.test.js diff --git a/src/core/config/action_reconciler.js b/src/core/config/action_reconciler.js new file mode 100644 index 0000000..5170f7f --- /dev/null +++ b/src/core/config/action_reconciler.js @@ -0,0 +1,303 @@ +// @ts-check + +import fs from 'node:fs' +import path from 'node:path' + +import { Attr, getLogger } from '../observability/index.js' + +/** + * @import { + * ActionContext, + * ActionHandler, + * ActionMarker, + * ActionMarkerStore, + * ActionOutcome, + * ActionReconciler, + * ClientActionStatus, + * CreateActionReconcilerOptions, + * DesiredAction, + * ReconcileActionResult, + * ReconcileInput, + * ReconcileReport, + * } from './types.d.ts' + */ + +/** + * The kernel state subdirectory the reconciler shares with the apply + * engine. Must match `CONTROL_DIRNAME` in `apply.js`: the marker is kernel + * surface and belongs beside `state.json`, not in a plugin state dir + * (LLP 0041 — "the action marker belongs here too, not in a plugin state + * dir — the reconciler is kernel surface"). + */ +const CONTROL_DIRNAME = 'config-control' + +/** + * The action-marker file. Namespaced per handler kind and keyed by request + * key; written atomically (tmp+rename, mode 0600) exactly like `state.json`. + * @ref LLP 0041#idempotency-and-completion-state [implements] — marker file config-control/client-actions.json, atomic tmp+rename, namespaced per handler kind + */ +const CLIENT_ACTIONS_BASENAME = 'client-actions.json' + +/** + * Build the generic, daemon-constructed action reconciler. It is the + * run-once / reconcile-on-config machinery and knows nothing about Claude + * vs Codex — only the {@link ActionHandler} interface. The daemon wires + * its `reconcile()` to the config-confirmation edge and the + * after-activation already-confirmed pass. + * + * @param {CreateActionReconcilerOptions} opts + * @returns {ActionReconciler} + * @ref LLP 0041#the-reconciler-component [implements] — createActionReconciler(opts) → { reconcile, readStatus }, constructed by the daemon like createConfigControl + * @ref LLP 0036 — central-config-driven client action seam (the decision this realizes) + */ +export function createActionReconciler(opts) { + const { stateRoot, handlers } = opts + const now = opts.now ?? Date.now + const log = opts.log ?? getLogger('action-reconciler') + const controlDir = path.join(stateRoot, CONTROL_DIRNAME) + const markerPath = path.join(controlDir, CLIENT_ACTIONS_BASENAME) + + /** @returns {ActionMarkerStore} */ + function readStore() { + return readMarkerStore(markerPath) + } + + /** @param {ActionMarkerStore} store */ + function writeStore(store) { + fs.mkdirSync(controlDir, { recursive: true, mode: 0o700 }) + const tmp = `${markerPath}.tmp.${process.pid}.${now()}` + fs.writeFileSync(tmp, JSON.stringify(store, null, 2) + '\n', { mode: 0o600 }) + fs.renameSync(tmp, markerPath) + } + + /** + * Level-triggered reconcile (LLP 0036): for each handler, diff `desired()` + * against the persisted markers and act only on the gap. A `done` marker + * short-circuits, so the pass is safe to call repeatedly and a run missed + * while probation was outstanding is recovered on the next call. + * + * @param {ReconcileInput} input + * @returns {Promise} + * @ref LLP 0041#the-reconciler-component [implements] — reconcile() is level-triggered: diff desired() against the marker, act only on the gap; a done marker short-circuits + */ + async function reconcile(input) { + /** @type {ActionContext} */ + const ctx = { config: input.config, backfills: input.backfills, now, log } + const store = readStore() + /** @type {ReconcileActionResult[]} */ + const results = [] + let mutated = false + + for (const handler of handlers) { + const kind = handler.kind + const markers = store[kind] ?? {} + + /** @type {DesiredAction[]} */ + let desired + try { + desired = handler.desired(ctx) ?? [] + } catch (err) { + // A handler whose detect step throws must not wedge the others. + log.error('client_action.desired_failed', { + [Attr.COMPONENT]: 'action-reconciler', + [Attr.OPERATION]: 'client_action.desired', + kind, + [Attr.STATUS]: 'failed', + [Attr.ERROR_KIND]: 'handler_desired_threw', + detail: err instanceof Error ? err.message : String(err), + }) + continue + } + + const desiredKeys = new Set(desired.map((d) => d.requestKey)) + + // Forward gap: run-once / retry the desired units not yet `done`. + for (const action of desired) { + const existing = markers[action.requestKey] + if (existing && existing.status === 'done') { + results.push({ kind, requestKey: action.requestKey, outcome: 'skipped' }) + continue + } + + const outcome = await runOutcome(() => handler.perform(action, ctx)) + const at = new Date(now()).toISOString() + if (outcome.status === 'done') { + /** @type {ActionMarker} */ + const marker = { + status: 'done', + request_key: action.requestKey, + at, + ...(typeof outcome.rows === 'number' ? { rows: outcome.rows } : {}), + ...(outcome.detail ?? {}), + } + markers[action.requestKey] = marker + results.push({ + kind, + requestKey: action.requestKey, + outcome: 'done', + ...(typeof outcome.rows === 'number' ? { rows: outcome.rows } : {}), + }) + log.info('client_action.done', { + [Attr.COMPONENT]: 'action-reconciler', + [Attr.OPERATION]: 'client_action.perform', + kind, + request_key: action.requestKey, + [Attr.STATUS]: 'ok', + ...(typeof outcome.rows === 'number' ? { rows: outcome.rows } : {}), + }) + } else { + // Not advanced to `done` on failure (LLP 0041 §failure is + // surfaced, not fatal): record `failed` + bump attempts so the + // next pass retries. Loud (its own status line) but not an outage. + const attempts = (typeof existing?.attempts === 'number' ? existing.attempts : 0) + 1 + const reason = outcome.reason ?? 'unknown' + /** @type {ActionMarker} */ + const marker = { + status: 'failed', + request_key: action.requestKey, + reason, + last_attempt: at, + attempts, + ...(outcome.detail ?? {}), + } + markers[action.requestKey] = marker + results.push({ kind, requestKey: action.requestKey, outcome: 'failed', reason, attempts }) + log.error('client_action.failed', { + [Attr.COMPONENT]: 'action-reconciler', + [Attr.OPERATION]: 'client_action.perform', + kind, + request_key: action.requestKey, + [Attr.STATUS]: 'failed', + [Attr.ERROR_KIND]: 'action_perform_failed', + attempts, + detail: reason, + }) + } + mutated = true + } + + // Reverse gap: only reversible handlers undo a previously-applied key + // the config no longer names (leave/detach). Run-once handlers + // (backfill) omit reverse() — imported data stays, the marker is kept, + // and this loop is skipped (LLP 0041 §Undo on leave). + const reverse = handler.reverse + if (typeof reverse === 'function') { + for (const requestKey of Object.keys(markers)) { + if (desiredKeys.has(requestKey)) continue + const marker = markers[requestKey] + if (!marker || marker.status === 'failed') { + // A failed marker for a no-longer-desired key never applied an + // effect, so there is nothing to undo — just drop it. + delete markers[requestKey] + mutated = true + continue + } + const outcome = await runOutcome(() => reverse(requestKey, ctx)) + if (outcome.status === 'done') { + delete markers[requestKey] + results.push({ kind, requestKey, outcome: 'reversed' }) + mutated = true + log.info('client_action.reversed', { + [Attr.COMPONENT]: 'action-reconciler', + [Attr.OPERATION]: 'client_action.reverse', + kind, + request_key: requestKey, + [Attr.STATUS]: 'ok', + }) + } else { + // Reverse failed: keep the marker so the next pass retries the + // undo; surface but do not escalate. + results.push({ kind, requestKey, outcome: 'failed', reason: outcome.reason ?? 'unknown' }) + log.error('client_action.reverse_failed', { + [Attr.COMPONENT]: 'action-reconciler', + [Attr.OPERATION]: 'client_action.reverse', + kind, + request_key: requestKey, + [Attr.STATUS]: 'failed', + [Attr.ERROR_KIND]: 'action_reverse_failed', + detail: outcome.reason ?? 'unknown', + }) + } + } + } + + // Persist a non-empty kind bucket only when it actually has markers, so + // a no-op handler never writes an empty namespace into the file. + if (Object.keys(markers).length > 0) { + store[kind] = markers + } else { + delete store[kind] + } + } + + if (mutated) writeStore(store) + return { results } + } + + /** @returns {ClientActionStatus} */ + function readStatus() { + return { byKind: readStore() } + } + + return { reconcile, readStatus } +} + +/** + * Invoke a handler hook and normalize a throw into a `failed` outcome, so a + * handler that rejects is treated identically to one that returns + * `{ status: 'failed' }` — the marker records the failure and the next pass + * retries. + * + * @param {() => Promise} fn + * @returns {Promise} + */ +async function runOutcome(fn) { + try { + const outcome = await fn() + if (outcome && (outcome.status === 'done' || outcome.status === 'failed')) return outcome + return { status: 'failed', reason: 'handler returned no outcome' } + } catch (err) { + return { status: 'failed', reason: err instanceof Error ? err.message : String(err) } + } +} + +/** + * Read the persisted marker store. ENOENT (no action has ever run) is the + * empty store; a non-object document is treated as empty. Mirrors + * `readControlState` in `apply.js`. + * + * @param {string} markerPath + * @returns {ActionMarkerStore} + */ +function readMarkerStore(markerPath) { + let raw + try { + raw = fs.readFileSync(markerPath, 'utf8') + } catch (err) { + if (/** @type {NodeJS.ErrnoException} */ (err).code === 'ENOENT') return {} + throw err + } + const parsed = JSON.parse(raw) + return parsed && typeof parsed === 'object' ? /** @type {ActionMarkerStore} */ (parsed) : {} +} + +/** + * Read-only view of the client-action markers for `hyp status` — usable + * from any process (the CLI is not the daemon), so it never constructs the + * reconciler or takes its handlers. Mirrors `readConfigControlStatus`. + * + * @param {{ stateRoot: string }} args + * @returns {ClientActionStatus} + * @ref LLP 0041#idempotency-and-completion-state [implements] — read-only marker view for the status surface, no engine construction + */ +export function readClientActionStatus({ stateRoot }) { + const markerPath = path.join(stateRoot, CONTROL_DIRNAME, CLIENT_ACTIONS_BASENAME) + /** @type {ActionMarkerStore} */ + let store = {} + try { + store = readMarkerStore(markerPath) + } catch { + // unreadable markers surface as empty — status is best-effort + } + return { byKind: store } +} diff --git a/src/core/config/types.d.ts b/src/core/config/types.d.ts index 849a225..15d53c1 100644 --- a/src/core/config/types.d.ts +++ b/src/core/config/types.d.ts @@ -8,6 +8,9 @@ import type { CapabilityName, ConfigRegistry, ValidationError, + BackfillRegistry, + PluginLogger, + JsonObject, } from '../../../collectivus-plugin-kernel-types.d.ts' /** @@ -267,4 +270,187 @@ export interface CreateConfigControlOptions { now?: () => number } +// ============================================================================= +// Client-action reconciler (LLP 0036 / LLP 0041) +// ============================================================================= + +/** + * Recorded state of a single reconciled action, persisted in + * `config-control/client-actions.json`. + * + * - `done` — run-once terminal state; the action is never auto-run again + * (the marker is what makes every subsequent boot cheap). See LLP 0036 + * §Idempotency. + * - `failed` — not terminal; the next reconcile pass retries it. + * - `applied` — current applied state of a reconciled/reversible handler + * (attach, future); `reverse()` runs on leave when the config stops + * naming the effect. + */ +export type ActionMarkerStatus = 'done' | 'failed' | 'applied' + +/** + * One persisted action marker, namespaced by handler `kind` then keyed by + * `request_key` inside `config-control/client-actions.json`. The key is a + * structured object (not a bare boolean) so a later refinement can add a + * high-water input without a format break (LLP 0036 §request-key, + * LLP 0041 §Idempotency-and-completion-state). Handlers may attach extra + * fields via `ActionOutcome.detail`. + */ +export interface ActionMarker { + status: ActionMarkerStatus + /** The reconciled unit's request key (echoed for self-describing files). */ + request_key: string + /** ISO time the action reached `done`. */ + at?: string + /** Rows written by a run-once import (recorded on `done`). */ + rows?: number + /** Human-readable failure reason (recorded on `failed`). */ + reason?: string + /** ISO time of the most recent attempt (recorded on `failed`). */ + last_attempt?: string + /** Attempts so far; bumped each `failed` pass (recorded on `failed`). */ + attempts?: number + /** Handler-specific extra fields merged from `ActionOutcome.detail`. */ + [extra: string]: unknown +} + +/** + * Persisted marker store: the whole `client-actions.json` document, + * namespaced by handler `kind` (e.g. `backfill`) then keyed by request key + * (e.g. the owning plugin name). + */ +export type ActionMarkerStore = Record> + +/** + * A unit the reconciler should converge, emitted by `ActionHandler.desired()`. + * `params` is handler-specific and not persisted — it is passed straight to + * `perform()` (e.g. backfill carries `{ plugin, windowDays }`). + */ +export interface DesiredAction { + requestKey: string + params?: Record +} + +/** + * Result of an `ActionHandler.perform()` / `reverse()` call. The reconciler + * turns this into the persisted {@link ActionMarker} (adding timestamps and + * the attempt counter); `detail` is merged onto the marker verbatim. + */ +export interface ActionOutcome { + /** `done` = the effect applied/reversed cleanly; `failed` = retry next pass. */ + status: 'done' | 'failed' + /** Rows written (run-once import); recorded on the `done` marker. */ + rows?: number + /** Failure reason; recorded on the `failed` marker. */ + reason?: string + /** Extra handler-specific fields merged into the persisted marker. */ + detail?: JsonObject +} + +/** + * Context handed to every handler hook on each pass. It is the + * {@link ReconcileInput} (effective config + kernel registries) augmented + * with the reconciler's injected clock and logger so a handler need not + * close over them itself. + */ +export interface ActionContext { + /** Effective (merged) config the daemon booted (LLP 0031). */ + config: HypAwareV2Config + /** Kernel backfill registry — `list()` yields enabled-or-not providers. */ + backfills: BackfillRegistry + /** Injectable clock (test seam). */ + now: () => number + log: PluginLogger +} + +/** + * A registered detect / perform / (optional) reverse triple — the unit the + * reconciler drives. The reconciler is generic: it knows nothing about + * Claude vs Codex, only this interface (LLP 0036 §Options-3, LLP 0041). + */ +export interface ActionHandler { + /** Marker namespace + status section key (e.g. `backfill`). */ + kind: string + /** + * Enumerate the units this handler wants reconciled, given the effective + * config + registries. Pure — no effects. + */ + desired(ctx: ActionContext): DesiredAction[] + /** Run the effect for one desired action (subprocess or in-proc). */ + perform(action: DesiredAction, ctx: ActionContext): Promise + /** + * Undo a previously-applied effect whose request key the config no longer + * names (leave/detach). Run-once handlers (backfill) omit this — imported + * data stays and the marker is kept. Reversible handlers (attach, future) + * implement it. + */ + reverse?(requestKey: string, ctx: ActionContext): Promise +} + +/** Arguments to one {@link ActionReconciler.reconcile} pass. */ +export interface ReconcileInput { + config: HypAwareV2Config + backfills: BackfillRegistry +} + +/** What the reconciler did with one (handler, requestKey) unit on a pass. */ +export interface ReconcileActionResult { + kind: string + requestKey: string + /** + * - `done` — `perform()` succeeded this pass; marker advanced to `done`. + * - `skipped` — a `done` marker already existed (run-once short-circuit). + * - `failed` — `perform()`/`reverse()` failed; marker recorded `failed`. + * - `reversed` — `reverse()` succeeded; marker removed. + */ + outcome: 'done' | 'skipped' | 'failed' | 'reversed' + rows?: number + reason?: string + attempts?: number +} + +/** Summary of one reconcile pass. */ +export interface ReconcileReport { + results: ReconcileActionResult[] +} + +/** + * Read-only client-action status for `hyp status`, usable from any process + * (it never constructs the reconciler). Mirrors `ConfigControlStatus`. + */ +export interface ClientActionStatus { + /** Persisted markers, namespaced by handler kind. Empty when none ran. */ + byKind: ActionMarkerStore +} + +/** + * Daemon-only handle to the action reconciler. Constructed like + * `createConfigControl`; the daemon wires its `reconcile()` to the + * config-confirmation edge and the after-activation already-confirmed pass. + */ +export interface ActionReconciler { + /** + * Level-triggered: for each handler, diff `desired()` against the + * persisted markers and act only on the gap (a missed run is recovered on + * the next pass). Safe to call repeatedly; a `done` marker short-circuits. + */ + reconcile(input: ReconcileInput): Promise + /** Current persisted markers (same shape as `readClientActionStatus`). */ + readStatus(): ClientActionStatus +} + +export interface CreateActionReconcilerOptions { + /** + * Kernel state root (`/hypaware`). The marker file lives at + * `/config-control/client-actions.json`, alongside the apply + * engine's `state.json` (LLP 0041 — the reconciler is kernel surface). + */ + stateRoot: string + /** Ordered handlers; v1 ships `[backfillHandler]`. */ + handlers: ActionHandler[] + /** Injectable clock (test seam); defaults to `Date.now`. */ + now?: () => number + log?: PluginLogger +} + export type { ConfigStageResult, ConfigApplyErrorKind } diff --git a/test/core/action-reconciler.test.js b/test/core/action-reconciler.test.js new file mode 100644 index 0000000..b24550d --- /dev/null +++ b/test/core/action-reconciler.test.js @@ -0,0 +1,341 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { + createActionReconciler, + readClientActionStatus, +} from '../../src/core/config/action_reconciler.js' + +/** + * @import { + * ActionContext, + * ActionHandler, + * ActionOutcome, + * DesiredAction, + * } from '../../src/core/config/types.d.ts' + */ + +/** A quiet logger so tests don't spam stderr. */ +const NOOP_LOG = { + debug() {}, + info() {}, + warn() {}, + error() {}, +} + +async function makeFixture() { + const tmp = await fsp.mkdtemp(path.join(os.tmpdir(), 'hyp-action-reconciler-')) + const stateRoot = path.join(tmp, 'hypaware') + return { tmp, stateRoot } +} + +/** A minimal reconcile input — handlers under test ignore these. */ +const INPUT = { + config: /** @type {any} */ ({ version: 2, plugins: [] }), + backfills: /** @type {any} */ ({ register() {}, get() { return undefined }, list() { return [] } }), +} + +function markerPath(stateRoot) { + return path.join(stateRoot, 'config-control', 'client-actions.json') +} + +function readMarkerFile(stateRoot) { + return JSON.parse(fs.readFileSync(markerPath(stateRoot), 'utf8')) +} + +/** + * A run-once handler whose `perform` counts calls. `desired()` returns one + * unit per configured request key. + * + * @param {{ kind?: string, keys?: string[], outcome?: ActionOutcome }} [opts] + */ +function countingHandler(opts = {}) { + const kind = opts.kind ?? 'backfill' + const keys = opts.keys ?? ['@hypaware/claude'] + /** @type {ActionHandler & { performCalls: number, desiredCalls: number }} */ + const handler = { + kind, + performCalls: 0, + desiredCalls: 0, + desired() { + handler.desiredCalls += 1 + return keys.map((requestKey) => ({ requestKey, params: { plugin: requestKey } })) + }, + async perform(action) { + handler.performCalls += 1 + return opts.outcome ?? { status: 'done', rows: 7 } + }, + } + return handler +} + +test('reconcile runs a desired action once and short-circuits on the done marker', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + const handler = countingHandler() + let clock = Date.parse('2026-06-25T00:00:00.000Z') + const reconciler = createActionReconciler({ + stateRoot, + handlers: [handler], + now: () => clock, + log: NOOP_LOG, + }) + + const first = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 1) + assert.deepEqual( + first.results.map((r) => [r.requestKey, r.outcome]), + [['@hypaware/claude', 'done']] + ) + + // Second pass: the done marker short-circuits — perform is not re-run. + clock += 1000 + const second = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 1, 'perform must not run again on a done marker') + assert.deepEqual( + second.results.map((r) => [r.requestKey, r.outcome]), + [['@hypaware/claude', 'skipped']] + ) + + const file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/claude'].status, 'done') + assert.equal(file.backfill['@hypaware/claude'].rows, 7) + assert.equal(file.backfill['@hypaware/claude'].request_key, '@hypaware/claude') + assert.equal(file.backfill['@hypaware/claude'].at, '2026-06-25T00:00:00.000Z') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a missed pass (no marker yet) runs on the next reconcile call', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + // Handler wants nothing on the first pass (the join hasn't confirmed), + // then names a unit on the second — the gap is picked up. + let active = false + /** @type {ActionHandler & { performCalls: number }} */ + const handler = { + kind: 'backfill', + performCalls: 0, + desired() { + return active ? [{ requestKey: '@hypaware/codex' }] : [] + }, + async perform() { + handler.performCalls += 1 + return { status: 'done', rows: 3 } + }, + } + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + + const first = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 0) + assert.deepEqual(first.results, []) + // No marker file written when nothing happened. + assert.equal(fs.existsSync(markerPath(stateRoot)), false) + + active = true + const second = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 1) + assert.equal(second.results[0].outcome, 'done') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('atomic marker read/write round-trips through readClientActionStatus and readStatus', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + const reconciler = createActionReconciler({ + stateRoot, + handlers: [countingHandler({ keys: ['@hypaware/claude', '@hypaware/codex'] })], + now: () => Date.parse('2026-06-25T12:00:00.000Z'), + log: NOOP_LOG, + }) + + // Empty before any pass — both the standalone reader and the handle agree. + assert.deepEqual(readClientActionStatus({ stateRoot }), { byKind: {} }) + assert.deepEqual(reconciler.readStatus(), { byKind: {} }) + + await reconciler.reconcile(INPUT) + + const standalone = readClientActionStatus({ stateRoot }) + const viaHandle = reconciler.readStatus() + assert.deepEqual(standalone, viaHandle) + assert.equal(standalone.byKind.backfill['@hypaware/claude'].status, 'done') + assert.equal(standalone.byKind.backfill['@hypaware/codex'].status, 'done') + + // File is mode 0600 and ends with a trailing newline (atomic-write idiom). + const raw = fs.readFileSync(markerPath(stateRoot), 'utf8') + assert.ok(raw.endsWith('}\n')) + const mode = fs.statSync(markerPath(stateRoot)).mode & 0o777 + assert.equal(mode, 0o600) + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a failed perform writes a failed marker (not done) and retries with bumped attempts', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + let shouldFail = true + /** @type {ActionHandler & { performCalls: number }} */ + const handler = { + kind: 'backfill', + performCalls: 0, + desired() { + return [{ requestKey: '@hypaware/codex' }] + }, + async perform() { + handler.performCalls += 1 + return shouldFail + ? { status: 'failed', reason: 'transcript dir missing' } + : { status: 'done', rows: 12 } + }, + } + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + + const p1 = await reconciler.reconcile(INPUT) + assert.equal(p1.results[0].outcome, 'failed') + let file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/codex'].status, 'failed') + assert.equal(file.backfill['@hypaware/codex'].reason, 'transcript dir missing') + assert.equal(file.backfill['@hypaware/codex'].attempts, 1) + + // A failed marker is not terminal — the next pass retries and bumps attempts. + const p2 = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 2) + assert.equal(p2.results[0].outcome, 'failed') + file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/codex'].attempts, 2) + + // Once it succeeds the marker flips to done and stops retrying. + shouldFail = false + const p3 = await reconciler.reconcile(INPUT) + assert.equal(p3.results[0].outcome, 'done') + file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/codex'].status, 'done') + assert.equal(file.backfill['@hypaware/codex'].rows, 12) + + const p4 = await reconciler.reconcile(INPUT) + assert.equal(handler.performCalls, 3, 'a done marker short-circuits subsequent passes') + assert.equal(p4.results[0].outcome, 'skipped') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a thrown perform is normalized to a failed marker', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + /** @type {ActionHandler} */ + const handler = { + kind: 'backfill', + desired() { + return [{ requestKey: '@hypaware/claude' }] + }, + async perform() { + throw new Error('spawn ENOENT') + }, + } + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + const report = await reconciler.reconcile(INPUT) + assert.equal(report.results[0].outcome, 'failed') + const file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/claude'].status, 'failed') + assert.equal(file.backfill['@hypaware/claude'].reason, 'spawn ENOENT') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a handler whose desired() throws does not wedge other handlers', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + /** @type {ActionHandler} */ + const bad = { + kind: 'attach', + desired() { + throw new Error('boom') + }, + async perform() { + return { status: 'done' } + }, + } + const good = countingHandler() + const reconciler = createActionReconciler({ stateRoot, handlers: [bad, good], log: NOOP_LOG }) + const report = await reconciler.reconcile(INPUT) + assert.equal(good.performCalls, 1) + assert.deepEqual( + report.results.map((r) => [r.kind, r.outcome]), + [['backfill', 'done']] + ) + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a reversible handler undoes a previously-applied key the config no longer names', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + let wanted = ['client-a'] + /** @type {ActionHandler & { reverseCalls: string[] }} */ + const handler = { + kind: 'attach', + reverseCalls: [], + desired() { + return wanted.map((requestKey) => ({ requestKey })) + }, + async perform() { + // Reversible handlers record an applied state; reuse `done` as the + // applied terminal for the test (the reconciler keys reverse off + // "present marker that isn't failed and is no longer desired"). + return { status: 'done' } + }, + async reverse(requestKey) { + handler.reverseCalls.push(requestKey) + return { status: 'done' } + }, + } + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + + await reconciler.reconcile(INPUT) + assert.equal(readMarkerFile(stateRoot).attach['client-a'].status, 'done') + + // Config no longer names client-a → reverse runs once and the marker is removed. + wanted = [] + const report = await reconciler.reconcile(INPUT) + assert.deepEqual(handler.reverseCalls, ['client-a']) + assert.equal(report.results[0].outcome, 'reversed') + assert.equal(readClientActionStatus({ stateRoot }).byKind.attach, undefined) + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + +test('a run-once handler never reverses a no-longer-desired done marker', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + let wanted = ['@hypaware/claude'] + const handler = countingHandler() + handler.desired = () => wanted.map((requestKey) => ({ requestKey })) + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + + await reconciler.reconcile(INPUT) + assert.equal(readMarkerFile(stateRoot).backfill['@hypaware/claude'].status, 'done') + + // Plugin disabled: a non-reversible handler keeps the marker (imported + // data stays; run-once still short-circuits if it is re-enabled later). + wanted = [] + const report = await reconciler.reconcile(INPUT) + assert.deepEqual(report.results, []) + assert.equal(readMarkerFile(stateRoot).backfill['@hypaware/claude'].status, 'done') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) From 5fa45d0a5728e99845eb1590277bc04aa1708183 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 20:47:56 -0700 Subject: [PATCH 05/10] Validate per-plugin backfill config in @hypaware/claude and @hypaware/codex (T5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `config_sections` manifest entry plus a ConfigRegistry section validator to the claude and codex adapters so each plugin's own `config.backfill` policy ({ on_join, window_days }) is validated by the owning plugin (LLP 0005 / LLP 0037). Plugin-local: no top-level `backfill` section and nothing new for core to validate. Unknown sibling keys (e.g. `proxy`) pass through untouched; the `backfill` block is checked strictly (on_join boolean, window_days positive integer, no unknown keys) so typos surface instead of being silently ignored. The validator registers via ctx.configRegistry.registerSection so the kernel runs it through runPerPluginSectionValidators. The central-locked on_join cannot be flipped by a colliding local plugin entry — that falls out of the existing LLP 0031 plugins[] merge model (covered by a test). Task-Id: T5 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../claude/hypaware.plugin.json | 6 + .../plugins-workspace/claude/src/config.js | 84 ++++++++++++++ .../plugins-workspace/claude/src/index.js | 12 ++ .../codex/hypaware.plugin.json | 6 + .../plugins-workspace/codex/src/config.js | 83 ++++++++++++++ .../plugins-workspace/codex/src/index.js | 12 ++ test/plugins/claude-config.test.js | 104 ++++++++++++++++++ test/plugins/codex-config.test.js | 73 ++++++++++++ 8 files changed, 380 insertions(+) create mode 100644 hypaware-core/plugins-workspace/claude/src/config.js create mode 100644 hypaware-core/plugins-workspace/codex/src/config.js create mode 100644 test/plugins/claude-config.test.js create mode 100644 test/plugins/codex-config.test.js diff --git a/hypaware-core/plugins-workspace/claude/hypaware.plugin.json b/hypaware-core/plugins-workspace/claude/hypaware.plugin.json index 6f10714..dce6c55 100644 --- a/hypaware-core/plugins-workspace/claude/hypaware.plugin.json +++ b/hypaware-core/plugins-workspace/claude/hypaware.plugin.json @@ -44,6 +44,12 @@ "name": "claude-and-otel-local", "summary": "Capture Claude Code + OTLP locally, export to Parquet under HYP_HOME/exports" } + ], + "config_sections": [ + { + "section": "claude", + "summary": "Claude adapter config, including the optional backfill-on-join policy { on_join, window_days }." + } ] } } diff --git a/hypaware-core/plugins-workspace/claude/src/config.js b/hypaware-core/plugins-workspace/claude/src/config.js new file mode 100644 index 0000000..a76c277 --- /dev/null +++ b/hypaware-core/plugins-workspace/claude/src/config.js @@ -0,0 +1,84 @@ +// @ts-check + +/** + * Config validation for the `@hypaware/claude` plugin's own `config` + * block. v1 validates only the optional `backfill` sub-object that drives + * backfill-on-join — `{ on_join, window_days }`. Every other key (e.g. + * `proxy`) passes through untouched so existing configs keep working; + * there is no top-level `backfill` section and nothing new for core to + * validate. + * + * Pure and dependency-free: it returns a `ValidationResult` so it plugs + * straight into `ctx.configRegistry.registerSection` and is callable from + * tests without spinning up observability. + * + * @import { ValidationError, ValidationResult } from '../../../../collectivus-plugin-kernel-types.d.ts' + */ + +/** Manifest `config_sections[].section` name this validator backs. */ +export const CLAUDE_CONFIG_SECTION = 'claude' + +/** + * Validate the `@hypaware/claude` plugin config slice. Only the optional + * `backfill` policy block is checked; unknown sibling keys are ignored so + * the validator stays additive over the existing config surface. + * + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — + * backfill policy ({ on_join, window_days }) lives in and is validated + * by the source plugin's own config section; the kernel reconciler adds + * no top-level schema. + * + * @param {unknown} value + * @returns {ValidationResult} + */ +export function validateClaudeConfig(value) { + if (value === undefined || value === null) return { ok: true } + if (typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, errors: [{ pointer: '', message: 'claude config must be an object' }] } + } + const raw = /** @type {Record} */ (value) + const errors = validateBackfillSection(raw.backfill, '/backfill') + if (errors.length > 0) return { ok: false, errors } + return { ok: true } +} + +/** + * Validate the optional `backfill` policy block shared by every + * backfill-capable source plugin: `on_join` (whether to import on join, + * boolean) and `window_days` (how far back, positive integer). Both are + * optional; unknown keys are rejected so a typo (`window_day`) surfaces + * instead of being silently ignored. Pure — the caller chooses where the + * returned pointers mount. + * + * @param {unknown} value + * @param {string} pointer JSON-pointer prefix for the `backfill` object + * @returns {ValidationError[]} + */ +export function validateBackfillSection(value, pointer) { + /** @type {ValidationError[]} */ + const errors = [] + if (value === undefined) return errors + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + errors.push({ pointer, message: 'backfill must be an object' }) + return errors + } + const raw = /** @type {Record} */ (value) + if (raw.on_join !== undefined && typeof raw.on_join !== 'boolean') { + errors.push({ pointer: `${pointer}/on_join`, message: 'backfill.on_join must be a boolean' }) + } + if (raw.window_days !== undefined) { + const days = raw.window_days + if (typeof days !== 'number' || !Number.isInteger(days) || days <= 0) { + errors.push({ + pointer: `${pointer}/window_days`, + message: 'backfill.window_days must be a positive integer', + }) + } + } + for (const key of Object.keys(raw)) { + if (key !== 'on_join' && key !== 'window_days') { + errors.push({ pointer: `${pointer}/${key}`, message: `unknown backfill key '${key}'` }) + } + } + return errors +} diff --git a/hypaware-core/plugins-workspace/claude/src/index.js b/hypaware-core/plugins-workspace/claude/src/index.js index 9f54761..8955a85 100644 --- a/hypaware-core/plugins-workspace/claude/src/index.js +++ b/hypaware-core/plugins-workspace/claude/src/index.js @@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url' import { Attr, getLogger, withSpan } from '../../../../src/core/observability/index.js' import { defaultConfigPath } from '../../../../src/core/config/schema.js' +import { CLAUDE_CONFIG_SECTION, validateClaudeConfig } from './config.js' import { attach, defaultSettingsPath, detach } from './settings.js' import { anthropicUpstreamPreset, createClaudeExchangeProjector } from './projector.js' import { createClaudeBackfillProvider } from './backfill.js' @@ -54,6 +55,17 @@ export function claudeSessionContextFile(ctx) { * @ref LLP 0016#knows-nothing-about-claude-or-codex [implements] — adapter requires the ai-gateway capability; registers client + upstream preset */ export async function activate(ctx) { + // Validate the plugin's own `config` block — currently just the + // optional `backfill` policy ({ on_join, window_days }) that drives + // backfill-on-join. Registered so the kernel runs it via + // `runPerPluginSectionValidators`; no top-level core schema change. + // @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — the source plugin owns and validates its `backfill` config + ctx.configRegistry.registerSection({ + plugin: PLUGIN_NAME, + section: CLAUDE_CONFIG_SECTION, + validate: validateClaudeConfig, + }) + /** @type {AiGatewayCapability} */ const gateway = ctx.requireCapability('hypaware.ai-gateway', '^2.0.0') diff --git a/hypaware-core/plugins-workspace/codex/hypaware.plugin.json b/hypaware-core/plugins-workspace/codex/hypaware.plugin.json index 389e522..aa392bf 100644 --- a/hypaware-core/plugins-workspace/codex/hypaware.plugin.json +++ b/hypaware-core/plugins-workspace/codex/hypaware.plugin.json @@ -31,6 +31,12 @@ }, "skills": [ { "name": "hypaware-query", "clients": ["codex"] } + ], + "config_sections": [ + { + "section": "codex", + "summary": "Codex adapter config, including the optional backfill-on-join policy { on_join, window_days }." + } ] } } diff --git a/hypaware-core/plugins-workspace/codex/src/config.js b/hypaware-core/plugins-workspace/codex/src/config.js new file mode 100644 index 0000000..215ce52 --- /dev/null +++ b/hypaware-core/plugins-workspace/codex/src/config.js @@ -0,0 +1,83 @@ +// @ts-check + +/** + * Config validation for the `@hypaware/codex` plugin's own `config` + * block. v1 validates only the optional `backfill` sub-object that drives + * backfill-on-join — `{ on_join, window_days }`. Every other key passes + * through untouched so existing configs keep working; there is no + * top-level `backfill` section and nothing new for core to validate. + * + * Pure and dependency-free: it returns a `ValidationResult` so it plugs + * straight into `ctx.configRegistry.registerSection` and is callable from + * tests without spinning up observability. + * + * @import { ValidationError, ValidationResult } from '../../../../collectivus-plugin-kernel-types.d.ts' + */ + +/** Manifest `config_sections[].section` name this validator backs. */ +export const CODEX_CONFIG_SECTION = 'codex' + +/** + * Validate the `@hypaware/codex` plugin config slice. Only the optional + * `backfill` policy block is checked; unknown sibling keys are ignored so + * the validator stays additive over the existing config surface. + * + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — + * backfill policy ({ on_join, window_days }) lives in and is validated + * by the source plugin's own config section; the kernel reconciler adds + * no top-level schema. + * + * @param {unknown} value + * @returns {ValidationResult} + */ +export function validateCodexConfig(value) { + if (value === undefined || value === null) return { ok: true } + if (typeof value !== 'object' || Array.isArray(value)) { + return { ok: false, errors: [{ pointer: '', message: 'codex config must be an object' }] } + } + const raw = /** @type {Record} */ (value) + const errors = validateBackfillSection(raw.backfill, '/backfill') + if (errors.length > 0) return { ok: false, errors } + return { ok: true } +} + +/** + * Validate the optional `backfill` policy block shared by every + * backfill-capable source plugin: `on_join` (whether to import on join, + * boolean) and `window_days` (how far back, positive integer). Both are + * optional; unknown keys are rejected so a typo (`window_day`) surfaces + * instead of being silently ignored. Pure — the caller chooses where the + * returned pointers mount. + * + * @param {unknown} value + * @param {string} pointer JSON-pointer prefix for the `backfill` object + * @returns {ValidationError[]} + */ +export function validateBackfillSection(value, pointer) { + /** @type {ValidationError[]} */ + const errors = [] + if (value === undefined) return errors + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + errors.push({ pointer, message: 'backfill must be an object' }) + return errors + } + const raw = /** @type {Record} */ (value) + if (raw.on_join !== undefined && typeof raw.on_join !== 'boolean') { + errors.push({ pointer: `${pointer}/on_join`, message: 'backfill.on_join must be a boolean' }) + } + if (raw.window_days !== undefined) { + const days = raw.window_days + if (typeof days !== 'number' || !Number.isInteger(days) || days <= 0) { + errors.push({ + pointer: `${pointer}/window_days`, + message: 'backfill.window_days must be a positive integer', + }) + } + } + for (const key of Object.keys(raw)) { + if (key !== 'on_join' && key !== 'window_days') { + errors.push({ pointer: `${pointer}/${key}`, message: `unknown backfill key '${key}'` }) + } + } + return errors +} diff --git a/hypaware-core/plugins-workspace/codex/src/index.js b/hypaware-core/plugins-workspace/codex/src/index.js index 626be84..5c535da 100644 --- a/hypaware-core/plugins-workspace/codex/src/index.js +++ b/hypaware-core/plugins-workspace/codex/src/index.js @@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url' import { Attr, getLogger, withSpan } from '../../../../src/core/observability/index.js' import { createCodexBackfillProvider } from './backfill.js' +import { CODEX_CONFIG_SECTION, validateCodexConfig } from './config.js' import { createCodexExchangeProjector } from './exchange-projector.js' import { attach, defaultConfigPath, detach } from './settings.js' @@ -35,6 +36,17 @@ const CHATGPT_UPSTREAM_NAME = 'chatgpt' * @ref LLP 0016#knows-nothing-about-claude-or-codex [implements] — adapter requires the ai-gateway capability; registers client + upstream presets */ export async function activate(ctx) { + // Validate the plugin's own `config` block — currently just the + // optional `backfill` policy ({ on_join, window_days }) that drives + // backfill-on-join. Registered so the kernel runs it via + // `runPerPluginSectionValidators`; no top-level core schema change. + // @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — the source plugin owns and validates its `backfill` config + ctx.configRegistry.registerSection({ + plugin: PLUGIN_NAME, + section: CODEX_CONFIG_SECTION, + validate: validateCodexConfig, + }) + /** @type {AiGatewayCapability} */ const gateway = ctx.requireCapability('hypaware.ai-gateway', '^2.0.0') diff --git a/test/plugins/claude-config.test.js b/test/plugins/claude-config.test.js new file mode 100644 index 0000000..fe5c21f --- /dev/null +++ b/test/plugins/claude-config.test.js @@ -0,0 +1,104 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { + CLAUDE_CONFIG_SECTION, + validateBackfillSection, + validateClaudeConfig, +} from '../../hypaware-core/plugins-workspace/claude/src/config.js' +import { createConfigRegistry } from '../../src/core/config/schema.js' +import { mergeConfigLayers } from '../../src/core/config/merge.js' + +test('validateClaudeConfig accepts an empty / absent config', () => { + assert.deepEqual(validateClaudeConfig(undefined), { ok: true }) + assert.deepEqual(validateClaudeConfig(null), { ok: true }) + assert.deepEqual(validateClaudeConfig({}), { ok: true }) +}) + +test('validateClaudeConfig leaves non-backfill keys (e.g. proxy) untouched', () => { + assert.deepEqual(validateClaudeConfig({ proxy: '@hypaware/ai-gateway' }), { ok: true }) +}) + +test('validateClaudeConfig accepts a full backfill block', () => { + assert.deepEqual( + validateClaudeConfig({ proxy: '@hypaware/ai-gateway', backfill: { on_join: true, window_days: 30 } }), + { ok: true } + ) + assert.deepEqual(validateClaudeConfig({ backfill: { on_join: false } }), { ok: true }) + assert.deepEqual(validateClaudeConfig({ backfill: {} }), { ok: true }) +}) + +test('validateClaudeConfig rejects a non-object config', () => { + const result = validateClaudeConfig('nope') + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.errors[0].pointer, '') +}) + +test('validateClaudeConfig rejects a malformed backfill block', () => { + /** @type {Array<[unknown, string]>} */ + const cases = [ + [{ backfill: [] }, '/backfill'], + [{ backfill: 7 }, '/backfill'], + [{ backfill: { on_join: 'yes' } }, '/backfill/on_join'], + [{ backfill: { window_days: 0 } }, '/backfill/window_days'], + [{ backfill: { window_days: -1 } }, '/backfill/window_days'], + [{ backfill: { window_days: 1.5 } }, '/backfill/window_days'], + [{ backfill: { window_days: '30' } }, '/backfill/window_days'], + [{ backfill: { bogus: true } }, '/backfill/bogus'], + ] + for (const [config, pointer] of cases) { + const result = validateClaudeConfig(config) + assert.equal(result.ok, false, `${JSON.stringify(config)} must fail`) + if (result.ok) continue + assert.equal(result.errors[0].pointer, pointer, `${JSON.stringify(config)} pointer`) + } +}) + +test('validateBackfillSection mounts pointers under the supplied prefix', () => { + assert.deepEqual(validateBackfillSection(undefined, '/backfill'), []) + const errors = validateBackfillSection({ on_join: 1 }, '/plugins/0/config/backfill') + assert.equal(errors.length, 1) + assert.equal(errors[0].pointer, '/plugins/0/config/backfill/on_join') +}) + +test('the registered claude section drives validatePluginConfig', () => { + const registry = createConfigRegistry() + registry.registerSection({ + plugin: '@hypaware/claude', + section: CLAUDE_CONFIG_SECTION, + validate: validateClaudeConfig, + }) + assert.deepEqual( + registry.validatePluginConfig('@hypaware/claude', { backfill: { on_join: true } }), + { ok: true } + ) + const bad = registry.validatePluginConfig('@hypaware/claude', { backfill: { on_join: 'nope' } }) + assert.equal(bad.ok, false) +}) + +test('a central-locked backfill.on_join cannot be flipped by a colliding local entry', () => { + // LLP 0031 merge model: plugins[] merges by name; a local entry that + // collides with a central-named plugin is dropped, so the operator's + // `on_join: false` survives and the user cannot re-enable it locally. + const central = { + version: 2, + plugins: [{ name: '@hypaware/claude', config: { backfill: { on_join: false } } }], + } + const local = { + version: 2, + plugins: [{ name: '@hypaware/claude', config: { backfill: { on_join: true } } }], + } + const merged = mergeConfigLayers(/** @type {any} */ (central), /** @type {any} */ (local)) + + assert.equal(merged.effective.plugins?.length, 1) + assert.deepEqual( + /** @type {any} */ (merged.effective.plugins?.[0].config).backfill, + { on_join: false } + ) + assert.deepEqual(merged.drops, [ + { section: 'plugins', key: '@hypaware/claude', reason: 'collides_with_central' }, + ]) +}) diff --git a/test/plugins/codex-config.test.js b/test/plugins/codex-config.test.js new file mode 100644 index 0000000..86eba39 --- /dev/null +++ b/test/plugins/codex-config.test.js @@ -0,0 +1,73 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' + +import { + CODEX_CONFIG_SECTION, + validateBackfillSection, + validateCodexConfig, +} from '../../hypaware-core/plugins-workspace/codex/src/config.js' +import { createConfigRegistry } from '../../src/core/config/schema.js' + +test('validateCodexConfig accepts an empty / absent config', () => { + assert.deepEqual(validateCodexConfig(undefined), { ok: true }) + assert.deepEqual(validateCodexConfig(null), { ok: true }) + assert.deepEqual(validateCodexConfig({}), { ok: true }) +}) + +test('validateCodexConfig accepts a full backfill block', () => { + assert.deepEqual( + validateCodexConfig({ backfill: { on_join: true, window_days: 30 } }), + { ok: true } + ) + assert.deepEqual(validateCodexConfig({ backfill: { on_join: false } }), { ok: true }) + assert.deepEqual(validateCodexConfig({ backfill: {} }), { ok: true }) +}) + +test('validateCodexConfig rejects a non-object config', () => { + const result = validateCodexConfig(42) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.errors[0].pointer, '') +}) + +test('validateCodexConfig rejects a malformed backfill block', () => { + /** @type {Array<[unknown, string]>} */ + const cases = [ + [{ backfill: [] }, '/backfill'], + [{ backfill: { on_join: 'yes' } }, '/backfill/on_join'], + [{ backfill: { window_days: 0 } }, '/backfill/window_days'], + [{ backfill: { window_days: -3 } }, '/backfill/window_days'], + [{ backfill: { window_days: 2.5 } }, '/backfill/window_days'], + [{ backfill: { bogus: true } }, '/backfill/bogus'], + ] + for (const [config, pointer] of cases) { + const result = validateCodexConfig(config) + assert.equal(result.ok, false, `${JSON.stringify(config)} must fail`) + if (result.ok) continue + assert.equal(result.errors[0].pointer, pointer, `${JSON.stringify(config)} pointer`) + } +}) + +test('validateBackfillSection mounts pointers under the supplied prefix', () => { + assert.deepEqual(validateBackfillSection(undefined, '/backfill'), []) + const errors = validateBackfillSection({ window_days: -1 }, '/plugins/0/config/backfill') + assert.equal(errors.length, 1) + assert.equal(errors[0].pointer, '/plugins/0/config/backfill/window_days') +}) + +test('the registered codex section drives validatePluginConfig', () => { + const registry = createConfigRegistry() + registry.registerSection({ + plugin: '@hypaware/codex', + section: CODEX_CONFIG_SECTION, + validate: validateCodexConfig, + }) + assert.deepEqual( + registry.validatePluginConfig('@hypaware/codex', { backfill: { window_days: 7 } }), + { ok: true } + ) + const bad = registry.validatePluginConfig('@hypaware/codex', { backfill: { window_days: 0 } }) + assert.equal(bad.ok, false) +}) From 7f20c7087d272610a401706097fcd052c903c86c Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 21:02:07 -0700 Subject: [PATCH 06/10] Backfill action handler: backfillHandler over selectProviders + per-plugin config (T3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the v1 instance of the generic client-action reconciler (LLP 0036 / LLP 0037 / LLP 0041 Part 2): src/core/config/action_backfill.js exporting createBackfillHandler({ spawn }) and the default backfillHandler the daemon (T4) constructs the reconciler with. desired() reuses the exact "enabled-in-config" predicate `hyp backfill` uses (selectProviders with no explicit names), then drops any provider whose owning plugin set backfill.on_join:false — the operator opt-out that rides the locked plugins[] entry (LLP 0041 consent gating). It emits the owning plugin name as the run-once requestKey (the per-(machine,provider) marker key) while carrying the provider name in params, because the CLI positional is the provider name (`hyp backfill claude`), not the plugin name. perform() resolves window_days -> --since (now - windowDays.days; omitted when absent so `hyp backfill` falls back to the retention window) and spawns `hyp backfill [--since ] --json` via the runSmoke spawn pattern. The spawn is async (not spawnSync) so a months-deep import never blocks the daemon event loop (LLP 0041 execution isolation), is injectable for tests, and inherits the daemon env (HYP_HOME). Exit 0 sums providers[].rows_written into a done outcome; a non-zero exit / spawn error yields failed so the reconciler retries next pass. Types (BackfillSpawn, BackfillSpawnResult, BackfillSpawnArgs, CreateBackfillHandlerOptions) land in src/core/config/types.d.ts. Tests cover the opt-out, window_days->--since resolution + retention fallback (argv asserted), row summing, failure paths, and an end-to-end run through the T1 reconciler (failed marker -> retry -> done -> run-once skip). Task-Id: T3 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/core/config/action_backfill.js | 245 +++++++++++++++++++++++++++ src/core/config/types.d.ts | 43 +++++ test/core/action-backfill.test.js | 262 +++++++++++++++++++++++++++++ 3 files changed, 550 insertions(+) create mode 100644 src/core/config/action_backfill.js create mode 100644 test/core/action-backfill.test.js diff --git a/src/core/config/action_backfill.js b/src/core/config/action_backfill.js new file mode 100644 index 0000000..093e513 --- /dev/null +++ b/src/core/config/action_backfill.js @@ -0,0 +1,245 @@ +// @ts-check + +import { Attr } from '../observability/index.js' +import { selectProviders } from '../commands/backfill.js' + +/** + * @import { + * ActionContext, + * ActionHandler, + * ActionOutcome, + * BackfillSpawn, + * BackfillSpawnResult, + * CreateBackfillHandlerOptions, + * DesiredAction, + * } from './types.d.ts' + * @import { PluginConfigInstance } from '../../../collectivus-plugin-kernel-types.d.ts' + */ + +const MS_PER_DAY = 86_400_000 + +/** + * The backfill action handler — the v1 instance of the generic client-action + * reconciler (LLP 0036 / LLP 0037). It is the run-once "import this client's + * local history once the central config that enabled it is confirmed" effect, + * realized as a subprocess `hyp backfill` launch so a months-deep import can + * never wedge the daemon tick loop or grow its heap (LLP 0041 §Execution + * isolation; see the parquet-in-daemon hazard). + * + * `desired()` is pure (it reads the effective config + the kernel backfill + * registry); `perform()` spawns the child off the tick loop. The spawn is + * injectable so tests assert the argv and the marker writes without a real + * child. + * + * @param {CreateBackfillHandlerOptions} [opts] + * @returns {ActionHandler} + * @ref LLP 0041#run-once-flow-backfill-handler [implements] — backfillHandler.desired() over selectProviders + per-plugin config.backfill; perform() resolves window_days->--since and spawns `hyp backfill --json` + * @ref LLP 0037 — backfill on join (the instance this realizes) + */ +export function createBackfillHandler(opts = {}) { + const spawn = opts.spawn ?? defaultBackfillSpawn + + return { + kind: 'backfill', + + /** + * Enumerate the (provider) units to import. Reuses the exact + * "enabled-in-config" predicate `hyp backfill` uses (`selectProviders` + * with no explicit names), then drops any provider whose owning plugin + * set `backfill.on_join: false` (the operator opt-out, which rides the + * locked `plugins[]` entry — there is no local override). Pure: no + * effects, no spawn. + * + * @param {ActionContext} ctx + * @returns {DesiredAction[]} + * @ref LLP 0041#consent-gating [constrained-by] — default-on; suppression is `backfill.on_join:false` in the locked central plugin entry, not a local override + */ + desired(ctx) { + const activePlugins = ctx.config.plugins ?? [] + const { providers } = selectProviders({ + requested: [], + available: ctx.backfills.list(), + activePlugins, + }) + const byPluginName = new Map( + activePlugins + .filter((p) => p && typeof p.name === 'string') + .map((p) => [p.name, p]) + ) + + /** @type {DesiredAction[]} */ + const desired = [] + for (const provider of providers) { + const policy = readBackfillPolicy(byPluginName.get(provider.plugin)) + // Default-on: only an explicit `on_join: false` opts out. + if (policy.onJoin === false) continue + desired.push({ + // The marker key is the owning plugin name — a per-(machine, + // provider) boolean (LLP 0041 §request-key). The CLI positional, + // however, is the *provider* name (`hyp backfill claude`, not + // `@hypaware/claude`), carried separately in params. + requestKey: provider.plugin, + params: { + provider: provider.name, + plugin: provider.plugin, + ...(policy.windowDays !== undefined ? { windowDays: policy.windowDays } : {}), + }, + }) + } + return desired + }, + + /** + * Run one import as a subprocess. Resolves `window_days` → `--since` + * (now − windowDays·days); when absent, omits `--since` so `hyp backfill` + * falls back to the configured retention window (LLP 0041 §Run-once flow, + * step 1). On exit 0 the `--json` payload's `providers[].rows_written` is + * summed into a `done` marker; a non-zero exit or spawn error yields a + * `failed` outcome the reconciler records and retries next pass. + * + * @param {DesiredAction} action + * @param {ActionContext} ctx + * @returns {Promise} + * @ref LLP 0041#run-once-flow-backfill-handler [implements] — spawn `hyp backfill [--since ] --json` (runSmoke spawn pattern), parse providers[].rows_written, never advance to done on non-zero exit + */ + async perform(action, ctx) { + const params = action.params ?? {} + const provider = typeof params.provider === 'string' ? params.provider : '' + if (provider.length === 0) { + return { status: 'failed', reason: 'backfill action missing provider name' } + } + const windowDays = + typeof params.windowDays === 'number' && Number.isFinite(params.windowDays) + ? params.windowDays + : undefined + const since = + windowDays !== undefined + ? new Date(ctx.now() - windowDays * MS_PER_DAY).toISOString() + : undefined + + const args = ['backfill', provider, ...(since ? ['--since', since] : []), '--json'] + + ctx.log.info('client_action.backfill_spawn', { + [Attr.COMPONENT]: 'action-backfill', + [Attr.OPERATION]: 'client_action.perform', + [Attr.PLUGIN]: typeof params.plugin === 'string' ? params.plugin : provider, + provider, + ...(since ? { since } : {}), + [Attr.STATUS]: 'ok', + }) + + /** @type {BackfillSpawnResult} */ + let result + try { + result = await spawn({ args, env: process.env }) + } catch (err) { + return { status: 'failed', reason: err instanceof Error ? err.message : String(err) } + } + + if (result.error) { + return { status: 'failed', reason: `hyp backfill spawn failed: ${result.error.message}` } + } + if (result.status !== 0) { + return { status: 'failed', reason: `hyp backfill exited with code ${result.status}` } + } + + const rows = parseRowsWritten(result.stdout) + // Exit 0 is authoritative: the import committed. A row count is + // best-effort — an unparseable payload still records `done` (rows + // omitted) rather than re-running a successful import. + return rows !== undefined ? { status: 'done', rows } : { status: 'done' } + }, + } +} + +/** + * The default `[backfillHandler]` the daemon constructs the reconciler with + * (T4 wiring). It uses the real async `hyp backfill` spawn; tests build their + * own via {@link createBackfillHandler} with an injected spawn. + * + * @type {ActionHandler} + */ +export const backfillHandler = createBackfillHandler() + +/* ------------------------------- Internals ------------------------------- */ + +/** + * Read the owning plugin entry's `backfill` policy block. Absent / malformed + * blocks degrade to defaults (on_join on, no window) — `desired()` is the + * reconcile path and must not throw on a config the plugin validator (T5) + * already accepted; a missing block is simply the default. + * + * @param {PluginConfigInstance | undefined} entry + * @returns {{ onJoin: boolean | undefined, windowDays: number | undefined }} + */ +function readBackfillPolicy(entry) { + const config = entry?.config + const backfill = + config && typeof config === 'object' && !Array.isArray(config) + ? /** @type {Record} */ (config).backfill + : undefined + if (!backfill || typeof backfill !== 'object' || Array.isArray(backfill)) { + return { onJoin: undefined, windowDays: undefined } + } + const raw = /** @type {Record} */ (backfill) + const onJoin = typeof raw.on_join === 'boolean' ? raw.on_join : undefined + const windowDays = + typeof raw.window_days === 'number' && Number.isInteger(raw.window_days) && raw.window_days > 0 + ? raw.window_days + : undefined + return { onJoin, windowDays } +} + +/** + * Sum `providers[].rows_written` out of a `hyp backfill --json` payload. + * Returns `undefined` when the payload is unparseable or shapeless so the + * caller can record a `done` marker without a row count. + * + * @param {string} stdout + * @returns {number | undefined} + */ +function parseRowsWritten(stdout) { + let parsed + try { + parsed = JSON.parse(stdout) + } catch { + return undefined + } + const providers = parsed && Array.isArray(parsed.providers) ? parsed.providers : undefined + if (!providers) return undefined + let rows = 0 + for (const p of providers) { + if (p && typeof p.rows_written === 'number' && Number.isFinite(p.rows_written)) { + rows += p.rows_written + } + } + return rows +} + +/** + * The real subprocess seam: spawn `process.execPath bin/hypaware.js ` + * resolved off `import.meta.url` (the `runSmoke` spawn pattern) and capture + * stdout. Async (not `spawnSync`) on purpose — a multi-minute import must not + * block the daemon's event loop (LLP 0041 §Execution isolation). Inherits the + * daemon's `env` so the child writes the same cache (`HYP_HOME`). + * + * @type {BackfillSpawn} + */ +async function defaultBackfillSpawn({ args, env }) { + const { spawn } = await import('node:child_process') + const { fileURLToPath } = await import('node:url') + const binPath = fileURLToPath(new URL('../../../bin/hypaware.js', import.meta.url)) + return await new Promise((resolve) => { + const child = spawn(process.execPath, [binPath, ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + env, + }) + let stdout = '' + child.stdout?.on('data', (chunk) => { stdout += chunk }) + // stderr is drained so the pipe buffer can't fill and stall the child; + // its content is not needed (the exit code drives the outcome). + child.stderr?.on('data', () => {}) + child.on('error', (error) => resolve({ status: null, stdout, error })) + child.on('close', (status) => resolve({ status, stdout })) + }) +} diff --git a/src/core/config/types.d.ts b/src/core/config/types.d.ts index 1aa5cf3..3686c1b 100644 --- a/src/core/config/types.d.ts +++ b/src/core/config/types.d.ts @@ -462,4 +462,47 @@ export interface CreateActionReconcilerOptions { log?: PluginLogger } +// ============================================================================= +// Backfill action handler (LLP 0037 / LLP 0041 Part 2) +// ============================================================================= + +/** + * Result of one spawned `hyp backfill` child. `status` is the exit code + * (`null` when the child was killed by a signal); `stdout` is the captured + * `--json` payload; `error` is set when the spawn itself failed (ENOENT, + * etc.). The reconciler turns a non-zero / errored result into a `failed` + * marker that the next pass retries (LLP 0041 §failure is surfaced). + */ +export interface BackfillSpawnResult { + status: number | null + stdout: string + error?: Error +} + +/** Arguments handed to the injectable backfill spawn seam. */ +export interface BackfillSpawnArgs { + /** + * The `hyp` argv after the bin path — e.g. + * `['backfill', 'claude', '--since', '', '--json']`. The default + * implementation prepends `process.execPath` and the resolved + * `bin/hypaware.js` path (the `runSmoke` spawn pattern). + */ + args: string[] + /** Environment for the child; the daemon's own env (notably `HYP_HOME`). */ + env: NodeJS.ProcessEnv +} + +/** + * The subprocess seam the backfill handler launches `hyp backfill` through. + * Injected in tests so the spawned argv + marker writes can be asserted + * without a real child (LLP 0041 — "testable with the spawn injected"). + */ +export type BackfillSpawn = (args: BackfillSpawnArgs) => Promise + +export interface CreateBackfillHandlerOptions { + /** Subprocess seam; defaults to a real async `hyp backfill` spawn. */ + spawn?: BackfillSpawn + log?: PluginLogger +} + export type { ConfigStageResult, ConfigApplyErrorKind } diff --git a/test/core/action-backfill.test.js b/test/core/action-backfill.test.js new file mode 100644 index 0000000..38eb6b2 --- /dev/null +++ b/test/core/action-backfill.test.js @@ -0,0 +1,262 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs' +import fsp from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { createBackfillHandler, backfillHandler } from '../../src/core/config/action_backfill.js' +import { createActionReconciler } from '../../src/core/config/action_reconciler.js' + +/** + * @import { ActionContext, BackfillSpawnArgs, BackfillSpawnResult } from '../../src/core/config/types.d.ts' + * @import { BackfillContribution } from '../../collectivus-plugin-kernel-types.d.ts' + */ + +/** A quiet logger so tests don't spam stderr. */ +const NOOP_LOG = { debug() {}, info() {}, warn() {}, error() {} } + +const FIXED_NOW = Date.parse('2026-06-25T00:00:00.000Z') + +/** + * A fake backfill registry over a fixed provider list. + * @param {Partial[]} list + */ +function fakeBackfills(list) { + const providers = /** @type {BackfillContribution[]} */ (list) + return { + register() {}, + get(name) { return providers.find((p) => p.name === name) }, + list() { return providers }, + } +} + +/** The claude provider as the kernel registers it (provider name != plugin name). */ +const CLAUDE_PROVIDER = { + name: 'claude', + plugin: '@hypaware/claude', + datasets: ['ai_gateway_messages'], + run() { return (async function* () {})() }, +} + +/** + * Build the ActionContext a handler hook receives. + * @param {{ plugins?: any[], providers?: Partial[] }} [opts] + * @returns {ActionContext} + */ +function makeCtx(opts = {}) { + return { + config: /** @type {any} */ ({ version: 2, plugins: opts.plugins ?? [] }), + backfills: /** @type {any} */ (fakeBackfills(opts.providers ?? [CLAUDE_PROVIDER])), + now: () => FIXED_NOW, + log: NOOP_LOG, + } +} + +/** + * A spawn seam that records every call and returns a scripted result list + * (the last entry repeats once exhausted). + * @param {BackfillSpawnResult[]} results + */ +function recordingSpawn(results) { + /** @type {BackfillSpawnArgs[]} */ + const calls = [] + let i = 0 + /** @param {BackfillSpawnArgs} args */ + async function spawn(args) { + calls.push(args) + const result = results[Math.min(i, results.length - 1)] + i += 1 + return result + } + return { spawn, calls } +} + +/** A successful `hyp backfill --json` payload with the given row count. */ +function jsonPayload(rows) { + return JSON.stringify({ + run_id: 'bf-test', + dry_run: false, + providers: [{ provider: 'claude', plugin: '@hypaware/claude', status: 'ok', rows_written: rows }], + }) +} + +test('the default backfillHandler is a backfill-kind ActionHandler', () => { + assert.equal(backfillHandler.kind, 'backfill') + assert.equal(typeof backfillHandler.desired, 'function') + assert.equal(typeof backfillHandler.perform, 'function') + // Run-once: backfill is not reversible (imported data stays). + assert.equal(backfillHandler.reverse, undefined) +}) + +test('desired() emits one action per enabled provider (default on_join, plugin->provider mapping)', () => { + const { spawn } = recordingSpawn([{ status: 0, stdout: jsonPayload(0) }]) + const handler = createBackfillHandler({ spawn }) + const desired = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: true, config: { proxy: '@hypaware/ai-gateway' } }] }) + ) + assert.deepEqual(desired, [ + { requestKey: '@hypaware/claude', params: { provider: 'claude', plugin: '@hypaware/claude' } }, + ]) +}) + +test('desired() honors an explicit on_join:false opt-out (no action)', () => { + const handler = createBackfillHandler({ spawn: recordingSpawn([]).spawn }) + const desired = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { on_join: false } } }] }) + ) + assert.deepEqual(desired, []) +}) + +test('desired() carries window_days through to params when present', () => { + const handler = createBackfillHandler({ spawn: recordingSpawn([]).spawn }) + const desired = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { window_days: 30 } } }] }) + ) + assert.deepEqual(desired, [ + { requestKey: '@hypaware/claude', params: { provider: 'claude', plugin: '@hypaware/claude', windowDays: 30 } }, + ]) +}) + +test('desired() excludes a provider whose owning plugin is not enabled', () => { + const handler = createBackfillHandler({ spawn: recordingSpawn([]).spawn }) + // Plugin present but disabled -> selectProviders drops it. + const disabled = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: false, config: {} }] }) + ) + assert.deepEqual(disabled, []) + // Plugin entirely absent from config -> also dropped. + const absent = handler.desired(makeCtx({ plugins: [] })) + assert.deepEqual(absent, []) +}) + +test('perform() resolves window_days to a --since flag (assert spawned argv)', async () => { + const { spawn, calls } = recordingSpawn([{ status: 0, stdout: jsonPayload(1234) }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude', plugin: '@hypaware/claude', windowDays: 30 } }, + makeCtx() + ) + assert.deepEqual(outcome, { status: 'done', rows: 1234 }) + assert.equal(calls.length, 1) + const expectedSince = new Date(FIXED_NOW - 30 * 86_400_000).toISOString() + assert.deepEqual(calls[0].args, ['backfill', 'claude', '--since', expectedSince, '--json']) +}) + +test('perform() omits --since when window_days is absent (retention fallback)', async () => { + const { spawn, calls } = recordingSpawn([{ status: 0, stdout: jsonPayload(5) }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude', plugin: '@hypaware/claude' } }, + makeCtx() + ) + assert.deepEqual(outcome, { status: 'done', rows: 5 }) + assert.deepEqual(calls[0].args, ['backfill', 'claude', '--json']) +}) + +test('perform() sums rows_written across providers in the --json payload', async () => { + const payload = JSON.stringify({ + providers: [ + { provider: 'claude', rows_written: 10 }, + { provider: 'claude-extra', rows_written: 7 }, + ], + }) + const { spawn } = recordingSpawn([{ status: 0, stdout: payload }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude' } }, + makeCtx() + ) + assert.deepEqual(outcome, { status: 'done', rows: 17 }) +}) + +test('perform() records done (without rows) on exit 0 with an unparseable payload', async () => { + const { spawn } = recordingSpawn([{ status: 0, stdout: 'not json' }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude' } }, + makeCtx() + ) + assert.deepEqual(outcome, { status: 'done' }) +}) + +test('perform() returns failed on a non-zero exit', async () => { + const { spawn } = recordingSpawn([{ status: 1, stdout: '' }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude' } }, + makeCtx() + ) + assert.equal(outcome.status, 'failed') + assert.match(String(outcome.reason), /exited with code 1/) +}) + +test('perform() returns failed on a spawn error', async () => { + const { spawn } = recordingSpawn([{ status: null, stdout: '', error: new Error('spawn ENOENT') }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude' } }, + makeCtx() + ) + assert.equal(outcome.status, 'failed') + assert.match(String(outcome.reason), /ENOENT/) +}) + +test('perform() guards against a missing provider name', async () => { + const { spawn, calls } = recordingSpawn([{ status: 0, stdout: jsonPayload(0) }]) + const handler = createBackfillHandler({ spawn }) + const outcome = await handler.perform({ requestKey: '@hypaware/claude', params: {} }, makeCtx()) + assert.equal(outcome.status, 'failed') + assert.equal(calls.length, 0, 'no child spawned when the provider name is missing') +}) + +test('driven through the reconciler: a failed perform writes a failed marker, then a retry flips to done', async () => { + const tmp = await fsp.mkdtemp(path.join(os.tmpdir(), 'hyp-action-backfill-')) + const stateRoot = path.join(tmp, 'hypaware') + try { + // First pass fails (exit 1); second pass succeeds with 42 rows. + const { spawn, calls } = recordingSpawn([ + { status: 1, stdout: '' }, + { status: 0, stdout: jsonPayload(42) }, + ]) + const handler = createBackfillHandler({ spawn }) + const input = { + config: /** @type {any} */ ({ + version: 2, + plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { window_days: 7 } } }], + }), + backfills: /** @type {any} */ (fakeBackfills([CLAUDE_PROVIDER])), + } + const reconciler = createActionReconciler({ + stateRoot, + handlers: [handler], + now: () => FIXED_NOW, + log: NOOP_LOG, + }) + + const p1 = await reconciler.reconcile(input) + assert.equal(p1.results[0].outcome, 'failed') + const markerPath = path.join(stateRoot, 'config-control', 'client-actions.json') + let marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')) + assert.equal(marker.backfill['@hypaware/claude'].status, 'failed') + assert.equal(marker.backfill['@hypaware/claude'].attempts, 1) + // The first spawn carried the resolved --since for a 7-day window. + const expectedSince = new Date(FIXED_NOW - 7 * 86_400_000).toISOString() + assert.deepEqual(calls[0].args, ['backfill', 'claude', '--since', expectedSince, '--json']) + + const p2 = await reconciler.reconcile(input) + assert.equal(p2.results[0].outcome, 'done') + marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')) + assert.equal(marker.backfill['@hypaware/claude'].status, 'done') + assert.equal(marker.backfill['@hypaware/claude'].rows, 42) + + // Run-once: a done marker short-circuits — no third spawn. + const p3 = await reconciler.reconcile(input) + assert.equal(p3.results[0].outcome, 'skipped') + assert.equal(calls.length, 2, 'no spawn after the done marker lands') + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) From 335202caaf1ce94efa867b314c9efb625342c895 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 21:07:46 -0700 Subject: [PATCH 07/10] T6: status surface for the client-action reconciler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `clientActions` section to `HypAwareStatusReport`, read via `readClientActionStatus` (no reconcile pass), surfacing per-provider backfill state as done / failed / pending / n-a. `done`/`failed` come straight from the persisted marker store (any request key); `pending`/`n-a` are derived for declared backfill targets (a plugin entry carrying its own `config.backfill` block) — suppressed (`on_join:false`) or inert (host never joined) → `n/a`, otherwise desired-but-unrun → `pending`. Wired into both the text and JSON status renderers. A `failed` entry is its own status line and is deliberately excluded from `overall === 'degraded'` (LLP 0041 §failure-is-surfaced-not-fatal) — it is not even a diagnostic, so it cannot reach the overall computation. Null when nothing applies, so the V1 status surface is unchanged on an ordinary host. New types `ClientActionState` / `ClientActionReport` / `ClientActionsReport` in src/core/daemon/types.d.ts. @ref LLP 0036 — central-config-driven client action seam @ref LLP 0041#idempotency-and-completion-state — marker-derived status view Task-Id: T6 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/core/cli/core_commands.js | 38 +++++ src/core/daemon/status.js | 112 ++++++++++++- src/core/daemon/types.d.ts | 53 ++++++ test/core/status-client-actions.test.js | 207 ++++++++++++++++++++++++ 4 files changed, 407 insertions(+), 3 deletions(-) create mode 100644 test/core/status-client-actions.test.js diff --git a/src/core/cli/core_commands.js b/src/core/cli/core_commands.js index 2f6c680..948d2f5 100644 --- a/src/core/cli/core_commands.js +++ b/src/core/cli/core_commands.js @@ -614,6 +614,21 @@ export function renderStatusJson({ report, clientNames, datasets, cacheRoot }) { bad_etag: report.remoteConfig.badEtag, } : null, + // Client-action reconciler state (LLP 0036 / 0041). Null until a + // backfill-on-join target is configured or a pass has run; a `failed` + // entry is informational and never affects `overall`. + client_actions: report.clientActions + ? report.clientActions.actions.map((a) => ({ + kind: a.kind, + request_key: a.requestKey, + state: a.state, + ...(a.rows !== undefined ? { rows: a.rows } : {}), + ...(a.at ? { at: a.at } : {}), + ...(a.reason ? { reason: a.reason } : {}), + ...(a.lastAttempt ? { last_attempt: a.lastAttempt } : {}), + ...(a.attempts !== undefined ? { attempts: a.attempts } : {}), + })) + : null, diagnostics: report.diagnostics.map((d) => ({ severity: d.severity, kind: d.kind, @@ -739,6 +754,29 @@ export function renderStatusText({ report, clientNames, datasets, cacheRoot, std } } + // Client-action reconciler section (LLP 0036 / 0041). Appears only once a + // backfill-on-join target is configured or a pass has run; a `failed` + // line is loud but informational — it never degrades `overall`. + if (report.clientActions && report.clientActions.actions.length > 0) { + stdout.write(' client actions:\n') + for (const a of report.clientActions.actions) { + let detail = '' + if (a.state === 'done') { + const bits = [] + if (a.rows !== undefined) bits.push(`${a.rows} rows`) + if (a.at) bits.push(`at ${a.at}`) + if (bits.length > 0) detail = ` (${bits.join(', ')})` + } else if (a.state === 'failed') { + const bits = [] + if (a.reason) bits.push(a.reason) + if (a.lastAttempt) bits.push(`last attempt ${a.lastAttempt}`) + if (a.attempts !== undefined) bits.push(`${a.attempts} attempt${a.attempts === 1 ? '' : 's'}`) + if (bits.length > 0) detail = ` (${bits.join(', ')})` + } + stdout.write(` - ${a.kind} ${a.requestKey} [${a.state}]${detail}\n`) + } + } + if (report.diagnostics.length > 0) { stdout.write(' diagnostics:\n') for (const d of report.diagnostics) { diff --git a/src/core/daemon/status.js b/src/core/daemon/status.js index 5b1d287..047a7d8 100644 --- a/src/core/daemon/status.js +++ b/src/core/daemon/status.js @@ -7,6 +7,7 @@ import process from 'node:process' import { defaultConfigPath, loadConfigFile } from '../config/schema.js' import { readConfigControlStatus, resolveCentralLayerPath } from '../config/apply.js' +import { readClientActionStatus } from '../config/action_reconciler.js' import { resolveLayeredConfig } from '../config/merge.js' import { devTelemetryDir, readObservabilityEnv } from '../observability/env.js' import { collectConfigErrors, diagnoseV1Config, validateConfig } from '../config/validate.js' @@ -34,8 +35,8 @@ import { /** * @import { HypAwareV2Config } from '../../../collectivus-plugin-kernel-types.d.ts' - * @import { ConfigControlStatus, ConfigLayerDrop, ConfigValidationError, V1Diagnostic } from '../config/types.d.ts' - * @import { ClientAttachReport, CollectStatusOptions, DaemonState, DaemonStatus, HypAwareStatusReport, ServiceState, SinkSnapshot, SourceSnapshot, StatusDiagnostic, StatusDiagnosticKind } from './types.d.ts' + * @import { ClientActionStatus, ConfigControlStatus, ConfigLayerDrop, ConfigValidationError, V1Diagnostic } from '../config/types.d.ts' + * @import { ClientActionReport, ClientActionsReport, ClientAttachReport, CollectStatusOptions, DaemonState, DaemonStatus, HypAwareStatusReport, ServiceState, SinkSnapshot, SourceSnapshot, StatusDiagnostic, StatusDiagnosticKind } from './types.d.ts' * @import { Dirent } from 'node:fs' * @import { PluginCatalog, ClientDescriptor } from '../plugin_catalog.js' * @import { LoadedManifest } from '../manifest.js' @@ -477,6 +478,19 @@ export async function collectHypAwareStatus(opts = {}) { }) } + // ----- client-action reconciler state (LLP 0036 / 0041) ----- + // Read-only marker view; `hyp status` never runs a reconcile pass. A + // failed backfill is surfaced here (its own section, below) but is + // deliberately NOT a degrading diagnostic — the gateway runs fine on a + // valid config (LLP 0041 §failure-is-surfaced-not-fatal). + // @ref LLP 0041#failure-is-surfaced-not-fatal [implements] — surface client-action failure as its own line, never an outage signal + /** @type {ClientActionsReport | null} */ + let clientActions = null + try { + const actionStatus = readClientActionStatus({ stateRoot }) + clientActions = buildClientActionsReport({ status: actionStatus, config, hasCentral }) + } catch { /* best-effort probe */ } + // ----- recent errors ----- const recentErrorCount = await countRecentErrors(devTelemetryDir(stateRoot)) if (recentErrorCount > 0) { @@ -492,7 +506,11 @@ export async function collectHypAwareStatus(opts = {}) { // "set up" should degrade overall — config errors, v1 inconsistencies, // and the "no config at all yet" case. `client_attach_missing` / // `recent_errors` stay informational so a perfectly-configured-but- - // not-yet-attached install can still report healthy. + // not-yet-attached install can still report healthy. A failed + // client-action (e.g. backfill-on-join) is likewise excluded — it has + // its own status line but never flips `overall` (LLP 0041 + // §failure-is-surfaced-not-fatal); note it is not even a diagnostic, so + // it cannot reach this computation. const degradingKinds = new Set(['config_missing', 'config_unreadable']) const overall = diagnostics.some((d) => d.severity === 'error') ? 'degraded' @@ -516,7 +534,95 @@ export async function collectHypAwareStatus(opts = {}) { diagnostics, overall, remoteConfig, + clientActions, + } +} + +/** + * Build the client-action reconciler section for `hyp status` from the + * persisted marker store and the effective config. Pure: it reads markers + * and config and never runs a reconcile pass (LLP 0041 — the status surface + * "reads the marker file, it never runs a pass"). Returns null when nothing + * applies so the V1 status surface is unchanged on an ordinary host. + * + * Per-provider state: + * - `done` / `failed` come straight from a persisted marker (any request + * key, even one whose plugin has since left the config). + * - `pending` / `n/a` are derived for *declared* backfill targets — a + * plugin entry carrying its own `config.backfill` block. Backfill + * capability is a runtime fact (a registered `BackfillContribution`, + * LLP 0041 §per-plugin-capability) the status collector cannot see + * without activating plugins, so the declared policy is the honest, + * provider-agnostic signal: `on_join: false` or a non-joined host → + * `n/a` (the reconciler is a no-op); otherwise desired-but-unrun → + * `pending`. + * + * @param {{ status: ClientActionStatus, config: HypAwareV2Config | null, hasCentral: boolean }} args + * @returns {ClientActionsReport | null} + * @ref LLP 0041#idempotency-and-completion-state [implements] — per-provider done/failed/pending/n-a derived from the per-handler/per-request-key marker store, no reconcile pass + */ +function buildClientActionsReport({ status, config, hasCentral }) { + /** @type {ClientActionReport[]} */ + const actions = [] + const byKind = status?.byKind ?? {} + + // Declared backfill targets: enabled plugin entries with their own + // `config.backfill` block (LLP 0037 — policy rides the owning plugin). + /** @type {Map} */ + const declared = new Map() + for (const entry of config?.plugins ?? []) { + if (entry.enabled === false) continue + const raw = entry.config?.backfill + if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + const onJoin = /** @type {Record} */ (raw).on_join !== false + declared.set(entry.name, { onJoin }) + } + } + + // Kinds to render: every kind the markers record, plus `backfill` when + // any target is declared (so a configured-but-unrun target still shows). + /** @type {Set} */ + const kinds = new Set(Object.keys(byKind)) + if (declared.size > 0) kinds.add('backfill') + + for (const kind of [...kinds].sort()) { + const markers = byKind[kind] ?? {} + /** @type {Set} */ + const keys = new Set(Object.keys(markers)) + if (kind === 'backfill') for (const name of declared.keys()) keys.add(name) + for (const requestKey of [...keys].sort()) { + const marker = markers[requestKey] + if (marker && marker.status === 'failed') { + actions.push({ + kind, + requestKey, + state: 'failed', + ...(typeof marker.reason === 'string' ? { reason: marker.reason } : {}), + ...(typeof marker.last_attempt === 'string' ? { lastAttempt: marker.last_attempt } : {}), + ...(typeof marker.attempts === 'number' ? { attempts: marker.attempts } : {}), + }) + } else if (marker) { + // `done` (run-once) or `applied` (reversible) — the effect is in place. + actions.push({ + kind, + requestKey, + state: 'done', + ...(typeof marker.rows === 'number' ? { rows: marker.rows } : {}), + ...(typeof marker.at === 'string' ? { at: marker.at } : {}), + }) + } else { + // No marker: a declared backfill target. Suppressed (on_join:false) + // or inert (host never joined → the reconciler is a no-op) → n/a; + // otherwise desired and simply not run yet → pending. + const decl = declared.get(requestKey) + const suppressed = decl ? !decl.onJoin : false + const state = suppressed || !hasCentral ? 'n/a' : 'pending' + actions.push({ kind, requestKey, state }) + } + } } + + return actions.length > 0 ? { actions } : null } /** diff --git a/src/core/daemon/types.d.ts b/src/core/daemon/types.d.ts index be27fea..7765e6f 100644 --- a/src/core/daemon/types.d.ts +++ b/src/core/daemon/types.d.ts @@ -88,6 +88,50 @@ export interface StatusDiagnostic { pointer?: string } +/** + * Display state of one reconciler client-action, derived for `hyp status` + * from the persisted marker store (LLP 0036 / 0041) plus the effective + * config — `hyp status` never runs a pass. A `failed` entry is + * informational: it never flips `overall` to `degraded` (the gateway runs + * fine on a valid config, LLP 0041 §failure-is-surfaced-not-fatal). + * + * - `done` — run-once effect completed (carries `rows` + `at`). + * - `failed` — last attempt failed; retried next pass (carries `reason`, + * `lastAttempt`, `attempts`). + * - `pending` — desired on this joined host but no marker yet. + * - `n/a` — suppressed (`on_join: false`) or inert (host never joined). + */ +export type ClientActionState = 'done' | 'failed' | 'pending' | 'n/a' + +/** One reconciler action's state for the status surface. */ +export interface ClientActionReport { + /** Handler kind / marker namespace, e.g. `backfill`. */ + kind: string + /** Request key — the owning plugin name for backfill (LLP 0041). */ + requestKey: string + state: ClientActionState + /** Rows imported (on `done`). */ + rows?: number + /** ISO time the action reached `done`. */ + at?: string + /** Failure reason (on `failed`). */ + reason?: string + /** ISO time of the most recent attempt (on `failed`). */ + lastAttempt?: string + /** Attempts so far (on `failed`). */ + attempts?: number +} + +/** + * Client-action reconciler state (LLP 0036 / 0041) read from the marker + * file for `hyp status`. The report field is null when nothing applies (no + * markers and no backfill-configured plugins), so the V1 status surface is + * unchanged on an ordinary host. + */ +export interface ClientActionsReport { + actions: ClientActionReport[] +} + /** Per-client attach state probed off the user's home directory. */ export interface ClientAttachReport { /** `claude` or `codex`. */ @@ -166,6 +210,15 @@ export interface HypAwareStatusReport { * never applied a remote config reports all-null fields. */ remoteConfig: ConfigControlStatus | null + /** + * Client-action reconciler state (LLP 0036 / 0041): per-provider + * backfill-on-join (and future reconciled actions), read from the marker + * file via `readClientActionStatus` — `hyp status` never runs a pass. + * Null when nothing applies, so the V1 status surface is unchanged. A + * `failed` entry is informational and is deliberately excluded from + * `overall === 'degraded'`. + */ + clientActions: ClientActionsReport | null } export interface CollectStatusOptions { diff --git a/test/core/status-client-actions.test.js b/test/core/status-client-actions.test.js new file mode 100644 index 0000000..afff3f9 --- /dev/null +++ b/test/core/status-client-actions.test.js @@ -0,0 +1,207 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { collectHypAwareStatus } from '../../src/core/daemon/status.js' +import { defaultConfigPath } from '../../src/core/config/schema.js' +import { centralSeedPath } from '../../src/core/config/apply.js' +import { renderStatusJson, renderStatusText } from '../../src/core/cli/core_commands.js' + +/** + * @import { ClientActionReport } from '../../src/core/daemon/types.d.ts' + */ + +// T6 — the client-action reconciler status surface (LLP 0036 / 0041). The +// collector reads the marker file (it never runs a pass) and derives a +// per-provider done/failed/pending/n-a section; a failed action is loud but +// never flips `overall` to `degraded`. +// @ref LLP 0041#failure-is-surfaced-not-fatal [tests] + +async function makeHome() { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-status-actions-')) + await fs.mkdir(path.join(hypHome, 'hypaware'), { recursive: true }) + return hypHome +} + +/** @param {string} hypHome */ +function env(hypHome) { + return { ...process.env, HYP_HOME: hypHome, HYP_CONFIG: '' } +} + +/** + * Write the reconciler marker file the way the daemon would, so the + * read-only status path has something to surface. + * + * @param {string} hypHome + * @param {Record>} byKind + */ +async function writeMarkers(hypHome, byKind) { + const dir = path.join(hypHome, 'hypaware', 'config-control') + await fs.mkdir(dir, { recursive: true }) + await fs.writeFile(path.join(dir, 'client-actions.json'), JSON.stringify(byKind, null, 2) + '\n') +} + +function makeBuf() { + let value = '' + return { write(/** @type {string} */ chunk) { value += String(chunk); return true }, text() { return value } } +} + +/** @param {ClientActionReport[]} actions */ +function byKey(actions) { + /** @type {Map} */ + const m = new Map() + for (const a of actions) m.set(a.requestKey, a) + return m +} + +test('mixed done/failed/pending/n-a reads cleanly off the marker store + config', async () => { + const hypHome = await makeHome() + const stateRoot = path.join(hypHome, 'hypaware') + + // Joined host: central enables the gateway plus two backfill-declaring + // plugins — claude (on_join true → pending until a pass runs) and codex + // (on_join false → suppressed → n/a). + const seedPath = centralSeedPath(stateRoot) + await fs.mkdir(path.dirname(seedPath), { recursive: true }) + await fs.writeFile(seedPath, JSON.stringify({ + version: 2, + plugins: [ + { name: '@hypaware/central' }, + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: true, window_days: 30 } } }, + { name: '@hypaware/codex', config: { backfill: { on_join: false } } }, + ], + sinks: { central: { plugin: '@hypaware/central', config: {} } }, + }) + '\n') + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ version: 2, plugins: [] }) + '\n') + + // Markers from a prior pass: a completed import and a failed one, keyed by + // plugin name. (Marker keys surface even if the plugin later left config.) + await writeMarkers(hypHome, { + backfill: { + '@acme/done-plugin': { status: 'done', request_key: '@acme/done-plugin', rows: 1234, at: '2026-06-25T00:00:00.000Z' }, + '@acme/failed-plugin': { status: 'failed', request_key: '@acme/failed-plugin', reason: 'transcript dir missing', last_attempt: '2026-06-25T01:00:00.000Z', attempts: 2 }, + }, + }) + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.ok(report.clientActions, 'clientActions section is populated') + const m = byKey(report.clientActions.actions) + + assert.equal(m.get('@acme/done-plugin')?.state, 'done') + assert.equal(m.get('@acme/done-plugin')?.rows, 1234) + assert.equal(m.get('@acme/done-plugin')?.at, '2026-06-25T00:00:00.000Z') + + assert.equal(m.get('@acme/failed-plugin')?.state, 'failed') + assert.equal(m.get('@acme/failed-plugin')?.reason, 'transcript dir missing') + assert.equal(m.get('@acme/failed-plugin')?.attempts, 2) + + assert.equal(m.get('@hypaware/claude')?.state, 'pending') + assert.equal(m.get('@hypaware/codex')?.state, 'n/a') // on_join:false → suppressed + + // Every entry is namespaced under the backfill handler kind. + assert.ok(report.clientActions.actions.every((a) => a.kind === 'backfill')) +}) + +test('a failed backfill does not flip overall to degraded', async () => { + const hypHome = await makeHome() + + // Minimal, otherwise-healthy config (gateway only — no client advisories) + // so the only notable state is the failed action marker. + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ + version: 2, + plugins: [{ name: '@hypaware/ai-gateway' }], + }) + '\n') + await writeMarkers(hypHome, { + backfill: { + '@hypaware/codex': { status: 'failed', request_key: '@hypaware/codex', reason: 'boom', last_attempt: '2026-06-25T01:00:00.000Z', attempts: 3 }, + }, + }) + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.equal(report.overall, 'healthy') + assert.ok(report.clientActions) + assert.equal(report.clientActions?.actions[0]?.state, 'failed') + // The failure is its own section, never a diagnostic. + assert.ok(!report.diagnostics.some((d) => d.message.includes('boom'))) +}) + +test('an ordinary host with no markers reports clientActions null (V1 surface unchanged)', async () => { + const hypHome = await makeHome() + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ + version: 2, + plugins: [{ name: '@hypaware/ai-gateway' }], + }) + '\n') + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.equal(report.clientActions, null) + + const json = renderStatusJson({ report, clientNames: [], datasets: [], cacheRoot: '/tmp/cache' }) + assert.equal(json.client_actions, null) + + const stdout = makeBuf() + renderStatusText({ report, clientNames: [], datasets: [], cacheRoot: '/tmp/cache', stdout }) + assert.doesNotMatch(stdout.text(), /client actions:/) +}) + +test('JSON renderer emits a stable client_actions block', async () => { + const hypHome = await makeHome() + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ + version: 2, + plugins: [{ name: '@hypaware/ai-gateway' }], + }) + '\n') + await writeMarkers(hypHome, { + backfill: { + '@hypaware/claude': { status: 'done', request_key: '@hypaware/claude', rows: 42, at: '2026-06-25T00:00:00.000Z' }, + '@hypaware/codex': { status: 'failed', request_key: '@hypaware/codex', reason: 'nope', last_attempt: '2026-06-25T02:00:00.000Z', attempts: 1 }, + }, + }) + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + const json = renderStatusJson({ report, clientNames: [], datasets: [], cacheRoot: '/tmp/cache' }) + + assert.ok(Array.isArray(json.client_actions)) + const rows = /** @type {any[]} */ (json.client_actions) + const claude = rows.find((r) => r.request_key === '@hypaware/claude') + const codex = rows.find((r) => r.request_key === '@hypaware/codex') + assert.deepEqual(claude, { kind: 'backfill', request_key: '@hypaware/claude', state: 'done', rows: 42, at: '2026-06-25T00:00:00.000Z' }) + assert.deepEqual(codex, { kind: 'backfill', request_key: '@hypaware/codex', state: 'failed', reason: 'nope', last_attempt: '2026-06-25T02:00:00.000Z', attempts: 1 }) +}) + +test('text renderer prints the client actions section with per-state detail', async () => { + const hypHome = await makeHome() + const stateRoot = path.join(hypHome, 'hypaware') + const seedPath = centralSeedPath(stateRoot) + await fs.mkdir(path.dirname(seedPath), { recursive: true }) + await fs.writeFile(seedPath, JSON.stringify({ + version: 2, + plugins: [ + { name: '@hypaware/central' }, + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: true } } }, + { name: '@hypaware/codex', config: { backfill: { on_join: false } } }, + ], + }) + '\n') + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ version: 2, plugins: [] }) + '\n') + await writeMarkers(hypHome, { + backfill: { + '@acme/done-plugin': { status: 'done', request_key: '@acme/done-plugin', rows: 7, at: '2026-06-25T00:00:00.000Z' }, + '@acme/failed-plugin': { status: 'failed', request_key: '@acme/failed-plugin', reason: 'transcript dir missing', last_attempt: '2026-06-25T01:00:00.000Z', attempts: 2 }, + }, + }) + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + const stdout = makeBuf() + renderStatusText({ report, clientNames: [], datasets: [], cacheRoot: '/tmp/cache', stdout }) + const text = stdout.text() + + assert.match(text, /client actions:/) + assert.match(text, /backfill @acme\/done-plugin\s+\[done\]\s+\(7 rows, at 2026-06-25T00:00:00\.000Z\)/) + assert.match(text, /backfill @acme\/failed-plugin\s+\[failed\]\s+\(transcript dir missing, last attempt 2026-06-25T01:00:00\.000Z, 2 attempts\)/) + assert.match(text, /backfill @hypaware\/claude\s+\[pending\]/) + assert.match(text, /backfill @hypaware\/codex\s+\[n\/a\]/) +}) From fea31172c990ef7e5a07869de35f6cd5201e1656 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 21:24:20 -0700 Subject: [PATCH 08/10] Daemon wiring for the client-action reconciler (T4) Wire the client-action reconciler (LLP 0036 / 0037 / 0041) into `runDaemon`. The daemon is the only host with `configControl`, so a reconciler attached here is daemon-only by construction. - Construct the reconciler with the v1 `[backfillHandler]` (injectable via `RunDaemonOptions.actionReconciler` as a test seam), passing `boot.config` (effective) and `boot.runtime.backfills` into each `reconcile()` pass. - Wire `configControl`'s `onConfirmed` hook to schedule a reconcile pass on the probation active->cleared edge; `apply.js` stays ignorant of the reconciler. An edge that races the tail of boot (before the scheduler is wired) is recovered by the after-activation pass. - Run the after-activation already-confirmed pass, gated on a present central layer and no active probation (a fresh join waits for the edge; a non-joined host stays a no-op). - Add `createReconcilePassScheduler`: a single-flight guard that runs each pass as its own async task off the tick loop, coalescing concurrent edges into exactly one rerun, and `settle()` awaited on shutdown so the daemon never exits mid-import. Tests: scheduler single-flight / off-tick / coalescing / throw-recovery units, plus daemon integration tests for the boot pass firing (central + no probation) and staying inert (no central layer; active probation). Task-Id: T4 Co-Authored-By: Claude Opus 4.8 (1M context) --- src/core/daemon/runtime.js | 164 ++++++++++++++++++ src/core/daemon/types.d.ts | 9 +- test/core/daemon-reconcile.test.js | 260 +++++++++++++++++++++++++++++ 3 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 test/core/daemon-reconcile.test.js diff --git a/src/core/daemon/runtime.js b/src/core/daemon/runtime.js index ffbccac..98c3d6b 100644 --- a/src/core/daemon/runtime.js +++ b/src/core/daemon/runtime.js @@ -13,6 +13,8 @@ import { import { readObservabilityEnv } from '../observability/env.js' import { createConfigControl } from '../config/apply.js' import { buildConfigApplyDeps } from '../config/apply_deps.js' +import { createActionReconciler } from '../config/action_reconciler.js' +import { backfillHandler } from '../config/action_backfill.js' import { bootKernel, resolveLayeredConfigForDaemon } from '../runtime/boot.js' import { createSinkDriver } from '../sinks/driver.js' import { materializeSinks } from '../sinks/materialize.js' @@ -134,6 +136,14 @@ export async function runDaemon(opts = {}) { const done = new Promise((resolve) => { resolveDone = resolve }) /** @type {(() => Promise) | null} */ let triggerReload = null + // Forward reference to the client-action reconcile scheduler. It can only + // be built after `boot` resolves (it needs the effective config + the + // kernel backfill registry), but the confirmation-edge hook below is wired + // into `configControl` before boot — so the hook calls through this ref and + // an edge that fires before the scheduler exists is recovered by the + // after-activation already-confirmed pass (mirrors `pendingRestart`). + /** @type {((reason: string) => void) | null} */ + let scheduleReconcile = null let healthyAtMs = 0 // PID file is written before any plugin activation: that way a @@ -171,6 +181,17 @@ export async function runDaemon(opts = {}) { pendingRestart = true } }, + // The confirmation edge (probation active→cleared on the first + // authenticated poll): the running config is now the confirmed one, so + // schedule one reconcile pass. The pull loop's immediate pull can race + // the tail of runDaemon, so an edge before the scheduler is wired is + // dropped here and recovered by the after-activation already-confirmed + // pass (probation is cleared by then) — same race handling as + // `pendingRestart`. + // @ref LLP 0041#when-the-reconciler-runs-lifecycle-integration [implements] — the daemon wires onConfirmed to schedule a reconcile pass per confirmation edge; apply.js stays ignorant of the reconciler + onConfirmed: () => { + if (scheduleReconcile) scheduleReconcile('confirm-edge') + }, }) const bootEval = await configControl.evaluateAtBoot() if (bootEval.action !== 'none') { @@ -260,6 +281,77 @@ export async function runDaemon(opts = {}) { configControl.attachApplyDeps(buildConfigApplyDeps({ stateRoot })) configControl.armProbationWatchdog() + // ----- Client-action reconciler (LLP 0036 / LLP 0037 / LLP 0041) ----- + // The daemon is the only host with `configControl`, so a reconciler + // attached here is daemon-only by construction: `hyp status` (a plain CLI + // boot) never performs a machine effect. v1 ships one handler, the + // run-once backfill-on-join. Constructed only after boot because a pass + // needs the effective config + the kernel backfill registry. + // @ref LLP 0041#the-reconciler-component [implements] — construct the reconciler with [backfillHandler] in the daemon + const actionReconciler = + opts.actionReconciler ?? + createActionReconciler({ + stateRoot, + handlers: [backfillHandler], + log: getLogger('action-reconciler'), + }) + + /** + * Run one reconcile pass against the effective config + backfill registry. + * Never throws — a failed handler is surfaced as a `failed` marker by the + * reconciler, and any unexpected error is logged here, so the single-flight + * scheduler's rerun loop is never aborted by a pass. + * @param {string} reason + */ + async function runReconcilePass(reason) { + const config = boot.config + // No effective config (neither layer present) → nothing to reconcile. + if (!config) return + await withSpan( + 'client_action.reconcile', + { + [Attr.COMPONENT]: 'daemon', + [Attr.OPERATION]: 'client_action.reconcile', + [Attr.DEV_RUN_ID]: runId, + hyp_reason: reason, + status: 'ok', + }, + async () => { + const report = await actionReconciler.reconcile({ + config, + backfills: boot.runtime.backfills, + }) + fileLog.info('daemon.reconcile_pass', { + hyp_reason: reason, + results: report.results.length, + }) + }, + { component: 'daemon' } + ).catch((err) => { + const message = err instanceof Error ? err.message : String(err) + fileLog.error('daemon.reconcile_failed', { hyp_reason: reason, message }) + }) + } + + // The single-flight guard: only one pass runs at a time, off the tick loop. + const reconcileScheduler = createReconcilePassScheduler({ + run: runReconcilePass, + log: fileLog, + }) + scheduleReconcile = reconcileScheduler.schedule + + // After-activation already-confirmed pass: if a central layer is present + // and the running config already cleared probation on a prior boot (no + // active probation marker now), run one pass to recover anything missed + // while a previous probation was outstanding. A fresh join — probation + // still active — instead waits for the `confirmPoll` edge above. A + // non-joined host has no central layer, so the reconciler stays a no-op. + // @ref LLP 0041#when-the-reconciler-runs-lifecycle-integration [implements] — after-activation already-confirmed pass, gated on a present central layer and no active probation + const bootControlStatus = await configControl.status() + if (boot.centralConfigPath != null && !bootControlStatus.probation) { + reconcileScheduler.schedule('boot-already-confirmed') + } + // ----- Materialize config-backed sinks ----- const sinkResult = await materializeSinks(boot.runtime, boot.config, { stateRoot, @@ -418,6 +510,10 @@ export async function runDaemon(opts = {}) { if (maintenanceInFlight) { await maintenanceInFlight } + // Let any in-flight reconcile pass finish so the daemon never exits + // mid-import — abandoning a pass would orphan the spawned `hyp backfill` + // child and interrupt the marker write. + await reconcileScheduler.settle() persist({ state: 'stopping' }) fileLog.info('daemon.stopping', { reason }) @@ -571,6 +667,74 @@ export async function runDaemon(opts = {}) { } } +/** + * Single-flight scheduler for client-action reconcile passes. + * + * Each confirmation edge — and the after-activation already-confirmed check — + * calls `schedule()`, which runs `run()` as its own async task **off the + * caller's stack**: `schedule()` returns synchronously, so a reconcile pass + * (which may spawn a multi-minute `hyp backfill` import) never delays the + * sink tick loop or the apply engine's confirm poll. Only one pass runs at a + * time; an edge that arrives while a pass is in flight sets a "re-run when + * done" flag, coalescing any number of edges during a pass into exactly one + * more pass. Coalescing is lossless because the reconciler is level-triggered + * — the next pass reads the latest config + markers and converges the gap. + * + * `settle()` resolves when no pass is in flight; the shutdown path awaits it + * so the daemon never exits mid-pass. + * + * @param {{ run: (reason: string) => Promise, log?: { error(message: string, attributes?: Record): void } }} args + * @returns {{ schedule: (reason: string) => void, settle: () => Promise }} + * @ref LLP 0041#when-the-reconciler-runs-lifecycle-integration [implements] — single-flight guard: one pass at a time, an edge during a pass re-runs once when done, and the pass is its own async task off the tick loop + */ +export function createReconcilePassScheduler({ run, log }) { + let running = false + let rerun = false + /** @type {Promise} */ + let idle = Promise.resolve() + /** @type {(() => void) | null} */ + let resolveIdle = null + + /** @param {string} reason */ + function schedule(reason) { + if (running) { + // A pass is already running off the tick loop; coalesce this edge into + // a single re-run rather than starting a concurrent pass. + rerun = true + return + } + running = true + idle = new Promise((resolve) => { resolveIdle = resolve }) + void pump(reason) + } + + /** @param {string} reason */ + async function pump(reason) { + let nextReason = reason + try { + do { + // Clear the flag before awaiting: any edge during this `run` flips it + // back on (the only interleaving point), driving exactly one re-run. + rerun = false + await run(nextReason) + nextReason = 'rerun' + } while (rerun) + } catch (err) { + log?.error('daemon.reconcile_pass_failed', { + [Attr.COMPONENT]: 'daemon', + message: err instanceof Error ? err.message : String(err), + }) + } finally { + running = false + const resolve = resolveIdle + resolveIdle = null + resolve?.() + } + } + + return { schedule, settle: () => idle } +} + /** * @param {number|undefined} value * @returns {number} diff --git a/src/core/daemon/types.d.ts b/src/core/daemon/types.d.ts index 7765e6f..9118c11 100644 --- a/src/core/daemon/types.d.ts +++ b/src/core/daemon/types.d.ts @@ -3,7 +3,7 @@ import type { CapabilityRegistry, QueryRegistry, } from '../../../collectivus-plugin-kernel-types.d.ts' -import type { ConfigControlStatus, ConfigLayerDrop, V1Diagnostic, ConfigValidationError } from '../config/types.d.ts' +import type { ActionReconciler, ConfigControlStatus, ConfigLayerDrop, V1Diagnostic, ConfigValidationError } from '../config/types.d.ts' import type { ExtendedSourceRegistry } from '../registry/sources.js' import type { ExtendedSinkRegistry } from '../registry/sinks.js' import type { KernelRuntime } from '../runtime/activation.js' @@ -450,6 +450,13 @@ export interface RunDaemonOptions { foreground?: boolean /** Temp directory root for sink materialization scratch files. */ tmpRoot?: string + /** + * Injected client-action reconciler (LLP 0041). Defaults to one built with + * the v1 `[backfillHandler]`; tests pass a fake to drive the boot + * already-confirmed pass and the confirmation-edge wiring without a real + * `hyp backfill` subprocess. + */ + actionReconciler?: ActionReconciler } export interface DaemonLogger { diff --git a/test/core/daemon-reconcile.test.js b/test/core/daemon-reconcile.test.js new file mode 100644 index 0000000..9e913f1 --- /dev/null +++ b/test/core/daemon-reconcile.test.js @@ -0,0 +1,260 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { createReconcilePassScheduler, runDaemon } from '../../src/core/daemon/runtime.js' +import { defaultConfigPath } from '../../src/core/config/schema.js' +import { centralSeedPath } from '../../src/core/config/apply.js' + +/** + * @import { ActionReconciler, ReconcileInput } from '../../src/core/config/types.d.ts' + */ + +/** Resolve on the next macrotask so a `void`-launched async pass can run. */ +function tick() { + return new Promise((resolve) => setImmediate(resolve)) +} + +/** + * Poll `predicate` until true or the deadline elapses. + * @param {() => boolean} predicate + * @param {number} [timeoutMs] + */ +async function waitFor(predicate, timeoutMs = 1000) { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (predicate()) return + await tick() + } + throw new Error('waitFor timed out') +} + +// --------------------------------------------------------------------------- +// Single-flight scheduler (the off-tick guard the daemon wires onConfirmed to) +// --------------------------------------------------------------------------- + +test('createReconcilePassScheduler runs exactly one pass per idle edge', async () => { + let runs = 0 + const sched = createReconcilePassScheduler({ run: async () => { runs++ } }) + + sched.schedule('edge-1') + await sched.settle() + assert.equal(runs, 1) + + // A second edge after the first pass settles runs again — one pass per edge. + sched.schedule('edge-2') + await sched.settle() + assert.equal(runs, 2) +}) + +test('createReconcilePassScheduler is single-flight and coalesces concurrent edges into one rerun', async () => { + let runs = 0 + /** @type {Array<() => void>} */ + const releases = [] + // Each pass blocks until its release is called, so the test controls the + // in-flight window deterministically. + const run = () => new Promise((resolve) => { + runs++ + releases.push(() => resolve(undefined)) + }) + const sched = createReconcilePassScheduler({ run }) + + sched.schedule('edge-1') // starts pass 1, blocks on its release + // schedule() returned while pass 1 is still in flight — proof the pass runs + // off the caller's stack (it never blocks the tick loop / confirm poll). + assert.equal(runs, 1) + + sched.schedule('edge-2') // in-flight → coalesced, no concurrent pass + sched.schedule('edge-3') // in-flight → still one coalesced rerun + assert.equal(runs, 1) + + // Finish pass 1: the coalesced edges drive exactly one more pass, not two. + releases.shift()?.() + await waitFor(() => runs === 2) + assert.equal(runs, 2) + + // Finish pass 2: no further reruns are pending, so the scheduler settles. + releases.shift()?.() + await sched.settle() + assert.equal(runs, 2) +}) + +test('createReconcilePassScheduler.settle resolves immediately when no pass is in flight', async () => { + const sched = createReconcilePassScheduler({ run: async () => {} }) + await sched.settle() // never scheduled — resolves without hanging + assert.ok(true) +}) + +test('createReconcilePassScheduler keeps scheduling after a pass throws', async () => { + let runs = 0 + const sched = createReconcilePassScheduler({ + run: async () => { runs++; throw new Error('boom') }, + log: { error() {} }, + }) + sched.schedule('edge-1') + await sched.settle() + assert.equal(runs, 1) + // A throw must not wedge the guard — the next edge still runs. + sched.schedule('edge-2') + await sched.settle() + assert.equal(runs, 2) +}) + +// --------------------------------------------------------------------------- +// Daemon wiring: the after-activation already-confirmed pass + gating +// --------------------------------------------------------------------------- + +/** + * A fake reconciler that records each `reconcile()` input. Lets the daemon + * tests assert whether (and with what) the boot pass ran without a real + * `hyp backfill` subprocess. + * @returns {{ reconciler: ActionReconciler, calls: ReconcileInput[] }} + */ +function makeFakeReconciler() { + /** @type {ReconcileInput[]} */ + const calls = [] + /** @type {ActionReconciler} */ + const reconciler = { + async reconcile(input) { + calls.push(input) + return { results: [] } + }, + readStatus() { + return { byKind: {} } + }, + } + return { reconciler, calls } +} + +test('runDaemon runs the boot already-confirmed pass when a central layer is present and no probation is active', async () => { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-reconcile-boot-')) + let handle + try { + const stateRoot = path.join(hypHome, 'hypaware') + // A central layer (join seed) with no applied slot ⇒ no probation marker. + const seedPath = centralSeedPath(stateRoot) + await fs.mkdir(path.dirname(seedPath), { recursive: true }) + await fs.writeFile(seedPath, JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile(configPath, JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const { reconciler, calls } = makeFakeReconciler() + handle = await runDaemon({ + hypHome, + configPath, + env: { ...process.env, HYP_HOME: hypHome }, + runId: 'reconcile-boot-test', + tickIntervalMs: 0, + installSignalHandlers: false, + actionReconciler: reconciler, + }) + + await waitFor(() => calls.length === 1) + assert.equal(calls.length, 1) + // The pass carries the effective config and the kernel backfill registry. + assert.ok(calls[0].config) + assert.ok(calls[0].backfills) + assert.equal(typeof calls[0].backfills.list, 'function') + } finally { + if (handle) { + await handle.stop() + await handle.done + } + await fs.rm(hypHome, { recursive: true, force: true }) + } +}) + +test('runDaemon does not run the boot pass on a non-joined host (no central layer)', async () => { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-reconcile-nocentral-')) + let handle + try { + // No seed, no applied slot ⇒ no central layer ⇒ the reconciler stays inert. + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile(configPath, JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const { reconciler, calls } = makeFakeReconciler() + handle = await runDaemon({ + hypHome, + configPath, + env: { ...process.env, HYP_HOME: hypHome }, + runId: 'reconcile-nocentral-test', + tickIntervalMs: 0, + installSignalHandlers: false, + actionReconciler: reconciler, + }) + + // Give any (erroneously) scheduled pass time to run, then assert none did. + await tick() + await tick() + assert.equal(calls.length, 0) + } finally { + if (handle) { + await handle.stop() + await handle.done + } + await fs.rm(hypHome, { recursive: true, force: true }) + } +}) + +test('runDaemon does not run the boot pass while probation is still active (fresh-join case)', async () => { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-reconcile-probation-')) + let handle + try { + const stateRoot = path.join(hypHome, 'hypaware') + const controlDir = path.join(stateRoot, 'config-control') + await fs.mkdir(controlDir, { recursive: true }) + + // An applied central slot 'a' under active, unexpired probation: the + // running config has NOT been confirmed yet, so the boot pass must wait + // for the confirmation edge rather than fire now. + const central = JSON.stringify({ version: 2, plugins: [] }) + '\n' + await fs.writeFile(path.join(controlDir, 'config.a.json'), central) + await fs.writeFile(path.join(controlDir, 'config.a.etag'), 'etag-1') + await fs.symlink('config.a.json', path.join(controlDir, 'active')) + const until = new Date(Date.now() + 10 * 24 * 60 * 60 * 1000).toISOString() + await fs.writeFile( + path.join(controlDir, 'state.json'), + JSON.stringify({ + probation: { + etag: 'etag-1', + applied_at: new Date().toISOString(), + until, + slot: 'a', + previous_slot: 'a', + }, + }) + '\n' + ) + + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile(configPath, JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const { reconciler, calls } = makeFakeReconciler() + handle = await runDaemon({ + hypHome, + configPath, + env: { ...process.env, HYP_HOME: hypHome }, + runId: 'reconcile-probation-test', + tickIntervalMs: 0, + installSignalHandlers: false, + actionReconciler: reconciler, + }) + + await tick() + await tick() + assert.equal(calls.length, 0) + } finally { + if (handle) { + await handle.stop() + await handle.done + } + await fs.rm(hypHome, { recursive: true, force: true }) + } +}) From 80172ebed99f07e4e86e7406b7256eae54835e21 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 22:32:37 -0700 Subject: [PATCH 09/10] Fix dual-review findings for client-action reconciler (PR #166) Addresses five actionable findings from the dual review, each correctness fix paired with a regression test that fails before and passes after. 1. [MAJOR] Backfill child spawned with process.env, not the daemon's resolved env. Thread the daemon env (HYP_HOME forced to hypHome) through ReconcileInput -> ActionContext -> spawn() so the import writes the same cache the daemon booted, even on the direct-runDaemon/hermetic path (LLP 0041 Run-once flow step 2). 2. [MAJOR] Per-plugin backfill validators were dead in production. Thread the live boot.runtime.configRegistry into buildConfigApplyDeps -> validateConfig so apply-time validation dispatches to the claude/codex config_sections validators. Also tighten readBackfillPolicy: a present non-boolean on_join (e.g. "false") is an opt-out, not fail-open. 3. [MAJOR] Cover the fresh-join confirm-edge path: a new daemon-reconcile test boots under active probation (no boot pass), drives the edge via the real configControl.confirmPoll() seam, and asserts exactly one pass runs with the effective config + backfill registry + resolved HYP_HOME. 4. [MINOR] Default-on backfill (enabled client, no explicit config.backfill) was invisible in hyp status. Align buildClientActionsReport with backfillHandler.desired() using the catalog client descriptors as the static backfill-provider proxy, gated on a joined host so a non-joined install keeps its V1 surface. 5. [MINOR] reconcile()'s marker read now tolerates a corrupt marker (unparseable -> empty store) like hyp status already does, so a corrupt file can't wedge all client actions. Tests: test/core/action-backfill.test.js (env + non-boolean on_join), test/core/action-reconciler.test.js (corrupt marker), test/core/daemon-reconcile.test.js (confirm edge), test/core/status-client-actions.test.js (default-on pending), test/core/config-apply-section-validators.test.js (live section validator). npm test green (1453 pass, 1 pre-existing skip); typecheck + lint clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/core/config/action_backfill.js | 31 ++++- src/core/config/action_reconciler.js | 21 +++- src/core/config/apply_deps.js | 17 ++- src/core/config/types.d.ts | 14 +++ src/core/daemon/runtime.js | 16 ++- src/core/daemon/status.js | 33 +++-- test/core/action-backfill.test.js | 50 +++++++- test/core/action-reconciler.test.js | 32 +++++ .../config-apply-section-validators.test.js | 117 ++++++++++++++++++ test/core/daemon-reconcile.test.js | 83 +++++++++++++ test/core/status-client-actions.test.js | 44 +++++++ 11 files changed, 436 insertions(+), 22 deletions(-) create mode 100644 test/core/config-apply-section-validators.test.js diff --git a/src/core/config/action_backfill.js b/src/core/config/action_backfill.js index 093e513..7d5c0eb 100644 --- a/src/core/config/action_backfill.js +++ b/src/core/config/action_backfill.js @@ -131,7 +131,12 @@ export function createBackfillHandler(opts = {}) { /** @type {BackfillSpawnResult} */ let result try { - result = await spawn({ args, env: process.env }) + // Spawn with the daemon's resolved env (HYP_HOME forced to the + // daemon's hypHome upstream in the reconcile input) — NOT + // `process.env`, which can name a different HYP_HOME on the + // direct-`runDaemon`/hermetic-smoke path and import into the wrong + // cache. @ref LLP 0041#run-once-flow-backfill-handler [constrained-by] + result = await spawn({ args, env: ctx.env }) } catch (err) { return { status: 'failed', reason: err instanceof Error ? err.message : String(err) } } @@ -164,10 +169,17 @@ export const backfillHandler = createBackfillHandler() /* ------------------------------- Internals ------------------------------- */ /** - * Read the owning plugin entry's `backfill` policy block. Absent / malformed - * blocks degrade to defaults (on_join on, no window) — `desired()` is the - * reconcile path and must not throw on a config the plugin validator (T5) - * already accepted; a missing block is simply the default. + * Read the owning plugin entry's `backfill` policy block. A *missing* block + * is the default (on_join on, no window) — `desired()` is the reconcile path + * and must not throw on a config the plugin validator (T5) already accepted. + * + * A block that is *present but malformed* must not fail open, though: a + * non-boolean `on_join` (e.g. the JSON typo `on_join: "false"`) is treated + * as an opt-out (`onJoin: false`), never as "default on". The operator + * clearly intended to set the flag, and a potentially months-deep import is + * the wrong thing to run on a malformed opt-out. (With the per-plugin + * validator now live — see apply/boot wiring — such a config is rejected at + * apply time anyway; this is the belt-and-braces reconcile-path read.) * * @param {PluginConfigInstance | undefined} entry * @returns {{ onJoin: boolean | undefined, windowDays: number | undefined }} @@ -182,7 +194,14 @@ function readBackfillPolicy(entry) { return { onJoin: undefined, windowDays: undefined } } const raw = /** @type {Record} */ (backfill) - const onJoin = typeof raw.on_join === 'boolean' ? raw.on_join : undefined + // Absent → default on (undefined). Present-and-boolean → that value. + // Present-but-non-boolean → opt-out (false): do not fail open. + const onJoin = + raw.on_join === undefined + ? undefined + : typeof raw.on_join === 'boolean' + ? raw.on_join + : false const windowDays = typeof raw.window_days === 'number' && Number.isInteger(raw.window_days) && raw.window_days > 0 ? raw.window_days diff --git a/src/core/config/action_reconciler.js b/src/core/config/action_reconciler.js index 5170f7f..8ccf16d 100644 --- a/src/core/config/action_reconciler.js +++ b/src/core/config/action_reconciler.js @@ -82,7 +82,7 @@ export function createActionReconciler(opts) { */ async function reconcile(input) { /** @type {ActionContext} */ - const ctx = { config: input.config, backfills: input.backfills, now, log } + const ctx = { config: input.config, backfills: input.backfills, env: input.env, now, log } const store = readStore() /** @type {ReconcileActionResult[]} */ const results = [] @@ -263,8 +263,16 @@ async function runOutcome(fn) { /** * Read the persisted marker store. ENOENT (no action has ever run) is the - * empty store; a non-object document is treated as empty. Mirrors - * `readControlState` in `apply.js`. + * empty store; an unparseable or non-object document is treated as empty. + * Mirrors `readControlState` in `apply.js`. + * + * Corruption tolerance is load-bearing: `reconcile()` reads the marker on + * every pass, so a `JSON.parse` throw here would wedge *all* client actions + * while `hyp status` (which already swallows the error in + * `readClientActionStatus`) reports clean. An empty store means the next + * pass simply re-derives the gap from `desired()` and rewrites a clean + * marker — losing only the (recoverable) completion record, never running a + * pass it should not. * * @param {string} markerPath * @returns {ActionMarkerStore} @@ -277,7 +285,12 @@ function readMarkerStore(markerPath) { if (/** @type {NodeJS.ErrnoException} */ (err).code === 'ENOENT') return {} throw err } - const parsed = JSON.parse(raw) + let parsed + try { + parsed = JSON.parse(raw) + } catch { + return {} + } return parsed && typeof parsed === 'object' ? /** @type {ActionMarkerStore} */ (parsed) : {} } diff --git a/src/core/config/apply_deps.js b/src/core/config/apply_deps.js index 6b303f9..6ac8756 100644 --- a/src/core/config/apply_deps.js +++ b/src/core/config/apply_deps.js @@ -10,7 +10,7 @@ import { installPlugin, loadLock } from '../plugin_install/install.js' import { getEntry } from '../plugin_install/lock.js' /** - * @import { PluginConfigInstance, PluginName, ValidationError } from '../../../collectivus-plugin-kernel-types.d.ts' + * @import { ConfigRegistry, PluginConfigInstance, PluginName, ValidationError } from '../../../collectivus-plugin-kernel-types.d.ts' * @import { ConfigApplyDeps, PinnedInstallResult } from './types.d.ts' */ @@ -22,11 +22,18 @@ import { getEntry } from '../plugin_install/lock.js' * bundled manifest set) and attached via * `configControl.attachApplyDeps()`. * - * @param {{ stateRoot: string, workspaceDir?: string }} opts + * The live `configRegistry` (the kernel registry the active plugins + * registered their `config_sections` validators into during boot) is + * threaded through so apply-time validation actually dispatches to those + * per-plugin validators. Omitting it makes them dead — a served config with + * a malformed plugin `config` block (e.g. claude/codex `backfill`) would be + * accepted instead of rejected (LLP 0037). + * + * @param {{ stateRoot: string, workspaceDir?: string, configRegistry?: ConfigRegistry }} opts * @returns {ConfigApplyDeps} */ export function buildConfigApplyDeps(opts) { - const { stateRoot, workspaceDir } = opts + const { stateRoot, workspaceDir, configRegistry } = opts const log = getLogger('config-control') /** @@ -56,6 +63,10 @@ export function buildConfigApplyDeps(opts) { const result = await validateConfig(shape.config, { knownPlugins: catalog.pluginMetadata, knownDatasets: catalog.knownDatasets, + // Pass the live registry so per-plugin `config_sections` validators run + // (LLP 0037). Absent (e.g. a non-daemon caller) it degrades to the + // cross-plugin checks only, exactly as before. + ...(configRegistry ? { configRegistry } : {}), }) return { ok: result.ok, errors: /** @type {ValidationError[]} */ (result.errors) } } diff --git a/src/core/config/types.d.ts b/src/core/config/types.d.ts index 3686c1b..3160f80 100644 --- a/src/core/config/types.d.ts +++ b/src/core/config/types.d.ts @@ -367,6 +367,13 @@ export interface ActionContext { config: HypAwareV2Config /** Kernel backfill registry — `list()` yields enabled-or-not providers. */ backfills: BackfillRegistry + /** + * The daemon's resolved environment, threaded down to any spawned child + * (notably `hyp backfill`). The daemon forces `HYP_HOME=hypHome` so the + * child imports into the *same* cache the daemon resolved — not whatever + * `process.env.HYP_HOME` happened to be (LLP 0041 §Run-once flow step 2). + */ + env: NodeJS.ProcessEnv /** Injectable clock (test seam). */ now: () => number log: PluginLogger @@ -400,6 +407,13 @@ export interface ActionHandler { export interface ReconcileInput { config: HypAwareV2Config backfills: BackfillRegistry + /** + * The daemon's resolved environment for any child a handler spawns. The + * daemon forces `HYP_HOME=hypHome` so a spawned `hyp backfill` writes the + * same cache the daemon resolved, even when `opts.env`/`opts.hypHome` + * diverge from `process.env` (the direct-`runDaemon`/hermetic-smoke path). + */ + env: NodeJS.ProcessEnv } /** What the reconciler did with one (handler, requestKey) unit on a pass. */ diff --git a/src/core/daemon/runtime.js b/src/core/daemon/runtime.js index 98c3d6b..b5ddd40 100644 --- a/src/core/daemon/runtime.js +++ b/src/core/daemon/runtime.js @@ -278,7 +278,15 @@ export async function runDaemon(opts = {}) { // sink's pull loop may deliver a document immediately after its // bootstrap, and `stage()` refuses to run without a validator. The // watchdog re-arms here on every relaunch that boots mid-probation. - configControl.attachApplyDeps(buildConfigApplyDeps({ stateRoot })) + // The live per-plugin config registry is threaded in so apply-time + // validation actually runs the section validators the active plugins + // registered (e.g. claude/codex `backfill` blocks). Without it the + // per-plugin validators are dead in production: a served config with a + // malformed `backfill` block would be accepted instead of rolled back. + // @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — apply-time validation dispatches to the source plugin's own config-section validator + configControl.attachApplyDeps( + buildConfigApplyDeps({ stateRoot, configRegistry: boot.runtime.configRegistry }) + ) configControl.armProbationWatchdog() // ----- Client-action reconciler (LLP 0036 / LLP 0037 / LLP 0041) ----- @@ -320,6 +328,12 @@ export async function runDaemon(opts = {}) { const report = await actionReconciler.reconcile({ config, backfills: boot.runtime.backfills, + // Thread the daemon's resolved env, forcing HYP_HOME to the + // hypHome this daemon actually booted against, so a spawned + // `hyp backfill` imports into the same cache rather than whatever + // process.env.HYP_HOME happened to be (LLP 0041 §Run-once flow). + // @ref LLP 0041#run-once-flow-backfill-handler [implements] — the child runs against the daemon's resolved HYP_HOME, not process.env + env: { ...env, HYP_HOME: hypHome }, }) fileLog.info('daemon.reconcile_pass', { hyp_reason: reason, diff --git a/src/core/daemon/status.js b/src/core/daemon/status.js index 047a7d8..04173aa 100644 --- a/src/core/daemon/status.js +++ b/src/core/daemon/status.js @@ -488,7 +488,15 @@ export async function collectHypAwareStatus(opts = {}) { let clientActions = null try { const actionStatus = readClientActionStatus({ stateRoot }) - clientActions = buildClientActionsReport({ status: actionStatus, config, hasCentral }) + // Backfill-capable plugins, derived statically from the catalog's client + // descriptors (claude/codex). Status cannot see the runtime backfill + // registry without activating plugins, so the client descriptors are the + // honest static proxy for "this enabled plugin imports on join". + /** @type {Set} */ + const backfillPlugins = new Set( + [...(catalog?.clientDescriptors?.values() ?? [])].map((d) => d.plugin) + ) + clientActions = buildClientActionsReport({ status: actionStatus, config, hasCentral, backfillPlugins }) } catch { /* best-effort probe */ } // ----- recent errors ----- @@ -557,25 +565,36 @@ export async function collectHypAwareStatus(opts = {}) { * `n/a` (the reconciler is a no-op); otherwise desired-but-unrun → * `pending`. * - * @param {{ status: ClientActionStatus, config: HypAwareV2Config | null, hasCentral: boolean }} args + * @param {{ status: ClientActionStatus, config: HypAwareV2Config | null, hasCentral: boolean, backfillPlugins?: Set }} args * @returns {ClientActionsReport | null} * @ref LLP 0041#idempotency-and-completion-state [implements] — per-provider done/failed/pending/n-a derived from the per-handler/per-request-key marker store, no reconcile pass */ -function buildClientActionsReport({ status, config, hasCentral }) { +function buildClientActionsReport({ status, config, hasCentral, backfillPlugins }) { /** @type {ClientActionReport[]} */ const actions = [] const byKind = status?.byKind ?? {} - - // Declared backfill targets: enabled plugin entries with their own - // `config.backfill` block (LLP 0037 — policy rides the owning plugin). + const backfillCapable = backfillPlugins ?? new Set() + + // Declared backfill targets: enabled plugin entries that drive + // backfill-on-join (LLP 0037 — policy rides the owning plugin). Two cases: + // 1. An explicit `config.backfill` block (any host). + // 2. *Default-on*: a known backfill provider with no explicit block — on + // a joined host `backfillHandler.desired()` still emits for it, so it + // is a real (pending) target. Status mirrors that here; without this + // the default-on case was invisible. It is gated on `hasCentral` so a + // non-joined host (where the reconciler never runs) keeps its + // V1-unchanged surface — a bare `claude`/`codex` install shows nothing. /** @type {Map} */ const declared = new Map() for (const entry of config?.plugins ?? []) { if (entry.enabled === false) continue const raw = entry.config?.backfill - if (raw && typeof raw === 'object' && !Array.isArray(raw)) { + const hasBlock = !!raw && typeof raw === 'object' && !Array.isArray(raw) + if (hasBlock) { const onJoin = /** @type {Record} */ (raw).on_join !== false declared.set(entry.name, { onJoin }) + } else if (hasCentral && backfillCapable.has(entry.name)) { + declared.set(entry.name, { onJoin: true }) } } diff --git a/test/core/action-backfill.test.js b/test/core/action-backfill.test.js index 38eb6b2..e29234d 100644 --- a/test/core/action-backfill.test.js +++ b/test/core/action-backfill.test.js @@ -20,6 +20,9 @@ const NOOP_LOG = { debug() {}, info() {}, warn() {}, error() {} } const FIXED_NOW = Date.parse('2026-06-25T00:00:00.000Z') +/** The HYP_HOME the daemon resolves and threads into the reconcile input. */ +const RESOLVED_HYP_HOME = '/resolved/hyp/home' + /** * A fake backfill registry over a fixed provider list. * @param {Partial[]} list @@ -50,6 +53,9 @@ function makeCtx(opts = {}) { return { config: /** @type {any} */ ({ version: 2, plugins: opts.plugins ?? [] }), backfills: /** @type {any} */ (fakeBackfills(opts.providers ?? [CLAUDE_PROVIDER])), + // The daemon threads its resolved env (HYP_HOME forced to the daemon's + // hypHome) down to the spawn; tests assert the child inherits it. + env: { ...process.env, HYP_HOME: RESOLVED_HYP_HOME }, now: () => FIXED_NOW, log: NOOP_LOG, } @@ -110,6 +116,22 @@ test('desired() honors an explicit on_join:false opt-out (no action)', () => { assert.deepEqual(desired, []) }) +test('desired() does not fail open on a non-boolean on_join (treats it as opt-out)', () => { + // The typo'd JSON string `"false"` is not a boolean; pre-fix it fell + // through to default-on and ran backfill anyway. It must now suppress. + const handler = createBackfillHandler({ spawn: recordingSpawn([]).spawn }) + const stringFalse = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { on_join: 'false' } } }] }) + ) + assert.deepEqual(stringFalse, [], 'on_join:"false" (string) must not run backfill') + // A truthy non-boolean is equally malformed and equally suppressed — + // only a real boolean true (or an absent flag) runs the import. + const numberOne = handler.desired( + makeCtx({ plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { on_join: 1 } } }] }) + ) + assert.deepEqual(numberOne, [], 'on_join:1 (number) must not run backfill') +}) + test('desired() carries window_days through to params when present', () => { const handler = createBackfillHandler({ spawn: recordingSpawn([]).spawn }) const desired = handler.desired( @@ -145,6 +167,29 @@ test('perform() resolves window_days to a --since flag (assert spawned argv)', a assert.deepEqual(calls[0].args, ['backfill', 'claude', '--since', expectedSince, '--json']) }) +test('perform() spawns with the daemon-resolved env (HYP_HOME), not process.env', async () => { + // Regression for the LLP 0041 §Run-once flow step 2 hazard: the child must + // import into the daemon's resolved HYP_HOME, not whatever process.env + // names. Diverge process.env.HYP_HOME from the ctx env so a regression + // (spawning with process.env) is unambiguous. + const original = process.env.HYP_HOME + process.env.HYP_HOME = '/some/other/process/home' + try { + const { spawn, calls } = recordingSpawn([{ status: 0, stdout: jsonPayload(1) }]) + const handler = createBackfillHandler({ spawn }) + await handler.perform( + { requestKey: '@hypaware/claude', params: { provider: 'claude', plugin: '@hypaware/claude' } }, + makeCtx() + ) + assert.equal(calls.length, 1) + assert.equal(calls[0].env.HYP_HOME, RESOLVED_HYP_HOME) + assert.notEqual(calls[0].env.HYP_HOME, process.env.HYP_HOME) + } finally { + if (original === undefined) delete process.env.HYP_HOME + else process.env.HYP_HOME = original + } +}) + test('perform() omits --since when window_days is absent (retention fallback)', async () => { const { spawn, calls } = recordingSpawn([{ status: 0, stdout: jsonPayload(5) }]) const handler = createBackfillHandler({ spawn }) @@ -228,6 +273,7 @@ test('driven through the reconciler: a failed perform writes a failed marker, th plugins: [{ name: '@hypaware/claude', enabled: true, config: { backfill: { window_days: 7 } } }], }), backfills: /** @type {any} */ (fakeBackfills([CLAUDE_PROVIDER])), + env: { ...process.env, HYP_HOME: RESOLVED_HYP_HOME }, } const reconciler = createActionReconciler({ stateRoot, @@ -242,9 +288,11 @@ test('driven through the reconciler: a failed perform writes a failed marker, th let marker = JSON.parse(fs.readFileSync(markerPath, 'utf8')) assert.equal(marker.backfill['@hypaware/claude'].status, 'failed') assert.equal(marker.backfill['@hypaware/claude'].attempts, 1) - // The first spawn carried the resolved --since for a 7-day window. + // The first spawn carried the resolved --since for a 7-day window, and + // the daemon-resolved HYP_HOME threaded through the reconcile input. const expectedSince = new Date(FIXED_NOW - 7 * 86_400_000).toISOString() assert.deepEqual(calls[0].args, ['backfill', 'claude', '--since', expectedSince, '--json']) + assert.equal(calls[0].env.HYP_HOME, RESOLVED_HYP_HOME) const p2 = await reconciler.reconcile(input) assert.equal(p2.results[0].outcome, 'done') diff --git a/test/core/action-reconciler.test.js b/test/core/action-reconciler.test.js index b24550d..536af8c 100644 --- a/test/core/action-reconciler.test.js +++ b/test/core/action-reconciler.test.js @@ -39,6 +39,7 @@ async function makeFixture() { const INPUT = { config: /** @type {any} */ ({ version: 2, plugins: [] }), backfills: /** @type {any} */ ({ register() {}, get() { return undefined }, list() { return [] } }), + env: process.env, } function markerPath(stateRoot) { @@ -254,6 +255,37 @@ test('a thrown perform is normalized to a failed marker', async () => { } }) +test('a corrupt marker file does not wedge reconcile (treated as empty, pass still runs)', async () => { + const { tmp, stateRoot } = await makeFixture() + try { + // Write garbage where the atomic marker store should be. `hyp status` + // already swallows this (readClientActionStatus), but reconcile() read + // it through a bare JSON.parse — a corrupt marker wedged ALL actions + // while status reported clean. It must now degrade to an empty store. + const controlDir = path.join(stateRoot, 'config-control') + fs.mkdirSync(controlDir, { recursive: true }) + fs.writeFileSync(path.join(controlDir, 'client-actions.json'), '{ this is not: json,,,') + + const handler = countingHandler() + const reconciler = createActionReconciler({ stateRoot, handlers: [handler], log: NOOP_LOG }) + + const report = await reconciler.reconcile(INPUT) + // The pass ran the desired unit instead of throwing on the corrupt file. + assert.equal(handler.performCalls, 1) + assert.equal(report.results[0].outcome, 'done') + // The corrupt file was overwritten with a clean, parseable marker store. + const file = readMarkerFile(stateRoot) + assert.equal(file.backfill['@hypaware/claude'].status, 'done') + // The standalone status reader agrees (both tolerate corruption). + assert.equal( + readClientActionStatus({ stateRoot }).byKind.backfill['@hypaware/claude'].status, + 'done' + ) + } finally { + await fsp.rm(tmp, { recursive: true, force: true }) + } +}) + test('a handler whose desired() throws does not wedge other handlers', async () => { const { tmp, stateRoot } = await makeFixture() try { diff --git a/test/core/config-apply-section-validators.test.js b/test/core/config-apply-section-validators.test.js new file mode 100644 index 0000000..4e21690 --- /dev/null +++ b/test/core/config-apply-section-validators.test.js @@ -0,0 +1,117 @@ +// @ts-check + +import test from 'node:test' +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' + +import { bootKernel } from '../../src/core/runtime/boot.js' +import { buildConfigApplyDeps } from '../../src/core/config/apply_deps.js' +import { defaultConfigPath } from '../../src/core/config/schema.js' + +/** + * Apply-time validation must dispatch to the per-plugin `config_sections` + * validators the active plugins register at activation (LLP 0037). The wiring + * is the live `configRegistry`, threaded from the booted runtime into + * `buildConfigApplyDeps`. Without it the validators are dead in production: a + * served central config with a malformed plugin `config` block (e.g. the + * claude/codex `backfill` policy) would be accepted instead of rolled back. + * + * These tests boot the real kernel so the claude validator registers exactly + * the way the daemon registers it — no hand-rolled registry. + */ + +/** @param {string} hypHome */ +function env(hypHome) { + return { ...process.env, HYP_HOME: hypHome, HYP_CONFIG: '' } +} + +/** Boot ai-gateway + claude from a local config so the claude section registers. */ +async function bootWithClaude() { + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-section-validators-')) + const stateRoot = path.join(hypHome, 'hypaware') + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile( + configPath, + JSON.stringify({ + version: 2, + plugins: [{ name: '@hypaware/ai-gateway' }, { name: '@hypaware/claude' }], + }) + '\n' + ) + const boot = await bootKernel({ hypHome, configPath, env: env(hypHome), mode: 'cli' }) + return { hypHome, stateRoot, boot, cleanup: () => fs.rm(hypHome, { recursive: true, force: true }) } +} + +test('apply validation rejects a malformed plugin backfill block via the live section validator', async () => { + const fx = await bootWithClaude() + try { + // The claude plugin must have registered its `config_sections` validator + // during activation — that is the registry the apply path now consults. + // (`list()` is on the concrete registry; the runtime types it as the + // narrower ConfigRegistry, so reach it through a local cast.) + const registry = /** @type {{ list(): Array<{ plugin: string }> }} */ ( + /** @type {unknown} */ (fx.boot.runtime.configRegistry) + ) + assert.ok( + registry.list().some((s) => s.plugin === '@hypaware/claude'), + 'claude registered its config section at activation' + ) + + const deps = buildConfigApplyDeps({ + stateRoot: fx.stateRoot, + configRegistry: fx.boot.runtime.configRegistry, + }) + + // `on_join: "false"` is the JSON typo the section validator rejects. + const badDoc = { + version: 2, + plugins: [ + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: 'false' } } }, + ], + } + const res = await deps.validateDocument(badDoc) + assert.equal(res.ok, false, 'a malformed backfill block must fail apply validation') + const kinds = /** @type {Array<{ errorKind?: string }>} */ (res.errors).map((e) => e.errorKind) + assert.ok( + kinds.includes('config_section_invalid'), + `expected a config_section_invalid error, got ${JSON.stringify(kinds)}` + ) + + // A well-formed backfill block validates cleanly through the same path. + const goodDoc = { + version: 2, + plugins: [ + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: false, window_days: 30 } } }, + ], + } + const ok = await deps.validateDocument(goodDoc) + assert.equal(ok.ok, true, JSON.stringify(ok.errors)) + } finally { + await fx.cleanup() + } +}) + +test('without the live registry the per-plugin validator is dead (the bug this fixes)', async () => { + const fx = await bootWithClaude() + try { + // Same malformed document, but apply deps built WITHOUT the registry — + // exactly the pre-fix call shape. The cross-plugin checks pass and the + // dead section validator never runs, so the bad block slips through. + const depsNoRegistry = buildConfigApplyDeps({ stateRoot: fx.stateRoot }) + const badDoc = { + version: 2, + plugins: [ + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: 'false' } } }, + ], + } + const res = await depsNoRegistry.validateDocument(badDoc) + assert.equal(res.ok, true, 'pre-fix shape accepts the malformed backfill block') + } finally { + await fx.cleanup() + } +}) diff --git a/test/core/daemon-reconcile.test.js b/test/core/daemon-reconcile.test.js index 9e913f1..31d3afa 100644 --- a/test/core/daemon-reconcile.test.js +++ b/test/core/daemon-reconcile.test.js @@ -258,3 +258,86 @@ test('runDaemon does not run the boot pass while probation is still active (fres await fs.rm(hypHome, { recursive: true, force: true }) } }) + +test('the confirmation edge during active probation drives exactly one reconcile pass (fresh-join path)', async () => { + // The primary LLP 0037 path: a fresh join boots under active probation + // (no boot pass — covered above) and the FIRST authenticated config poll + // clears probation, firing the confirmation edge that schedules backfill. + // Previously only the no-fire half was tested; this drives the edge through + // the real configControl seam and asserts the pass actually runs once. + const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hypaware-reconcile-confirm-')) + let handle + try { + const stateRoot = path.join(hypHome, 'hypaware') + const controlDir = path.join(stateRoot, 'config-control') + await fs.mkdir(controlDir, { recursive: true }) + + // Applied central slot 'a' under active, unexpired probation (the + // fresh-join case): the boot pass must NOT fire yet. + const central = JSON.stringify({ version: 2, plugins: [] }) + '\n' + await fs.writeFile(path.join(controlDir, 'config.a.json'), central) + await fs.writeFile(path.join(controlDir, 'config.a.etag'), 'etag-1') + await fs.symlink('config.a.json', path.join(controlDir, 'active')) + const until = new Date(Date.now() + 10 * 24 * 60 * 60 * 1000).toISOString() + await fs.writeFile( + path.join(controlDir, 'state.json'), + JSON.stringify({ + probation: { + etag: 'etag-1', + applied_at: new Date().toISOString(), + until, + slot: 'a', + previous_slot: 'a', + }, + }) + '\n' + ) + + const configPath = defaultConfigPath(hypHome) + await fs.mkdir(path.dirname(configPath), { recursive: true }) + await fs.writeFile(configPath, JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const { reconciler, calls } = makeFakeReconciler() + handle = await runDaemon({ + hypHome, + configPath, + env: { ...process.env, HYP_HOME: hypHome }, + runId: 'reconcile-confirm-test', + tickIntervalMs: 0, + installSignalHandlers: false, + actionReconciler: reconciler, + }) + + // Probation is active → no boot pass. + await tick() + await tick() + assert.equal(calls.length, 0, 'no pass while probation is outstanding') + + // Drive the confirmation edge through the same configControl seam the + // central plugin's poll loop uses in production: probation active→cleared + // fires onConfirmed → schedules exactly one reconcile pass. + const configControl = /** @type {{ confirmPoll(): void } | undefined} */ ( + handle.runtime.configControl + ) + assert.ok(configControl, 'the daemon runtime exposes the configControl seam') + configControl.confirmPoll() + + await waitFor(() => calls.length === 1) + // A second confirmPoll is a no-op (probation already cleared) — no extra pass. + configControl.confirmPoll() + await tick() + await tick() + assert.equal(calls.length, 1, 'exactly one pass per confirmation edge') + + // The pass carried the effective config + the kernel backfill registry, + // and the daemon's resolved HYP_HOME threaded into the input. + assert.ok(calls[0].config) + assert.equal(typeof calls[0].backfills.list, 'function') + assert.equal(calls[0].env.HYP_HOME, hypHome) + } finally { + if (handle) { + await handle.stop() + await handle.done + } + await fs.rm(hypHome, { recursive: true, force: true }) + } +}) diff --git a/test/core/status-client-actions.test.js b/test/core/status-client-actions.test.js index afff3f9..94ef0cf 100644 --- a/test/core/status-client-actions.test.js +++ b/test/core/status-client-actions.test.js @@ -107,6 +107,50 @@ test('mixed done/failed/pending/n-a reads cleanly off the marker store + config' assert.ok(report.clientActions.actions.every((a) => a.kind === 'backfill')) }) +test('a default-on backfill target (enabled client, no explicit block) shows pending on a joined host', async () => { + // Regression: backfillHandler.desired() emits for an enabled provider even + // with no `config.backfill` block (default-on). Status used to require an + // explicit block, so the default-on case was invisible. On a joined host + // it must now surface as pending. + const hypHome = await makeHome() + const stateRoot = path.join(hypHome, 'hypaware') + + const seedPath = centralSeedPath(stateRoot) + await fs.mkdir(path.dirname(seedPath), { recursive: true }) + await fs.writeFile(seedPath, JSON.stringify({ + version: 2, + plugins: [ + { name: '@hypaware/central' }, + { name: '@hypaware/ai-gateway' }, + // No `config.backfill` block at all → default-on. + { name: '@hypaware/claude' }, + ], + sinks: { central: { plugin: '@hypaware/central', config: {} } }, + }) + '\n') + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.ok(report.clientActions, 'the default-on target is surfaced') + const m = byKey(report.clientActions.actions) + assert.equal(m.get('@hypaware/claude')?.state, 'pending') + assert.equal(m.get('@hypaware/claude')?.kind, 'backfill') + // ai-gateway is enabled but is not a backfill provider — it must not appear. + assert.equal(m.has('@hypaware/ai-gateway'), false) +}) + +test('a default-on client on a NON-joined host keeps the V1 surface (no spurious action)', async () => { + // The reconciler never runs on a non-joined host, so a bare local claude + // install must not grow a new status line. + const hypHome = await makeHome() + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ + version: 2, + plugins: [{ name: '@hypaware/ai-gateway' }, { name: '@hypaware/claude' }], + }) + '\n') + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.equal(report.clientActions, null) +}) + test('a failed backfill does not flip overall to degraded', async () => { const hypHome = await makeHome() From b2613f7a87fa60afe4fe1398390302477273b1c9 Mon Sep 17 00:00:00 2001 From: Phillip Cunliffe Date: Thu, 25 Jun 2026 23:45:51 -0700 Subject: [PATCH 10/10] Round-2 fixes for client-action reconciler (PR #166) Close the three residual review findings: 1. Status/reconciler on_join inconsistency. status.js read the policy inline as `on_join !== false`, treating a malformed `on_join: "false"` (string) as default-on (pending forever), while the reconciler's readBackfillPolicy treats a non-boolean on_join as opt-out. Extract the tri-state read into a shared src/core/config/backfill_policy.js and use it in BOTH action_backfill.js and status.js so they can never disagree. Regression: status renders n/a (not pending) for a malformed on_join on a joined host. 2. Boot re-validation configRegistry. At boot.js the merge-time validate runs during config resolution, BEFORE activatePlugins registers any config_sections validators, so the runtime registry is empty there. Threading it would be a no-op giving false confidence. Documented why it is intentionally omitted; apply-time validation is the populated gate. 3. Introduce-new-plugin apply validation. A central config that first introduces a backfill-capable plugin (e.g. @hypaware/claude) skipped its config.backfill validation because the live registry only carries validators for already-active plugins. Plugins now expose a side-effect-free `configSection` export; apply discovers introduced plugins' validators from disk (never runs activate(), which would clobber live module singletons like ai-gateway's runtime) and routes each plugin to live-or-discovered. Tests boot WITHOUT claude/codex and prove an introduced malformed backfill block is rejected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plugins-workspace/claude/src/index.js | 13 +++ .../plugins-workspace/codex/src/index.js | 13 +++ src/core/config/action_backfill.js | 43 +------- src/core/config/apply_deps.js | 82 +++++++++++++- src/core/config/backfill_policy.js | 54 ++++++++++ .../config/discover_section_validators.js | 102 ++++++++++++++++++ src/core/daemon/status.js | 7 +- src/core/runtime/boot.js | 10 ++ .../config-apply-section-validators.test.js | 89 +++++++++++++-- test/core/status-client-actions.test.js | 30 ++++++ 10 files changed, 386 insertions(+), 57 deletions(-) create mode 100644 src/core/config/backfill_policy.js create mode 100644 src/core/config/discover_section_validators.js diff --git a/hypaware-core/plugins-workspace/claude/src/index.js b/hypaware-core/plugins-workspace/claude/src/index.js index 8955a85..d546e97 100644 --- a/hypaware-core/plugins-workspace/claude/src/index.js +++ b/hypaware-core/plugins-workspace/claude/src/index.js @@ -24,6 +24,19 @@ const CLIENT_NAME = 'claude' const UPSTREAM_NAME = 'anthropic' const FALLBACK_BIN_PATH = fileURLToPath(new URL('../../../../bin/hypaware.js', import.meta.url)) +/** + * The plugin's `config_sections` validator, surfaced as a side-effect-free + * export so the kernel apply path can validate this plugin's `config` block + * (the `backfill` policy) *before* the plugin is ever activated — e.g. a + * central config that first introduces `@hypaware/claude`. It is the same + * registration `activate()` hands `ctx.configRegistry.registerSection`; + * importing this module never runs `activate()`, so discovery is safe. + * + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — the plugin owns + exposes its own `backfill` validator + * @type {{ section: string, validate: typeof validateClaudeConfig }} + */ +export const configSection = { section: CLAUDE_CONFIG_SECTION, validate: validateClaudeConfig } + /** * Resolve the canonical session-context state file the Claude hook * appends to and the projector reads from. Centralised so attach() diff --git a/hypaware-core/plugins-workspace/codex/src/index.js b/hypaware-core/plugins-workspace/codex/src/index.js index 5c535da..5fd3e57 100644 --- a/hypaware-core/plugins-workspace/codex/src/index.js +++ b/hypaware-core/plugins-workspace/codex/src/index.js @@ -20,6 +20,19 @@ const CLIENT_NAME = 'codex' const UPSTREAM_NAME = 'openai' const CHATGPT_UPSTREAM_NAME = 'chatgpt' +/** + * The plugin's `config_sections` validator, surfaced as a side-effect-free + * export so the kernel apply path can validate this plugin's `config` block + * (the `backfill` policy) *before* the plugin is ever activated — e.g. a + * central config that first introduces `@hypaware/codex`. It is the same + * registration `activate()` hands `ctx.configRegistry.registerSection`; + * importing this module never runs `activate()`, so discovery is safe. + * + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — the plugin owns + exposes its own `backfill` validator + * @type {{ section: string, validate: typeof validateCodexConfig }} + */ +export const configSection = { section: CODEX_CONFIG_SECTION, validate: validateCodexConfig } + /** * Activate the `@hypaware/codex` adapter plugin. * diff --git a/src/core/config/action_backfill.js b/src/core/config/action_backfill.js index 7d5c0eb..5520a11 100644 --- a/src/core/config/action_backfill.js +++ b/src/core/config/action_backfill.js @@ -2,6 +2,7 @@ import { Attr } from '../observability/index.js' import { selectProviders } from '../commands/backfill.js' +import { readBackfillPolicy } from './backfill_policy.js' /** * @import { @@ -13,7 +14,6 @@ import { selectProviders } from '../commands/backfill.js' * CreateBackfillHandlerOptions, * DesiredAction, * } from './types.d.ts' - * @import { PluginConfigInstance } from '../../../collectivus-plugin-kernel-types.d.ts' */ const MS_PER_DAY = 86_400_000 @@ -168,47 +168,6 @@ export const backfillHandler = createBackfillHandler() /* ------------------------------- Internals ------------------------------- */ -/** - * Read the owning plugin entry's `backfill` policy block. A *missing* block - * is the default (on_join on, no window) — `desired()` is the reconcile path - * and must not throw on a config the plugin validator (T5) already accepted. - * - * A block that is *present but malformed* must not fail open, though: a - * non-boolean `on_join` (e.g. the JSON typo `on_join: "false"`) is treated - * as an opt-out (`onJoin: false`), never as "default on". The operator - * clearly intended to set the flag, and a potentially months-deep import is - * the wrong thing to run on a malformed opt-out. (With the per-plugin - * validator now live — see apply/boot wiring — such a config is rejected at - * apply time anyway; this is the belt-and-braces reconcile-path read.) - * - * @param {PluginConfigInstance | undefined} entry - * @returns {{ onJoin: boolean | undefined, windowDays: number | undefined }} - */ -function readBackfillPolicy(entry) { - const config = entry?.config - const backfill = - config && typeof config === 'object' && !Array.isArray(config) - ? /** @type {Record} */ (config).backfill - : undefined - if (!backfill || typeof backfill !== 'object' || Array.isArray(backfill)) { - return { onJoin: undefined, windowDays: undefined } - } - const raw = /** @type {Record} */ (backfill) - // Absent → default on (undefined). Present-and-boolean → that value. - // Present-but-non-boolean → opt-out (false): do not fail open. - const onJoin = - raw.on_join === undefined - ? undefined - : typeof raw.on_join === 'boolean' - ? raw.on_join - : false - const windowDays = - typeof raw.window_days === 'number' && Number.isInteger(raw.window_days) && raw.window_days > 0 - ? raw.window_days - : undefined - return { onJoin, windowDays } -} - /** * Sum `providers[].rows_written` out of a `hyp backfill --json` payload. * Returns `undefined` when the payload is unparseable or shapeless so the diff --git a/src/core/config/apply_deps.js b/src/core/config/apply_deps.js index 6ac8756..aaa29ff 100644 --- a/src/core/config/apply_deps.js +++ b/src/core/config/apply_deps.js @@ -3,6 +3,7 @@ import { Attr, getLogger } from '../observability/index.js' import { parseConfigShape } from './schema.js' import { validateConfig } from './validate.js' +import { discoverConfigSectionValidators } from './discover_section_validators.js' import { buildPluginCatalog } from '../plugin_catalog.js' import { discoverBundledPlugins } from '../runtime/bundled.js' import { discoverInstalledPlugins } from '../runtime/installed.js' @@ -10,7 +11,8 @@ import { installPlugin, loadLock } from '../plugin_install/install.js' import { getEntry } from '../plugin_install/lock.js' /** - * @import { ConfigRegistry, PluginConfigInstance, PluginName, ValidationError } from '../../../collectivus-plugin-kernel-types.d.ts' + * @import { ConfigRegistry, HypAwareV2Config, PluginConfigInstance, PluginName, ValidationError, ValidationResult } from '../../../collectivus-plugin-kernel-types.d.ts' + * @import { LoadedManifest } from '../manifest.js' * @import { ConfigApplyDeps, PinnedInstallResult } from './types.d.ts' */ @@ -60,13 +62,25 @@ export function buildConfigApplyDeps(opts) { [...bundled.loaded, ...bundled.excluded], installed.loaded ) + + // Per-plugin `config_sections` validation (LLP 0037). The live registry + // only carries validators for *already-active* plugins, so a central + // config that first introduces a backfill-capable plugin (e.g. + // `@hypaware/claude`) would otherwise skip its `config.backfill` + // validation. Discover the section validators for any introduced plugin + // from disk (side-effect-free — never runs `activate()`) and route each + // plugin to the right source: live for active, discovered for introduced. + const allManifests = [...bundled.loaded, ...bundled.excluded, ...installed.loaded] + const sectionRegistry = await buildSectionRegistry({ + document: shape.config, + live: configRegistry, + allManifests, + }) + const result = await validateConfig(shape.config, { knownPlugins: catalog.pluginMetadata, knownDatasets: catalog.knownDatasets, - // Pass the live registry so per-plugin `config_sections` validators run - // (LLP 0037). Absent (e.g. a non-daemon caller) it degrades to the - // cross-plugin checks only, exactly as before. - ...(configRegistry ? { configRegistry } : {}), + ...(sectionRegistry ? { configRegistry: sectionRegistry } : {}), }) return { ok: result.ok, errors: /** @type {ValidationError[]} */ (result.errors) } } @@ -153,3 +167,61 @@ export function buildConfigApplyDeps(opts) { return { validateDocument, installPinnedPlugins } } + +/** + * Build the `ConfigRegistry` the document's per-plugin section validation + * should run against. Active plugins are served by the live registry (the + * validators actually registered at activation); plugins the document + * *introduces* are served by validators discovered from disk so their + * `config` block is validated before they ever activate (LLP 0037). Returns + * `undefined` when there is nothing to validate against (no live registry and + * no introduced plugin), preserving the prior "cross-plugin checks only" + * degradation for non-daemon callers with a document that introduces nothing. + * + * @param {{ + * document: HypAwareV2Config, + * live: ConfigRegistry | undefined, + * allManifests: LoadedManifest[], + * }} args + * @returns {Promise} + */ +async function buildSectionRegistry({ document, live, allManifests }) { + // Plugin names the live registry already validates. `list()` is on the + // concrete registry; the type narrows it to `ConfigRegistry`, so reach it + // through a local cast. + const liveList = /** @type {{ list?: () => Array<{ plugin: string }> }} */ (live ?? {}) + const liveSections = new Set((liveList.list?.() ?? []).map((s) => s.plugin)) + + const introduced = (document.plugins ?? []) + .filter((p) => p && p.enabled !== false && !liveSections.has(p.name)) + .map((p) => p.name) + + let discovered + if (introduced.length > 0) { + const want = new Set(introduced) + discovered = await discoverConfigSectionValidators({ + manifests: allManifests.filter((m) => want.has(/** @type {PluginName} */ (m.manifest.name))), + }) + } + + if (!live && !discovered) return undefined + if (!discovered) return live + if (!live) return discovered + + /** @type {ConfigRegistry} */ + const composite = { + // This is a read-only validation view; nothing registers through it. + registerSection() {}, + /** + * @param {PluginName} name + * @param {unknown} config + * @returns {ValidationResult} + */ + validatePluginConfig(name, config) { + return liveSections.has(name) + ? live.validatePluginConfig(name, config) + : discovered.validatePluginConfig(name, config) + }, + } + return composite +} diff --git a/src/core/config/backfill_policy.js b/src/core/config/backfill_policy.js new file mode 100644 index 0000000..e4b66f9 --- /dev/null +++ b/src/core/config/backfill_policy.js @@ -0,0 +1,54 @@ +// @ts-check + +/** + * @import { PluginConfigInstance } from '../../../collectivus-plugin-kernel-types.d.ts' + */ + +/** + * Read a plugin entry's `backfill` policy block (LLP 0037) as a + * tri-state. The single source of truth for interpreting the block — + * shared by the reconciler (`action_backfill.js`, which decides whether to + * run an import) and the status surface (`status.js`, which renders + * `pending`/`n/a`) so the two can never disagree on what a given block + * means. + * + * A *missing* block is the default (on_join on → `onJoin: undefined`, no + * window): the reconcile path must not throw on a config the plugin + * validator (LLP 0037) already accepted. + * + * A block that is *present but malformed* must not fail open, though: a + * non-boolean `on_join` (e.g. the JSON typo `on_join: "false"`) is treated + * as an opt-out (`onJoin: false`), never as "default on". The operator + * clearly intended to set the flag, and a potentially months-deep import is + * the wrong thing to run on a malformed opt-out. (With the per-plugin + * validator now live — see apply/boot wiring — such a config is rejected at + * apply time anyway; this is the belt-and-braces read both consumers share.) + * + * @param {PluginConfigInstance | undefined} entry + * @returns {{ onJoin: boolean | undefined, windowDays: number | undefined }} + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [constrained-by] — backfill policy ({ on_join, window_days }) is owned by the plugin; the kernel only reads it + */ +export function readBackfillPolicy(entry) { + const config = entry?.config + const backfill = + config && typeof config === 'object' && !Array.isArray(config) + ? /** @type {Record} */ (config).backfill + : undefined + if (!backfill || typeof backfill !== 'object' || Array.isArray(backfill)) { + return { onJoin: undefined, windowDays: undefined } + } + const raw = /** @type {Record} */ (backfill) + // Absent → default on (undefined). Present-and-boolean → that value. + // Present-but-non-boolean → opt-out (false): do not fail open. + const onJoin = + raw.on_join === undefined + ? undefined + : typeof raw.on_join === 'boolean' + ? raw.on_join + : false + const windowDays = + typeof raw.window_days === 'number' && Number.isInteger(raw.window_days) && raw.window_days > 0 + ? raw.window_days + : undefined + return { onJoin, windowDays } +} diff --git a/src/core/config/discover_section_validators.js b/src/core/config/discover_section_validators.js new file mode 100644 index 0000000..3aa65a5 --- /dev/null +++ b/src/core/config/discover_section_validators.js @@ -0,0 +1,102 @@ +// @ts-check + +import path from 'node:path' +import { pathToFileURL } from 'node:url' + +import { Attr, getLogger } from '../observability/index.js' +import { createConfigRegistry } from './schema.js' + +/** + * @import { ConfigRegistry, ConfigSectionRegistration, PluginName } from '../../../collectivus-plugin-kernel-types.d.ts' + * @import { LoadedManifest } from '../manifest.js' + */ + +/** + * Discover the per-plugin `config_sections` validators a set of plugins + * expose, WITHOUT activating them. A plugin opts in by exporting a + * `configSection` (`{ section, validate }`) from its manifest entrypoint — + * the same registration it hands `ctx.configRegistry.registerSection` at + * activation, surfaced as a side-effect-free export. + * + * Why not just run `activate()`? Because activation is unsafe to run + * ad-hoc: a plugin's `activate()` can mutate module-global singletons that + * the *live* process shares (e.g. `@hypaware/ai-gateway`'s + * `setAiGatewayRuntime`), so re-running it outside a real boot would corrupt + * the running daemon. Importing a module to read an export runs only its + * top-level code (which boot imports anyway) and never `activate()`. + * + * Used by the apply path so a central config that *introduces* a + * backfill-capable plugin (e.g. `@hypaware/claude`) has its `config.backfill` + * block validated even though that plugin isn't active yet — closing the gap + * where the live registry only carries validators for already-active plugins + * (LLP 0037). + * + * Best-effort: an entrypoint that can't be imported, or that exports no + * `configSection`, contributes nothing and is logged but never throws — + * discovery must never fail an apply on its own; real activation at the next + * boot remains the backstop. + * + * @param {{ manifests: LoadedManifest[] }} args + * @returns {Promise} + * @ref LLP 0037#per-plugin-config-kernel-generic-reconciler [implements] — discover the owning plugin's `backfill` validator for not-yet-active plugins, side-effect-free + */ +export async function discoverConfigSectionValidators({ manifests }) { + const registry = createConfigRegistry() + const log = getLogger('config') + + for (const entry of manifests) { + // Only import plugins that declare a config section in their manifest. + // This keeps discovery from importing entrypoints that can't contribute + // a validator anyway. + const declared = entry.manifest.contributes?.config_sections + if (!Array.isArray(declared) || declared.length === 0) continue + + try { + const abs = path.resolve(entry.rootDir, entry.manifest.entrypoint) + const mod = await import(pathToFileURL(abs).href) + for (const reg of exportedSections(mod)) { + registry.registerSection({ + plugin: /** @type {PluginName} */ (entry.manifest.name), + section: reg.section, + validate: reg.validate, + }) + } + } catch (err) { + log.warn('config.section_discovery_failed', { + [Attr.COMPONENT]: 'config', + [Attr.PLUGIN]: entry.manifest.name, + message: err instanceof Error ? err.message : String(err), + }) + } + } + + return registry +} + +/** + * Pull the validator registration(s) a plugin entrypoint exports. Accepts a + * single `configSection` or an array `configSections`; ignores malformed + * shapes so a typo in a plugin export can never crash discovery. + * + * @param {Record} mod + * @returns {Array<{ section: string, validate: ConfigSectionRegistration['validate'] }>} + */ +function exportedSections(mod) { + /** @type {Array<{ section: string, validate: ConfigSectionRegistration['validate'] }>} */ + const out = [] + const candidates = [ + mod.configSection, + ...(Array.isArray(mod.configSections) ? mod.configSections : []), + ] + for (const c of candidates) { + if (!c || typeof c !== 'object') continue + const reg = /** @type {Record} */ (c) + if (typeof reg.section === 'string' && reg.section.length > 0 && typeof reg.validate === 'function') { + out.push({ + section: reg.section, + validate: /** @type {ConfigSectionRegistration['validate']} */ (reg.validate), + }) + } + } + return out +} diff --git a/src/core/daemon/status.js b/src/core/daemon/status.js index 04173aa..3fff0b0 100644 --- a/src/core/daemon/status.js +++ b/src/core/daemon/status.js @@ -8,6 +8,7 @@ import process from 'node:process' import { defaultConfigPath, loadConfigFile } from '../config/schema.js' import { readConfigControlStatus, resolveCentralLayerPath } from '../config/apply.js' import { readClientActionStatus } from '../config/action_reconciler.js' +import { readBackfillPolicy } from '../config/backfill_policy.js' import { resolveLayeredConfig } from '../config/merge.js' import { devTelemetryDir, readObservabilityEnv } from '../observability/env.js' import { collectConfigErrors, diagnoseV1Config, validateConfig } from '../config/validate.js' @@ -591,7 +592,11 @@ function buildClientActionsReport({ status, config, hasCentral, backfillPlugins const raw = entry.config?.backfill const hasBlock = !!raw && typeof raw === 'object' && !Array.isArray(raw) if (hasBlock) { - const onJoin = /** @type {Record} */ (raw).on_join !== false + // Use the shared tri-state read so status can never disagree with the + // reconciler about what a block means: a malformed `on_join` (e.g. the + // string "false") is an opt-out, not default-on. `onJoin: undefined` + // (block present, `on_join` absent) is default-on → not suppressed. + const onJoin = readBackfillPolicy(entry).onJoin !== false declared.set(entry.name, { onJoin }) } else if (hasCentral && backfillCapable.has(entry.name)) { declared.set(entry.name, { onJoin: true }) diff --git a/src/core/runtime/boot.js b/src/core/runtime/boot.js index df6facc..80ab9d3 100644 --- a/src/core/runtime/boot.js +++ b/src/core/runtime/boot.js @@ -336,6 +336,16 @@ export async function resolveLayeredConfigFromDisk({ stateRoot, configPath, know const merged = resolveLayeredConfig({ central: centralConfig, local: localConfig, + // No `configRegistry` here on purpose. This merge-time validation runs + // during config *resolution* — before `activatePlugins`, which is when + // each plugin registers its `config_sections` validator. At this point in + // boot the runtime's `configRegistry` exists but is *empty*, so threading + // it would dispatch `runPerPluginSectionValidators` against zero + // registered sections: a no-op that gives false confidence. Per-plugin + // section validation is enforced where the registry is actually populated + // — the daemon's apply path (`buildConfigApplyDeps`), which also discovers + // validators for plugins a document introduces but that aren't active yet. + // Boot's merge stays limited to the cross-plugin/structural checks. validate: (cfg) => collectConfigErrors(cfg, { ...(knownPlugins ? { knownPlugins } : {}), ...(knownDatasets ? { knownDatasets } : {}), diff --git a/test/core/config-apply-section-validators.test.js b/test/core/config-apply-section-validators.test.js index 4e21690..359f116 100644 --- a/test/core/config-apply-section-validators.test.js +++ b/test/core/config-apply-section-validators.test.js @@ -27,8 +27,13 @@ function env(hypHome) { return { ...process.env, HYP_HOME: hypHome, HYP_CONFIG: '' } } -/** Boot ai-gateway + claude from a local config so the claude section registers. */ -async function bootWithClaude() { +/** + * Boot a kernel from a local config with the given plugin list, returning the + * booted runtime so apply deps can be built against the live registry. + * + * @param {string[]} pluginNames + */ +async function bootWith(pluginNames) { const hypHome = await fs.mkdtemp(path.join(os.tmpdir(), 'hyp-section-validators-')) const stateRoot = path.join(hypHome, 'hypaware') const configPath = defaultConfigPath(hypHome) @@ -37,13 +42,18 @@ async function bootWithClaude() { configPath, JSON.stringify({ version: 2, - plugins: [{ name: '@hypaware/ai-gateway' }, { name: '@hypaware/claude' }], + plugins: pluginNames.map((name) => ({ name })), }) + '\n' ) const boot = await bootKernel({ hypHome, configPath, env: env(hypHome), mode: 'cli' }) return { hypHome, stateRoot, boot, cleanup: () => fs.rm(hypHome, { recursive: true, force: true }) } } +/** Boot ai-gateway + claude from a local config so the claude section registers. */ +function bootWithClaude() { + return bootWith(['@hypaware/ai-gateway', '@hypaware/claude']) +} + test('apply validation rejects a malformed plugin backfill block via the live section validator', async () => { const fx = await bootWithClaude() try { @@ -95,12 +105,71 @@ test('apply validation rejects a malformed plugin backfill block via the live se } }) -test('without the live registry the per-plugin validator is dead (the bug this fixes)', async () => { - const fx = await bootWithClaude() +test('apply validates a backfill block for a plugin the document INTRODUCES but is not active yet', async () => { + // Round-2 regression: the live registry only carries validators for + // *already-active* plugins. A central config that first introduces a + // backfill-capable plugin (the realistic join/fleet path) would skip its + // `config.backfill` validation. The apply path now discovers the introduced + // plugin's section validator from disk (side-effect-free — never activates + // it), so the malformed block is rejected, not silently accepted. + // + // Boot WITHOUT claude/codex so neither section is in the live registry. + const fx = await bootWith(['@hypaware/ai-gateway']) + try { + const registry = /** @type {{ list(): Array<{ plugin: string }> }} */ ( + /** @type {unknown} */ (fx.boot.runtime.configRegistry) + ) + const live = registry.list().map((s) => s.plugin) + assert.ok( + !live.includes('@hypaware/claude') && !live.includes('@hypaware/codex'), + `neither client section should be live-registered, got ${JSON.stringify(live)}` + ) + + const deps = buildConfigApplyDeps({ + stateRoot: fx.stateRoot, + configRegistry: fx.boot.runtime.configRegistry, + }) + + // A doc that first introduces claude + codex, claude's backfill malformed. + const badDoc = { + version: 2, + plugins: [ + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: 'false', window_days: -3 } } }, + { name: '@hypaware/codex' }, + ], + } + const res = await deps.validateDocument(badDoc) + assert.equal(res.ok, false, 'an introduced plugin with a malformed backfill block must be rejected') + const kinds = /** @type {Array<{ errorKind?: string }>} */ (res.errors).map((e) => e.errorKind) + assert.ok( + kinds.includes('config_section_invalid'), + `expected a config_section_invalid error, got ${JSON.stringify(kinds)}` + ) + + // The same introduce-claude/codex doc with well-formed blocks validates. + const goodDoc = { + version: 2, + plugins: [ + { name: '@hypaware/ai-gateway' }, + { name: '@hypaware/claude', config: { backfill: { on_join: false, window_days: 30 } } }, + { name: '@hypaware/codex', config: { backfill: { on_join: true } } }, + ], + } + const ok = await deps.validateDocument(goodDoc) + assert.equal(ok.ok, true, JSON.stringify(ok.errors)) + } finally { + await fx.cleanup() + } +}) + +test('introduced-plugin discovery rejects a malformed block even without the live registry', async () => { + // Even with NO live registry passed (a non-daemon caller), the apply path + // discovers the introduced plugin's validator from disk and rejects the bad + // block. (Before round-2 this exact shape silently accepted it — the + // per-plugin validator was dead without the live registry.) + const fx = await bootWith(['@hypaware/ai-gateway']) try { - // Same malformed document, but apply deps built WITHOUT the registry — - // exactly the pre-fix call shape. The cross-plugin checks pass and the - // dead section validator never runs, so the bad block slips through. const depsNoRegistry = buildConfigApplyDeps({ stateRoot: fx.stateRoot }) const badDoc = { version: 2, @@ -110,7 +179,9 @@ test('without the live registry the per-plugin validator is dead (the bug this f ], } const res = await depsNoRegistry.validateDocument(badDoc) - assert.equal(res.ok, true, 'pre-fix shape accepts the malformed backfill block') + assert.equal(res.ok, false, 'disk discovery rejects the malformed block with no live registry') + const kinds = /** @type {Array<{ errorKind?: string }>} */ (res.errors).map((e) => e.errorKind) + assert.ok(kinds.includes('config_section_invalid'), JSON.stringify(kinds)) } finally { await fx.cleanup() } diff --git a/test/core/status-client-actions.test.js b/test/core/status-client-actions.test.js index 94ef0cf..9ddecf3 100644 --- a/test/core/status-client-actions.test.js +++ b/test/core/status-client-actions.test.js @@ -107,6 +107,36 @@ test('mixed done/failed/pending/n-a reads cleanly off the marker store + config' assert.ok(report.clientActions.actions.every((a) => a.kind === 'backfill')) }) +test('a malformed on_join block renders n/a (not pending) on a joined host', async () => { + // Regression (round-2): a *present but malformed* `on_join` (the JSON typo + // `on_join: "false"`) is an opt-out, exactly as `backfillHandler.desired()` + // reads it — so the reconciler never writes a marker and the honest state is + // `n/a`. Status used to read `on_join !== false` inline, so the string + // "false" (!== the boolean false) showed `pending` forever. Both consumers + // now share `readBackfillPolicy`, so they agree. + const hypHome = await makeHome() + const stateRoot = path.join(hypHome, 'hypaware') + + const seedPath = centralSeedPath(stateRoot) + await fs.mkdir(path.dirname(seedPath), { recursive: true }) + await fs.writeFile(seedPath, JSON.stringify({ + version: 2, + plugins: [ + { name: '@hypaware/central' }, + { name: '@hypaware/ai-gateway' }, + // Malformed opt-out: the string "false", not the boolean false. + { name: '@hypaware/claude', config: { backfill: { on_join: 'false' } } }, + ], + sinks: { central: { plugin: '@hypaware/central', config: {} } }, + }) + '\n') + await fs.writeFile(defaultConfigPath(hypHome), JSON.stringify({ version: 2, plugins: [] }) + '\n') + + const report = await collectHypAwareStatus({ env: env(hypHome) }) + assert.ok(report.clientActions, 'the malformed-opt-out target is surfaced') + const m = byKey(report.clientActions.actions) + assert.equal(m.get('@hypaware/claude')?.state, 'n/a') +}) + test('a default-on backfill target (enabled client, no explicit block) shows pending on a joined host', async () => { // Regression: backfillHandler.desired() emits for an enabled provider even // with no `config.backfill` block (default-on). Status used to require an