From 5db66242f2af89054378571c1bced67815314dfd Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Tue, 25 Aug 2026 07:29:34 -0700 Subject: [PATCH 01/37] docs(architecture): define model lifecycle intelligence --- .../0005-dashboard-in-page-routing-reveal.md | 27 ++- ...ge-scorecard-local-transcript-analytics.md | 23 +- ...-capability-driven-integration-adapters.md | 12 +- docs/adr/0017-opencode-host.md | 12 +- docs/adr/0032-model-lifecycle-intelligence.md | 198 ++++++++++++++++++ docs/adr/README.md | 11 + docs/ddd/README.md | 5 + docs/ddd/context-map.md | 35 +++- docs/ddd/integration-management.md | 24 ++- docs/ddd/model-lifecycle-intelligence.md | 198 ++++++++++++++++++ docs/ddd/observability.md | 19 +- docs/ddd/routing-and-orchestration.md | 19 ++ docs/ddd/ubiquitous-language.md | 30 +++ 13 files changed, 574 insertions(+), 39 deletions(-) create mode 100644 docs/adr/0032-model-lifecycle-intelligence.md create mode 100644 docs/ddd/model-lifecycle-intelligence.md diff --git a/docs/adr/0005-dashboard-in-page-routing-reveal.md b/docs/adr/0005-dashboard-in-page-routing-reveal.md index e817910..b30fded 100644 --- a/docs/adr/0005-dashboard-in-page-routing-reveal.md +++ b/docs/adr/0005-dashboard-in-page-routing-reveal.md @@ -2,9 +2,10 @@ - **Status:** Implemented - **Date:** 2026-07-23 -- **Updated:** 2026-08-04 -- **Update note:** Kept the read-only routing reveal and canonical GA configuration while - consolidating the dashboard into three primary areas with one shared secondary navigation rail. +- **Updated:** 2026-08-25 +- **Update note:** Reconciled the implemented five-area shell introduced by ADR-0025 and ADR-0026. + ADR-0032 is Accepted but not implemented; when it ships, Models will be a secondary Usage + destination and the Dashboard will remain read-only and network-silent on ordinary reads. - **Deciders:** agentic-kit maintainers > **GA amendment:** the read-only dashboard decision remains. References below to compatibility @@ -18,18 +19,22 @@ At the time of this decision, `ak dashboard` was a single-page, poll-every-5s, grid grouped by subsystem, a `#history` strip — all fed by shelling `ak status --json` (`src/lib/dashboard-server.mjs`). It is health/status oriented. -> **Current implementation note (2026-08-04):** the read-only and loopback boundaries remain. The -> page now has three primary areas—Overview, Usage, and Observability—and one fixed, left-aligned +> **Current implementation note (2026-08-25):** the read-only and loopback boundaries remain. The +> page now has five primary areas—About, Overview, Usage, Observability, and System—and one fixed, +> left-aligned > secondary rail. It also has user-configurable status polling, lazy Usage reads, and an SSE-driven > Observability view. Page, styles, browser client, Observability, and request/session security live > under `src/lib/dashboard/`; `dashboard-server.mjs` is the HTTP composition root. -**2026-08-04 information-architecture amendment:** Overview absorbs the former health-oriented +**2026-08-25 information-architecture amendment:** About supplies the component directory and +System supplies machine-footprint views under ADR-0026 and ADR-0025. Overview absorbs the former +health-oriented peer tabs as **Summary**, **Hosts & Routing**, **Providers**, **Runtime**, and **Intelligence**. Usage owns **Scorecard**, **Limits**, **Findings**, **Sessions**, and **Transcript**. Observability -owns **Live** and **History**. The secondary row remains in one stable location across all three -areas. Canonical hashes are `#overview/{view}`, `#usage/{view-or-session-id}`, and -`#observability/{live,history}`. Every destination has a visible heading and description. Primary +owns **Live** and **History**. System and About retain the secondary destinations documented by +their governing ADRs. The secondary row remains in one stable location across all five areas. +Canonical hashes are rooted at `#about`, `#overview`, `#usage`, `#observability`, and `#system`. +Every destination has a visible heading and description. Primary and secondary tab lists use roving focus: Left/Right activates the adjacent tab with wrapping, and Home/End activates the first/last tab. @@ -55,7 +60,7 @@ Surface routing via **in-page reveal**, not a new page or tab: - Preserves the single-page, health-first idiom; routing is an enhancement reached by an intuitive in-page link, not a replacement. -- Presents only three stable primary choices while keeping Overview's status domains one keyboard +- Preserves five stable primary choices while keeping Overview's status domains one keyboard action away in the shared secondary rail. - Provides durable, hierarchical deep-link vocabulary without adding routes, servers, or a second navigation component per area. @@ -67,7 +72,7 @@ Surface routing via **in-page reveal**, not a new page or tab: ## References - `src/lib/dashboard-server.mjs` (`renderPage`, `#cards`, `#history`/`renderHistory`, `PREF`, `shellOutStatus`) -- `src/lib/dashboard/page.mjs` and `src/lib/dashboard/client.mjs` (three-area shell, shared secondary +- `src/lib/dashboard/page.mjs` and `src/lib/dashboard/client.mjs` (five-area shell, shared secondary rail, canonical hashes, headings, and keyboard semantics) - [Dashboard user guide](../DASHBOARD.md) - Mockup: ak dashboard — Routing panel; ADR-0001, ADR-0003 diff --git a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md index 33a9622..3b8e0be 100644 --- a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md +++ b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md @@ -2,12 +2,15 @@ - **Status:** Implemented - **Date:** 2026-07-25 -- **Updated:** 2026-08-24 -- **Update note:** Issue #170 added backward-compatible parsing for legacy Codex messages and +- **Updated:** 2026-08-25 +- **Update note:** Reconciled Usage with the implemented five-area Dashboard. ADR-0032 is Accepted + but not implemented; its planned Models destination will consume structured observed-model facts + without moving transcript indexing, session history, or usage aggregates out of this context. + Issue #170 added backward-compatible parsing for legacy Codex messages and `item_completed` envelopes, bumped the derived-index schema to force reparse of stale zero-turn records, added Codex parse-yield diagnostics, and separated first-billed-day session counts from token-bearing active-day counts. The earlier OpenRouter account-analytics cache boundary for issue #59, - aligned Usage with the dashboard's shared three-area navigation, and documented independent + aligned Usage with the dashboard's shared navigation, and documented independent host, inference-provider, provenance, and model facts in session rows. ADR-0023 subsequently classified SQLite source failures and made transient OpenCode failures preserve last-good records with explicit degraded source health instead of becoming observed zero usage; the Usage UI now @@ -77,12 +80,12 @@ Three properties of the data force most of the design: ADR-0007 split `admin` from `dashboard` along **network egress**: `dashboard` promises silence, `admin` promises reach. Usage analytics reads **local files only** and makes **zero network calls**. It therefore sits squarely inside the dashboard's existing offline-first contract and ships as one -of the dashboard's three primary areas, not a new server. +of the dashboard's five primary areas, not a new server. -> **Current implementation note (2026-08-04):** The dashboard exposes exactly three primary areas: -> Overview, Usage, and Observability. One fixed, left-aligned secondary rail provides the current -> area's destinations. Usage still loads lazily and remains separate from the live transcript -> tailers. +> **Current implementation note (2026-08-25):** The dashboard exposes five primary areas: About, +> Overview, Usage, Observability, and System. One fixed, left-aligned secondary rail provides the +> current area's destinations. Usage still loads lazily and remains separate from the live +> transcript tailers. Usage carries five in-page views — **Scorecard**, **Limits**, **Findings**, **Sessions**, and **Transcript** — deep-linked as `#usage/score`, `#usage/limits`, `#usage/findings`, @@ -93,6 +96,10 @@ Left/Right Arrow activates the adjacent destination with wrapping, while Home an first and last destination. This reuses ADR-0005's in-page reveal idiom without adding navigation concepts. +> **Planned ADR-0032 amendment:** Models will become a sixth secondary Usage destination after the +> model-lifecycle contract is implemented. It will read inventory and lifecycle projections; it +> will not make Historical Usage own catalogues, entitlement, route impact, or model quality. + **2026-08-04 session-identity amendment:** the compact session badge identifies the execution host only. Its adjacent disclosure control expands evidence without navigating away from the session list. The details report execution host, inference provider, provider evidence/provenance, and diff --git a/docs/adr/0016-capability-driven-integration-adapters.md b/docs/adr/0016-capability-driven-integration-adapters.md index 7e31a02..7a23ee4 100644 --- a/docs/adr/0016-capability-driven-integration-adapters.md +++ b/docs/adr/0016-capability-driven-integration-adapters.md @@ -4,7 +4,7 @@ [ADR-0020](0020-ga-stable-surfaces.md); closed-registry clause superseded by [ADR-0029](0029-host-adapter-extension-point.md) - **Date:** 2026-07-28 -- **Updated:** 2026-08-20 +- **Updated:** 2026-08-25 - **Update note:** Added read-only Codex plugin-hook compatibility facts, runtime-selected Ruflo project-memory store proofs, and the non-correlatable OpenRouter account-analytics boundary; removed the pre-GA compatibility command, @@ -12,10 +12,12 @@ to declare its setup trust posture and changes for host-neutral preflight. Phase 0 consistency pass (2026-08-14): host adapters gained a required `enabledByDefault` boolean and the three enabled-host default literals now - derive from it via `defaultHostMap()` (F-15); the `observability` axis is - recorded as terminal — validation metadata with referential integrity only, - deliberately not a dispatch surface, no collector loop exists (F-12); the - non-throwing `validateBinding` is wired into `ak host status` as per-entry + derive from it via `defaultHostMap()` (F-15); the `observability` axis remains validation + metadata for implemented integrations: no general-purpose collector loop + exists (F-12). ADR-0032 now accepts one narrow future exception: Model lifecycle intelligence + may select catalogue descriptors and dispatch only to built-in, bounded source adapters; it does + not turn descriptors into arbitrary executable plugins or claim that behavior is implemented. + The non-throwing `validateBinding` is wired into `ak host status` as per-entry warnings (F-16); and the integrations migrator derives each host's native default provider from the provider registry's host-login entries instead of a literal map, inferring no binding at all for hosts without one (F-13). diff --git a/docs/adr/0017-opencode-host.md b/docs/adr/0017-opencode-host.md index 63c8e27..218c40c 100644 --- a/docs/adr/0017-opencode-host.md +++ b/docs/adr/0017-opencode-host.md @@ -3,8 +3,11 @@ - **Status:** Accepted; compatibility references amended by [ADR-0020](0020-ga-stable-surfaces.md) - **Date:** 2026-07-28 -- **Updated:** 2026-08-17 -- **Update note:** Clarified that the AQE boundary applies to inference-provider routing, not +- **Updated:** 2026-08-25 +- **Update note:** ADR-0032 accepts, but has not yet implemented, project/provider-scoped OpenCode + model discovery through a bounded descriptor-driven source adapter and an explicit online refresh. + That future reader does not change OpenCode's opt-in, non-primary, non-AQE routing boundary. + Clarified that the AQE boundary applies to inference-provider routing, not AQE's upstream OpenCode platform assets, and recorded the implemented OpenCode transcript, token, observed-cost, and provider-id analytics path. ADR-0023 adds classified SQLite source health, preserves last-good OpenCode usage when a present store is temporarily unreadable, @@ -20,6 +23,11 @@ > configuration. Historical references to the removed compatibility executor do not describe a > supported 4.0 surface. +**Planned model-lifecycle amendment (2026-08-25):** OpenCode's configured model references, +project/provider-scoped `models` output, and explicitly refreshed catalogue may become evidence +sources under ADR-0032. Public discovery will not prove current-project entitlement, and ordinary +status or Dashboard reads will not invoke OpenCode or perform network refresh. + ## Context ADR-0016 separates execution hosts, inference providers, projections, observability, diff --git a/docs/adr/0032-model-lifecycle-intelligence.md b/docs/adr/0032-model-lifecycle-intelligence.md new file mode 100644 index 0000000..7fc9c2b --- /dev/null +++ b/docs/adr/0032-model-lifecycle-intelligence.md @@ -0,0 +1,198 @@ +# ADR-0032 — Model lifecycle intelligence from provenance-aware local evidence + +- **Status:** Accepted (implementation planned) +- **Date:** 2026-08-25 +- **Updated:** 2026-08-25 +- **Update note:** Accepted the domain semantics, source-adapter boundary, snapshot policy, + command ownership, privacy rules, and Dashboard placement. No implementation is claimed yet. +- **Deciders:** agentic-kit maintainers +- **Related:** [issue #110](https://github.com/pacphi/agentic-kit/issues/110), + [ADR-0001](0001-one-routing-policy-many-projections.md), + [ADR-0005](0005-dashboard-in-page-routing-reveal.md), + [ADR-0009](0009-usage-scorecard-local-transcript-analytics.md), + [ADR-0016](0016-capability-driven-integration-adapters.md), + [ADR-0017](0017-opencode-host.md), + [ADR-0020](0020-ga-stable-surfaces.md), + [ADR-0021](0021-inference-provider-provenance.md), and + [ADR-0023](0023-fail-closed-operations-and-explicit-degradation.md) + +## Context + +Agentic Kit knows the configured activity routes, their generated projections, and models observed +in local session evidence. It also carries a dated curated model table. Those facts cannot answer a +model's lifecycle state: a configured alias can move without the configuration changing; a public +catalogue entry does not prove account entitlement; a failed catalogue read does not prove removal; +and a first-party migration target does not prove equal quality or lower cost. + +Claude Code, Codex, OpenCode, and local providers expose different configuration, catalogue, cache, +policy, and runtime shapes. Treating any one as the universal schema would collapse independent +claims and make upstream schema drift look like model churn. The same model reference can also be +consumed by canonical routes, escalation rungs, host projections, Agentic QE, Ruflo, and future +Route Intelligence evidence. A lifecycle change can invalidate any of those consumers without +authorizing Agentic Kit to rewrite them. + +The existing boundaries remain load-bearing: + +- top-level `kit.json.routing` is the one routing policy; +- host and inference-provider identity are independent; +- native evidence crosses source-specific anti-corruption adapters; +- unknown and degraded evidence remain visible rather than becoming false or zero; and +- the Dashboard is local, protected, read-only, and network-silent on ordinary reads. + +## Decision + +### 1. Establish a Model lifecycle intelligence bounded context + +Model lifecycle intelligence owns normalized model inventory snapshots, lifecycle and +compatibility edges, trustworthy snapshot diffs, consumer impact, and read-model projections. It +consumes configuration intent, integration descriptors, Historical Usage and Observability +evidence, and host/provider catalogue evidence. + +It does not own routing policy, host/provider configuration, transcript indexing, model quality +evaluation, or Ruflo/Agentic-QE routing. It may diagnose those consumers and produce a read-only +swap plan, but mutation remains on the canonical `ak host pick` surface. + +### 2. Keep identity, state, and evidence independent + +A model identity is scoped by execution host, inference provider, concrete model id, and a +non-identifying scope id. A mutable local digest participates when the provider exposes one. +Reasoning effort and service tier belong to a route binding or observed execution variant; the +model record may state which variants it supports, but those settings do not create unrelated base +model identities. + +The following dimensions remain independent: + +- configured; +- effective after precedence and alias resolution; +- observed in structured local evidence; +- discoverable in the active scope; +- entitled for the active account or profile; +- allowed by managed or user policy; +- routable through the complete host/provider/auth/capability path; +- lifecycle state; and +- recommended by a named first-party or evidence-backed source. + +Every field and graph edge carries an evidence reference naming its source, class, capture time, +freshness, completeness, and scope. Evidence strength never leaks from one field to another. In +particular, observed use may establish entitlement for that observed path, but it does not establish +the completeness of a catalogue. + +### 3. Translate native catalogues through descriptor-driven source adapters + +Integration observability descriptors may identify catalogue sources for a host or provider. A +bounded Model lifecycle collector loop selects those descriptors and dispatches to explicit +source-specific adapters. Command code does not branch on host ids. A descriptor is metadata, not +an executable supplied by the host, and external adapters without a supported catalogue descriptor +report `unsupported` rather than receiving an inferred capability. + +The initial adapter set covers: + +- Claude configured values, aliases, overrides, policy allowlists, and configured gateway + discovery while leaving unsupported entitlement unknown; +- Codex's host-owned model cache or stable model-list protocol behind schema/version guards; +- OpenCode's project/provider-scoped model list and separately authorized online refresh; and +- Ollama catalogue, digest, and runtime evidence through the same normalized contract. + +Interactive picker scraping and inference probes are excluded. All subprocess calls use literal +argument arrays, bounded timeouts, output-size limits, and no shell interpolation. + +### 4. Persist sanitized, bounded snapshots and advance baselines conservatively + +A `CatalogSnapshot` contains a schema version, id, capture time, scope, source states, normalized +models, bindings, and diagnostics. Source states are `complete`, `partial`, `stale`, `unavailable`, +`unsupported`, or `unsupported-schema`. Snapshots are rebuildable operational evidence, never +canonical configuration. + +History retains at most 32 baseline-eligible snapshots per scope and no snapshot older than 90 +days. An implementation may retain a newer degraded diagnostic snapshot outside the comparison +baseline, but it may not displace the last eligible baseline. Scope identity includes host, +provider, host/source schema version, and keyed non-identifying account/profile/project +fingerprints. Snapshots from different scopes are never compared as one lifecycle sequence. + +An authoritative first-party retirement or removal signal may create a tombstone immediately. +Without one, removal requires absence from two consecutive complete snapshots in the same stable +scope. Partial, stale, unavailable, or schema-invalid sources never create removals and never +advance the comparison baseline. + +### 5. Represent lifecycle and compatibility as typed edges + +Lifecycle edges include `resolves-to`, `first-party-migration`, and `same-family-newer`. +Compatibility edges include `mechanically-compatible`, `tier-up`, `tier-down`, and +`specialized-alternative`. Each edge carries provenance and scope. + +Mechanical compatibility requires an expressible host/provider transport, required modality/tool +capabilities, supported route variant, policy allowance, and established entitlement or an explicit +warning that blocks the compatibility claim. Unknown required evidence yields `unknown`, not a +compatible edge. + +Only Route Intelligence may contribute `evidence-backed-equivalent`, `cheaper-equivalent`, or +`premium-justified` claims. When no such evidence exists, Model lifecycle intelligence says +`quality unknown`. Alias-target or relevant capability changes keep historical Route Intelligence +evidence visible but mark it stale; invalidation never deletes its audit history. + +### 6. Ship one read-only command family + +The stable noun is plural: `ak models`. + +- `ak models status` reads the latest local snapshot and current local bindings; +- `ak models refresh` collects local config, caches, protocols, and observed evidence; +- `ak models refresh --online` is the only model-catalogue network boundary; +- `ak models diff` compares eligible same-scope snapshots; +- `ak models explain` discloses a model or alias evidence chain; and +- `ak models plan` reports affected routes, projections, Agentic QE/Ruflo consumers, and stale + evidence, then emits a copyable canonical `ak host pick --route ...` command when expressible. + +There is no `ak models apply`. A future transactional swap requires a separate decision and must +still mutate only canonical routing intent with preview, confirmation, verification, and undo. +`ak status --deep` remains local and does not perform remote model refresh. + +### 7. Add cache-only status and Dashboard read models + +`ak status` gains one cache-only model-health row. The Dashboard keeps its five primary areas — +About, Overview, Usage, Observability, and System — and adds Models as a secondary destination +under Usage plus a compact Overview summary. The Models destination presents attention, host +inventory, change history, consumers, swap impact, and evidence disclosure. + +Normal status and Dashboard reads never refresh a catalogue or invoke a model. The Dashboard cannot +apply a plan. All model counts, badges, and warnings link to their evidence state, freshness, +completeness, and scope. + +### 8. Protect private scope and configuration facts + +Credentials, auth tokens, prompts, reasoning traces, and raw private provider configuration never +enter snapshots or Dashboard payloads. Account, profile, project, gateway deployment, and private +endpoint identities use a stable keyed fingerprint derived from an owner-only per-install secret. +Normal display uses sanitized identifiers; exact local disclosure is limited to an explicit CLI +request. + +Snapshot files are owner-only and atomically replaced. Native cache/protocol data is untrusted and +subject to byte, schema, enum, and timeout bounds. Dashboard delivery retains loopback binding, +session-token authorization, CSP/origin protections, `no-store`, and secret scanning. + +## Consequences + +- Model existence, access, policy, use, lifecycle, and recommendation can disagree honestly without + collapsing to one availability boolean. +- Failed discovery cannot manufacture a mass removal or erase the last trustworthy baseline. +- Host-native schema changes degrade one source behind its adapter instead of corrupting the domain + model. +- `kit.json.routing`, `ak host pick`, Ruflo, Agentic QE, and Route Intelligence keep their existing + ownership; inventory is a diagnostic and planning consumer. +- The local cache adds bounded disk state and a per-install fingerprint secret that uninstall and + privacy documentation must account for. +- Supporting a new host catalogue requires a descriptor, an anti-corruption adapter, fixtures, and + explicit evidence semantics; host identity alone grants nothing. + +## Acceptance conditions + +The decision may be marked Implemented only when: + +1. all independent state dimensions and evidence references survive human and JSON projections; +2. Claude, Codex, OpenCode, and local-provider fixtures normalize deterministically; +3. partial or cross-scope snapshots cannot create removals or advance the baseline; +4. alias, migration, capability, visibility, and local-digest changes diff correctly; +5. plans enumerate canonical routes and independent Agentic QE/Ruflo consumers without mutation; +6. ordinary CLI/Dashboard reads are proven network-silent and token-silent; +7. snapshot and Dashboard payloads pass credential, prompt, private-id, and path disclosure checks; +8. Dashboard keyboard, responsive, and screen-reader contracts pass; and +9. exact-head project, Agentic QE, privacy, security, and release gates are recorded. diff --git a/docs/adr/README.md b/docs/adr/README.md index c6d2bb9..d2e93e7 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -39,6 +39,7 @@ Consequences**, and cites the grounded source it rests on where relevant. | [0028](0028-local-openai-compatible-providers.md) | One generic local OpenAI-compatible provider, not a vendor enumeration | Accepted | | [0029](0029-host-adapter-extension-point.md) | External host adapters: declarative manifest, subprocess hooks | Accepted (experimental contract) | | [0031](0031-capability-graduation-and-upstream-requests.md) | Capability graduation: earned parity for external adapters, and the upstream request path | Accepted (governance; implementation staged) | +| [0032](0032-model-lifecycle-intelligence.md) | Model lifecycle intelligence from provenance-aware local evidence | Accepted (implementation planned) | Theme: ADRs **0001–0006** define **dual-host LLM routing and leadership** — how `ak` lets ruflo route each development activity (architecture, implementation, testing, review, …) to the right host (Claude @@ -225,3 +226,13 @@ agentic-qe's closed enum; being a native ruflo backend is ruflo's `ENABLE_*` mod tracked capability requests with honest interim behaviour rather than pretended support. Accepted as a governance decision; the machinery (the trust CLI, external execution, tiered conformance, the grant store) is staged and self-graded in the ADR's implementation-status table. + +**0032** accepts Model lifecycle intelligence as a new bounded context while leaving implementation +planned. It keeps configured, effective, observed, discoverable, entitled, policy-allowed, +routable, lifecycle, and recommended state independent; normalizes host/provider catalogues through +bounded descriptor-driven source adapters; and persists sanitized same-scope snapshots whose +baselines advance only on sufficiently complete evidence. Diffs, explanations, and swap plans are +read-only. Canonical route mutation remains `ak host pick`, first-party migration remains distinct +from quality, and only Route Intelligence may claim evidence-backed equivalence. When implemented, +the feature adds `ak models`, a cache-only status row, an Overview summary, and Models beneath Usage +without changing the Dashboard's five primary areas. diff --git a/docs/ddd/README.md b/docs/ddd/README.md index 6e86f39..7b52c00 100644 --- a/docs/ddd/README.md +++ b/docs/ddd/README.md @@ -11,6 +11,7 @@ describe the current system unless a section is explicitly marked as future work | [Context map](context-map.md) | Bounded contexts, ownership, and relationships | | [Integration management](integration-management.md) | Hosts, inference providers, bindings, capabilities, lifecycle, facts, and ownership | | [Routing and orchestration](routing-and-orchestration.md) | Activities, routes, leadership, escalation, projections, and canonical `ak run` execution | +| [Model lifecycle intelligence](model-lifecycle-intelligence.md) | Model identities, evidence dimensions, catalogue snapshots, lifecycle diff, and read-only impact plans | | [Observability](observability.md) | Evidence acquisition, observed-session aggregates, replay, and dashboard delivery | | [Project intelligence](project-intelligence.md) | Pattern store, learning counters, reasoning-graph size, and live delivery for Overview's Intelligence view | @@ -32,6 +33,10 @@ A change that introduces or changes a domain concept should: 4. add executable coverage for new invariants; and 5. update operational documentation when user-visible behavior changes. +An Accepted ADR may define a target contract before its implementation exists. Such a domain +document must label planned commands, collectors, or read models explicitly and must not present +them as shipped behavior. + The GA model keeps host enablement under `integrations.hosts`, integration ownership under `integrations`, and activity intent under top-level `routing`. These persisted locations do not change the canonical distinction between a host and an inference provider. diff --git a/docs/ddd/context-map.md b/docs/ddd/context-map.md index a57816a..5d36aee 100644 --- a/docs/ddd/context-map.md +++ b/docs/ddd/context-map.md @@ -13,12 +13,12 @@ Configuration Intent Native Evidence ----> Evidence Acquisition ----> Canonical Evidence | +-------------------------+----------------------+ - | | - v v - Observability Historical Usage - | | | - | +----> Workspace Snapshot Cache | - | | | + | | | + v v v + Observability Model Lifecycle Intelligence Historical Usage + | | | | + | +----> Workspace +----------------------+ + | Snapshot Cache | +-----------------------+----> Dashboard Delivery <----+ ^ Project State (.claude-flow/*) ----> Project Intelligence-+ @@ -71,6 +71,18 @@ Owns transcript indexing, session history, token and cost aggregation, classific findings. It may share a host-qualified session identity with Observability, but its aggregate and cache are separate from the live event store. +### Model lifecycle intelligence + +Owns the planned normalized inventory of configured, effective, observed, discoverable, entitled, +policy-allowed, routable, and lifecycle model facts; sanitized same-scope snapshots; trustworthy +diffs; lifecycle and compatibility edges; and read-only consumer impact plans. It consumes +Configuration Intent, integration descriptors, source-adapted native catalogues, and structured +observed facts from Historical Usage and Observability. It does not own transcripts, route +mutation, model quality, or downstream router policy. + +See [Model lifecycle intelligence](model-lifecycle-intelligence.md). ADR-0032 is Accepted; this +context remains a target contract until that ADR is marked Implemented. + ### Workspace snapshot cache Owns the bounded, owner-only last safe `SessionWorkspace` value per host-qualified session. It is @@ -139,6 +151,13 @@ and credential policy is distinct from the offline-first dashboard and integrati | Native evidence | Evidence acquisition | Source-specific anti-corruption adapters | | Evidence acquisition | Observability | Versioned canonical events | | Evidence acquisition | Historical usage | Normalized transcript and provider evidence | +| Configuration intent | Model lifecycle intelligence | Canonical route, escalation, binding, and projection references | +| Integration management | Model lifecycle intelligence | Capability and catalogue descriptors plus bounded host/provider facts | +| Evidence acquisition | Model lifecycle intelligence | Source-adapted catalogue, policy, lifecycle, and runtime evidence | +| Historical usage | Model lifecycle intelligence | Structured observed host/provider/model facts; transcript ownership stays upstream | +| Observability | Model lifecycle intelligence | Structured recent execution identity; live-event ownership stays upstream | +| Model lifecycle intelligence | Routing and orchestration | Read-only lifecycle diagnostics and copyable canonical route actions | +| Model lifecycle intelligence | Dashboard delivery | Sanitized cache-only inventory, changes, consumers, and plan read models | | Observability | Dashboard delivery | Read-model snapshots, deltas, and selected evidence | | Observability | Workspace snapshot cache | Last safe metadata-only session workspace capture | | Workspace snapshot cache | Dashboard delivery | Inert last-recorded History context after restart | @@ -175,6 +194,10 @@ observed before the split was made explicit. - User intent does not establish observed reality. - A host observation does not establish inference-provider identity. - Dashboard presentation cannot upgrade provenance. +- Model lifecycle inventory cannot turn discovery into entitlement, compatibility into quality, or + a read-only plan into route mutation. +- Snapshot comparison requires stable scope and sufficient source completeness; degraded evidence + never creates removal. - Historical usage and live topology share identifiers, not aggregate ownership. - Network egress occurs only in commands and contexts whose contract explicitly permits it. - Every project count is rendered with the scope that produced it; two contexts may report diff --git a/docs/ddd/integration-management.md b/docs/ddd/integration-management.md index 63b2e1f..4034c14 100644 --- a/docs/ddd/integration-management.md +++ b/docs/ddd/integration-management.md @@ -27,8 +27,10 @@ Host -------- ProviderBinding -------- InferenceProvider +-- host capabilities ``` -The four adapter registries are closed, validated, built-in code. A provider binding is persisted -relationship data, not a fifth executable adapter family. +The four built-in adapter registries are validated code. ADR-0029 additionally admits an +experimental, hash-consented, subprocess-only external host overlay; it does not turn provider, +projection, or observability descriptors into arbitrary in-process plugins. A provider binding is +persisted relationship data, not a fifth executable adapter family. ### Host @@ -62,6 +64,10 @@ protocols, and non-loopback plaintext HTTP. - Adding OpenRouter does not make it a host. - Provider claims are rendered only when the provider declares and evidence supports them. - Compatibility collections are derived from registries. +- An observability descriptor grants no executable behavior by itself. Under Accepted ADR-0032, a + future Model lifecycle collector may select catalogue descriptors and dispatch only to an + explicit bounded source-adapter implementation. External hosts without a supported catalogue + descriptor remain `unsupported`. ## Integration facts @@ -86,6 +92,17 @@ Billing follows the access path. Anthropic or OpenAI host login may be subscript their API keys are metered. Ollama is local and may be priced at exact zero only after provider identity is established. +## Planned model catalogue evidence + +ADR-0032 accepts a narrow, read-only use of integration observability descriptors. Catalogue +descriptors identify source ownership, evidence fields, local/online mode, and host/provider scope. +They remain declarative. The Model lifecycle bounded context owns collector dispatch, native-schema +translation, completeness, freshness, and snapshot policy. + +This planned reader does not widen the managed projection lifecycle: catalogue refresh never calls +`apply`, never invokes a model, and never changes a binding. Online discovery is separately +authorized by `ak models refresh --online`; normal status and Dashboard reads remain cache-only. + ## Managed projection lifecycle Every managed configuration projection follows: @@ -149,3 +166,6 @@ envelopes and never reads the retired paths. 8. Undo cannot overwrite drift. 9. Existing and future configuration survives additive migration. 10. Status, setup, sync, verify, and uninstall consume the same normalized model. +11. Catalogue descriptor identity alone executes nothing and proves neither entitlement nor + routability. +12. Model lifecycle collection cannot mutate integration intent or native projections. diff --git a/docs/ddd/model-lifecycle-intelligence.md b/docs/ddd/model-lifecycle-intelligence.md new file mode 100644 index 0000000..1cd415a --- /dev/null +++ b/docs/ddd/model-lifecycle-intelligence.md @@ -0,0 +1,198 @@ +# Model Lifecycle Intelligence Domain + +This document defines the bounded context accepted by +[ADR-0032](../adr/0032-model-lifecycle-intelligence.md). It describes the target contract; until +ADR-0032 is Implemented, commands and read models named here are planned rather than shipped. + +## Purpose + +Model lifecycle intelligence answers which models are configured, selected, observed, discoverable, +usable, changing, and consequential to local consumers. It preserves the evidence and uncertainty +behind each answer and produces read-only status, diff, explanation, and swap-impact projections. + +It does not select the best model, mutate routing, own host/provider configuration, or replace the +Ruflo, Agentic QE, or Route Intelligence routers. + +## Context boundary + +```text +Configuration Intent ------> binding collector --------+ +Integration descriptors ---> catalogue adapters -------+ +Historical Usage ----------> observed-use collector ---+--> CatalogSnapshot +Host/provider evidence -----> source adapters ----------+ | + +--> ModelChange + +--> ConsumerImpact + +--> SwapPlan + +--> read models +``` + +Model lifecycle intelligence consumes facts from four neighboring contexts: + +- Configuration Intent supplies canonical routes, escalation rungs, integration bindings, and + provider configuration references. +- Integration Management supplies capability and observability descriptors, projection identity, + and bounded host/provider facts. +- Evidence Acquisition supplies source-adapted catalogue, policy, lifecycle, and runtime evidence. +- Historical Usage and Observability supply structured observed host/provider/model facts without + surrendering ownership of transcripts, sessions, or live events. + +Its outputs are advisory inputs to Routing and Orchestration, Dashboard Delivery, Agentic QE/Ruflo +diagnostics, and Route Intelligence. + +## Aggregate and value objects + +### ModelIdentity + +```text +ModelIdentity + host + provider + modelId + scopeId + digest? +``` + +Identity is host- and scope-qualified because equal strings can name different deployments, +accounts, projects, gateways, or local model bytes. `digest` participates only where a source +establishes a mutable local artifact digest. Reasoning effort and service tier belong to a binding +or execution variant, not to the base identity. + +### ModelRecord + +A ModelRecord carries display identity, aliases, lifecycle state, supported variants and +capabilities, optional pricing metadata, and field-level evidence references. No record-level +confidence may silently strengthen a weaker field. + +The independent state dimensions are configured, effective, observed, discoverable, entitled, +policy allowed, routable, lifecycle, and recommended. Each is `true`, `false`, or `unknown` where +applicable and names the evidence that established it. + +### ModelBinding + +```text +ModelBinding + consumer + activity? + host + provider + configured reference + effective concrete identity? + execution variant? + provenance + evidenceRefs[] +``` + +Consumers include canonical routes, escalation rungs, host projections, Agentic QE overrides and +fallbacks, Ruflo candidates, and Route Intelligence cohorts. A configured alias remains in the +binding even after an effective concrete target is resolved. + +### CatalogSource + +A source identifies its host or provider, local or online collection mode, source/schema version, +capture time, non-identifying scope fingerprint, status, completeness, and diagnostics. Status is +one of `complete`, `partial`, `stale`, `unavailable`, `unsupported`, or `unsupported-schema`. + +### CatalogSnapshot + +The aggregate root is a sanitized, immutable snapshot: + +```text +CatalogSnapshot + schemaVersion + snapshotId + capturedAt + scope + sources[] + models[] + bindings[] + diagnostics[] +``` + +Changes, opportunities, and plans are derived from snapshots plus current bindings; they are not +written back as catalogue truth. + +### LifecycleEdge and CompatibilityEdge + +Lifecycle relations are `resolves-to`, `first-party-migration`, and `same-family-newer`. +Compatibility relations are `mechanically-compatible`, `tier-up`, `tier-down`, and +`specialized-alternative`. Every edge has source, scope, confidence, and evidence references. + +`evidence-backed-equivalent`, `cheaper-equivalent`, and `premium-justified` belong to Route +Intelligence. This context can preserve and invalidate those imported claims but cannot create them. + +### ModelChange + +A change names its kind, subject, before/after values, severity, scope, and evidence. Additions, +visibility, alias, lifecycle, capability, reasoning, context, and digest changes require comparable +same-scope evidence. Removal additionally requires an authoritative signal or two consecutive +complete same-scope absences. + +### SwapPlan + +A SwapPlan is a read-only impact report. Each item links a source binding to affected canonical +routes, projections, Agentic QE/Ruflo consumers, compatibility blockers, and evidence that becomes +stale. When expressible, it supplies a copyable `ak host pick --route ...` command. It has no apply +operation. + +## Evidence rules + +Evidence classes, strongest first for the field they actually establish, are: + +1. observed successful execution with concrete host/provider/model identity; +2. host-owned entitled catalogue or explicit lifecycle metadata; +3. host-owned discoverable catalogue/cache in the active scope; +4. managed policy/configuration after precedence resolution; +5. canonical Agentic Kit intent and generated projection; +6. provider-published public catalogue; +7. dated locally curated metadata; +8. inferred family relationship; and +9. unknown. + +Strength is field-local. A successful execution proves that path worked at that time; it does not +prove catalogue completeness, future entitlement, or global provider identity. Negative evidence +requires completeness and stable scope. + +## Snapshot lifecycle + +- Retain at most 32 baseline-eligible snapshots per scope and no snapshot older than 90 days. +- Advance a baseline only from a sufficiently complete snapshot in the same stable scope. +- Preserve the last eligible baseline when a later collection is partial, stale, unavailable, or + schema-invalid. +- Compare account/profile/project scopes only when their keyed fingerprints match. +- Allow an authoritative first-party removal immediately; otherwise require two consecutive + complete absences. +- Keep stale recommendation history auditable after alias, capability, provider, host-version, or + harness changes. + +## Discovery contract + +The collector selects catalogue observability descriptors and invokes a bounded source adapter. +The command layer never switches on a hard-coded host id. Descriptors authorize no arbitrary code: +built-in adapters own parsing and external hosts without a supported descriptor report +`unsupported`. + +Local refresh reads configuration, local caches/protocols, and observed evidence. Online refresh is +separate and explicit. Neither path invokes a model or sends a prompt. + +## Privacy and delivery + +Snapshots exclude credentials, raw provider configuration, prompts, reasoning traces, and raw +private deployment identities. An owner-only per-install secret keys scope and private-identity +fingerprints. Snapshot replacement is atomic and owner-only. Native inputs have size, timeout, +schema, and enum bounds. + +CLI status and Dashboard reads are cache-only. The Dashboard receives sanitized read models behind +its existing loopback, session-token, origin, CSP, and `no-store` boundary and cannot apply a plan. + +## Invariants + +1. Host, provider, model, scope, and execution variant remain separate facts. +2. Configured, effective, observed, discoverable, entitled, allowed, and routable never collapse. +3. Every fact and edge carries source, freshness, completeness, and scope. +4. Unknown or degraded evidence never becomes removal, unavailability, or compatibility. +5. Snapshots from different scopes never produce lifecycle churn. +6. Inventory never mutates canonical routing or downstream router state. +7. First-party migration is not a quality or economic recommendation. +8. Route Intelligence evidence stays visible but stale after invalidation. +9. Ordinary reads make no network request and consume no inference tokens. +10. Private configuration and transcript content never enter snapshots or aggregate APIs. diff --git a/docs/ddd/observability.md b/docs/ddd/observability.md index 9cfdcfa..4f6b693 100644 --- a/docs/ddd/observability.md +++ b/docs/ddd/observability.md @@ -569,22 +569,31 @@ The UI receives only projection DTOs: - `health`: sanitized adapter status and aggregate counters; - `cursor` and `schemaVersion`. -The dashboard shell has exactly three primary areas: `Overview`, `Usage`, and `Observability`. One -fixed, left-aligned secondary navigation rail remains in the same location while its contents change: +The dashboard shell has five primary areas: `About`, `Overview`, `Usage`, `Observability`, and +`System`. One fixed, left-aligned secondary navigation rail remains in the same location while its +contents change: ```text Overview → Summary | Hosts & Routing | Providers | Runtime | Intelligence Usage → Scorecard | Limits | Findings | Sessions | Transcript Observability → Live | History +System → Summary | Advisory | Sessions | Storage | Runtime | Catalog | Projects ``` +About is a continuous editorial directory with secondary section anchors. ADR-0032 accepts a +planned Models destination beneath Usage; it is not part of the implemented navigation until that +ADR is marked Implemented. Model lifecycle read models may consume sanitized observed identity, +but Observability retains ownership of live sessions, events, topology, and replay. + Navigation state has canonical hierarchical hashes: ```text +#about/{hosts,engine,quality,kit,configured} #overview/{summary,hosts,providers,runtime,intelligence} #usage/{score,limits,findings,sessions,transcript} #usage/{sessionId} #observability/{live,history} +#system/{summary,advisory,sessions,storage,runtime,catalog,projects} ``` Each destination owns a visible heading and concise description. The primary and secondary controls @@ -812,7 +821,7 @@ process memory limits. independently draggable and pinnable. - Hover/focus descriptions, persistent selection detail, and Legend / Help explain every unit and the available next interaction. -- Exactly three primary areas share one fixed, left-aligned secondary rail; canonical hashes, +- Five primary areas share one fixed, left-aligned secondary rail; canonical hashes, headings, descriptions, roving Left/Right focus, and Home/End behavior match each destination. - Collapsing Session Stream preserves its connection and local choice, expands Agent activity, leaves a keyboard-accessible restore rail, and remains compact when the layout stacks. @@ -848,8 +857,8 @@ The server subscribes before taking the initial snapshot and reconciles buffered closing the snapshot-to-subscribe race. A slow-client queue overflow discards queued frames and sends a reset snapshot after drain. -The implemented dashboard shell exposes three primary areas and one shared secondary rail. It emits -the canonical Overview, Usage, and Observability hashes above, gives every view a heading and +The implemented dashboard shell exposes five primary areas and one shared secondary rail. It emits +the canonical About, Overview, Usage, Observability, and System hashes, gives every view a heading and description, and implements Left/Right/Home/End tab semantics. Observability's locally persisted Session Stream chevron changes layout without changing subscription ownership. Usage session rows show a host-only badge and reveal provider/provenance/model facts in their own detail strip. diff --git a/docs/ddd/routing-and-orchestration.md b/docs/ddd/routing-and-orchestration.md index 33b65b5..b91d8c0 100644 --- a/docs/ddd/routing-and-orchestration.md +++ b/docs/ddd/routing-and-orchestration.md @@ -101,6 +101,23 @@ two Ollama bindings do not create two Ollama providers, and configured host/mode prove which inference vendor served a session. OpenCode runs only through `ak run` after its host adapter declares the required capability. +## Model lifecycle boundary + +ADR-0032 accepts a separate Model lifecycle intelligence context. It may read canonical routes, +escalation rungs, and generated projections to diagnose affected consumers. Its `SwapPlan` is a +read-only projection and may emit a copyable `ak host pick --route ...` action; it cannot apply that +action or become another routing policy. + +The curated `RETIRED_MODELS` behavior described above remains the implemented routing rule until a +separate implementation deliberately changes it. Future catalogue discovery does not feed that +automatic substitution path merely because a model is hidden, missing, stale, or deprecated. +Inventory lifecycle facts diagnose and plan; only authoritative evidence under an implemented +contract may justify changing dispatch behavior. + +First-party migration and mechanical compatibility are not quality claims. Only imported Route +Intelligence evidence can call a candidate equivalent, cheaper, or worth a premium. Alias or +capability changes mark that evidence stale while preserving its audit history. + ## Cost safety Automatic seeding targets only known subscription-backed or local execution paths. Metered @@ -121,3 +138,5 @@ must distinguish per-token price from measured or expected per-task cost. 6. Escalation is explicit, ordered, and per route. 7. Automatic seeding cannot introduce a metered provider path. 8. `ak run` is the sole executor for materialized activity plans. +9. Model lifecycle inventory may diagnose routes but cannot mutate or execute them. +10. Catalogue absence never enters the curated retirement substitution path by inference. diff --git a/docs/ddd/ubiquitous-language.md b/docs/ddd/ubiquitous-language.md index 89be81a..64c9b5e 100644 --- a/docs/ddd/ubiquitous-language.md +++ b/docs/ddd/ubiquitous-language.md @@ -60,6 +60,30 @@ missing price. `Dual-host` describes two enabled peer hosts, not an execution command and not evidence that two inference vendors served a workflow. Generalized execution belongs to `ak run`. +## Model lifecycle language + +These terms define the Accepted target contract in ADR-0032. They are not shipped command claims +until that ADR becomes Implemented. + +| Term | Meaning | +|------|---------| +| Model identity | Host-, provider-, model-id-, and scope-qualified inference target, plus a digest when local bytes are mutable and evidenced | +| Model scope | Non-identifying account/profile/project/source boundary within which catalogue snapshots are comparable | +| Execution variant | Binding- or execution-level reasoning effort, service tier, modality, or similar setting; not a separate base model identity | +| Model binding | One consumer's configured reference and, when established, effective concrete model identity with provenance | +| Catalog source | Host/provider-native configuration, cache, protocol, or catalogue input with version, scope, freshness, completeness, and diagnostics | +| Catalog snapshot | Sanitized immutable inventory of source states, model records, bindings, scope, and diagnostics at one capture time | +| Baseline-eligible snapshot | Sufficiently complete same-scope snapshot permitted to replace the prior lifecycle comparison baseline | +| Model change | Evidence-backed difference between comparable snapshots; removal needs authoritative evidence or repeated complete absence | +| Lifecycle edge | Typed alias resolution, first-party migration, or same-family-newer relationship with provenance and scope | +| Compatibility edge | Typed mechanical swap relationship; it is not a quality or economic recommendation | +| Consumer impact | Read-only link from a lifecycle fact to affected routes, projections, Agentic QE/Ruflo consumers, or Route Intelligence evidence | +| Swap plan | Read-only impact report and copyable canonical route action; never an independent routing policy or apply operation | + +Configured, effective, observed, discoverable, entitled, policy allowed, routable, lifecycle, and +recommended are separate model-state dimensions. `Unknown` in one dimension cannot be filled from +another. A first-party migration is a supported lifecycle edge, not proof of equivalence. + ## Project intelligence language | Term | Meaning | @@ -148,6 +172,10 @@ runtime state is a chip word, never a prose word. See - Qualify **projection** as configuration projection or read-model projection when ambiguity is possible. - Qualify **adapter** as integration adapter or source adapter when ambiguity is possible. +- Say **catalogue source** for model discovery evidence and **catalog snapshot** for the normalized, + sanitized local record; neither is canonical routing policy. +- Say **compatible candidate** only when required mechanical facts are established. Reserve + **cheaper equivalent** and **premium justified** for Route Intelligence evidence. - Do not infer an inference provider from a transcript host alone. - Do not replace an unknown fact with a convenient default. - Say **System** for the dashboard area and the command; say **Machine footprint** only for the @@ -161,6 +189,8 @@ runtime state is a chip word, never a prose word. See providers. - `ak system [--deep] [--json]` renders the Machine footprint collector; `ak about [--category] [--json]` renders the Component directory. Both are read-only twins of a dashboard area. +- ADR-0032 reserves `ak models` for the planned read-only model inventory, refresh, diff, explain, + and plan family. Route mutation remains `ak host pick`; there is no accepted `ak models apply`. - `kit.json.integrations.hosts` records enabled hosts. Top-level `routing` records `version`, `primaryHost`, and per-activity `routes`; route entries use `provenance` and `escalation`. - Derived exports in `hosts.mjs`, `providers.mjs`, and `routing.mjs` are views, not independent From 024d4974f1efbef4461d3a1db6746eaaca9831cb Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Tue, 25 Aug 2026 07:31:30 -0700 Subject: [PATCH 02/37] feat(models): add lifecycle evidence core --- src/lib/model-inventory/contracts.mjs | 237 +++++++++++++++++++ src/lib/model-inventory/diff.mjs | 124 ++++++++++ src/lib/model-inventory/impact.mjs | 190 +++++++++++++++ src/lib/model-inventory/index.mjs | 5 + src/lib/model-inventory/read-model.mjs | 63 +++++ src/lib/model-inventory/store.mjs | 116 +++++++++ tests/kit/model-inventory-contracts.test.mjs | 79 +++++++ tests/kit/model-inventory-diff.test.mjs | 84 +++++++ tests/kit/model-inventory-impact.test.mjs | 133 +++++++++++ tests/kit/model-inventory-store.test.mjs | 89 +++++++ 10 files changed, 1120 insertions(+) create mode 100644 src/lib/model-inventory/contracts.mjs create mode 100644 src/lib/model-inventory/diff.mjs create mode 100644 src/lib/model-inventory/impact.mjs create mode 100644 src/lib/model-inventory/index.mjs create mode 100644 src/lib/model-inventory/read-model.mjs create mode 100644 src/lib/model-inventory/store.mjs create mode 100644 tests/kit/model-inventory-contracts.test.mjs create mode 100644 tests/kit/model-inventory-diff.test.mjs create mode 100644 tests/kit/model-inventory-impact.test.mjs create mode 100644 tests/kit/model-inventory-store.test.mjs diff --git a/src/lib/model-inventory/contracts.mjs b/src/lib/model-inventory/contracts.mjs new file mode 100644 index 0000000..8790a90 --- /dev/null +++ b/src/lib/model-inventory/contracts.mjs @@ -0,0 +1,237 @@ +import { immutable } from '../adapters/schema.mjs'; + +export const MODEL_INVENTORY_SCHEMA_VERSION = 1; +export const MODEL_DIMENSIONS = Object.freeze([ + 'configured', 'effective', 'observed', 'discoverable', + 'entitled', 'policyAllowed', 'routable', 'recommended', +]); +export const SOURCE_STATUSES = Object.freeze([ + 'complete', 'partial', 'stale', 'unavailable', 'unsupported', 'unsupported-schema', +]); +export const EVIDENCE_CLASSES = Object.freeze([ + 'configured', 'observed', 'catalog', 'runtime', 'first-party', 'inferred', 'unknown', +]); +export const LIFECYCLE_STATES = Object.freeze([ + 'active', 'preview', 'hidden', 'deprecated', 'retiring', 'removed', 'unknown', +]); +export const CONSUMER_STATES = Object.freeze([ + 'configured', 'reported', 'runtime-proven', 'unknown', +]); + +const plain = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const text = (value, field, { nullable = false, max = 512 } = {}) => { + if (nullable && value == null) return null; + if (typeof value !== 'string' || value.length === 0 || value.length > max) { + throw new TypeError(`${field} must be a non-empty string${nullable ? ' or null' : ''}`); + } + return value; +}; +const iso = (value, field, { nullable = true } = {}) => { + if (nullable && value == null) return null; + const ms = Date.parse(value); + if (!Number.isFinite(ms)) throw new TypeError(`${field} must be an ISO timestamp${nullable ? ' or null' : ''}`); + return new Date(ms).toISOString(); +}; +const enumValue = (value, allowed, field) => { + if (!allowed.includes(value)) throw new TypeError(`${field} must be one of: ${allowed.join(', ')}`); + return value; +}; +const strings = (value, field) => { + if (value == null) return []; + if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string' || !entry)) { + throw new TypeError(`${field} must be an array of non-empty strings`); + } + return [...new Set(value)]; +}; + +export function modelIdentityKey({ host, provider = null, modelId, scopeId }) { + return [ + text(host, 'model.key.host', { max: 128 }), + provider == null ? '' : text(provider, 'model.key.provider', { max: 256 }), + text(modelId, 'model.key.modelId', { max: 256 }), + text(scopeId, 'model.key.scopeId', { max: 256 }), + ].map((part) => encodeURIComponent(part)).join('|'); +} + +export function normalizeEvidence(value, index = 0) { + if (!plain(value)) throw new TypeError(`evidence[${index}] must be an object`); + const id = text(value.id ?? `evidence-${index}`, `evidence[${index}].id`, { max: 256 }); + const completeness = value.completeness ?? 'unknown'; + if (!['complete', 'partial', 'unknown'].includes(completeness)) { + throw new TypeError(`evidence[${index}].completeness must be complete, partial, or unknown`); + } + const freshness = value.freshness ?? 'unknown'; + if (!['fresh', 'stale', 'unknown'].includes(freshness)) { + throw new TypeError(`evidence[${index}].freshness must be fresh, stale, or unknown`); + } + return immutable({ + id, + field: text(value.field, `evidence[${index}].field`, { max: 256 }), + source: text(value.source, `evidence[${index}].source`, { max: 256 }), + class: enumValue(value.class ?? 'unknown', EVIDENCE_CLASSES, `evidence[${index}].class`), + capturedAt: iso(value.capturedAt, `evidence[${index}].capturedAt`), + freshness, + completeness, + scopeFingerprint: value.scopeFingerprint == null + ? null : text(value.scopeFingerprint, `evidence[${index}].scopeFingerprint`, { max: 256 }), + refs: strings(value.refs, `evidence[${index}].refs`), + }); +} + +function normalizeDimension(value, field) { + const input = plain(value) ? value : { value }; + if (input.value !== true && input.value !== false && input.value !== null && input.value !== undefined) { + throw new TypeError(`${field}.value must be true, false, or null`); + } + return immutable({ + value: input.value ?? null, + evidenceRefs: strings(input.evidenceRefs, `${field}.evidenceRefs`), + }); +} + +function normalizeAlias(value, index) { + if (!plain(value)) throw new TypeError(`aliases[${index}] must be an object`); + return immutable({ + name: text(value.name, `aliases[${index}].name`, { max: 256 }), + resolvesTo: value.resolvesTo == null + ? null : text(value.resolvesTo, `aliases[${index}].resolvesTo`, { max: 256 }), + observedAt: iso(value.observedAt, `aliases[${index}].observedAt`), + evidenceRefs: strings(value.evidenceRefs, `aliases[${index}].evidenceRefs`), + }); +} + +export function normalizeModelRecord(value) { + if (!plain(value)) throw new TypeError('model must be an object'); + if (!plain(value.key)) throw new TypeError('model.key must be an object'); + const key = { + host: text(value.key.host, 'model.key.host', { max: 128 }), + provider: value.key.provider == null + ? null : text(value.key.provider, 'model.key.provider', { max: 256 }), + modelId: text(value.key.modelId, 'model.key.modelId', { max: 256 }), + scopeId: text(value.key.scopeId, 'model.key.scopeId', { max: 256 }), + }; + const evidence = (value.evidence ?? []).map(normalizeEvidence); + const evidenceIds = new Set(evidence.map(({ id }) => id)); + if (evidenceIds.size !== evidence.length) throw new TypeError('model.evidence contains duplicate ids'); + const dimensions = Object.fromEntries(MODEL_DIMENSIONS.map((name) => [ + name, normalizeDimension(value.dimensions?.[name], `model.dimensions.${name}`), + ])); + for (const [name, dimension] of Object.entries(dimensions)) { + for (const ref of dimension.evidenceRefs) { + if (!evidenceIds.has(ref)) throw new TypeError(`model.dimensions.${name} references unknown evidence ${ref}`); + } + } + const lifecycle = plain(value.lifecycle) ? value.lifecycle : {}; + return immutable({ + key: immutable(key), + identity: modelIdentityKey(key), + displayName: value.displayName == null ? key.modelId + : text(value.displayName, 'model.displayName', { max: 256 }), + aliases: (value.aliases ?? []).map(normalizeAlias), + visibility: value.visibility == null ? 'unknown' + : text(value.visibility, 'model.visibility', { max: 64 }), + variant: immutable(structuredClone(plain(value.variant) ? value.variant : {})), + lifecycle: immutable({ + state: enumValue(lifecycle.state ?? 'unknown', LIFECYCLE_STATES, 'model.lifecycle.state'), + replacement: lifecycle.replacement == null ? null + : text(lifecycle.replacement, 'model.lifecycle.replacement', { max: 256 }), + notice: lifecycle.notice == null ? null + : text(lifecycle.notice, 'model.lifecycle.notice', { max: 1_024 }), + effectiveAt: iso(lifecycle.effectiveAt, 'model.lifecycle.effectiveAt'), + evidenceRefs: strings(lifecycle.evidenceRefs, 'model.lifecycle.evidenceRefs'), + }), + capabilities: immutable(structuredClone(plain(value.capabilities) ? value.capabilities : {})), + dimensions: immutable(dimensions), + evidence, + }); +} + +export function normalizeBindingRecord(value, index = 0) { + if (!plain(value)) throw new TypeError(`bindings[${index}] must be an object`); + const consumerState = value.consumerState ?? 'unknown'; + return immutable({ + id: text(value.id ?? `binding-${index}`, `bindings[${index}].id`, { max: 256 }), + consumer: text(value.consumer, `bindings[${index}].consumer`, { max: 256 }), + consumerState: enumValue(consumerState, CONSUMER_STATES, `bindings[${index}].consumerState`), + activity: value.activity == null ? null + : text(value.activity, `bindings[${index}].activity`, { max: 128 }), + host: value.host == null ? null : text(value.host, `bindings[${index}].host`, { max: 128 }), + provider: value.provider == null ? null + : text(value.provider, `bindings[${index}].provider`, { max: 256 }), + configured: value.configured == null ? null + : text(value.configured, `bindings[${index}].configured`, { max: 256 }), + effective: value.effective == null ? null + : text(value.effective, `bindings[${index}].effective`, { max: 256 }), + provenance: enumValue(value.provenance ?? 'unknown', + ['observed', 'configured', 'inferred', 'unknown'], `bindings[${index}].provenance`), + drift: value.drift === true, + evidenceRefs: strings(value.evidenceRefs, `bindings[${index}].evidenceRefs`), + }); +} + +export function normalizeSourceResult(value, index = 0) { + if (!plain(value)) throw new TypeError(`sources[${index}] must be an object`); + const status = enumValue(value.status, SOURCE_STATUSES, `sources[${index}].status`); + return immutable({ + id: text(value.id, `sources[${index}].id`, { max: 256 }), + status, + complete: status === 'complete' && value.complete !== false, + capturedAt: iso(value.capturedAt, `sources[${index}].capturedAt`), + sourceVersion: value.sourceVersion == null ? null + : text(value.sourceVersion, `sources[${index}].sourceVersion`, { max: 256 }), + schemaVersion: value.schemaVersion ?? null, + scopeFingerprint: value.scopeFingerprint == null ? null + : text(value.scopeFingerprint, `sources[${index}].scopeFingerprint`, { max: 256 }), + diagnostics: strings(value.diagnostics, `sources[${index}].diagnostics`), + }); +} + +export function normalizeScope(value) { + if (!plain(value)) throw new TypeError('snapshot.scope must be an object'); + const profiles = plain(value.profileFingerprints) ? value.profileFingerprints : {}; + return immutable({ + fingerprint: text(value.fingerprint, 'snapshot.scope.fingerprint', { max: 256 }), + machine: value.machine == null ? null : text(value.machine, 'snapshot.scope.machine', { max: 256 }), + project: value.project == null ? null : text(value.project, 'snapshot.scope.project', { max: 256 }), + hosts: strings(value.hosts, 'snapshot.scope.hosts'), + profileFingerprints: immutable(Object.fromEntries(Object.entries(profiles).map(([host, fingerprint]) => [ + text(host, 'snapshot.scope.profileFingerprints host', { max: 128 }), + text(fingerprint, `snapshot.scope.profileFingerprints.${host}`, { max: 256 }), + ]))), + }); +} + +export function normalizeSnapshot(value) { + if (!plain(value)) throw new TypeError('snapshot must be an object'); + if (value.schemaVersion !== MODEL_INVENTORY_SCHEMA_VERSION) { + throw new TypeError(`unsupported model inventory schemaVersion ${String(value.schemaVersion)}`); + } + const models = (value.models ?? []).map(normalizeModelRecord); + if (new Set(models.map(({ identity }) => identity)).size !== models.length) { + throw new TypeError('snapshot.models contains duplicate identities'); + } + const bindings = (value.bindings ?? []).map(normalizeBindingRecord); + if (new Set(bindings.map(({ id }) => id)).size !== bindings.length) { + throw new TypeError('snapshot.bindings contains duplicate ids'); + } + return immutable({ + schemaVersion: MODEL_INVENTORY_SCHEMA_VERSION, + snapshotId: text(value.snapshotId, 'snapshot.snapshotId', { max: 256 }), + capturedAt: iso(value.capturedAt, 'snapshot.capturedAt', { nullable: false }), + scope: normalizeScope(value.scope), + sources: (value.sources ?? []).map(normalizeSourceResult), + models, + bindings, + changes: immutable(structuredClone(Array.isArray(value.changes) ? value.changes : [])), + opportunities: immutable(structuredClone(Array.isArray(value.opportunities) ? value.opportunities : [])), + diagnostics: strings(value.diagnostics, 'snapshot.diagnostics'), + }); +} + +export function isCompleteStableSnapshot(value) { + let snapshot; + try { snapshot = normalizeSnapshot(value); } catch { return false; } + return snapshot.sources.length > 0 && snapshot.sources.every((source) => + source.status === 'complete' && source.complete + && source.scopeFingerprint === snapshot.scope.fingerprint); +} diff --git a/src/lib/model-inventory/diff.mjs b/src/lib/model-inventory/diff.mjs new file mode 100644 index 0000000..c386bba --- /dev/null +++ b/src/lib/model-inventory/diff.mjs @@ -0,0 +1,124 @@ +import { isCompleteStableSnapshot, normalizeSnapshot } from './contracts.mjs'; + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])); +} + +const equal = (a, b) => JSON.stringify(canonical(a)) === JSON.stringify(canonical(b)); +const evidenceRefs = (model, prefix) => model.evidence + .filter((entry) => entry.field === prefix || entry.field.startsWith(`${prefix}.`)) + .map(({ id }) => id); + +function change(kind, model, before, after, { + severity = 'info', field = kind, provisional = false, +} = {}) { + return { + kind, + subject: model.identity, + before: structuredClone(before), + after: structuredClone(after), + severity, + provisional, + evidenceRefs: evidenceRefs(model, field), + }; +} + +function aliasTargets(model) { + return Object.fromEntries(model.aliases.map(({ name, resolvesTo }) => [name, resolvesTo])); +} + +function fieldChanges(before, after, provisional) { + const out = []; + const oldAliases = aliasTargets(before); + const newAliases = aliasTargets(after); + for (const name of new Set([...Object.keys(oldAliases), ...Object.keys(newAliases)])) { + if (oldAliases[name] !== newAliases[name]) { + out.push(change('alias-target-changed', after, + { name, resolvesTo: oldAliases[name] ?? null }, + { name, resolvesTo: newAliases[name] ?? null }, + { severity: 'warn', field: 'aliases', provisional })); + } + } + if (!equal(before.lifecycle, after.lifecycle)) { + out.push(change('lifecycle-changed', after, before.lifecycle, after.lifecycle, + { severity: after.lifecycle.state === 'removed' ? 'fail' : 'warn', field: 'lifecycle', provisional })); + } + if (before.visibility !== after.visibility) { + out.push(change('visibility-changed', after, before.visibility, after.visibility, + { severity: 'warn', field: 'visibility', provisional })); + } + const capabilityNames = new Set([ + ...Object.keys(before.capabilities), ...Object.keys(after.capabilities), + ]); + for (const field of capabilityNames) { + if (!equal(before.capabilities[field], after.capabilities[field])) { + out.push(change('capability-changed', after, + { field, value: before.capabilities[field] ?? null }, + { field, value: after.capabilities[field] ?? null }, + { severity: 'warn', field: `capabilities.${field}`, provisional })); + } + } + return out; +} + +export function diffSnapshots(beforeValue, afterValue, { + absenceCounts = {}, authoritativeRemovals = [], removalThreshold = 2, +} = {}) { + const before = normalizeSnapshot(beforeValue); + const after = normalizeSnapshot(afterValue); + if (before.scope.fingerprint !== after.scope.fingerprint) { + return { + comparable: false, + reason: 'scope-changed', + beforeSnapshotId: before.snapshotId, + afterSnapshotId: after.snapshotId, + changes: [], + diagnostics: ['snapshot scopes differ; lifecycle comparison refused'], + }; + } + + const complete = isCompleteStableSnapshot(before) && isCompleteStableSnapshot(after); + const oldModels = new Map(before.models.map((model) => [model.identity, model])); + const newModels = new Map(after.models.map((model) => [model.identity, model])); + const changes = []; + for (const [identity, model] of newModels) { + const prior = oldModels.get(identity); + if (!prior) { + changes.push(change('model-added', model, null, model.key, + { field: 'key', provisional: !complete })); + continue; + } + changes.push(...fieldChanges(prior, model, !complete)); + } + if (complete) { + const authoritative = new Set(authoritativeRemovals); + for (const [identity, model] of oldModels) { + if (!newModels.has(identity)) { + const absenceCount = Number(absenceCounts[identity] ?? 1); + const removed = authoritative.has(identity) || absenceCount >= removalThreshold; + changes.push(change(removed ? 'model-removed' : 'model-missing', model, model.key, null, + { severity: 'warn', field: 'key', provisional: !removed })); + } + } + } + + const diagnostics = []; + if (!complete) diagnostics.push('incomplete or stale evidence suppressed model removals'); + if (changes.some(({ kind }) => kind === 'model-missing')) { + diagnostics.push(`model absence is provisional until ${removalThreshold} complete same-scope snapshots`); + } + for (const source of after.sources.filter(({ status }) => status !== 'complete')) { + diagnostics.push(`${source.id}: ${source.status}`); + } + return { + comparable: true, + reason: null, + beforeSnapshotId: before.snapshotId, + afterSnapshotId: after.snapshotId, + changes, + diagnostics: [...new Set(diagnostics)], + }; +} diff --git a/src/lib/model-inventory/impact.mjs b/src/lib/model-inventory/impact.mjs new file mode 100644 index 0000000..b5d57a8 --- /dev/null +++ b/src/lib/model-inventory/impact.mjs @@ -0,0 +1,190 @@ +import { modelIdentityKey, normalizeSnapshot } from './contracts.mjs'; + +function parseSelector(selector) { + if (selector && typeof selector === 'object' && !Array.isArray(selector)) return selector; + if (typeof selector !== 'string' || !selector) throw new TypeError('model selector is required'); + const split = selector.indexOf(':'); + return split < 0 ? { modelId: selector } + : { host: selector.slice(0, split), modelId: selector.slice(split + 1) }; +} + +function selectedModels(snapshot, selector) { + const query = parseSelector(selector); + if (query.identity) return snapshot.models.filter(({ identity }) => identity === query.identity); + return snapshot.models.filter((model) => + (!query.host || model.key.host === query.host) + && (!query.provider || model.key.provider === query.provider) + && (!query.modelId || model.key.modelId === query.modelId) + && (!query.scopeId || model.key.scopeId === query.scopeId)); +} + +function bindingReferences(binding, model) { + if (binding.host && binding.host !== model.key.host) return false; + if (binding.provider && model.key.provider && binding.provider !== model.key.provider) return false; + const refs = [binding.configured, binding.effective].filter(Boolean); + const ids = new Set([ + model.key.modelId, + model.key.provider ? `${model.key.provider}/${model.key.modelId}` : null, + ...model.aliases.map(({ name }) => name), + ].filter(Boolean)); + return refs.some((ref) => ids.has(ref)); +} + +/** @param {any} snapshotValue @param {{models?: any[]}} [options] */ +export function consumerDiagnostics(snapshotValue, options = {}) { + const { models } = options; + const snapshot = normalizeSnapshot(snapshotValue); + const selected = models ?? snapshot.models; + return snapshot.bindings + .filter((binding) => selected.some((model) => bindingReferences(binding, model))) + .filter((binding) => binding.consumer.startsWith('aqe:') || binding.consumer.startsWith('ruflo:')) + .map((binding) => ({ + bindingId: binding.id, + consumer: binding.consumer, + state: binding.consumerState, + drift: binding.drift, + configured: binding.configured, + effective: binding.effective, + evidenceRefs: binding.evidenceRefs, + diagnostic: binding.drift + ? `${binding.consumer} differs from canonical routing` + : binding.consumerState === 'runtime-proven' + ? `${binding.consumer} is runtime-proven` + : `${binding.consumer} is ${binding.consumerState}; runtime availability is not proven`, + })); +} + +export function explainModel(snapshotValue, selector) { + const snapshot = normalizeSnapshot(snapshotValue); + const models = selectedModels(snapshot, selector); + if (models.length === 0) return { found: false, reason: 'model-not-found', matches: [] }; + return { + found: true, + ambiguous: models.length > 1, + matches: models.map((model) => ({ + identity: model.identity, + key: model.key, + displayName: model.displayName, + aliases: model.aliases, + dimensions: model.dimensions, + lifecycle: model.lifecycle, + capabilities: model.capabilities, + evidence: model.evidence, + bindings: snapshot.bindings.filter((binding) => bindingReferences(binding, model)), + })), + consumers: consumerDiagnostics(snapshot, { models }), + }; +} + +function compatibility(source, target) { + const blockers = []; + const warnings = []; + for (const dimension of ['discoverable', 'entitled', 'policyAllowed', 'routable']) { + const value = target.dimensions[dimension].value; + if (value === false) blockers.push(`${dimension} is false`); + else if (value === null) blockers.push(`${dimension} is unknown`); + } + if (target.lifecycle.state === 'removed') blockers.push('target lifecycle is removed'); + else if (['deprecated', 'retiring'].includes(target.lifecycle.state)) { + warnings.push(`target lifecycle is ${target.lifecycle.state}`); + } + if (source) { + for (const [name, required] of Object.entries(source.capabilities)) { + if (required === true && target.capabilities[name] !== true) { + blockers.push(`required capability ${name} is not proven on target`); + } + } + } + return { + mechanicallyCompatible: blockers.length === 0, + blockers, + warnings, + quality: { + state: 'unknown', + claim: null, + reason: 'model inventory does not establish quality, equivalence, or lower cost', + }, + }; +} + +const shellQuote = (value) => `'${String(value).replaceAll("'", "'\"'\"'")}'`; + +/** + * @param {any} snapshotValue + * @param {{activity: string, from?: any, to: any}} options + */ +export function planModelChange(snapshotValue, options) { + const { activity, from = null, to } = options ?? /** @type {any} */ ({}); + if (typeof activity !== 'string' || !activity) throw new TypeError('activity is required'); + const snapshot = normalizeSnapshot(snapshotValue); + const targets = selectedModels(snapshot, to); + if (targets.length !== 1) { + return { + plannable: false, + reason: targets.length ? 'target-ambiguous' : 'target-not-found', + matches: targets.map(({ identity }) => identity), + }; + } + const target = targets[0]; + const sources = from == null ? [] : selectedModels(snapshot, from); + if (from != null && sources.length !== 1) { + return { + plannable: false, + reason: sources.length ? 'source-ambiguous' : 'source-not-found', + matches: sources.map(({ identity }) => identity), + }; + } + const source = sources[0] ?? null; + const affectedBindings = snapshot.bindings.filter((binding) => + binding.activity === activity && (!source || bindingReferences(binding, source))); + const assessment = compatibility(source, target); + const routeSpec = `${activity}:${target.key.host}:${target.key.modelId}`; + const warnings = [...assessment.warnings]; + if (!affectedBindings.length) warnings.push(`no canonical binding for activity ${activity} matched the source`); + if (target.key.provider == null) warnings.push('target inference provider is unknown'); + const invalidationMarkers = affectedBindings.map((binding) => ({ + kind: 'route-intelligence-stale', + consumer: '#109', + bindingId: binding.id, + reason: 'concrete model identity changes', + retainHistory: true, + evidenceRefs: binding.evidenceRefs, + })); + return { + plannable: assessment.mechanicallyCompatible, + readOnly: true, + activity, + from: source?.key ?? null, + to: target.key, + affectedBindings, + consumerDiagnostics: consumerDiagnostics(snapshot, { + models: source ? [source] : [target], + }), + compatibility: { ...assessment, warnings }, + invalidationMarkers, + action: assessment.mechanicallyCompatible ? { + command: `ak host pick --route ${shellQuote(routeSpec)}`, + executed: false, + commandMutates: true, + mutationSurface: 'canonical-routing-policy', + requiresExplicitUserAction: true, + note: 'copyable canonical-policy action; this plan does not execute it', + } : null, + }; +} + +export function impactGraph(snapshotValue) { + const snapshot = normalizeSnapshot(snapshotValue); + return { + nodes: snapshot.models.map((model) => ({ id: model.identity, kind: 'model', key: model.key })) + .concat(snapshot.bindings.map((binding) => ({ id: binding.id, kind: 'consumer', consumer: binding.consumer }))), + edges: snapshot.bindings.flatMap((binding) => snapshot.models + .filter((model) => bindingReferences(binding, model)) + .map((model) => ({ + from: binding.id, + to: modelIdentityKey(model.key), + kind: 'consumes', + evidenceRefs: binding.evidenceRefs, + }))), + }; +} diff --git a/src/lib/model-inventory/index.mjs b/src/lib/model-inventory/index.mjs new file mode 100644 index 0000000..faab62f --- /dev/null +++ b/src/lib/model-inventory/index.mjs @@ -0,0 +1,5 @@ +export * from './contracts.mjs'; +export * from './store.mjs'; +export * from './diff.mjs'; +export * from './impact.mjs'; +export * from './read-model.mjs'; diff --git a/src/lib/model-inventory/read-model.mjs b/src/lib/model-inventory/read-model.mjs new file mode 100644 index 0000000..f2db6d4 --- /dev/null +++ b/src/lib/model-inventory/read-model.mjs @@ -0,0 +1,63 @@ +import { immutable } from '../adapters/schema.mjs'; +import { normalizeSnapshot } from './contracts.mjs'; + +/** @param {any} snapshotValue @param {{changes?: any[]|{changes?: any[]}}} [options] */ +export function createModelReadModel(snapshotValue, options = {}) { + const { changes } = options; + const snapshot = normalizeSnapshot(snapshotValue); + const changeRows = Array.isArray(changes) ? changes + : Array.isArray(changes?.changes) ? changes.changes : snapshot.changes; + const configuredBindings = snapshot.bindings.filter(({ configured }) => configured != null); + const observed = snapshot.models.filter(({ dimensions }) => dimensions.observed.value === true); + const migrations = snapshot.models.filter(({ lifecycle }) => lifecycle.replacement != null); + const aliasChanges = changeRows.filter(({ kind }) => kind === 'alias-target-changed'); + const staleSources = snapshot.sources.filter(({ status }) => status !== 'complete'); + const driftedConsumers = snapshot.bindings.filter(({ drift }) => drift); + const attention = [ + ...staleSources.map((source) => ({ kind: 'source', severity: 'warn', subject: source.id, reason: source.status })), + ...migrations.map((model) => ({ + kind: 'migration', severity: 'warn', subject: model.identity, + reason: `${model.lifecycle.state} → ${model.lifecycle.replacement}`, + })), + ...aliasChanges.map((entry) => ({ kind: 'alias', severity: entry.severity, subject: entry.subject, reason: entry.kind })), + ...driftedConsumers.map((binding) => ({ kind: 'consumer', severity: 'warn', subject: binding.id, reason: 'projection drift' })), + ]; + return immutable({ + schemaVersion: snapshot.schemaVersion, + snapshotId: snapshot.snapshotId, + capturedAt: snapshot.capturedAt, + scope: snapshot.scope, + counts: { + models: snapshot.models.length, + configured: configuredBindings.length, + observed: observed.length, + migrations: migrations.length, + aliasChanges: aliasChanges.length, + staleSources: staleSources.length, + driftedConsumers: driftedConsumers.length, + }, + sources: snapshot.sources, + models: snapshot.models, + bindings: snapshot.bindings, + changes: changeRows, + attention, + diagnostics: snapshot.diagnostics, + }); +} + +export function summarizeModelHealth(snapshotValue, options = {}) { + const model = createModelReadModel(snapshotValue, options); + const level = model.attention.some(({ severity }) => severity === 'fail') ? 'fail' + : model.attention.length ? 'warn' : 'ok'; + const sourceAt = model.sources.map(({ capturedAt }) => capturedAt).filter(Boolean).sort().at(-1) + ?? model.capturedAt; + return immutable({ + level, + message: `${model.counts.configured} configured · ${model.counts.observed} observed · ` + + `${model.counts.migrations} migrations · ${model.counts.aliasChanges} alias changes · catalog ${sourceAt}`, + fix: model.counts.staleSources ? 'ak models refresh' : model.counts.migrations || model.counts.aliasChanges + ? 'ak models diff' : null, + counts: model.counts, + capturedAt: model.capturedAt, + }); +} diff --git a/src/lib/model-inventory/store.mjs b/src/lib/model-inventory/store.mjs new file mode 100644 index 0000000..6e8ac67 --- /dev/null +++ b/src/lib/model-inventory/store.mjs @@ -0,0 +1,116 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { configDir } from '../paths.mjs'; +import { + MODEL_INVENTORY_SCHEMA_VERSION, isCompleteStableSnapshot, normalizeSnapshot, +} from './contracts.mjs'; + +export const MODEL_STORE_SCHEMA_VERSION = 1; +export const MAX_MODEL_SNAPSHOTS = 32; +export const MODEL_SNAPSHOT_RETENTION_MS = 90 * 86_400_000; + +export const modelInventoryPath = () => path.join(configDir(), 'model-inventory.json'); + +const emptyStore = () => ({ + schemaVersion: MODEL_STORE_SCHEMA_VERSION, + inventorySchemaVersion: MODEL_INVENTORY_SCHEMA_VERSION, + updatedAt: null, + baselineByScope: {}, + snapshots: [], +}); + +function normalizeStore(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || value.schemaVersion !== MODEL_STORE_SCHEMA_VERSION + || value.inventorySchemaVersion !== MODEL_INVENTORY_SCHEMA_VERSION + || !Array.isArray(value.snapshots)) return emptyStore(); + const snapshots = []; + for (const raw of value.snapshots) { + try { snapshots.push(normalizeSnapshot(raw)); } catch { /* one bad record does not hide valid history */ } + } + const known = new Set(snapshots.map(({ snapshotId }) => snapshotId)); + const baselineByScope = Object.fromEntries(Object.entries(value.baselineByScope ?? {}) + .filter(([scope, id]) => typeof scope === 'string' && typeof id === 'string' && known.has(id))); + return { + schemaVersion: MODEL_STORE_SCHEMA_VERSION, + inventorySchemaVersion: MODEL_INVENTORY_SCHEMA_VERSION, + updatedAt: typeof value.updatedAt === 'string' ? value.updatedAt : null, + baselineByScope, + snapshots, + }; +} + +export function readModelStore({ file = modelInventoryPath(), fsImpl = fs } = {}) { + try { + return normalizeStore(JSON.parse(fsImpl.readFileSync(file, 'utf8'))); + } catch { + return emptyStore(); + } +} + +function pruneStore(store, now) { + const cutoff = now - MODEL_SNAPSHOT_RETENTION_MS; + const byScope = new Map(); + for (const snapshot of store.snapshots + .filter((entry) => Date.parse(entry.capturedAt) >= cutoff) + .sort((a, b) => Date.parse(a.capturedAt) - Date.parse(b.capturedAt))) { + const scope = snapshot.scope.fingerprint; + const values = byScope.get(scope) ?? []; + values.push(snapshot); + byScope.set(scope, values.slice(-MAX_MODEL_SNAPSHOTS)); + } + const snapshots = [...byScope.values()].flat() + .sort((a, b) => Date.parse(a.capturedAt) - Date.parse(b.capturedAt)); + const ids = new Set(snapshots.map(({ snapshotId }) => snapshotId)); + const baselineByScope = Object.fromEntries(Object.entries(store.baselineByScope) + .filter(([, snapshotId]) => ids.has(snapshotId))); + return { ...store, snapshots, baselineByScope }; +} + +export function writeModelStore(value, { + file = modelInventoryPath(), fsImpl = fs, now = Date.now(), +} = {}) { + const store = pruneStore(normalizeStore(value), now); + store.updatedAt = new Date(now).toISOString(); + fsImpl.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.${now}.tmp`; + try { + fsImpl.writeFileSync(tmp, JSON.stringify(store), { mode: 0o600 }); + fsImpl.renameSync(tmp, file); + try { fsImpl.chmodSync(file, 0o600); } catch { /* best effort on filesystems without modes */ } + } catch (error) { + try { fsImpl.rmSync(tmp, { force: true }); } catch { /* preserve the original error */ } + throw error; + } + return store; +} + +export function appendModelSnapshot(value, options = {}) { + const snapshot = normalizeSnapshot(value); + const now = options.now ?? Date.now(); + const store = readModelStore(options); + store.snapshots = store.snapshots.filter(({ snapshotId }) => snapshotId !== snapshot.snapshotId); + store.snapshots.push(snapshot); + if (isCompleteStableSnapshot(snapshot)) { + store.baselineByScope[snapshot.scope.fingerprint] = snapshot.snapshotId; + } + return writeModelStore(store, { ...options, now }); +} + +export function snapshotById(store, snapshotId) { + return store?.snapshots?.find((snapshot) => snapshot.snapshotId === snapshotId) ?? null; +} + +export function baselineFor(store, scopeFingerprint) { + const id = store?.baselineByScope?.[scopeFingerprint]; + return id ? snapshotById(store, id) : null; +} + +/** @param {any} store @param {{scopeFingerprint?: string}} [options] */ +export function latestSnapshot(store, options = {}) { + const { scopeFingerprint } = options; + const values = (store?.snapshots ?? []) + .filter((snapshot) => !scopeFingerprint || snapshot.scope.fingerprint === scopeFingerprint); + return values.reduce((latest, snapshot) => !latest + || Date.parse(snapshot.capturedAt) > Date.parse(latest.capturedAt) ? snapshot : latest, null); +} diff --git a/tests/kit/model-inventory-contracts.test.mjs b/tests/kit/model-inventory-contracts.test.mjs new file mode 100644 index 0000000..d8e56d4 --- /dev/null +++ b/tests/kit/model-inventory-contracts.test.mjs @@ -0,0 +1,79 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + MODEL_INVENTORY_SCHEMA_VERSION, modelIdentityKey, normalizeModelRecord, + normalizeSnapshot, normalizeSourceResult, +} from '../../src/lib/model-inventory/contracts.mjs'; + +const AT = '2026-08-25T12:00:00.000Z'; + +function evidence(id, field, klass) { + return { + id, field, source: `fixture:${field}`, class: klass, capturedAt: AT, + freshness: 'fresh', completeness: 'complete', scopeFingerprint: 'scope-a', + }; +} + +function model(overrides = {}) { + return { + key: { host: 'codex', provider: 'openai', modelId: 'gpt-x', scopeId: 'scope-a' }, + dimensions: { + configured: { value: true, evidenceRefs: ['configured'] }, + observed: { value: false, evidenceRefs: ['observed'] }, + entitled: { value: null, evidenceRefs: [] }, + }, + evidence: [ + evidence('configured', 'dimensions.configured', 'configured'), + evidence('observed', 'dimensions.observed', 'observed'), + ], + ...overrides, + }; +} + +test('model contract preserves independent dimensions and per-field evidence', () => { + const value = normalizeModelRecord(model()); + assert.equal(value.dimensions.configured.value, true); + assert.equal(value.dimensions.observed.value, false); + assert.equal(value.dimensions.entitled.value, null); + assert.equal(value.dimensions.routable.value, null); + assert.deepEqual(value.dimensions.configured.evidenceRefs, ['configured']); + assert.equal(value.evidence[0].field, 'dimensions.configured'); + assert.equal(value.evidence[0].class, 'configured'); + assert.equal(Object.isFrozen(value), true); +}); + +test('host, provider, concrete model, and scope are all part of identity', () => { + const base = { host: 'opencode', provider: 'openrouter', modelId: 'vendor/model', scopeId: 'project-a' }; + assert.notEqual(modelIdentityKey(base), modelIdentityKey({ ...base, provider: 'gateway' })); + assert.notEqual(modelIdentityKey(base), modelIdentityKey({ ...base, scopeId: 'project-b' })); + assert.notEqual(modelIdentityKey(base), modelIdentityKey({ ...base, host: 'codex' })); +}); + +test('dimension evidence references must resolve within the model record', () => { + assert.throws(() => normalizeModelRecord(model({ + dimensions: { configured: { value: true, evidenceRefs: ['missing'] } }, + })), /unknown evidence missing/); +}); + +test('source completeness cannot be asserted by a partial or stale result', () => { + const partial = normalizeSourceResult({ + id: 'codex-cache', status: 'partial', complete: true, capturedAt: AT, + scopeFingerprint: 'scope-a', + }); + assert.equal(partial.complete, false); + const complete = normalizeSourceResult({ + id: 'codex-cache', status: 'complete', capturedAt: AT, scopeFingerprint: 'scope-a', + }); + assert.equal(complete.complete, true); +}); + +test('snapshot rejects duplicate model identities without collapsing records', () => { + const raw = { + schemaVersion: MODEL_INVENTORY_SCHEMA_VERSION, + snapshotId: 'snapshot-a', capturedAt: AT, + scope: { fingerprint: 'scope-a', hosts: ['codex'] }, + sources: [{ id: 'codex-cache', status: 'complete', capturedAt: AT, scopeFingerprint: 'scope-a' }], + models: [model(), model()], bindings: [], + }; + assert.throws(() => normalizeSnapshot(raw), /duplicate identities/); +}); diff --git a/tests/kit/model-inventory-diff.test.mjs b/tests/kit/model-inventory-diff.test.mjs new file mode 100644 index 0000000..5d71148 --- /dev/null +++ b/tests/kit/model-inventory-diff.test.mjs @@ -0,0 +1,84 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { diffSnapshots } from '../../src/lib/model-inventory/diff.mjs'; +import { modelIdentityKey } from '../../src/lib/model-inventory/contracts.mjs'; + +const AT = '2026-08-25T12:00:00.000Z'; + +function model(id, overrides = {}) { + return { + key: { host: 'codex', provider: 'openai', modelId: id, scopeId: 'scope-a' }, + aliases: [], lifecycle: { state: 'active' }, capabilities: { tools: true }, + dimensions: {}, evidence: [], ...overrides, + }; +} + +function snapshot(id, models, { scope = 'scope-a', status = 'complete' } = {}) { + return { + schemaVersion: 1, snapshotId: id, capturedAt: AT, + scope: { fingerprint: scope, hosts: ['codex'] }, + sources: [{ id: 'catalog', status, capturedAt: AT, scopeFingerprint: scope }], + models, bindings: [], changes: [], opportunities: [], diagnostics: [], + }; +} + +test('identical same-scope snapshots produce no lifecycle changes', () => { + const before = snapshot('before', [model('gpt-a')]); + const after = snapshot('after', [model('gpt-a')]); + assert.deepEqual(diffSnapshots(before, after).changes, []); +}); + +test('complete same-scope snapshots require repeated absence before removal', () => { + const before = snapshot('before', [model('gpt-a')]); + const after = snapshot('after', [model('gpt-b')]); + const first = diffSnapshots(before, after); + assert.deepEqual(first.changes.map(({ kind }) => kind).sort(), ['model-added', 'model-missing']); + assert.equal(first.changes.find(({ kind }) => kind === 'model-missing').provisional, true); + + const identity = modelIdentityKey(before.models[0].key); + const result = diffSnapshots( + before, + after, + { absenceCounts: { [identity]: 2 } }, + ); + assert.deepEqual(result.changes.map(({ kind }) => kind).sort(), ['model-added', 'model-removed']); + assert.equal(result.changes.every(({ provisional }) => provisional === false), true); +}); + +test('partial or stale evidence suppresses removals', () => { + for (const status of ['partial', 'stale']) { + const result = diffSnapshots( + snapshot('before', [model('gpt-a')]), + snapshot(`after-${status}`, [], { status }), + ); + assert.equal(result.changes.some(({ kind }) => kind === 'model-removed'), false); + assert.match(result.diagnostics.join(' '), /suppressed model removals/); + } +}); + +test('alias targets, lifecycle, visibility, and capabilities diff independently', () => { + const before = model('gpt-a', { + aliases: [{ name: 'default', resolvesTo: 'gpt-a' }], + visibility: 'list', lifecycle: { state: 'active' }, capabilities: { tools: true, vision: false }, + }); + const after = model('gpt-a', { + aliases: [{ name: 'default', resolvesTo: 'gpt-b' }], + visibility: 'hidden', lifecycle: { state: 'retiring', replacement: 'gpt-b' }, + capabilities: { tools: true, vision: true }, + }); + const kinds = diffSnapshots(snapshot('before', [before]), snapshot('after', [after])) + .changes.map(({ kind }) => kind); + assert.deepEqual(kinds.sort(), [ + 'alias-target-changed', 'capability-changed', 'lifecycle-changed', 'visibility-changed', + ]); +}); + +test('cross-scope comparison is refused rather than reported as mass churn', () => { + const result = diffSnapshots( + snapshot('before', [model('gpt-a')]), + snapshot('after', [model('gpt-b')], { scope: 'scope-b' }), + ); + assert.equal(result.comparable, false); + assert.equal(result.reason, 'scope-changed'); + assert.deepEqual(result.changes, []); +}); diff --git a/tests/kit/model-inventory-impact.test.mjs b/tests/kit/model-inventory-impact.test.mjs new file mode 100644 index 0000000..8396cd6 --- /dev/null +++ b/tests/kit/model-inventory-impact.test.mjs @@ -0,0 +1,133 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + consumerDiagnostics, explainModel, impactGraph, planModelChange, +} from '../../src/lib/model-inventory/impact.mjs'; +import { createModelReadModel, summarizeModelHealth } from '../../src/lib/model-inventory/read-model.mjs'; + +const AT = '2026-08-25T12:00:00.000Z'; + +function evidence(id, field) { + return { + id, field, source: 'codex-cache', class: 'catalog', capturedAt: AT, + freshness: 'fresh', completeness: 'complete', scopeFingerprint: 'scope-a', + }; +} + +function model(id, { dimensions = {}, capabilities = {}, lifecycle = { state: 'active' } } = {}) { + const evidenceRows = [ + evidence(`${id}-routable`, 'dimensions.routable'), + evidence(`${id}-lifecycle`, 'lifecycle'), + ]; + return { + key: { host: 'codex', provider: 'openai', modelId: id, scopeId: 'scope-a' }, + dimensions: { + routable: { value: true, evidenceRefs: [`${id}-routable`] }, + discoverable: { value: true }, entitled: { value: true }, policyAllowed: { value: true }, + ...dimensions, + }, + capabilities, lifecycle: { ...lifecycle, evidenceRefs: [`${id}-lifecycle`] }, + evidence: evidenceRows, + }; +} + +function fixture({ targetDimensions = {} } = {}) { + const oldModel = model('gpt-old', { capabilities: { tools: true } }); + const newModel = model('gpt-new', { + dimensions: targetDimensions, capabilities: { tools: true }, + lifecycle: { state: 'active' }, + }); + return { + schemaVersion: 1, snapshotId: 'snapshot-a', capturedAt: AT, + scope: { fingerprint: 'scope-a', hosts: ['codex'] }, + sources: [{ id: 'codex-cache', status: 'complete', capturedAt: AT, scopeFingerprint: 'scope-a' }], + models: [oldModel, newModel], + bindings: [ + { + id: 'route-implementation', consumer: 'route:implementation', activity: 'implementation', + host: 'codex', provider: 'openai', configured: 'gpt-old', effective: 'gpt-old', + provenance: 'configured', consumerState: 'configured', evidenceRefs: ['route-config'], + }, + { + id: 'aqe-coder', consumer: 'aqe:agent:coder', activity: 'implementation', + host: 'codex', provider: 'openai', configured: 'gpt-old', effective: 'gpt-old', + provenance: 'configured', consumerState: 'reported', drift: true, evidenceRefs: ['aqe-config'], + }, + { + id: 'ruflo-openai', consumer: 'ruflo:provider:openai', activity: 'implementation', + host: 'codex', provider: 'openai', configured: 'gpt-old', effective: 'gpt-old', + provenance: 'observed', consumerState: 'runtime-proven', evidenceRefs: ['ruflo-runtime'], + }, + ], + changes: [], opportunities: [], diagnostics: [], + }; +} + +test('explain returns field evidence and every bound consumer without upgrading provenance', () => { + const result = explainModel(fixture(), 'codex:gpt-old'); + assert.equal(result.found, true); + assert.equal(result.matches[0].dimensions.routable.evidenceRefs[0], 'gpt-old-routable'); + assert.deepEqual(result.matches[0].bindings.map(({ id }) => id), [ + 'route-implementation', 'aqe-coder', 'ruflo-openai', + ]); + assert.deepEqual(result.consumers.map(({ state }) => state), ['reported', 'runtime-proven']); +}); + +test('plan is read-only, emits the canonical action, and keeps quality unknown', () => { + const result = planModelChange(fixture(), { + activity: 'implementation', from: 'codex:gpt-old', to: 'codex:gpt-new', + }); + assert.equal(result.plannable, true); + assert.equal(result.readOnly, true); + assert.match(result.action.command, /^ak host pick --route /); + assert.match(result.action.command, /implementation:codex:gpt-new/); + assert.equal(result.action.executed, false); + assert.equal(result.action.commandMutates, true); + assert.equal(result.action.requiresExplicitUserAction, true); + assert.equal(result.compatibility.quality.state, 'unknown'); + assert.equal(result.compatibility.quality.claim, null); + assert.equal(result.invalidationMarkers.length, 3); + assert.equal(result.invalidationMarkers.every((marker) => + marker.consumer === '#109' && marker.retainHistory), true); +}); + +test('known entitlement, policy, or routability failure blocks a mechanical plan', () => { + for (const dimension of ['entitled', 'policyAllowed', 'routable']) { + const result = planModelChange(fixture({ targetDimensions: { [dimension]: { value: false } } }), { + activity: 'implementation', from: 'codex:gpt-old', to: 'codex:gpt-new', + }); + assert.equal(result.plannable, false, dimension); + assert.equal(result.action, null, dimension); + assert.match(result.compatibility.blockers.join(' '), new RegExp(dimension)); + } +}); + +test('unknown required availability evidence blocks a mechanical compatibility claim', () => { + for (const dimension of ['discoverable', 'entitled', 'policyAllowed', 'routable']) { + const result = planModelChange(fixture({ targetDimensions: { [dimension]: { value: null } } }), { + activity: 'implementation', from: 'codex:gpt-old', to: 'codex:gpt-new', + }); + assert.equal(result.plannable, false, dimension); + assert.match(result.compatibility.blockers.join(' '), new RegExp(`${dimension} is unknown`)); + } +}); + +test('AQE and Ruflo diagnostics distinguish configured/reported/runtime-proven state', () => { + const diagnostics = consumerDiagnostics(fixture()); + assert.deepEqual(diagnostics.map(({ consumer, state, drift }) => [consumer, state, drift]), [ + ['aqe:agent:coder', 'reported', true], + ['ruflo:provider:openai', 'runtime-proven', false], + ]); +}); + +test('impact graph and read model expose consumers and attention without mutation claims', () => { + const snapshot = fixture(); + const graph = impactGraph(snapshot); + assert.equal(graph.edges.length, 3); + const readModel = createModelReadModel(snapshot, { + changes: [{ kind: 'alias-target-changed', severity: 'warn', subject: 'alias' }], + }); + assert.equal(readModel.counts.aliasChanges, 1); + assert.equal(readModel.counts.driftedConsumers, 1); + assert.equal(summarizeModelHealth(snapshot).level, 'warn'); +}); diff --git a/tests/kit/model-inventory-store.test.mjs b/tests/kit/model-inventory-store.test.mjs new file mode 100644 index 0000000..17efab2 --- /dev/null +++ b/tests/kit/model-inventory-store.test.mjs @@ -0,0 +1,89 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + MAX_MODEL_SNAPSHOTS, appendModelSnapshot, baselineFor, latestSnapshot, readModelStore, +} from '../../src/lib/model-inventory/store.mjs'; + +const DAY = 86_400_000; +const NOW = Date.parse('2026-08-25T12:00:00.000Z'); + +function snapshot(id, at, { scope = 'scope-a', status = 'complete' } = {}) { + return { + schemaVersion: 1, snapshotId: id, capturedAt: new Date(at).toISOString(), + scope: { fingerprint: scope, hosts: ['codex'] }, + sources: [{ + id: 'codex-cache', status, capturedAt: new Date(at).toISOString(), + scopeFingerprint: scope, + }], + models: [], bindings: [], changes: [], opportunities: [], diagnostics: [], + }; +} + +function sandbox() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ak-model-store-')); + return { dir, file: path.join(dir, 'model-inventory.json') }; +} + +test('append writes atomically with private permissions and advances a complete stable baseline', () => { + const sb = sandbox(); + const store = appendModelSnapshot(snapshot('complete-a', NOW), { file: sb.file, now: NOW }); + assert.equal(baselineFor(store, 'scope-a').snapshotId, 'complete-a'); + assert.equal(latestSnapshot(store).snapshotId, 'complete-a'); + assert.equal(fs.readdirSync(sb.dir).some((name) => name.endsWith('.tmp')), false); + if (process.platform !== 'win32') assert.equal(fs.statSync(sb.file).mode & 0o777, 0o600); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); + +test('partial snapshot is retained but cannot replace the complete baseline', () => { + const sb = sandbox(); + appendModelSnapshot(snapshot('complete-a', NOW - DAY), { file: sb.file, now: NOW }); + const store = appendModelSnapshot(snapshot('partial-a', NOW, { status: 'partial' }), { + file: sb.file, now: NOW, + }); + assert.equal(latestSnapshot(store).snapshotId, 'partial-a'); + assert.equal(baselineFor(store, 'scope-a').snapshotId, 'complete-a'); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); + +test('history is bounded to 32 snapshots and 90 days', () => { + const sb = sandbox(); + appendModelSnapshot(snapshot('expired', NOW - 91 * DAY), { file: sb.file, now: NOW }); + for (let i = 0; i < MAX_MODEL_SNAPSHOTS + 3; i++) { + appendModelSnapshot(snapshot(`recent-${i}`, NOW - (MAX_MODEL_SNAPSHOTS + 3 - i) * 1_000), { + file: sb.file, now: NOW, + }); + } + const store = readModelStore({ file: sb.file }); + assert.equal(store.snapshots.length, MAX_MODEL_SNAPSHOTS); + assert.equal(store.snapshots.some(({ snapshotId }) => snapshotId === 'expired'), false); + assert.equal(store.snapshots.at(-1).snapshotId, `recent-${MAX_MODEL_SNAPSHOTS + 2}`); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); + +test('the 32-snapshot retention bound applies independently per scope', () => { + const sb = sandbox(); + for (const scope of ['scope-a', 'scope-b']) { + for (let i = 0; i < MAX_MODEL_SNAPSHOTS + 1; i++) { + appendModelSnapshot(snapshot(`${scope}-${i}`, NOW - (MAX_MODEL_SNAPSHOTS - i) * 1_000, { scope }), { + file: sb.file, now: NOW, + }); + } + } + const store = readModelStore({ file: sb.file }); + assert.equal(store.snapshots.filter(({ scope }) => scope.fingerprint === 'scope-a').length, + MAX_MODEL_SNAPSHOTS); + assert.equal(store.snapshots.filter(({ scope }) => scope.fingerprint === 'scope-b').length, + MAX_MODEL_SNAPSHOTS); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); + +test('missing or corrupt store degrades to an empty readable store', () => { + const sb = sandbox(); + assert.deepEqual(readModelStore({ file: sb.file }).snapshots, []); + fs.writeFileSync(sb.file, '{broken'); + assert.deepEqual(readModelStore({ file: sb.file }).baselineByScope, {}); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); From 6bae4ab02bf8eb1650406da93a415c54b5d9d70f Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Tue, 25 Aug 2026 07:43:09 -0700 Subject: [PATCH 03/37] feat(models): discover host model evidence --- src/lib/adapters/registries.mjs | 41 ++++- src/lib/model-inventory/bindings.mjs | 89 +++++++++++ src/lib/model-inventory/discovery/claude.mjs | 68 ++++++++ src/lib/model-inventory/discovery/codex.mjs | 101 ++++++++++++ src/lib/model-inventory/discovery/index.mjs | 93 +++++++++++ src/lib/model-inventory/discovery/ollama.mjs | 49 ++++++ .../model-inventory/discovery/opencode.mjs | 80 ++++++++++ src/lib/model-inventory/index.mjs | 1 + src/lib/model-inventory/observed.mjs | 52 +++++++ src/lib/model-inventory/refresh.mjs | 147 ++++++++++++++++++ src/lib/model-inventory/store.mjs | 24 +++ .../claude/managed-settings.json | 3 + .../model-inventory/claude/settings.json | 7 + .../model-inventory/codex/models-cache.json | 19 +++ .../fixtures/model-inventory/ollama/list.txt | 3 + .../model-inventory/opencode/models.txt | 2 + tests/kit/adapter-registries.test.mjs | 26 ++++ .../kit/model-discovery-claude-codex.test.mjs | 83 ++++++++++ .../model-discovery-opencode-ollama.test.mjs | 78 ++++++++++ tests/kit/model-inventory-collect.test.mjs | 101 ++++++++++++ tests/kit/model-inventory-store.test.mjs | 11 ++ 21 files changed, 1077 insertions(+), 1 deletion(-) create mode 100644 src/lib/model-inventory/bindings.mjs create mode 100644 src/lib/model-inventory/discovery/claude.mjs create mode 100644 src/lib/model-inventory/discovery/codex.mjs create mode 100644 src/lib/model-inventory/discovery/index.mjs create mode 100644 src/lib/model-inventory/discovery/ollama.mjs create mode 100644 src/lib/model-inventory/discovery/opencode.mjs create mode 100644 src/lib/model-inventory/observed.mjs create mode 100644 src/lib/model-inventory/refresh.mjs create mode 100644 tests/fixtures/model-inventory/claude/managed-settings.json create mode 100644 tests/fixtures/model-inventory/claude/settings.json create mode 100644 tests/fixtures/model-inventory/codex/models-cache.json create mode 100644 tests/fixtures/model-inventory/ollama/list.txt create mode 100644 tests/fixtures/model-inventory/opencode/models.txt create mode 100644 tests/kit/model-discovery-claude-codex.test.mjs create mode 100644 tests/kit/model-discovery-opencode-ollama.test.mjs create mode 100644 tests/kit/model-inventory-collect.test.mjs diff --git a/src/lib/adapters/registries.mjs b/src/lib/adapters/registries.mjs index 9c76fa0..e0629c6 100644 --- a/src/lib/adapters/registries.mjs +++ b/src/lib/adapters/registries.mjs @@ -81,6 +81,28 @@ export function validateObservabilityAdapter(value) { return immutable(structuredClone(value)); } +export function validateModelDiscoveryAdapter(value) { + assertRecord(value, 'modelDiscovery'); + const allowed = new Set(['id', 'ownerType', 'ownerId', 'transport', 'command', 'network', 'schema']); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new TypeError(`modelDiscovery has unknown field ${key}`); + } + assertId(value.id, 'modelDiscovery.id'); + assertEnum(value.ownerType, ['host', 'provider'], 'modelDiscovery.ownerType'); + assertId(value.ownerId, 'modelDiscovery.ownerId'); + assertEnum(value.transport, ['file', 'command'], 'modelDiscovery.transport'); + assertEnum(value.network, ['never', 'local', 'explicit'], 'modelDiscovery.network'); + if (typeof value.schema !== 'string' || !value.schema) throw new TypeError('modelDiscovery.schema is required'); + if (value.transport === 'command') { + if (typeof value.command !== 'string' || !/^[A-Za-z0-9._-]+$/.test(value.command)) { + throw new TypeError('modelDiscovery.command must be an executable name'); + } + } else if (value.command !== undefined) { + throw new TypeError('file modelDiscovery cannot declare command'); + } + return immutable(structuredClone(value)); +} + export function validateHostAdapter(value, { projections, observability, } = /** @type {any} */ ({})) { @@ -152,6 +174,13 @@ const OBSERVABILITY_MAP = registryFrom([ { id: 'openrouter-metadata', kind: 'usage', evidence: ['provider', 'model', 'billing'] }, ], validateObservabilityAdapter, 'observability'); +const MODEL_DISCOVERY_MAP = registryFrom([ + { id: 'claude-config', ownerType: 'host', ownerId: 'claude', transport: 'file', network: 'never', schema: 'claude-settings-v1' }, + { id: 'codex-cache', ownerType: 'host', ownerId: 'codex', transport: 'file', network: 'never', schema: 'codex-model-cache-v1' }, + { id: 'opencode-models', ownerType: 'host', ownerId: 'opencode', transport: 'command', command: 'opencode', network: 'explicit', schema: 'opencode-models-lines-v1' }, + { id: 'ollama-catalog', ownerType: 'provider', ownerId: 'ollama', transport: 'command', command: 'ollama', network: 'local', schema: 'ollama-list-v1' }, +], validateModelDiscoveryAdapter, 'model discovery'); + const hostEntries = [ { id: 'claude', label: 'Claude Code', enabledByDefault: true, @@ -332,6 +361,7 @@ export const PROJECTION_REGISTRY = immutable(Object.values(PROJECTION_MAP)); // are cross-checked against this set) — deliberately NOT a dispatch surface; // no collector loop maps an observability id to a live collector (F-12). export const OBSERVABILITY_REGISTRY = immutable(Object.values(OBSERVABILITY_MAP)); +export const MODEL_DISCOVERY_REGISTRY = immutable(Object.values(MODEL_DISCOVERY_MAP)); export const HOST_REGISTRY = immutable(Object.values(HOST_MAP)); export const PROVIDER_REGISTRY = immutable(Object.values(PROVIDER_MAP)); @@ -346,6 +376,7 @@ const registryErrors = validateRegistries({ providers: Object.values(PROVIDER_MAP), projections: Object.values(PROJECTION_MAP), observability: Object.values(OBSERVABILITY_MAP), + modelDiscovery: Object.values(MODEL_DISCOVERY_MAP), }); if (registryErrors.length) { throw new Error(`adapter registries invalid: ${registryErrors.map((e) => `${e.path} (${e.code})`).join('; ')}`); @@ -406,7 +437,7 @@ export function validateActivityHost(id, hosts = HOST_REGISTRY) { export function validateRegistries(registries) { const errors = []; - const axes = ['hosts', 'providers', 'projections', 'observability']; + const axes = ['hosts', 'providers', 'projections', 'observability', 'modelDiscovery']; for (const axis of axes) { const seen = new Set(); for (const [index, entry] of (registries?.[axis] ?? []).entries()) { @@ -451,5 +482,13 @@ export function validateRegistries(registries) { }); } } + const hosts = new Set((registries?.hosts ?? []).map((entry) => entry.id)); + const providers = new Set((registries?.providers ?? []).map((entry) => entry.id)); + for (const [index, descriptor] of (registries?.modelDiscovery ?? []).entries()) { + const owners = descriptor.ownerType === 'host' ? hosts : providers; + if (!owners.has(descriptor.ownerId)) errors.push({ + path: `modelDiscovery[${index}].ownerId`, code: `unknown-${descriptor.ownerType}`, value: descriptor.ownerId, + }); + } return errors; } diff --git a/src/lib/model-inventory/bindings.mjs b/src/lib/model-inventory/bindings.mjs new file mode 100644 index 0000000..ea6ea3a --- /dev/null +++ b/src/lib/model-inventory/bindings.mjs @@ -0,0 +1,89 @@ +import { createHash } from 'node:crypto'; + +const plain = (value) => value && typeof value === 'object' && !Array.isArray(value); +const bounded = (value) => typeof value === 'string' && value.length > 0 && value.length <= 512 ? value : null; +const bindingId = (parts) => `binding:${createHash('sha256').update(parts.join('\n')).digest('hex').slice(0, 16)}`; + +function record({ consumer, source, host = null, provider = null, modelRef = null, activity = null, + provenance = 'configured', variant = {}, index = null }) { + const evidenceClass = provenance === 'observed' ? 'observed' : 'configured'; + const consumerState = consumer.startsWith('aqe:') || consumer.startsWith('ruflo:') ? 'reported' + : provenance === 'observed' ? 'runtime-proven' : 'configured'; + return { + id: bindingId([consumer, source, host, provider, modelRef, activity, index].map(String)), + consumer, source, host, provider, modelRef, activity, variant, + configured: modelRef, effective: null, consumerState, drift: false, evidenceRefs: [], + evidenceClass, provenance: evidenceClass, + }; +} + +export function collectModelBindings({ + config = {}, aqeConfig, rufloConfig, +} = /** @type {any} */ ({})) { + const bindings = []; + const diagnostics = []; + const routes = config?.routing?.routes; + if (routes !== undefined && !plain(routes)) diagnostics.push({ code: 'invalid-routing-schema' }); + for (const [activity, route] of Object.entries(plain(routes) ? routes : {})) { + if (!plain(route) || !bounded(route.host)) { + diagnostics.push({ code: 'invalid-route', activity }); + continue; + } + bindings.push(record({ + consumer: `route:${activity}`, source: 'kit.json', host: route.host, modelRef: bounded(route.model), + activity, provenance: route.provenance, variant: { reasoningEffort: bounded(route.reasoningEffort) }, + })); + for (const [index, rung] of (Array.isArray(route.escalation) ? route.escalation : []).entries()) { + if (!plain(rung) || !bounded(rung.host)) { + diagnostics.push({ code: 'invalid-escalation', activity, index }); + continue; + } + bindings.push(record({ + consumer: `route:${activity}:escalation:${index}`, source: 'kit.json', host: rung.host, + modelRef: bounded(rung.model), activity, provenance: route.provenance, index, + variant: { reasoningEffort: bounded(rung.reasoningEffort) }, + })); + } + } + for (const [index, binding] of (Array.isArray(config?.integrations?.bindings) + ? config.integrations.bindings : []).entries()) { + if (!plain(binding) || !bounded(binding.host)) { + diagnostics.push({ code: 'invalid-integration-binding', index }); + continue; + } + bindings.push(record({ + consumer: `integration:${index}`, source: 'kit.json', host: binding.host, + provider: bounded(binding.provider), modelRef: bounded(binding.model), index, + variant: { reasoningEffort: bounded(binding.reasoningEffort) }, + })); + } + if (plain(aqeConfig)) { + if (bounded(aqeConfig.defaultProvider)) bindings.push(record({ + consumer: 'aqe:default', source: '.agentic-qe/llm-config.json', provider: aqeConfig.defaultProvider, + })); + const fallbackEntries = Array.isArray(aqeConfig.fallbackChain) ? aqeConfig.fallbackChain + : Array.isArray(aqeConfig.fallbackChain?.entries) ? aqeConfig.fallbackChain.entries : []; + for (const [index, item] of fallbackEntries.entries()) { + const entry = typeof item === 'string' ? { provider: item } : item; + if (plain(entry) && bounded(entry.provider)) bindings.push(record({ + consumer: `aqe:fallback:${index}`, source: '.agentic-qe/llm-config.json', provider: entry.provider, + modelRef: bounded(entry.model) ?? (Array.isArray(entry.models) ? bounded(entry.models[0]) : null), index, + })); + } + for (const [agent, entry] of Object.entries(plain(aqeConfig.agentOverrides) ? aqeConfig.agentOverrides : {})) { + if (plain(entry) && bounded(entry.provider)) bindings.push(record({ + consumer: `aqe:agent:${agent}`, source: '.agentic-qe/llm-config.json', provider: entry.provider, + modelRef: bounded(entry.model), activity: agent, + })); + } + } + const rufloCandidates = Array.isArray(rufloConfig?.candidates) ? rufloConfig.candidates + : Array.isArray(rufloConfig?.providers?.models) ? rufloConfig.providers.models : []; + for (const [index, candidate] of rufloCandidates.entries()) { + if (plain(candidate) && (bounded(candidate.provider) || bounded(candidate.model))) bindings.push(record({ + consumer: `ruflo:candidate:${index}`, source: 'ruflo', provider: bounded(candidate.provider) ?? bounded(candidate.id), + modelRef: bounded(candidate.model) ?? bounded(candidate.id), index, + })); + } + return { status: diagnostics.length ? 'partial' : 'complete', bindings, diagnostics }; +} diff --git a/src/lib/model-inventory/discovery/claude.mjs b/src/lib/model-inventory/discovery/claude.mjs new file mode 100644 index 0000000..694c5b4 --- /dev/null +++ b/src/lib/model-inventory/discovery/claude.mjs @@ -0,0 +1,68 @@ +import { + MAX_CONFIG_BYTES, MAX_MODELS, diagnostic, modelRecord, sourceRecord, +} from './index.mjs'; + +const ALIAS_ENV = Object.freeze({ + sonnet: 'ANTHROPIC_DEFAULT_SONNET_MODEL', + opus: 'ANTHROPIC_DEFAULT_OPUS_MODEL', + haiku: 'ANTHROPIC_DEFAULT_HAIKU_MODEL', +}); + +function parseSettings(raw, field) { + if (raw === undefined || raw === null || raw === '') return {}; + if (typeof raw === 'object' && !Array.isArray(raw)) return structuredClone(raw); + const text = String(raw); + if (Buffer.byteLength(text) > MAX_CONFIG_BYTES) throw new TypeError(`${field}-too-large`); + let parsed; + try { parsed = JSON.parse(text); } catch { throw new TypeError(`${field}-invalid-json`); } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new TypeError(`${field}-schema`); + return parsed; +} + +const boundedModel = (value) => typeof value === 'string' && value.length > 0 && value.length <= 256 + ? value : null; + +export function discoverClaude({ + settingsRaw, managedSettingsRaw, capturedAt, scope = {}, scopeKey, environment = {}, +} = /** @type {any} */ ({})) { + let settings; + let managed; + try { + settings = parseSettings(settingsRaw, 'settings'); + managed = parseSettings(managedSettingsRaw, 'managed-settings'); + } catch (error) { + const source = sourceRecord({ id: 'claude-config', owner: 'claude', scope, scopeKey, capturedAt, complete: false, status: 'unsupported-schema', schema: 'claude-settings-v1', diagnostics: ['unsupported-schema'] }); + return { status: 'unsupported-schema', source, models: [], diagnostics: [diagnostic('unsupported-schema', error.message)] }; + } + + const env = { ...(settings.env && typeof settings.env === 'object' ? settings.env : {}), ...environment }; + const configured = boundedModel(managed.model) ?? boundedModel(environment.ANTHROPIC_MODEL) + ?? boundedModel(settings.model); + const allowed = Array.isArray(managed.availableModels) + ? managed.availableModels.filter(boundedModel).slice(0, MAX_MODELS) : []; + const complete = !Array.isArray(managed.availableModels) || managed.availableModels.length <= MAX_MODELS; + const source = sourceRecord({ id: 'claude-config', owner: 'claude', scope, scopeKey, capturedAt, complete, schema: 'claude-settings-v1' }); + const records = new Map(); + const add = (reference, states = {}) => { + const alias = Object.hasOwn(ALIAS_ENV, reference) ? reference : null; + const target = alias ? boundedModel(env[ALIAS_ENV[alias]]) ?? reference : reference; + if (!boundedModel(target)) return; + const prior = records.get(target); + const aliases = prior?.aliases ?? []; + if (alias && !aliases.some((entry) => entry.name === alias)) { + aliases.push({ name: alias, resolvesTo: target, provenance: 'configured', observedAt: source.capturedAt }); + } + records.set(target, modelRecord({ + host: 'claude', provider: null, modelId: target, scopeId: source.scopeId, + aliases, source, states: { ...(prior?.states ?? {}), ...states, entitled: 'unknown' }, + })); + }; + for (const model of allowed) add(model, { policyAllowed: true, discoverable: true }); + if (configured) add(configured, { configured: true, effective: true, + policyAllowed: allowed.length ? allowed.includes(configured) || allowed.includes(records.get(configured)?.aliases?.[0]?.name) : 'unknown' }); + + return { + status: complete ? 'complete' : 'partial', source, models: [...records.values()], + diagnostics: complete ? [] : [diagnostic('model-cap', `availableModels exceeds ${MAX_MODELS}`)], + }; +} diff --git a/src/lib/model-inventory/discovery/codex.mjs b/src/lib/model-inventory/discovery/codex.mjs new file mode 100644 index 0000000..594e29c --- /dev/null +++ b/src/lib/model-inventory/discovery/codex.mjs @@ -0,0 +1,101 @@ +import { + MAX_COMMAND_BYTES, MAX_MODELS, diagnostic, modelRecord, sourceRecord, +} from './index.mjs'; + +const VISIBILITY = new Set(['list', 'visible', 'hide', 'hidden']); +const REASONING = new Set(['minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'ultra']); +const bounded = (value, max = 256) => typeof value === 'string' && value.length > 0 && value.length <= max ? value : null; + +function parseCache(raw) { + if (raw && typeof raw === 'object' && !Array.isArray(raw)) return structuredClone(raw); + const text = String(raw ?? ''); + if (Buffer.byteLength(text) > MAX_COMMAND_BYTES) throw new TypeError('cache-too-large'); + let parsed; + try { parsed = JSON.parse(text); } catch { throw new TypeError('cache-invalid-json'); } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new TypeError('cache-schema'); + return parsed; +} + +function parseConfig(raw) { + if (raw === undefined || raw === null || raw === '') return {}; + const text = String(raw); + if (Buffer.byteLength(text) > MAX_COMMAND_BYTES) throw new TypeError('config-too-large'); + const result = {}; + let topLevel = true; + for (const line of text.split(/\r?\n/)) { + const clean = line.replace(/\s+#.*$/, '').trim(); + if (!clean) continue; + if (/^\[/.test(clean)) { topLevel = false; continue; } + if (!topLevel) continue; + const match = clean.match(/^(model|model_provider|model_reasoning_effort)\s*=\s*["']([^"']{1,256})["']\s*$/); + if (match) result[match[1]] = match[2]; + } + return result; +} + +export function discoverCodex({ + cacheRaw, configRaw, capturedAt, scope = {}, scopeKey, now = Date.now(), maxAgeMs = 7 * 86_400_000, +} = /** @type {any} */ ({})) { + let cache; + let config; + try { cache = parseCache(cacheRaw); config = parseConfig(configRaw); } catch (error) { + const source = sourceRecord({ id: 'codex-cache', owner: 'codex', scope, scopeKey, capturedAt, complete: false, status: 'unsupported-schema', schema: 'codex-model-cache-v1', diagnostics: ['unsupported-schema'] }); + return { status: 'unsupported-schema', source, models: [], diagnostics: [diagnostic('unsupported-schema', error.message)] }; + } + if (!Array.isArray(cache.models)) { + const source = sourceRecord({ id: 'codex-cache', owner: 'codex', scope, scopeKey, capturedAt, complete: false, status: 'unsupported-schema', schema: 'codex-model-cache-v1', diagnostics: ['unsupported-schema'] }); + return { status: 'unsupported-schema', source, models: [], diagnostics: [diagnostic('unsupported-schema', 'models must be an array')] }; + } + const fetchedAt = Date.parse(cache.fetched_at ?? cache.fetchedAt ?? ''); + const stale = Number.isFinite(fetchedAt) && now - fetchedAt > maxAgeMs; + let complete = cache.models.length <= MAX_MODELS; + const source = sourceRecord({ + id: 'codex-cache', owner: 'codex', scope, scopeKey, capturedAt: capturedAt ?? (Number.isFinite(fetchedAt) ? new Date(fetchedAt).toISOString() : undefined), + complete, schema: `codex-model-cache-v1${bounded(cache.client_version, 32) ? `@${cache.client_version}` : ''}`, + freshness: stale ? 'stale' : 'current', + }); + const diagnostics = []; + const models = []; + for (const [index, raw] of cache.models.slice(0, MAX_MODELS).entries()) { + const modelId = bounded(raw?.slug ?? raw?.id); + const visibility = raw?.visibility ?? 'list'; + if (!modelId || !VISIBILITY.has(visibility)) { + complete = false; + diagnostics.push(diagnostic('invalid-model-schema', `models[${index}] is invalid`)); + continue; + } + const reasoningEfforts = (Array.isArray(raw.supported_reasoning_levels) ? raw.supported_reasoning_levels : []) + .map((entry) => typeof entry === 'string' ? entry : entry?.effort) + .filter((effort) => REASONING.has(effort)); + const replacementId = bounded(raw.upgrade?.model ?? raw.upgrade?.model_id); + models.push(modelRecord({ + host: 'codex', provider: null, modelId, scopeId: source.scopeId, + displayName: bounded(raw.display_name) ?? modelId, source, + variant: { + reasoningEfforts: [...new Set(reasoningEfforts)], + contextWindow: Number.isInteger(raw.context_window) && raw.context_window > 0 ? raw.context_window : null, + }, + lifecycle: replacementId + ? { state: 'retiring', replacement: { modelId: replacementId, edge: 'first-party-migration' } } + : { state: visibility === 'hide' || visibility === 'hidden' ? 'hidden' : 'active', replacement: null }, + states: { + configured: config.model === modelId, effective: config.model === modelId, + discoverable: visibility === 'list' || visibility === 'visible', entitled: 'unknown', + }, + })); + } + if (bounded(config.model) && !models.some((model) => model.identity.modelId === config.model)) { + models.push(modelRecord({ + host: 'codex', provider: bounded(config.model_provider), modelId: config.model, + scopeId: source.scopeId, source, + variant: { reasoningEffort: REASONING.has(config.model_reasoning_effort) ? config.model_reasoning_effort : null }, + states: { configured: true, effective: true, discoverable: 'unknown', entitled: 'unknown' }, + })); + } + if (cache.models.length > MAX_MODELS) diagnostics.push(diagnostic('model-cap', `models exceeds ${MAX_MODELS}`)); + source.complete = complete; + source.status = complete ? (stale ? 'stale' : 'complete') : 'partial'; + source.diagnostics = diagnostics.map(({ code }) => code); + for (const model of models) model.evidence[0].completeness = complete ? 'complete' : 'partial'; + return { status: complete ? (stale ? 'stale' : 'complete') : 'partial', source, models, diagnostics }; +} diff --git a/src/lib/model-inventory/discovery/index.mjs b/src/lib/model-inventory/discovery/index.mjs new file mode 100644 index 0000000..6673384 --- /dev/null +++ b/src/lib/model-inventory/discovery/index.mjs @@ -0,0 +1,93 @@ +import { createHmac } from 'node:crypto'; + +export const MAX_CONFIG_BYTES = 1024 * 1024; +export const MAX_COMMAND_BYTES = 2 * 1024 * 1024; +export const MAX_MODELS = 2048; + +/** Non-identifying, stable scope key. Raw account/project/profile values never leave this function. */ +export function scopeFingerprint(owner, scope = {}, key) { + if (typeof key !== 'string' || key.length < 16) throw new TypeError('scope fingerprint key is required'); + const stable = Object.entries(scope ?? {}) + .filter(([, value]) => value !== undefined && value !== null) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => `${key}=${String(value)}`) + .join('\n'); + return `scope:${createHmac('sha256', key).update(`${owner}\n${stable || 'default'}`).digest('hex').slice(0, 16)}`; +} + +export function sourceRecord({ + id, owner, scope, scopeKey, capturedAt, complete, schema, freshness = 'current', status = null, diagnostics = [], +}) { + const fingerprint = scopeFingerprint(owner, scope, scopeKey); + return { + id, owner, schema, schemaVersion: schema, sourceVersion: null, + capturedAt: capturedAt ?? new Date().toISOString(), + scopeId: fingerprint, scopeFingerprint: fingerprint, + complete: Boolean(complete), freshness, + status: status ?? (complete ? (freshness === 'stale' ? 'stale' : 'complete') : 'partial'), + diagnostics, + }; +} + +export function diagnostic(code, message) { + return { code, message: String(message ?? code).slice(0, 240) }; +} + +export function modelRecord({ host, provider = null, modelId, scopeId, displayName = null, aliases = [], + variant = {}, lifecycle = /** @type {any} */ ({ state: 'unknown', replacement: null }), + states = /** @type {any} */ ({}), source }) { + const safeHost = host || 'unknown'; + const evidenceId = `${source.id}:${modelId}:${source.capturedAt}`.slice(0, 256); + const evidenceClass = source.id === 'usage-index' ? 'observed' + : source.id.includes('config') ? 'configured' : 'catalog'; + const normalizedStates = { + configured: false, effective: false, observed: false, discoverable: false, + entitled: 'unknown', policyAllowed: 'unknown', routable: 'unknown', recommended: false, + ...states, + }; + const dimensions = Object.fromEntries(Object.entries(normalizedStates).map(([name, value]) => [name, { + value: value === 'unknown' ? null : Boolean(value), + evidenceRefs: value === 'unknown' ? [] : [evidenceId], + }])); + const replacement = lifecycle?.replacement && typeof lifecycle.replacement === 'object' + ? lifecycle.replacement.modelId : lifecycle?.replacement ?? null; + return { + key: { host: safeHost, provider, modelId, scopeId }, + identity: { host: safeHost, provider, modelId, scopeId }, + displayName: displayName || modelId, + aliases: aliases.map((alias) => ({ + name: alias.name, resolvesTo: alias.resolvesTo ?? null, observedAt: alias.observedAt ?? null, + evidenceRefs: alias.evidenceRefs ?? [evidenceId], + })), + visibility: states.discoverable === true ? 'visible' : states.discoverable === false ? 'hidden' : 'unknown', + variant, + lifecycle: { state: lifecycle?.state ?? 'unknown', replacement, notice: lifecycle?.notice ?? null, + effectiveAt: lifecycle?.effectiveAt ?? null, evidenceRefs: [evidenceId] }, + capabilities: {}, dimensions, + states: normalizedStates, + evidence: [{ + id: evidenceId, field: 'catalog', source: source.id, class: evidenceClass, + capturedAt: source.capturedAt, freshness: source.freshness === 'stale' ? 'stale' : 'fresh', + completeness: source.complete ? 'complete' : 'partial', scopeFingerprint: scopeId, refs: [], + }], + }; +} + +// Executable dispatch is intentionally separate from adapters/registries.mjs's +// immutable metadata. Dynamic imports keep the pure parser modules independently testable. +export const DISCOVERY_DISPATCH = Object.freeze({ + claude: async (options) => (await import('./claude.mjs')).discoverClaude(options), + codex: async (options) => (await import('./codex.mjs')).discoverCodex(options), + opencode: async (options) => options?.raw !== undefined + ? (await import('./opencode.mjs')).discoverOpenCode(options) + : (await import('./opencode.mjs')).collectOpenCode(options), + ollama: async (options) => options?.raw !== undefined + ? (await import('./ollama.mjs')).discoverOllama(options) + : (await import('./ollama.mjs')).collectOllama(options), +}); + +export async function discoverModels(owner, options = {}) { + const collector = DISCOVERY_DISPATCH[owner]; + if (!collector) throw new TypeError(`unsupported model discovery owner: ${String(owner)}`); + return collector(options); +} diff --git a/src/lib/model-inventory/discovery/ollama.mjs b/src/lib/model-inventory/discovery/ollama.mjs new file mode 100644 index 0000000..4fcfcf2 --- /dev/null +++ b/src/lib/model-inventory/discovery/ollama.mjs @@ -0,0 +1,49 @@ +import { run } from '../../exec.mjs'; +import { + MAX_COMMAND_BYTES, MAX_MODELS, diagnostic, modelRecord, sourceRecord, +} from './index.mjs'; + +const TOKEN = /^[A-Za-z0-9][A-Za-z0-9._:+@/-]*$/; + +export function discoverOllama({ raw, capturedAt, scope = {}, scopeKey } = /** @type {any} */ ({})) { + const text = String(raw ?? ''); + const source = sourceRecord({ id: 'ollama-catalog', owner: 'ollama', scope, scopeKey, capturedAt, complete: true, schema: 'ollama-list-v1' }); + if (Buffer.byteLength(text) > MAX_COMMAND_BYTES) { + source.complete = false; + source.status = 'unsupported'; + source.diagnostics = ['output-too-large']; + return { status: 'unsupported', source, models: [], diagnostics: [diagnostic('output-too-large', `output exceeds ${MAX_COMMAND_BYTES}`)] }; + } + const diagnostics = []; + const models = []; + const lines = text.split(/\r?\n/).filter(Boolean); + for (const [index, line] of lines.entries()) { + if (index === 0 && /^NAME\s+ID\s+/i.test(line)) continue; + const [modelId, digest] = line.trim().split(/\s+/); + if (!TOKEN.test(modelId ?? '') || !/^[a-f0-9]{6,128}$/i.test(digest ?? '')) { + diagnostics.push(diagnostic('invalid-model-row', `line ${index + 1} is invalid`)); + continue; + } + if (models.length >= MAX_MODELS) break; + models.push(modelRecord({ + host: null, provider: 'ollama', modelId, scopeId: source.scopeId, source, + variant: { digest }, states: { discoverable: true, entitled: 'unknown', routable: 'unknown' }, + })); + } + const complete = diagnostics.length === 0 && models.length < MAX_MODELS; + source.complete = complete; + source.status = complete ? 'complete' : 'partial'; + source.diagnostics = diagnostics.map(({ code }) => code); + return { status: complete ? 'complete' : 'partial', source, models, diagnostics }; +} + +export async function collectOllama({ + runner = run, capturedAt, scope = {}, scopeKey, timeout = 15_000, +} = /** @type {any} */ ({})) { + const result = await runner('ollama', ['list'], { timeout, maxBuffer: MAX_COMMAND_BYTES, shell: false }); + if (result.code !== 0) { + const source = sourceRecord({ id: 'ollama-catalog', owner: 'ollama', scope, scopeKey, capturedAt, complete: false, status: 'unavailable', schema: 'ollama-list-v1', diagnostics: ['command-failed'] }); + return { status: 'unavailable', source, models: [], diagnostics: [diagnostic('command-failed', result.stderr || 'ollama list failed')] }; + } + return discoverOllama({ raw: result.stdout, capturedAt, scope, scopeKey }); +} diff --git a/src/lib/model-inventory/discovery/opencode.mjs b/src/lib/model-inventory/discovery/opencode.mjs new file mode 100644 index 0000000..b17a2b9 --- /dev/null +++ b/src/lib/model-inventory/discovery/opencode.mjs @@ -0,0 +1,80 @@ +import { run } from '../../exec.mjs'; +import { + MAX_COMMAND_BYTES, MAX_MODELS, diagnostic, modelRecord, sourceRecord, +} from './index.mjs'; + +const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:+@-]*(?:\/[A-Za-z0-9][A-Za-z0-9._:+@/-]*)?$/; + +function configuredRefs(raw) { + if (raw === undefined || raw === null || raw === '') return []; + const text = typeof raw === 'string' ? raw : JSON.stringify(raw); + if (Buffer.byteLength(text) > MAX_COMMAND_BYTES) throw new TypeError('config-too-large'); + let config; + try { config = typeof raw === 'string' ? JSON.parse(raw) : structuredClone(raw); } catch { throw new TypeError('config-invalid-json'); } + if (!config || typeof config !== 'object' || Array.isArray(config)) throw new TypeError('config-schema'); + const refs = [config.model]; + for (const agent of Object.values(config.agent && typeof config.agent === 'object' ? config.agent : {})) { + if (agent && typeof agent === 'object') refs.push(agent.model); + } + return [...new Set(refs.filter((value) => typeof value === 'string' && MODEL_ID.test(value) && value.length <= 512))]; +} + +export function discoverOpenCode({ raw, configRaw, capturedAt, scope = {}, scopeKey, online = false } = /** @type {any} */ ({})) { + const text = String(raw ?? ''); + const source = sourceRecord({ id: 'opencode-models', owner: 'opencode', scope, scopeKey, capturedAt, complete: true, schema: 'opencode-models-lines-v1' }); + if (Buffer.byteLength(text) > MAX_COMMAND_BYTES) { + source.complete = false; + source.status = 'unsupported'; + source.diagnostics = ['output-too-large']; + return { status: 'unsupported', source, models: [], diagnostics: [diagnostic('output-too-large', `output exceeds ${MAX_COMMAND_BYTES}`)], networkUsed: online }; + } + const diagnostics = []; + let configured = []; + try { configured = configuredRefs(configRaw); } catch (error) { + source.complete = false; + source.status = 'partial'; + diagnostics.push(diagnostic('unsupported-config-schema', error.message)); + } + const ids = []; + for (const [index, line] of text.split(/\r?\n/).entries()) { + const id = line.trim(); + if (!id) continue; + if (!MODEL_ID.test(id) || id.length > 512) { + diagnostics.push(diagnostic('invalid-model-id', `line ${index + 1} is invalid`)); + continue; + } + if (!ids.includes(id)) ids.push(id); + if (ids.length >= MAX_MODELS) break; + } + const complete = diagnostics.length === 0 && ids.length < MAX_MODELS; + source.complete = complete; + source.status = complete ? 'complete' : 'partial'; + source.diagnostics = diagnostics.map(({ code }) => code); + const allIds = [...new Set([...ids, ...configured])]; + const models = allIds.map((qualified) => { + const slash = qualified.indexOf('/'); + const provider = slash > 0 ? qualified.slice(0, slash) : null; + const modelId = slash > 0 ? qualified.slice(slash + 1) : qualified; + return modelRecord({ + host: 'opencode', provider, modelId, scopeId: source.scopeId, displayName: qualified, source, + states: { + configured: configured.includes(qualified), effective: configRaw !== undefined && configured[0] === qualified, + discoverable: ids.includes(qualified) ? true : 'unknown', entitled: 'unknown', + }, + }); + }); + return { status: complete ? 'complete' : 'partial', source, models, diagnostics, networkUsed: online }; +} + +export async function collectOpenCode({ + runner = run, online = false, provider, configRaw, capturedAt, scope = {}, scopeKey, timeout = 30_000, +} = /** @type {any} */ ({})) { + const providerArg = typeof provider === 'string' && provider.length <= 256 ? provider : null; + const args = ['models', ...(providerArg ? [providerArg] : []), ...(online ? ['--refresh'] : [])]; + const result = await runner('opencode', args, { timeout, maxBuffer: MAX_COMMAND_BYTES, shell: false }); + if (result.code !== 0) { + const source = sourceRecord({ id: 'opencode-models', owner: 'opencode', scope, scopeKey, capturedAt, complete: false, status: 'unavailable', schema: 'opencode-models-lines-v1', diagnostics: ['command-failed'] }); + return { status: 'unavailable', source, models: [], diagnostics: [diagnostic('command-failed', result.stderr || 'opencode models failed')], networkUsed: online }; + } + return discoverOpenCode({ raw: result.stdout, configRaw, capturedAt, scope, scopeKey, online }); +} diff --git a/src/lib/model-inventory/index.mjs b/src/lib/model-inventory/index.mjs index faab62f..5917739 100644 --- a/src/lib/model-inventory/index.mjs +++ b/src/lib/model-inventory/index.mjs @@ -3,3 +3,4 @@ export * from './store.mjs'; export * from './diff.mjs'; export * from './impact.mjs'; export * from './read-model.mjs'; +export * from './refresh.mjs'; diff --git a/src/lib/model-inventory/observed.mjs b/src/lib/model-inventory/observed.mjs new file mode 100644 index 0000000..d757aea --- /dev/null +++ b/src/lib/model-inventory/observed.mjs @@ -0,0 +1,52 @@ +import { readIndex } from '../usage-index.mjs'; +import { diagnostic, modelRecord, scopeFingerprint, sourceRecord } from './discovery/index.mjs'; + +const bounded = (value) => typeof value === 'string' && value.length > 0 && value.length <= 512 ? value : null; + +export async function collectObservedModels({ + readIndexFn = readIndex, indexOptions = {}, scope = {}, scopeKey, days = 365, +} = {}) { + let aggregate; + try { aggregate = await readIndexFn({ days, ...indexOptions }); } catch (error) { + const capturedAt = new Date().toISOString(); + return { + status: 'unavailable', generatedAt: capturedAt, models: [], + source: sourceRecord({ id: 'usage-index', owner: 'usage', scope, scopeKey, capturedAt, + complete: false, status: 'unavailable', schema: 'usage-index-v6', diagnostics: ['usage-index-unavailable'] }), + diagnostics: [diagnostic('usage-index-unavailable', error?.message ?? error)], + }; + } + const map = new Map(); + for (const session of Array.isArray(aggregate?.sessions) ? aggregate.sessions : []) { + const host = bounded(session.host) ?? 'unknown'; + const provider = bounded(session.provider); + for (const modelId of Array.isArray(session.models) ? session.models : []) { + if (!bounded(modelId)) continue; + const key = `${host}\0${provider ?? ''}\0${modelId}`; + const prior = map.get(key); + if (prior) { prior.observations++; continue; } + const source = { + id: 'usage-index', capturedAt: aggregate.generatedAt ?? new Date().toISOString(), + complete: true, freshness: 'current', + }; + const record = modelRecord({ + host, provider, modelId, scopeId: scopeFingerprint(host, scope, scopeKey), source, + states: { observed: true, entitled: 'unknown' }, + }); + record.observations = 1; + record.evidence[0].providerProvenance = session.providerProvenance ?? 'unknown'; + map.set(key, record); + } + } + const degraded = Object.values(aggregate?.sourceHealth ?? {}).some((health) => health?.status === 'degraded'); + const capturedAt = aggregate?.generatedAt ?? new Date().toISOString(); + const source = sourceRecord({ + id: 'usage-index', owner: 'usage', scope, scopeKey, capturedAt, complete: !degraded, + status: degraded ? 'partial' : 'complete', schema: 'usage-index-v6', + diagnostics: degraded ? ['usage-source-degraded'] : [], + }); + return { + status: degraded ? 'partial' : 'complete', generatedAt: capturedAt, + models: [...map.values()], source, diagnostics: [], sourceHealth: aggregate?.sourceHealth ?? {}, + }; +} diff --git a/src/lib/model-inventory/refresh.mjs b/src/lib/model-inventory/refresh.mjs new file mode 100644 index 0000000..e2de5fa --- /dev/null +++ b/src/lib/model-inventory/refresh.mjs @@ -0,0 +1,147 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import * as paths from '../paths.mjs'; +import { run } from '../exec.mjs'; +import { discoverModels } from './discovery/index.mjs'; +import { collectModelBindings } from './bindings.mjs'; +import { collectObservedModels } from './observed.mjs'; +import { readOrCreateModelScopeKey } from './store.mjs'; +import { MODEL_INVENTORY_SCHEMA_VERSION, modelIdentityKey, normalizeSnapshot } from './contracts.mjs'; +import { scopeFingerprint } from './discovery/index.mjs'; + +const DEFAULT_OWNERS = Object.freeze(['claude', 'codex', 'opencode', 'ollama']); +const CONTACT = Object.freeze({ + claude: 'Claude local settings', codex: 'Codex local model cache', + opencode: 'opencode catalog', ollama: 'local Ollama daemon', +}); + +function readOptional(file, readFileFn) { + try { return readFileFn(file, 'utf8'); } catch { return undefined; } +} + +export async function refreshModelDiscovery({ + owners = DEFAULT_OWNERS, online = false, runner = run, readFileFn = fs.readFileSync, + cwd = process.cwd(), inputs = {}, capturedAt, scope = {}, scopeKey, +} = /** @type {any} */ ({})) { + const fingerprintKey = scopeKey ?? readOrCreateModelScopeKey(); + const results = {}; + const contacts = []; + for (const owner of owners) { + if (!DEFAULT_OWNERS.includes(owner)) throw new TypeError(`unsupported model discovery owner: ${String(owner)}`); + contacts.push(CONTACT[owner]); + const ownerScope = { ...scope, ...(owner === 'opencode' ? { project: cwd } : {}) }; + let options = { capturedAt, scope: ownerScope, scopeKey: fingerprintKey }; + if (owner === 'claude') { + options = { + ...options, + settingsRaw: inputs.claude?.settingsRaw ?? readOptional(paths.claudeSettingsPath(), readFileFn), + managedSettingsRaw: inputs.claude?.managedSettingsRaw, + environment: inputs.claude?.environment ?? {}, + }; + } else if (owner === 'codex') { + options = { + ...options, + cacheRaw: inputs.codex?.cacheRaw ?? readOptional(path.join(paths.codexDir(), 'models_cache.json'), readFileFn), + configRaw: inputs.codex?.configRaw ?? readOptional(path.join(paths.codexDir(), 'config.toml'), readFileFn), + }; + } else if (owner === 'opencode') { + options = { + ...options, runner, online, provider: inputs.opencode?.provider, + configRaw: inputs.opencode?.configRaw ?? readOptional(path.join(cwd, 'opencode.json'), readFileFn), + }; + } else { + options = { ...options, runner }; + } + results[owner] = await discoverModels(owner, options); + } + return { generatedAt: capturedAt ?? new Date().toISOString(), online, contacts, results }; +} + +export async function collectModelInventory({ + config = {}, aqeConfig, rufloConfig, discoveryOptions = {}, readIndexFn, indexOptions, scope = {}, scopeKey, +} = /** @type {any} */ ({})) { + const fingerprintKey = scopeKey ?? readOrCreateModelScopeKey(); + const [discovery, observed] = await Promise.all([ + refreshModelDiscovery({ ...discoveryOptions, scope: discoveryOptions.scope ?? scope, scopeKey: fingerprintKey }), + collectObservedModels({ readIndexFn, indexOptions, scope, scopeKey: fingerprintKey }), + ]); + return { + generatedAt: new Date().toISOString(), discovery, + bindings: collectModelBindings({ config, aqeConfig, rufloConfig }), observed, + }; +} + +function mergeModels(records) { + const merged = new Map(); + for (const record of records) { + const identity = modelIdentityKey(record.key); + const prior = merged.get(identity); + if (!prior) { merged.set(identity, structuredClone(record)); continue; } + const evidence = [...prior.evidence, ...record.evidence] + .filter((entry, index, all) => all.findIndex(({ id }) => id === entry.id) === index); + const dimensions = {}; + for (const name of Object.keys(prior.dimensions)) { + const values = [prior.dimensions[name], record.dimensions[name]]; + const value = values.some((entry) => entry?.value === true) ? true + : values.every((entry) => entry?.value === false) ? false : null; + dimensions[name] = { + value, + evidenceRefs: [...new Set(values.flatMap((entry) => entry?.evidenceRefs ?? []))], + }; + } + merged.set(identity, { + ...prior, + displayName: prior.displayName || record.displayName, + aliases: [...prior.aliases, ...record.aliases] + .filter((entry, index, all) => all.findIndex(({ name, resolvesTo }) => name === entry.name && resolvesTo === entry.resolvesTo) === index), + variant: { ...prior.variant, ...record.variant }, + lifecycle: record.lifecycle?.state !== 'unknown' ? record.lifecycle : prior.lifecycle, + capabilities: { ...prior.capabilities, ...record.capabilities }, + dimensions, evidence, + }); + } + return [...merged.values()]; +} + +export function composeModelSnapshot(collection, { + scope = {}, scopeKey, capturedAt = collection?.generatedAt ?? new Date().toISOString(), +} = {}) { + const discoveryResults = Object.values(collection?.discovery?.results ?? {}); + const profileFingerprints = Object.fromEntries(discoveryResults + .filter((result) => result?.source?.scopeFingerprint) + .map((result) => [result.source.owner ?? result.source.id, result.source.scopeFingerprint])); + const hosts = Object.keys(collection?.discovery?.results ?? {}).sort(); + const fingerprint = scopeFingerprint('inventory', { ...scope, hosts: hosts.join(',') }, scopeKey); + const sources = [...discoveryResults.map(({ source }) => source), collection?.observed?.source] + .filter(Boolean) + .map((source) => ({ ...source, scopeFingerprint: fingerprint, scopeId: fingerprint })); + const models = mergeModels([ + ...discoveryResults.flatMap(({ models }) => models ?? []), + ...(collection?.observed?.models ?? []), + ]); + const bindings = collection?.bindings?.bindings ?? []; + const diagnostics = [ + ...discoveryResults.flatMap(({ diagnostics }) => diagnostics ?? []), + ...(collection?.observed?.diagnostics ?? []), + ...(collection?.bindings?.diagnostics ?? []), + ].map((entry) => typeof entry === 'string' ? entry : entry.code).filter(Boolean); + const digestInput = JSON.stringify({ capturedAt, fingerprint, sources, models, bindings }); + return normalizeSnapshot({ + schemaVersion: MODEL_INVENTORY_SCHEMA_VERSION, + snapshotId: `models:${createHash('sha256').update(digestInput).digest('hex').slice(0, 20)}`, + capturedAt, + scope: { fingerprint, machine: null, project: null, hosts, profileFingerprints }, + sources, models, bindings, changes: [], opportunities: [], diagnostics: [...new Set(diagnostics)], + }); +} + +export async function collectModelSnapshot(options = {}) { + const scopeKey = options.scopeKey ?? readOrCreateModelScopeKey(); + const capturedAt = options.discoveryOptions?.capturedAt ?? new Date().toISOString(); + const collection = await collectModelInventory({ + ...options, scopeKey, + discoveryOptions: { ...options.discoveryOptions, capturedAt }, + }); + return composeModelSnapshot(collection, { scope: options.scope, scopeKey, capturedAt }); +} diff --git a/src/lib/model-inventory/store.mjs b/src/lib/model-inventory/store.mjs index 6e8ac67..0ffbed7 100644 --- a/src/lib/model-inventory/store.mjs +++ b/src/lib/model-inventory/store.mjs @@ -1,5 +1,6 @@ import fs from 'node:fs'; import path from 'node:path'; +import { randomBytes } from 'node:crypto'; import { configDir } from '../paths.mjs'; import { MODEL_INVENTORY_SCHEMA_VERSION, isCompleteStableSnapshot, normalizeSnapshot, @@ -10,6 +11,29 @@ export const MAX_MODEL_SNAPSHOTS = 32; export const MODEL_SNAPSHOT_RETENTION_MS = 90 * 86_400_000; export const modelInventoryPath = () => path.join(configDir(), 'model-inventory.json'); +export const modelScopeKeyPath = () => path.join(configDir(), 'model-scope.key'); + +export function readOrCreateModelScopeKey({ + file = modelScopeKeyPath(), fsImpl = fs, randomBytesFn = randomBytes, +} = {}) { + try { + const existing = String(fsImpl.readFileSync(file, 'utf8')).trim(); + if (/^[a-f0-9]{64}$/i.test(existing)) return existing.toLowerCase(); + } catch { /* create a new key below */ } + const value = randomBytesFn(32).toString('hex'); + if (!/^[a-f0-9]{64}$/i.test(value)) throw new TypeError('invalid generated model scope key'); + fsImpl.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = `${file}.${process.pid}.${Date.now()}.tmp`; + try { + fsImpl.writeFileSync(tmp, `${value}\n`, { mode: 0o600, flag: 'wx' }); + fsImpl.renameSync(tmp, file); + try { fsImpl.chmodSync(file, 0o600); } catch { /* best effort */ } + } catch (error) { + try { fsImpl.rmSync(tmp, { force: true }); } catch { /* preserve original */ } + throw error; + } + return value; +} const emptyStore = () => ({ schemaVersion: MODEL_STORE_SCHEMA_VERSION, diff --git a/tests/fixtures/model-inventory/claude/managed-settings.json b/tests/fixtures/model-inventory/claude/managed-settings.json new file mode 100644 index 0000000..7a8cebe --- /dev/null +++ b/tests/fixtures/model-inventory/claude/managed-settings.json @@ -0,0 +1,3 @@ +{ + "availableModels": ["sonnet", "claude-opus-5"] +} diff --git a/tests/fixtures/model-inventory/claude/settings.json b/tests/fixtures/model-inventory/claude/settings.json new file mode 100644 index 0000000..2f8fed6 --- /dev/null +++ b/tests/fixtures/model-inventory/claude/settings.json @@ -0,0 +1,7 @@ +{ + "model": "sonnet", + "env": { + "ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-sonnet-5-20260801", + "ANTHROPIC_API_KEY": "must-never-appear" + } +} diff --git a/tests/fixtures/model-inventory/codex/models-cache.json b/tests/fixtures/model-inventory/codex/models-cache.json new file mode 100644 index 0000000..4275b32 --- /dev/null +++ b/tests/fixtures/model-inventory/codex/models-cache.json @@ -0,0 +1,19 @@ +{ + "fetched_at": "2026-08-25T12:00:00.000Z", + "client_version": "0.146.0", + "models": [ + { + "slug": "gpt-5.6-terra", + "display_name": "GPT-5.6 Terra", + "visibility": "list", + "context_window": 272000, + "supported_reasoning_levels": [{"effort": "low"}, {"effort": "high"}] + }, + { + "slug": "gpt-5.4", + "display_name": "GPT-5.4", + "visibility": "hide", + "upgrade": {"model": "gpt-5.6-terra"} + } + ] +} diff --git a/tests/fixtures/model-inventory/ollama/list.txt b/tests/fixtures/model-inventory/ollama/list.txt new file mode 100644 index 0000000..90ac78c --- /dev/null +++ b/tests/fixtures/model-inventory/ollama/list.txt @@ -0,0 +1,3 @@ +NAME ID SIZE MODIFIED +qwen3-coder:latest 9e3f6a12abcd 18 GB 2 hours ago +deepseek-r1:8b 71aa22bb33cc 5.2 GB 3 days ago diff --git a/tests/fixtures/model-inventory/opencode/models.txt b/tests/fixtures/model-inventory/opencode/models.txt new file mode 100644 index 0000000..3758b32 --- /dev/null +++ b/tests/fixtures/model-inventory/opencode/models.txt @@ -0,0 +1,2 @@ +anthropic/claude-sonnet-5 +openrouter/z-ai/glm-5 diff --git a/tests/kit/adapter-registries.test.mjs b/tests/kit/adapter-registries.test.mjs index 8ead813..bc15350 100644 --- a/tests/kit/adapter-registries.test.mjs +++ b/tests/kit/adapter-registries.test.mjs @@ -5,8 +5,10 @@ import { PROVIDER_REGISTRY, PROJECTION_REGISTRY, OBSERVABILITY_REGISTRY, + MODEL_DISCOVERY_REGISTRY, validateRegistries, validateHostAdapter, + validateModelDiscoveryAdapter, defaultHostMap, assertValidBinding, } from '../../src/lib/adapters/index.mjs'; @@ -28,6 +30,30 @@ test('the built-in registries satisfy their own contract', () => { }), []); }); +test('model discovery registry is immutable metadata, never executable dispatch', () => { + assert.deepEqual(MODEL_DISCOVERY_REGISTRY.map((entry) => entry.id), [ + 'claude-config', 'codex-cache', 'opencode-models', 'ollama-catalog', + ]); + for (const entry of MODEL_DISCOVERY_REGISTRY) { + assert.equal(Object.isFrozen(entry), true); + assert.equal(Object.values(entry).some((value) => typeof value === 'function'), false); + assert.match(entry.ownerType, /^(host|provider)$/); + assert.match(entry.network, /^(never|local|explicit)$/); + } + assert.throws(() => { MODEL_DISCOVERY_REGISTRY[0].id = 'changed'; }, TypeError); +}); + +test('model discovery descriptor validation rejects ambiguous owners and unsafe command metadata', () => { + assert.throws(() => validateModelDiscoveryAdapter({ + id: 'ambiguous', ownerType: 'host', ownerId: 'claude', provider: 'anthropic', + transport: 'file', network: 'never', schema: 'v1', + }), /unknown field provider/); + assert.throws(() => validateModelDiscoveryAdapter({ + id: 'unsafe', ownerType: 'host', ownerId: 'opencode', transport: 'command', + command: 'opencode models', network: 'explicit', schema: 'v1', + }), /command must be an executable name/); +}); + // qe-court A3: the cross-axis invariants now run AT module construction — a // host edit that violates canDriveSession → canBePrimary/canRouteActivities // (or any other cross-axis rule) makes the import itself throw instead of diff --git a/tests/kit/model-discovery-claude-codex.test.mjs b/tests/kit/model-discovery-claude-codex.test.mjs new file mode 100644 index 0000000..bb3aff5 --- /dev/null +++ b/tests/kit/model-discovery-claude-codex.test.mjs @@ -0,0 +1,83 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { discoverClaude } from '../../src/lib/model-inventory/discovery/claude.mjs'; +import { discoverCodex } from '../../src/lib/model-inventory/discovery/codex.mjs'; +import { normalizeModelRecord, normalizeSourceResult } from '../../src/lib/model-inventory/contracts.mjs'; + +const FIX = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../fixtures/model-inventory'); +const fixture = (...parts) => fs.readFileSync(path.join(FIX, ...parts), 'utf8'); +const SCOPE_KEY = '0123456789abcdef0123456789abcdef'; + +test('Claude preserves alias and concrete resolution while entitlement stays unknown', () => { + const result = discoverClaude({ + settingsRaw: fixture('claude', 'settings.json'), + managedSettingsRaw: fixture('claude', 'managed-settings.json'), + capturedAt: '2026-08-25T13:00:00.000Z', scope: { profile: 'default', project: '/private/repo' }, scopeKey: SCOPE_KEY, + }); + assert.equal(result.status, 'complete'); + assert.match(result.source.scopeId, /^scope:[a-f0-9]{16}$/); + assert.equal(JSON.stringify(result).includes('/private/repo'), false); + assert.equal(JSON.stringify(result).includes('must-never-appear'), false); + const selected = result.models.find((model) => model.aliases.some((alias) => alias.name === 'sonnet')); + assert.equal(selected.identity.modelId, 'claude-sonnet-5-20260801'); + assert.equal(selected.states.configured, true); + assert.equal(selected.states.effective, true); + assert.equal(selected.states.entitled, 'unknown'); + assert.equal(result.models.find((model) => model.identity.modelId === 'claude-opus-5').states.policyAllowed, true); + assert.doesNotThrow(() => result.models.map(normalizeModelRecord)); + assert.doesNotThrow(() => normalizeSourceResult(result.source)); +}); + +test('Claude rejects oversized or invalid settings as unsupported without leaking raw input', () => { + const huge = '{"model":"' + 'x'.repeat(1_100_000) + '"}'; + for (const raw of [huge, '{not-json']) { + const result = discoverClaude({ settingsRaw: raw, scopeKey: SCOPE_KEY }); + assert.equal(result.status, 'unsupported-schema'); + assert.deepEqual(result.models, []); + assert.equal(JSON.stringify(result).includes('not-json'), false); + } +}); + +test('Codex parses visibility, reasoning variants, and first-party migration metadata', () => { + const result = discoverCodex({ + cacheRaw: fixture('codex', 'models-cache.json'), + now: Date.parse('2026-08-25T13:00:00.000Z'), scope: { profile: 'default' }, scopeKey: SCOPE_KEY, + }); + assert.equal(result.status, 'complete'); + assert.equal(result.models.length, 2); + const terra = result.models.find((model) => model.identity.modelId === 'gpt-5.6-terra'); + assert.deepEqual(terra.variant.reasoningEfforts, ['low', 'high']); + assert.equal(terra.variant.contextWindow, 272000); + assert.equal(terra.states.entitled, 'unknown'); + const old = result.models.find((model) => model.identity.modelId === 'gpt-5.4'); + assert.equal(old.lifecycle.state, 'retiring'); + assert.equal(old.lifecycle.replacement, 'gpt-5.6-terra'); + assert.equal(old.states.discoverable, false); + assert.doesNotThrow(() => result.models.map(normalizeModelRecord)); + assert.doesNotThrow(() => normalizeSourceResult(result.source)); +}); + +test('Codex includes configured top-level model evidence without parsing unrelated TOML tables', () => { + const result = discoverCodex({ + cacheRaw: JSON.stringify({ models: [] }), scopeKey: SCOPE_KEY, + configRaw: 'model = "gpt-private"\nmodel_provider = "openai"\nmodel_reasoning_effort = "high"\n[profiles.other]\nmodel = "ignored"\n', + }); + assert.equal(result.models.length, 1); + assert.equal(result.models[0].identity.modelId, 'gpt-private'); + assert.equal(result.models[0].identity.provider, 'openai'); + assert.equal(result.models[0].states.configured, true); + assert.equal(result.models[0].variant.reasoningEffort, 'high'); +}); + +test('Codex schema and enum guards degrade explicitly and never manufacture models', () => { + const badSchema = discoverCodex({ cacheRaw: JSON.stringify({ models: 'all' }), scopeKey: SCOPE_KEY }); + assert.equal(badSchema.status, 'unsupported-schema'); + assert.match(badSchema.diagnostics[0].code, /schema/); + const badEnum = discoverCodex({ cacheRaw: JSON.stringify({ models: [{ slug: 'gpt-x', visibility: 'maybe' }] }), scopeKey: SCOPE_KEY }); + assert.equal(badEnum.status, 'partial'); + assert.deepEqual(badEnum.models, []); + assert.equal(badEnum.source.complete, false); +}); diff --git a/tests/kit/model-discovery-opencode-ollama.test.mjs b/tests/kit/model-discovery-opencode-ollama.test.mjs new file mode 100644 index 0000000..c3a939b --- /dev/null +++ b/tests/kit/model-discovery-opencode-ollama.test.mjs @@ -0,0 +1,78 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { collectOpenCode, discoverOpenCode } from '../../src/lib/model-inventory/discovery/opencode.mjs'; +import { collectOllama, discoverOllama } from '../../src/lib/model-inventory/discovery/ollama.mjs'; +import { DISCOVERY_DISPATCH, discoverModels } from '../../src/lib/model-inventory/discovery/index.mjs'; +import { normalizeModelRecord, normalizeSourceResult } from '../../src/lib/model-inventory/contracts.mjs'; + +const FIX = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../fixtures/model-inventory'); +const fixture = (...parts) => fs.readFileSync(path.join(FIX, ...parts), 'utf8'); +const SCOPE_KEY = '0123456789abcdef0123456789abcdef'; + +test('OpenCode normalizes provider-qualified ids in project scope', () => { + const result = discoverOpenCode({ raw: fixture('opencode', 'models.txt'), scope: { project: '/private/repo' }, scopeKey: SCOPE_KEY }); + assert.equal(result.status, 'complete'); + assert.deepEqual(result.models.map((model) => [model.identity.provider, model.identity.modelId]), [ + ['anthropic', 'claude-sonnet-5'], ['openrouter', 'z-ai/glm-5'], + ]); + assert.equal(JSON.stringify(result).includes('/private/repo'), false); + assert.equal(result.models.every((model) => model.states.entitled === 'unknown'), true); + assert.doesNotThrow(() => result.models.map(normalizeModelRecord)); + assert.doesNotThrow(() => normalizeSourceResult(result.source)); +}); + +test('OpenCode preserves configured global and agent model evidence', () => { + const result = discoverOpenCode({ + raw: 'anthropic/claude-sonnet-5\n', scopeKey: SCOPE_KEY, + configRaw: JSON.stringify({ model: 'openrouter/z-ai/glm-5', agent: { review: { model: 'anthropic/claude-opus-5' } } }), + }); + assert.equal(result.models.find((model) => model.displayName === 'openrouter/z-ai/glm-5').states.effective, true); + assert.equal(result.models.find((model) => model.displayName === 'anthropic/claude-opus-5').states.configured, true); + assert.equal(result.models.find((model) => model.displayName === 'anthropic/claude-opus-5').states.discoverable, 'unknown'); +}); + +test('OpenCode uses literal argv and refresh is the only online boundary', async () => { + const calls = []; + const runner = async (command, args, options) => { + calls.push({ command, args, options }); + return { code: 0, stdout: fixture('opencode', 'models.txt'), stderr: '' }; + }; + await collectOpenCode({ runner, online: false, provider: 'x; touch /tmp/nope', scopeKey: SCOPE_KEY }); + await collectOpenCode({ runner, online: true, provider: 'anthropic', scopeKey: SCOPE_KEY }); + assert.deepEqual(calls[0].args, ['models', 'x; touch /tmp/nope']); + assert.deepEqual(calls[1].args, ['models', 'anthropic', '--refresh']); + assert.equal(calls.every((call) => call.options.shell === false), true); +}); + +test('Ollama parses local names and digests without claiming entitlement', () => { + const result = discoverOllama({ raw: fixture('ollama', 'list.txt'), scopeKey: SCOPE_KEY }); + assert.equal(result.status, 'complete'); + assert.deepEqual(result.models.map((model) => model.identity.modelId), ['qwen3-coder:latest', 'deepseek-r1:8b']); + assert.equal(result.models[0].variant.digest, '9e3f6a12abcd'); + assert.equal(result.models[0].states.entitled, 'unknown'); + assert.doesNotThrow(() => result.models.map(normalizeModelRecord)); + assert.doesNotThrow(() => normalizeSourceResult(result.source)); +}); + +test('Ollama collector invokes list only and caps untrusted output', async () => { + const calls = []; + const runner = async (command, args, options) => { + calls.push({ command, args, options }); + return { code: 0, stdout: 'x'.repeat(3_000_000), stderr: '' }; + }; + const result = await collectOllama({ runner, scopeKey: SCOPE_KEY }); + assert.deepEqual(calls[0].args, ['list']); + assert.equal(calls[0].options.shell, false); + assert.equal(result.status, 'unsupported'); +}); + +test('dispatch map is separate from immutable registry metadata', async () => { + assert.deepEqual(Object.keys(DISCOVERY_DISPATCH), ['claude', 'codex', 'opencode', 'ollama']); + assert.equal(typeof DISCOVERY_DISPATCH.opencode, 'function'); + const result = await discoverModels('opencode', { raw: fixture('opencode', 'models.txt'), scopeKey: SCOPE_KEY }); + assert.equal(result.models.length, 2); + await assert.rejects(discoverModels('unknown-host'), /unsupported model discovery owner/); +}); diff --git a/tests/kit/model-inventory-collect.test.mjs b/tests/kit/model-inventory-collect.test.mjs new file mode 100644 index 0000000..d5d0e46 --- /dev/null +++ b/tests/kit/model-inventory-collect.test.mjs @@ -0,0 +1,101 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { collectModelBindings } from '../../src/lib/model-inventory/bindings.mjs'; +import { collectObservedModels } from '../../src/lib/model-inventory/observed.mjs'; +import { collectModelInventory, composeModelSnapshot, refreshModelDiscovery } from '../../src/lib/model-inventory/refresh.mjs'; +import { isCompleteStableSnapshot, normalizeBindingRecord, normalizeModelRecord } from '../../src/lib/model-inventory/contracts.mjs'; + +const SCOPE_KEY = '0123456789abcdef0123456789abcdef'; + +test('binding collection enumerates routes and escalation without inventing provider identity', () => { + const result = collectModelBindings({ config: { routing: { routes: { + testing: { host: 'codex', model: 'gpt-5.6-terra', provenance: 'user', reasoningEffort: 'high', escalation: [ + { host: 'claude', model: 'sonnet' }, + ] }, + } } } }); + assert.equal(result.status, 'complete'); + assert.deepEqual(result.bindings.map((binding) => ({ + activity: binding.activity, host: binding.host, provider: binding.provider, + modelRef: binding.modelRef, consumer: binding.consumer, + })), [ + { activity: 'testing', host: 'codex', provider: null, modelRef: 'gpt-5.6-terra', consumer: 'route:testing' }, + { activity: 'testing', host: 'claude', provider: null, modelRef: 'sonnet', consumer: 'route:testing:escalation:0' }, + ]); + assert.equal(result.bindings[0].variant.reasoningEffort, 'high'); + assert.doesNotThrow(() => result.bindings.map(normalizeBindingRecord)); +}); + +test('binding collection includes AQE and Ruflo as independently sourced consumers', () => { + const result = collectModelBindings({ + config: { routing: { routes: {} } }, + aqeConfig: { defaultProvider: 'codex', fallbackChain: [{ provider: 'openrouter', model: 'z-ai/glm-5' }], agentOverrides: { tester: { provider: 'codex', model: 'gpt-5.6-terra' } } }, + rufloConfig: { candidates: [{ provider: 'openrouter', model: 'z-ai/glm-5', price: 0.1 }] }, + }); + assert.deepEqual(new Set(result.bindings.map((binding) => binding.consumer)), new Set(['aqe:default', 'aqe:fallback:0', 'aqe:agent:tester', 'ruflo:candidate:0'])); + assert.equal(result.bindings.find((binding) => binding.consumer === 'ruflo:candidate:0').evidenceClass, 'configured'); +}); + +test('observed collection reuses readIndex and emits no prompt, title, or transcript content', async () => { + const calls = []; + const result = await collectObservedModels({ + readIndexFn: async (options) => { + calls.push(options); + return { generatedAt: '2026-08-25T13:00:00.000Z', sourceHealth: { codex: { status: 'ok' } }, sessions: [ + { id: 'secret-id', host: 'codex', provider: 'openai', providerProvenance: 'observed', models: ['gpt-5.6-terra'], title: 'PRIVATE PROMPT', turns: ['PRIVATE'] }, + { id: 'other', host: 'codex', provider: 'openai', providerProvenance: 'observed', models: ['gpt-5.6-terra', 'gpt-5.4'] }, + ] }; + }, + indexOptions: { roots: { codex: '/fixtures' } }, scope: { project: '/private/repo' }, scopeKey: SCOPE_KEY, + }); + assert.equal(calls.length, 1); + assert.equal(result.status, 'complete'); + assert.equal(result.models.find((model) => model.identity.modelId === 'gpt-5.6-terra').observations, 2); + assert.doesNotThrow(() => result.models.map(normalizeModelRecord)); + const wire = JSON.stringify(result); + for (const secret of ['PRIVATE PROMPT', 'PRIVATE', 'secret-id', '/private/repo']) assert.equal(wire.includes(secret), false); +}); + +test('refresh names contacts, never invokes a model, and preserves per-source failure', async () => { + const calls = []; + const runner = async (command, args) => { + calls.push([command, args]); + if (command === 'opencode') return { code: 1, stdout: '', stderr: 'catalog unavailable' }; + return { code: 0, stdout: 'NAME ID SIZE MODIFIED\nqwen:latest abcdef 1 GB now\n', stderr: '' }; + }; + const result = await refreshModelDiscovery({ owners: ['opencode', 'ollama'], online: true, runner, scopeKey: SCOPE_KEY }); + assert.deepEqual(result.contacts, ['opencode catalog', 'local Ollama daemon']); + assert.equal(result.results.opencode.status, 'unavailable'); + assert.equal(result.results.ollama.status, 'complete'); + assert.equal(calls.some(([, args]) => args.some((arg) => /prompt|chat|run|generate/.test(arg))), false); +}); + +test('combined collection keeps discovery, configured, and observed evidence separate', async () => { + const result = await collectModelInventory({ + config: { routing: { routes: { testing: { host: 'codex', model: 'gpt-x', provenance: 'user' } } } }, + discoveryOptions: { owners: [], online: false }, scopeKey: SCOPE_KEY, + readIndexFn: async () => ({ generatedAt: '2026-08-25T13:00:00.000Z', sessions: [] }), + }); + assert.equal(result.bindings.bindings[0].evidenceClass, 'configured'); + assert.deepEqual(result.discovery.results, {}); + assert.deepEqual(result.observed.models, []); +}); + +test('snapshot composition emits a normalized same-scope stable baseline', async () => { + const collection = await collectModelInventory({ + config: { routing: { routes: { testing: { host: 'codex', model: 'gpt-x' } } } }, + discoveryOptions: { + owners: ['codex'], scopeKey: SCOPE_KEY, + inputs: { codex: { cacheRaw: JSON.stringify({ models: [{ slug: 'gpt-x', visibility: 'list' }] }), configRaw: '' } }, + }, + readIndexFn: async () => ({ generatedAt: '2026-08-25T13:00:00.000Z', sessions: [] }), + scopeKey: SCOPE_KEY, + }); + const snapshot = composeModelSnapshot(collection, { + scopeKey: SCOPE_KEY, capturedAt: '2026-08-25T13:00:00.000Z', scope: { project: '/private/repo' }, + }); + assert.equal(isCompleteStableSnapshot(snapshot), true); + assert.equal(snapshot.models.length, 1); + assert.equal(snapshot.bindings.length, 1); + assert.equal(JSON.stringify(snapshot).includes('/private/repo'), false); + assert.equal(snapshot.sources.every((source) => source.scopeFingerprint === snapshot.scope.fingerprint), true); +}); diff --git a/tests/kit/model-inventory-store.test.mjs b/tests/kit/model-inventory-store.test.mjs index 17efab2..b85610b 100644 --- a/tests/kit/model-inventory-store.test.mjs +++ b/tests/kit/model-inventory-store.test.mjs @@ -5,6 +5,7 @@ import os from 'node:os'; import path from 'node:path'; import { MAX_MODEL_SNAPSHOTS, appendModelSnapshot, baselineFor, latestSnapshot, readModelStore, + readOrCreateModelScopeKey, } from '../../src/lib/model-inventory/store.mjs'; const DAY = 86_400_000; @@ -87,3 +88,13 @@ test('missing or corrupt store degrades to an empty readable store', () => { assert.deepEqual(readModelStore({ file: sb.file }).baselineByScope, {}); fs.rmSync(sb.dir, { recursive: true, force: true }); }); + +test('scope fingerprints use a stable private per-install key', () => { + const sb = sandbox(); + const file = path.join(sb.dir, 'model-scope.key'); + const key = readOrCreateModelScopeKey({ file, randomBytesFn: () => Buffer.alloc(32, 0xab) }); + assert.equal(key, 'ab'.repeat(32)); + assert.equal(readOrCreateModelScopeKey({ file, randomBytesFn: () => Buffer.alloc(32) }), key); + if (process.platform !== 'win32') assert.equal(fs.statSync(file).mode & 0o777, 0o600); + fs.rmSync(sb.dir, { recursive: true, force: true }); +}); From 1b28ded4f6ba1f58ef357696286aa5d052b99e11 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Tue, 25 Aug 2026 07:45:53 -0700 Subject: [PATCH 04/37] feat(models): expose lifecycle CLI and status --- bin/agentic-kit.mjs | 4 +- src/commands/models.mjs | 199 ++++++++++++++++++++++++++++++ src/commands/status.mjs | 14 +++ tests/kit/models-command.test.mjs | 88 +++++++++++++ 4 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 src/commands/models.mjs create mode 100644 tests/kit/models-command.test.mjs diff --git a/bin/agentic-kit.mjs b/bin/agentic-kit.mjs index 737126d..eed88bc 100755 --- a/bin/agentic-kit.mjs +++ b/bin/agentic-kit.mjs @@ -22,6 +22,7 @@ const PORCELAIN = Object.assign(Object.create(null), { dashboard: () => import('../src/commands/x/dashboard.mjs'), admin: () => import('../src/commands/x/admin.mjs'), usage: () => import('../src/commands/usage.mjs'), + models: () => import('../src/commands/models.mjs'), system: () => import('../src/commands/system.mjs'), about: () => import('../src/commands/about.mjs'), run: () => import('../src/commands/run.mjs'), @@ -52,6 +53,7 @@ Usage (ak = alias of agentic-kit): ak dashboard open the local web dashboard (localhost; auto-opens browser) [--port N] [--no-open] ak admin maintainer-only telemetry admin (localhost; GitHub/npm egress) [--port N] [--no-open] ak usage inspect/refresh offline provider analytics [status|refresh openrouter] + ak models inspect/refresh model lifecycle evidence [status|refresh|diff|explain|plan] ak system what this stack occupies on your machine [--deep] [--json] ak about what agentic-kit installs and configures, and why [--category N] ak run execute a host-neutral activity pipeline [template "task"] [--dry-run] @@ -177,7 +179,7 @@ async function main() { // setup and host own complete mutation/reporting flows. Running the generic // nudge after a declined trust preflight could write version-cache state and // violate their "before any changes" boundary. - if (!values.json && !values['dry-run'] && !['sync', 'usage', 'setup', 'host', 'ruflo-mcp'].includes(cmd)) { + if (!values.json && !values['dry-run'] && !['sync', 'usage', 'models', 'setup', 'host', 'ruflo-mcp'].includes(cmd)) { try { const { driftReport } = await import('../src/lib/versions.mjs'); for (const r of await driftReport()) { diff --git a/src/commands/models.mjs b/src/commands/models.mjs new file mode 100644 index 0000000..11103da --- /dev/null +++ b/src/commands/models.mjs @@ -0,0 +1,199 @@ +import { heading, info, ok, warn, dim } from '../lib/output.mjs'; +import { loadKitConfig } from '../lib/config.mjs'; +import { aqeRouterFile } from '../lib/providers.mjs'; +import { readJson } from '../lib/settings.mjs'; +import { + appendModelSnapshot, baselineFor, collectModelSnapshot, createModelReadModel, + diffSnapshots, explainModel, latestSnapshot, modelInventoryPath, planModelChange, + readModelStore, snapshotById, summarizeModelHealth, +} from '../lib/model-inventory/index.mjs'; + +export const options = { + json: { type: 'boolean', default: false }, + host: { type: 'string' }, + all: { type: 'boolean', default: false }, + online: { type: 'boolean', default: false }, + since: { type: 'string' }, + activity: { type: 'string' }, + from: { type: 'string' }, + to: { type: 'string' }, + 'dry-run': { type: 'boolean', default: false }, +}; + +export const help = `ak models — model lifecycle evidence, changes, and swap impact + +Every command except refresh reads the private local snapshot cache and performs +no network requests. Refresh contacts only the named local/configured sources; +--online additionally permits OpenCode to refresh its catalog. + +Usage: + ak models status [--host claude|codex|opencode|ollama] [--json] + ak models refresh [--host HOST|--all] [--online] [--dry-run] + ak models diff [FROM_SNAPSHOT [TO_SNAPSHOT]] [--json] + ak models explain HOST:MODEL [--json] + ak models plan --activity ACTIVITY [--from HOST:MODEL] --to HOST:MODEL [--json] + +The plan command is read-only. It may print a copyable canonical routing command, +but never changes routing, AQE, Ruflo, or provider configuration.`; + +const ALL_OWNERS = Object.freeze(['claude', 'codex', 'opencode', 'ollama']); + +function selectedOwners(flags, cfg) { + if (flags.host) { + const owners = [...new Set(String(flags.host).split(',').map((value) => value.trim()).filter(Boolean))]; + for (const owner of owners) if (!ALL_OWNERS.includes(owner)) throw new TypeError(`unsupported model host: ${owner}`); + return owners; + } + if (flags.all) return [...ALL_OWNERS]; + const enabled = Object.entries(cfg?.integrations?.hosts ?? {}) + .filter(([, value]) => value === true).map(([owner]) => owner).filter((owner) => ALL_OWNERS.includes(owner)); + return enabled.length ? enabled : ['claude']; +} + +function printJson(value) { console.log(JSON.stringify(value, null, 2)); } + +function visibleSnapshot(snapshot, host) { + if (!snapshot || !host) return snapshot; + return { ...snapshot, models: snapshot.models.filter((model) => model.key.host === host) }; +} + +function selectedPair(store, positionals, flags) { + const fromId = flags.from ?? positionals[1]; + const toId = flags.to ?? positionals[2]; + const after = toId ? snapshotById(store, toId) : latestSnapshot(store); + const before = fromId ? snapshotById(store, fromId) + : after ? baselineFor(store, after.scope.fingerprint) : null; + return { before, after, fromId, toId }; +} + +function noSnapshot(flags, cacheFile) { + const result = { status: 'empty', cacheFile, snapshot: null, hint: 'ak models refresh' }; + if (flags.json) printJson(result); + else { + heading('ak models — offline lifecycle inventory'); + info('No local model snapshot yet.'); + info('Refresh explicitly: ak models refresh'); + } + return 0; +} + +/** @param {{flags: Record, positionals: string[], deps?: Record}} input */ +export async function run({ flags, positionals, deps = {} }) { + const action = positionals[0] ?? 'status'; + const cacheFile = deps.cacheFile ?? modelInventoryPath(); + const readStore = deps.readStore ?? readModelStore; + const append = deps.append ?? appendModelSnapshot; + const collect = deps.collect ?? collectModelSnapshot; + const loadConfig = deps.loadConfig ?? loadKitConfig; + const cfg = loadConfig(); + + if (action === 'refresh') { + const owners = selectedOwners(flags, cfg); + if (flags['dry-run']) { + const result = { dryRun: true, action, owners, online: flags.online, network: false, writes: false, cacheFile }; + if (flags.json) printJson(result); + else { + heading('ak models — refresh plan (dry-run)'); + info(`Would inspect: ${owners.join(', ')}.`); + info(flags.online ? 'OpenCode catalog refresh would be permitted.' : 'No online catalog refresh would be permitted.'); + info(dim('No source was contacted and no file was written.')); + } + return 0; + } + const aqeConfig = (deps.readJson ?? readJson)((deps.aqeFile ?? aqeRouterFile)(process.cwd())); + const snapshot = await collect({ + config: cfg, aqeConfig, rufloConfig: cfg, scope: { project: process.cwd() }, + discoveryOptions: { owners, online: flags.online, cwd: process.cwd() }, + }); + const store = append(snapshot, { file: cacheFile }); + const result = { status: 'refreshed', cacheFile, contacts: owners, online: flags.online, + snapshot: createModelReadModel(snapshot), retainedSnapshots: store.snapshots.length }; + if (flags.json) printJson(result); + else { + const health = summarizeModelHealth(snapshot); + ok(`Model inventory refreshed: ${snapshot.models.length} model(s) · ${snapshot.sources.length} source(s)`); + info(health.message); + info(dim(`private cache: ${cacheFile}`)); + } + return 0; + } + + const store = readStore({ file: cacheFile }); + const latest = latestSnapshot(store); + if (!latest) return noSnapshot(flags, cacheFile); + + if (action === 'status') { + const snapshot = visibleSnapshot(latest, flags.host); + const since = flags.since ? Date.parse(flags.since) : null; + const history = store.snapshots.filter((entry) => entry.scope.fingerprint === latest.scope.fingerprint + && (!Number.isFinite(since) || Date.parse(entry.capturedAt) >= since)); + const result = { status: 'cached', cacheFile, health: summarizeModelHealth(snapshot), + inventory: createModelReadModel(snapshot), history: history.map(({ snapshotId, capturedAt }) => ({ snapshotId, capturedAt })) }; + if (flags.json) printJson(result); + else { + heading('ak models — offline lifecycle inventory'); + const health = result.health; + (health.level === 'ok' ? ok : warn)(health.message); + for (const source of snapshot.sources) info(`${source.id}: ${source.status} · ${source.capturedAt}`); + info(dim(`snapshot ${snapshot.snapshotId} · ${history.length} retained same-scope capture(s)`)); + } + return 0; + } + + if (action === 'diff') { + const { before, after, fromId, toId } = selectedPair(store, positionals, flags); + if (!before || !after) { + const missing = !before ? fromId ?? 'same-scope baseline' : toId ?? 'latest snapshot'; + if (flags.json) printJson({ comparable: false, reason: 'snapshot-not-found', missing }); + else warn(`Cannot diff: ${missing} not found.`); + return 1; + } + const result = diffSnapshots(before, after); + if (flags.json) printJson(result); + else { + heading(`ak models diff — ${before.snapshotId} → ${after.snapshotId}`); + if (!result.comparable) warn(result.diagnostics.join('; ')); + else if (!result.changes.length) ok('No model lifecycle changes.'); + else for (const change of result.changes) info(`${change.kind}: ${change.subject}${change.provisional ? ' (provisional)' : ''}`); + for (const message of result.diagnostics) info(dim(message)); + } + return result.comparable ? 0 : 1; + } + + if (action === 'explain') { + const selector = positionals[1] ?? flags.to; + if (!selector) { warn('usage: ak models explain HOST:MODEL'); return 2; } + const result = explainModel(latest, selector); + if (flags.json) printJson(result); + else if (!result.found) warn(`Model not found: ${selector}`); + else { + heading(`ak models explain — ${selector}`); + for (const match of result.matches) { + info(`${match.key.host}${match.key.provider ? `/${match.key.provider}` : ''}: ${match.key.modelId}`); + for (const [name, dimension] of Object.entries(match.dimensions)) info(` ${name}: ${dimension.value ?? 'unknown'}`); + info(` lifecycle: ${match.lifecycle.state}${match.lifecycle.replacement ? ` → ${match.lifecycle.replacement}` : ''}`); + } + } + return result.found ? 0 : 1; + } + + if (action === 'plan') { + const activity = flags.activity; + const to = flags.to ?? positionals[1]; + if (!activity || !to) { warn('usage: ak models plan --activity ACTIVITY [--from HOST:MODEL] --to HOST:MODEL'); return 2; } + const result = planModelChange(latest, { activity, from: flags.from, to }); + if (flags.json) printJson(result); + else { + heading(`ak models plan — ${activity}`); + if (!result.plannable) warn(`No mechanical plan: ${result.reason ?? result.compatibility?.blockers?.join('; ')}`); + else { + ok('Mechanical compatibility is supported by current evidence. Quality equivalence remains unknown.'); + info(`Copy to apply explicitly: ${result.action.command}`); + } + } + return result.plannable ? 0 : 1; + } + + warn('usage: ak models status|refresh|diff|explain|plan'); + return 2; +} diff --git a/src/commands/status.mjs b/src/commands/status.mjs index ba2c88c..4e7348f 100644 --- a/src/commands/status.mjs +++ b/src/commands/status.mjs @@ -33,6 +33,7 @@ import { statuslineDrift } from '../lib/codex-statusline.mjs'; import { inspectCodexPlugins } from '../lib/codex-plugins.mjs'; import { projectMemoryStatus } from '../lib/project-memory.mjs'; import { removedAgentGaps, upstreamFixAvailable } from '../lib/scaffold.mjs'; +import { latestSnapshot, readModelStore, summarizeModelHealth } from '../lib/model-inventory/index.mjs'; export const options = { json: { type: 'boolean', default: false }, @@ -247,6 +248,19 @@ export async function collect({ pkgRoot, cwd = process.cwd() }) { const cfg = loadKitConfig(); const integrationFacts = await collectIntegrationFacts({ cwd, cfg }); + // Cache-only model lifecycle summary. Discovery and network access belong + // exclusively to `ak models refresh`. + try { + const snapshot = latestSnapshot(readModelStore()); + if (!snapshot) rows.push(row('models', 'warn', 'no local model inventory yet', 'run `ak models refresh`')); + else { + const health = summarizeModelHealth(snapshot); + rows.push(row('models', health.level, health.message, health.fix)); + } + } catch (error) { + rows.push(row('models', 'warn', `model inventory unavailable: ${error.message}`, 'run `ak models refresh`')); + } + // versions try { for (const r of await driftReport()) { diff --git a/tests/kit/models-command.test.mjs b/tests/kit/models-command.test.mjs new file mode 100644 index 0000000..b22bc8c --- /dev/null +++ b/tests/kit/models-command.test.mjs @@ -0,0 +1,88 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { run } from '../../src/commands/models.mjs'; + +const capturedAt = '2026-08-25T13:00:00.000Z'; +const snapshot = { + schemaVersion: 1, snapshotId: 'models:test', capturedAt, + scope: { fingerprint: 'scope:test', hosts: ['codex'] }, + sources: [{ id: 'codex-cache', status: 'complete', complete: true, capturedAt, scopeFingerprint: 'scope:test' }], + models: [{ + key: { host: 'codex', provider: 'openai', modelId: 'gpt-x', scopeId: 'scope:host' }, + displayName: 'gpt-x', aliases: [], visibility: 'visible', variant: {}, + lifecycle: { state: 'active', replacement: null, evidenceRefs: ['ev'] }, capabilities: {}, + dimensions: Object.fromEntries(['configured', 'effective', 'observed', 'discoverable', 'entitled', 'policyAllowed', 'routable', 'recommended'] + .map((name) => [name, { value: ['configured', 'effective', 'discoverable', 'entitled', 'policyAllowed', 'routable'].includes(name) ? true : null, evidenceRefs: ['ev'] }])), + evidence: [{ id: 'ev', field: 'catalog', source: 'codex-cache', class: 'catalog', capturedAt, + freshness: 'fresh', completeness: 'complete', scopeFingerprint: 'scope:host', refs: [] }], + }], + bindings: [], changes: [], opportunities: [], diagnostics: [], +}; + +const cfg = { integrations: { hosts: { claude: true, codex: true, opencode: false } } }; +const store = { snapshots: [snapshot], baselineByScope: { 'scope:test': snapshot.snapshotId } }; + +async function capture(fn) { + const lines = []; + const original = console.log; + console.log = (...args) => lines.push(args.join(' ')); + try { return { code: await fn(), output: lines.join('\n') }; } finally { console.log = original; } +} + +test('models status is a cache-only read', async () => { + let collected = 0; + const result = await capture(() => run({ + flags: { json: true }, positionals: ['status'], + deps: { loadConfig: () => cfg, readStore: () => store, collect: async () => { collected++; } }, + })); + assert.equal(result.code, 0); + assert.equal(collected, 0); + assert.equal(JSON.parse(result.output).inventory.snapshotId, 'models:test'); +}); + +test('models refresh dry-run contacts nothing and writes nothing', async () => { + let collected = 0; + let appended = 0; + const result = await capture(() => run({ + flags: { json: true, 'dry-run': true, all: true }, positionals: ['refresh'], + deps: { + loadConfig: () => cfg, collect: async () => { collected++; }, append: () => { appended++; }, + }, + })); + assert.equal(result.code, 0); + assert.equal(collected, 0); + assert.equal(appended, 0); + assert.deepEqual(JSON.parse(result.output).owners, ['claude', 'codex', 'opencode', 'ollama']); +}); + +test('models refresh is the sole collection and snapshot write boundary', async () => { + let options; + let appended; + const result = await capture(() => run({ + flags: { json: true, online: true, host: 'codex' }, positionals: ['refresh'], + deps: { + loadConfig: () => cfg, readJson: () => null, aqeFile: () => '/fixture/aqe.json', + collect: async (value) => { options = value; return snapshot; }, + append: (value) => { appended = value; return store; }, + }, + })); + assert.equal(result.code, 0); + assert.deepEqual(options.discoveryOptions.owners, ['codex']); + assert.equal(options.discoveryOptions.online, true); + assert.equal(appended.snapshotId, 'models:test'); +}); + +test('models explain and plan remain read-only', async () => { + const explain = await capture(() => run({ + flags: { json: true }, positionals: ['explain', 'codex:gpt-x'], + deps: { loadConfig: () => cfg, readStore: () => store }, + })); + assert.equal(JSON.parse(explain.output).found, true); + const plan = await capture(() => run({ + flags: { json: true, activity: 'testing', to: 'codex:gpt-x' }, positionals: ['plan'], + deps: { loadConfig: () => cfg, readStore: () => store }, + })); + const value = JSON.parse(plan.output); + assert.equal(value.readOnly, true); + assert.equal(value.action.executed, false); +}); From 9b2b1a752930a954e2f934c9517ac68beee8a616 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Tue, 25 Aug 2026 07:50:19 -0700 Subject: [PATCH 05/37] feat(dashboard): add model lifecycle evidence view --- src/lib/dashboard-server.mjs | 33 +++++++++++++++-- src/lib/dashboard/client.mjs | 69 +++++++++++++++++++++++++++++++++--- src/lib/dashboard/page.mjs | 46 ++++++++++++++++++------ src/lib/dashboard/styles.mjs | 31 ++++++++++++++++ tests/dashboard.test.cjs | 43 +++++++++++++++++++--- 5 files changed, 202 insertions(+), 20 deletions(-) diff --git a/src/lib/dashboard-server.mjs b/src/lib/dashboard-server.mjs index be49050..4eb82cb 100644 --- a/src/lib/dashboard-server.mjs +++ b/src/lib/dashboard-server.mjs @@ -672,7 +672,7 @@ function lazyLive(liveOptions = {}) { * intelClientBuffer?: number, intelMaxClients?: number, * discoverProjects?: () => Array<{ path: string, label: string, source?: string }>, * machineWideIntel?: (projects: Array) => any, - * system?: any, systemOptions?: any }} [opts] + * models?: any, system?: any, systemOptions?: any }} [opts] * @returns {Promise<{ url: string, urlWithToken: string, port: number, token: string, close: () => Promise }>} */ export function startDashboard({ @@ -681,10 +681,30 @@ export function startDashboard({ liveOptions = {}, liveIdleMs = 30_000, transcripts, transcriptOptions = {}, transcriptClientBuffer = 64, transcriptMaxClients = 16, intelWatch, intelClientBuffer = 256, intelMaxClients = 32, - discoverProjects, machineWideIntel, system, systemOptions = {}, + discoverProjects, machineWideIntel, models, system, systemOptions = {}, } = {}) { const provide = fetchStatus || shellOutStatus(cwd); const usageApi = usage || lazyUsage(); + // Cache-only and lazy: model discovery is exclusively owned by + // `ak models refresh`; opening the dashboard never contacts a host/catalog. + const provideModels = typeof models === 'function' ? models : models ? async () => models : async () => { + const [{ readModelStore, latestSnapshot, baselineFor }, { diffSnapshots }, { createModelReadModel }] = await Promise.all([ + import('./model-inventory/store.mjs'), import('./model-inventory/diff.mjs'), + import('./model-inventory/read-model.mjs'), + ]); + const store = readModelStore(); + const snapshot = latestSnapshot(store); + if (!snapshot) return { status: 'empty', snapshot: null, history: [], hint: 'ak models refresh' }; + const baseline = baselineFor(store, snapshot.scope.fingerprint); + const diff = baseline ? diffSnapshots(baseline, snapshot) : { changes: [], diagnostics: [] }; + return { + status: 'cached', snapshot: createModelReadModel(snapshot, { changes: diff }), + history: store.snapshots.filter((entry) => entry.scope.fingerprint === snapshot.scope.fingerprint) + .map(({ snapshotId, capturedAt }) => ({ snapshotId, capturedAt })), + comparison: { baseline: baseline?.snapshotId ?? null, latest: snapshot.snapshotId, + comparable: diff.comparable ?? false, diagnostics: diff.diagnostics ?? [] }, + }; + }; // Injectable like `usage`: tests must never spawn a real codex or read the // real ~/.config through this route. Lazy for the same reason lazyUsage is. // enabledHosts drives quota.mjs's F-10 labeling (any OTHER enabled host with @@ -1370,6 +1390,15 @@ export function startDashboard({ // ── Usage (ADR-0009). Lazy: nothing below runs until the tab is opened. ── + if (url === '/api/models') { + try { + sendJson(res, 200, await provideModels()); + } catch (e) { + sendJson(res, 500, { error: String(e && e.message || e) }); + } + return; + } + // Rollups only. Dropping the top-level sessions[] is NOT sufficient on its // own: projectTree[].rows holds the SAME object references, so every session // still shipped and the "order of magnitude" saving was really about 20%. diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index 2d82ce6..65514ef 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -102,7 +102,7 @@ export const JS = ` var OVERVIEW_VIEWS=["summary","hosts","providers","runtime","intel"]; var SYSTEM_VIEWS=["summary","advisory","sessions","storage","runtime","catalog","projects"]; var ABOUT_SECTIONS=["hosts","engine","quality","kit","configured"]; - var VIEWS=["score","limits","findings","sessions","transcript"]; + var VIEWS=["score","limits","findings","sessions","models","transcript"]; var CAT=${CAT_JS}; ${catOf.toString()} @@ -776,6 +776,7 @@ export const JS = ` renderHistory(buildHistoryView(data)); renderRouting(data.routing); renderModels(data.routing); + renderModelSummary(data.rows||[]); positionThumb(); // badges can change segment widths } @@ -922,7 +923,10 @@ export const JS = ` var btn=document.getElementById("poll-now"); if(btn)btn.classList.add("spin"); var jobs=[pollStatus()]; - if(activeTab==="usage")jobs.push(loadUsage(true)); + if(activeTab==="usage"){ + jobs.push(loadUsage(true)); + if(usageView==="models")jobs.push(loadModelLifecycle(true)); + } // The Runtime view is a live census — processes, CPU, RSS, daemon ages — // and it used to load ONCE when the System tab was first opened, so its // "live" figures could sit unchanged for an entire session while the @@ -999,6 +1003,7 @@ export const JS = ` // ══ Usage tab ══════════════════════════════════════════════════════════════ var USAGE=null, usageLoaded=false, usageBusy=false, TRANSCRIPT=null; + var MODELS=null,modelsBusy=false; function fmtUsd(n){ n=Number(n)||0; @@ -1112,13 +1117,22 @@ export const JS = ` .then(function(d){TRANSCRIPT=d&&!d.error?{id:id,meta:d.meta,turns:d.turns||[]}:{id:id,error:(d&&d.error)||"unreadable"};}); } + function loadModelLifecycle(force){ + if(modelsBusy||(!force&&MODELS))return Promise.resolve(); + modelsBusy=true; + return fetch("/api/models",{cache:"no-store",headers:authHeaders()}) + .then(function(r){return r.json();}).then(function(d){MODELS=d;}) + .catch(function(){MODELS={error:"model inventory unavailable"};}) + .then(function(){modelsBusy=false;renderModelLifecycle();}); + } + function setUsageView(v,session){ usageView=v; if(session!==undefined)usageSession=session; - var headings={score:["Usage scorecard","Token consumption, API-equivalent cost, efficiency, and trends."],limits:["Provider limits","Current provider windows, reset timing, and available capacity."],findings:["Usage findings","Actionable anomalies, efficiency opportunities, and evidence-backed recommendations."],sessions:["Session usage","Browse retained sessions by project, category, duration, tokens, and cost."],transcript:["Transcript detail","Inspect the selected session's locally retained, server-masked evidence."]},heading=headings[v]||headings.score; + var headings={score:["Usage scorecard","Token consumption, API-equivalent cost, efficiency, and trends."],limits:["Provider limits","Current provider windows, reset timing, and available capacity."],findings:["Usage findings","Actionable anomalies, efficiency opportunities, and evidence-backed recommendations."],sessions:["Session usage","Browse retained sessions by project, category, duration, tokens, and cost."],models:["Model lifecycle","Host-scoped inventory, change history, consumers, and evidence-backed swap impact."],transcript:["Transcript detail","Inspect the selected session's locally retained, server-masked evidence."]},heading=headings[v]||headings.score; document.getElementById("usage-view-title").textContent=heading[0];document.getElementById("usage-view-description").textContent=heading[1]; var btns=document.querySelectorAll("#usage-seg [data-view]"); - for(var i=0;ithis session has no readable turns.'; } + function renderModelSummary(rows){ + var row=(rows||[]).find(function(value){return value&&value.subsystem==="models";}); + var copy=document.getElementById("mli-summary-copy"),state=document.getElementById("mli-summary-state"); + if(!copy||!state)return; + var level=row&&row.level||"warn"; + copy.textContent=row&&row.message||"No cached inventory yet"; + state.setAttribute("data-level",level); + state.innerHTML=''+esc(level==="ok"?"current":level==="fail"?"attention":"review"); + } + + function mliState(model,name){ + var value=model&&model.dimensions&&model.dimensions[name]&&model.dimensions[name].value; + var state=value===true?"yes":value===false?"no":"unknown"; + return ''+(state==="yes"?"yes":state==="no"?"no":"unknown")+""; + } + + function renderModelLifecycle(){ + if(!MODELS)return; + var empty=MODELS.error||MODELS.status==="empty"||!MODELS.snapshot; + var snap=MODELS.snapshot||{},models=snap.models||[],attention=snap.attention||[],bindings=snap.bindings||[]; + var badge=document.getElementById("mli-attention-n"); + if(badge){badge.hidden=!attention.length;badge.textContent=attention.length?String(attention.length):"";} + document.getElementById("mli-asof").textContent=empty?"not captured":("captured "+String(snap.capturedAt||"").replace("T"," ").replace(".000Z","Z")); + document.getElementById("mli-attention").innerHTML=empty + ?'
'+esc(MODELS.error||"No model inventory yet. Run ak models refresh explicitly.")+"
" + :attention.map(function(item){return '
'+esc(item.kind)+" · "+esc(item.reason)+"
";}).join(""); + document.getElementById("mli-models").innerHTML=models.map(function(model){ + var key=model.key||{},life=model.lifecycle||{}; + return ''+esc(key.modelId||"unknown")+''+esc(key.host||"unknown")+(key.provider?" / "+esc(key.provider):"")+"" + +""+mliState(model,"configured")+""+mliState(model,"effective")+""+mliState(model,"observed") + +""+mliState(model,"discoverable")+""+mliState(model,"entitled")+""+mliState(model,"policyAllowed") + +""+mliState(model,"routable")+''+esc(life.state||"unknown")+(life.replacement?" → "+esc(life.replacement):"")+""; + }).join("")||'
No model records in this snapshot.
'; + var changes=snap.changes||[]; + document.getElementById("mli-history-note").textContent=(MODELS.history||[]).length+" retained snapshot"+((MODELS.history||[]).length===1?"":"s"); + document.getElementById("mli-history").innerHTML='
'+(changes.map(function(change){return '
'+esc(change.kind)+'
'+esc(change.subject)+""+esc(change.provisional?"provisional":"established")+"
";}).join("")||'
No same-scope lifecycle changes.
')+"
"; + document.getElementById("mli-consumers").innerHTML='
'+(bindings.map(function(binding){return '
'+esc(binding.consumer)+'
'+esc(binding.activity||binding.host||binding.provider||"consumer")+''+esc(binding.consumerState)+" · "+esc(binding.configured||"model not pinned")+"
";}).join("")||'
No configured model consumers.
')+"
"; + document.getElementById("mli-impact").innerHTML=bindings.length + ?'
→'+bindings.length+' consumer'+(bindings.length===1?"":"s")+' may be affected by a concrete model swap. Run ak models plan --activity ACTIVITY --to HOST:MODEL for evidence-backed compatibility and a copyable action.
' + :'
No bound consumers to assess. A plan will remain read-only and report the missing binding.
'; + document.getElementById("mli-sources").innerHTML=(snap.sources||[]).map(function(source){return ''+esc(source.id)+" · "+esc(source.status)+"";}).join("")||'
No source evidence.
'; + } + function renderUsage(){ if(!USAGE)return; if(USAGE.error){ @@ -1748,6 +1807,8 @@ export const JS = ` if(b)setUsageView(b.getAttribute("data-view")); }); if(seg)seg.addEventListener("keydown",function(e){if(!/^(ArrowLeft|ArrowRight|Home|End)$/.test(e.key))return;var i=VIEWS.indexOf(usageView);i=e.key==="Home"?0:e.key==="End"?VIEWS.length-1:(i+(e.key==="ArrowRight"?1:VIEWS.length-1))%VIEWS.length;setUsageView(VIEWS[i]);var b=seg.querySelector('[data-view="'+VIEWS[i]+'"]');if(b)b.focus();e.preventDefault();}); + var summary=document.getElementById("mli-summary"); + if(summary)summary.addEventListener("click",function(e){e.preventDefault();setTab("usage");setUsageView("models");}); var chips=document.getElementById("usage-days"); if(chips)chips.addEventListener("click",function(e){ var b=e.target.closest?e.target.closest("[data-days]"):null; diff --git a/src/lib/dashboard/page.mjs b/src/lib/dashboard/page.mjs index 67299a0..1dd0e9a 100644 --- a/src/lib/dashboard/page.mjs +++ b/src/lib/dashboard/page.mjs @@ -156,11 +156,12 @@ export function renderPage({ name, version }) {