From c4bbef8b1f818397a420f7f5f3fe4dc1d0c95412 Mon Sep 17 00:00:00 2001 From: Chris Phillipson Date: Mon, 24 Aug 2026 13:28:26 -0700 Subject: [PATCH] feat: add cross-host telemetry coverage --- docs/DASHBOARD.md | 7 + docs/TRANSCRIPTS.md | 71 ++++++--- docs/USAGE-SCORECARD-METRICS.md | 44 ++++-- ...ge-scorecard-local-transcript-analytics.md | 32 +++- ...sed-operations-and-explicit-degradation.md | 11 ++ src/lib/dashboard/client.mjs | 49 +++++- src/lib/dashboard/page.mjs | 4 + src/lib/dashboard/styles.mjs | 13 ++ src/lib/usage-index.mjs | 73 ++++++++- src/lib/usage-telemetry.mjs | 140 ++++++++++++++++++ tests/dashboard.test.cjs | 14 +- tests/kit/dashboard-usage-telemetry.test.mjs | 25 ++++ tests/kit/usage-index-opencode.test.mjs | 6 +- tests/kit/usage-index-v6.test.mjs | 9 +- tests/kit/usage-index.test.mjs | 12 +- tests/kit/usage-telemetry.test.mjs | 61 ++++++++ 16 files changed, 525 insertions(+), 46 deletions(-) create mode 100644 src/lib/usage-telemetry.mjs create mode 100644 tests/kit/dashboard-usage-telemetry.test.mjs create mode 100644 tests/kit/usage-telemetry.test.mjs diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index fd48493..644965e 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -179,6 +179,13 @@ control, and the original masked value never reaches the browser. See [ADR-0009](adr/0009-usage-scorecard-local-transcript-analytics.md) for the full evidence and pricing contract. +The Scorecard view also shows a host-neutral **telemetry coverage** panel for Claude, Codex +transcript evidence, and OpenCode. It reports parsed units and observed prompt/response totals, plus +capability states (`supported`, `unsupported`, or `unavailable`). A readable source with no observed +activity is a measured zero; an absent, degraded, or old API response is disclosed as unavailable or +not reported rather than rendered as zero. The Codex transcript card does not merge the separate +`codexLedger` corrective source into its coverage counts. + ## Observability Observability separates navigation scope from playback state: diff --git a/docs/TRANSCRIPTS.md b/docs/TRANSCRIPTS.md index f98d6a9..7db66f8 100644 --- a/docs/TRANSCRIPTS.md +++ b/docs/TRANSCRIPTS.md @@ -36,10 +36,10 @@ rewritten; rule 3 of the module header, `usage-index.mjs:22-29`): | Host | Store | Discovered by | |---|---|---| -| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:876-886`) — exactly one level of project directories | +| Claude Code | `~/.claude/projects//.jsonl` | `listClaude` (`usage-index.mjs:916-926`) — exactly one level of project directories | | Codex CLI | `~/.codex/sessions///
/rollout--.jsonl` | `listCodex` (`usage-index.mjs:891-904`) — the `yyyy/mm/dd` tree walk | -Roots come from `defaultRoots()` (`usage-index.mjs:750-754`) and are injectable +Roots come from `defaultRoots()` (`usage-index.mjs:908-912`) and are injectable for tests. A malformed line is skipped, never fatal (`jsonLines`, `usage-index.mjs:381-387` — one corrupt line must not cost a whole file). @@ -90,6 +90,39 @@ human prompts, model responses, or existing tool metrics. Codex tool calls and tool outputs therefore still travel in event types the parser does not surface as turns — a fidelity gap, not an attribution bug. +### 1.3 Cross-host telemetry capability contract + +The public host APIs are richer than the historical readers in this module, and +they are not interchangeable transcript schemas. This distinction is verified +against the public surfaces available on **2026-08-24**: + +| Host | Public evidence | Historical adapter contract in this repository | +|---|---|---| +| Claude Code | Hooks expose `transcript_path`, `tool_name`, tool input/results, and `tool_use_id`; its monitoring surface also documents `claude_code.tool` spans and tool-result events ([hooks reference](https://code.claude.com/docs/en/hooks), [monitoring](https://code.claude.com/docs/en/monitoring-usage)) | `prompts`, `responses`, and normalized `toolCalls` are supported; command/file/MCP/collaboration subcategories remain unclaimed until their cross-host semantics are specified | +| Codex | The public `codex app-server` protocol documents typed `userMessage`, `agentMessage`, `commandExecution`, `fileChange`, `mcpToolCall`, and `collabToolCall` items ([app-server protocol](https://github.com/openai/codex/blob/main/codex-rs/app-server/README.md)) | Only prompt/response items are normalized today. The rollout parser records unknown item kinds diagnostically; Codex activity categories are `unsupported` in this historical adapter, not measured zero | +| OpenCode | The public SDK returns session messages with `parts`, and its public message model includes tool invocation parts ([SDK](https://github.com/anomalyco/opencode/blob/dev/packages/web/src/content/docs/sdk.mdx), [message model](https://github.com/anomalyco/opencode/blob/dev/packages/opencode/src/session/message.ts)) | `prompts`, `responses`, and persisted `toolCalls` are supported; command/file/MCP/collaboration subcategories remain unclaimed | + +`sourceHealth..diagnostics.common` is the additive, host-neutral +coverage envelope. `unitsSeen` counts discovered session candidates in the +requested window; `unitsParsed` counts candidates parsed successfully; +`unitsWithUsage`, `unitsWithPrompts`, and `unitsWithResponses` count parsed +units carrying each kind of evidence; `prompts` and `responses` are observed +totals. `sourceHealth..capabilities` uses three states: + +`unknownKinds` is capped at 32 distinct wire kinds; additional occurrences are +retained in `unknownKindOverflow` so future schema growth cannot expand the +diagnostics payload without limit. + +* `supported` means this historical adapter can produce the category; +* `unsupported` means the category is intentionally not claimed by this adapter; +* `unavailable` means the adapter supports the category in principle, but the + source is absent or degraded for this scan. + +Therefore a supported source with zero observations is different from an +absent/degraded source, and neither is silently converted into a host-specific +metric. Existing status/reason fields and Codex diagnostic keys remain in place +for compatibility; the common envelope and capability matrix are additive. + --- ## 2. Two read paths: scan vs reader @@ -154,20 +187,20 @@ story is [Appendix A](#appendix-a--fix-history).) Two deliberate subtleties: -- **`kind` is broader than `prompt` on the image-only edge.** An image-only +* **`kind` is broader than `prompt` on the image-only edge.** An image-only paste has no text block, so `isHumanPrompt` returns `false` (it is not *counted* as a text prompt) — but it **is** the person acting, and `userTurnKind` returns `'prompt'` for it. "Not countable as a text prompt" and "not the human" are different claims. -- **Harness-output envelopes are excluded from the prompt *count* too.** +* **Harness-output envelopes are excluded from the prompt *count* too.** `isHumanPrompt` shares `HARNESS_OUTPUT_RE`, so a session's `prompts` figure never counts stdout dumps or task notifications as things the person said (`SCHEMA_VERSION` 5, `usage-index.mjs:47-51`; the correction this shipped with is in [Appendix A](#appendix-a--fix-history)). -- **`tool-result` outranks `context`**: a `tool_result` block on an `isMeta` +* **`tool-result` outranks `context`**: a `tool_result` block on an `isMeta` entry is still tool feedback. -Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:710`) +Codex user turns are `kind: 'prompt'` by construction (`usage-index.mjs:767-775`) — rollouts only record real prompts as `user_message` events (§1.2). Coverage: `tests/kit/usage-index.test.mjs` — "user-role turns carry a kind" @@ -185,8 +218,8 @@ transcript content leaves the module, and every step is a gate: 1. **Id grammar before any filesystem access** — `VALID_ID` (`/^[A-Za-z0-9._-]{1,128}$/`, `usage-index.mjs:95`) rejects traversal - shapes with `ERR_INVALID_SESSION_ID` (`usage-index.mjs:1508-1512`). -2. **Locate by id** across both roots (`locate`, `usage-index.mjs:1518`), + shapes with `ERR_INVALID_SESSION_ID` (`usage-index.mjs:1568-1572`). +2. **Locate by id** across both roots (`locate`, `usage-index.mjs:1578`), consulting the scan cache when present but never requiring it — `readSession` works with no prior `buildIndex`. 3. **Realpath containment** (`usage-index.mjs:1587-1601`) — the resolved file @@ -213,11 +246,11 @@ Every turn body is passed through `maskSecrets` (`usage-index.mjs:208` — the 23 secret shapes) **server-side, before serialization**, then length-capped at `MAX_TURN_CHARS` (40,000, `usage-index.mjs:89`) with the marker appended -(`usage-index.mjs:1661-1670`). Two invariants: +(`usage-index.mjs:1721-1729`). Two invariants: -- **Presence is the signal.** `truncated`/`originalChars` are emitted only +* **Presence is the signal.** `truncated`/`originalChars` are emitted only when the slice fired, so a complete turn cannot be misread as abridged. -- **`originalChars` is measured after masking** — it describes loss due to +* **`originalChars` is measured after masking** — it describes loss due to truncation alone, never a raw-file length. The two kinds of withholding keep distinct vocabulary end-to-end: masking @@ -352,25 +385,25 @@ and [#59](https://github.com/pacphi/agentic-kit/issues/59). The main body describes only current behavior; this appendix records what was wrong before, for the curious. -- **User-role turns rendered as "you" (fixed 2026-07-26).** Before `kind` +* **User-role turns rendered as "you" (fixed 2026-07-26).** Before `kind` existed, the Transcript view labelled every user-role turn as the person. On the reference session that misattributed 276 tool results and 6 harness context injections — ~93% of its "you" turns (§3.1's measured split). The turn-`kind` machinery in §3 is the fix. -- **Prompt counts included harness output (SCHEMA_VERSION 5).** +* **Prompt counts included harness output (SCHEMA_VERSION 5).** `isHumanPrompt` once counted `harness-output` envelopes as human prompts — 32 claimed vs 20 real on the reference session. Cached session records carried the inflated counts, hence the wholesale `SCHEMA_VERSION` 5 cache invalidation (`usage-index.mjs:48-51`). -- **Session expander fields shipped but unrendered.** The per-session fields +* **Session expander fields shipped but unrendered.** The per-session fields §6.1's expander now renders (classification `basis` + confidence, the token split, flags) once travelled on the wire and rendered nowhere. -- **Transcript header once showed a hardcoded `$0.00`.** `readSession`'s +* **Transcript header once showed a hardcoded `$0.00`.** `readSession`'s assembled `meta` left `cost` undefined, and `fmtUsd(undefined)` renders the truthy string `"$0.00"` — a fixed-looking zero on a panel whose whole subject is cost. `meta.cost` is now priced via `sessionCost()` from the - same per-model usage rows `aggregate()` uses (`usage-index.mjs:1651`). -- **Aggregate-side incidents** (the v4/v5 cache bumps, the Codex parsing + same per-model usage rows `aggregate()` uses (`usage-index.mjs:1691`). +* **Aggregate-side incidents** (the v4/v5 cache bumps, the Codex parsing defects) are recorded in `USAGE-SCORECARD-METRICS.md` Appendix A. --- @@ -386,10 +419,10 @@ kind-attribution behavior is pinned by unit tests at both layers — parser **Against real data** (this machine's real stores, 2026-07-26): -- A real Claude session (this feature's own working session, 884 turns): +* A real Claude session (this feature's own working session, 884 turns): `{ prompt: 20, context: 6, 'tool-result': 276, assistant: 588 }`, zero user turns missing `kind`. -- A real Codex rollout (8 user turns): every one `kind: 'prompt'`, as §1.2 +* A real Codex rollout (8 user turns): every one `kind: 'prompt'`, as §1.2 predicts. --- diff --git a/docs/USAGE-SCORECARD-METRICS.md b/docs/USAGE-SCORECARD-METRICS.md index 838f0dc..4ddc021 100644 --- a/docs/USAGE-SCORECARD-METRICS.md +++ b/docs/USAGE-SCORECARD-METRICS.md @@ -101,6 +101,26 @@ token-bearing files but zero normalized responses is degraded as responses is degraded as `parse-yield-partial`, rather than reported as healthy empty or complete usage. +The additive `sourceHealth..diagnostics.common` envelope makes coverage +comparable without pretending the hosts have the same wire format: it reports +discovered and parsed units, units with usage/prompts/responses, observed prompt +and response totals, warnings, and unknown kinds. The companion +`sourceHealth..capabilities` matrix distinguishes `supported`, +`unsupported`, and `unavailable`. This is intentionally the historical adapter +contract, not a claim that the hosts lack richer public APIs: Claude documents +tool hooks and OpenTelemetry tool spans ([hooks](https://code.claude.com/docs/en/hooks), +[monitoring](https://code.claude.com/docs/en/monitoring-usage)); Codex documents +typed command, file-change, MCP, and collaboration items in app-server +([protocol](https://github.com/openai/codex/blob/main/codex-rs/app-server/README.md)); +and OpenCode documents session `parts` and tool invocation parts ([SDK](https://github.com/anomalyco/opencode/blob/dev/packages/web/src/content/docs/sdk.mdx), +[message model](https://github.com/anomalyco/opencode/blob/dev/packages/opencode/src/session/message.ts)). +Those richer activity categories remain unclaimed by the scorecard until a +cross-host taxonomy, nested-agent policy, and deduplication rule are accepted. + +Unknown wire kinds are bounded to 32 distinct names; `unknownKindOverflow` +retains the number of additional occurrences without allowing transcript data +to expand the diagnostics payload without limit. + The current persisted field named `provider` identifies which host transcript parser produced a session row; it is not sufficient evidence of the inference provider. The Proposed model in [ADR-0016](adr/0016-capability-driven-integration-adapters.md) separates host, provider, @@ -134,9 +154,9 @@ responses = Σ over included sessions of session.responses **Source:** - Filter: a parsed record with zero assistant turns is dropped entirely — "no - assistant turn → not a session" (`usage-index.mjs:1083`) — and a record whose + assistant turn → not a session" (`usage-index.mjs:1123`) — and a record whose last activity falls outside the requested window is dropped too - (`usage-index.mjs:1084`). + (`usage-index.mjs:1124`). - `responses` accumulation: Claude increments per assistant message (`usage-index.mjs:568-571`); Codex increments per `agent_message` event (`usage-index.mjs:653-662`). @@ -219,7 +239,7 @@ already in effect on the given day, comparing ISO date strings lexicographically so no `Date` parsing is involved and the module stays clock-free. -`aggregate()` passes each usage row's own `day` (`usage-index.mjs:1100-1103`), which +`aggregate()` passes each usage row's own `day` (`usage-index.mjs:1129-1130`), which it already has because rows are keyed by `(day, model)`. **This is the whole point:** tokens metered in August must still read as August's rate when the panel is opened in December. Pricing by *today's* date instead would restate a @@ -409,7 +429,7 @@ session data, and each needs its own fix: - `mergeIntervals()` (`usage-index.mjs:89-114`) — the pure union primitive, sorts intervals and merges any two that overlap **or exactly touch** - (`s <= curEnd`, `usage-index.mjs:105`), returning total covered seconds + (`s <= curEnd`, `usage-index.mjs:133`), returning total covered seconds rounded to the nearest second. - `activeIntervals()` (`usage-index.mjs:427-438`) — splits one session's sorted timestamp list into sub-intervals wherever a gap exceeds @@ -470,12 +490,12 @@ byDay[day].sessionsActive = count of distinct sessions with any usage row that d **Source:** the day key is the row's own `row.day`, computed once at parse time as **local calendar day**, not UTC -(`usage-index.mjs:598`/`usage-index.mjs:784` call `localDay(at)`) — so a +(`usage-index.mjs:602`/`usage-index.mjs:798` call `localDay(at)`) — so a session that runs from 23:58 local to 00:05 local is billed to the day its *first* row landed on (test: `tests/kit/usage-index.test.mjs:634`, "a session that opens before midnight is counted on its first billed day"). Accumulation: -`byDay[row.day].cost += rowCost` (`usage-index.mjs:1111`). Bar height: +`byDay[row.day].cost += rowCost` (`usage-index.mjs:1151`). Bar height: `h = maxDay ? max(2, cost/maxDay*100) : 2` (`dashboard/client.mjs`) — every non-empty day gets a visually nonzero bar (floor of 2%), so a very cheap day is never rendered as invisible. @@ -589,11 +609,11 @@ excluded subagent-replay session still shows up as "used," at zero cost, rather than vanishing. `byModel[...].responses` is populated from `row.responses` -(`usage-index.mjs:1114`), which in turn comes from the `responses` field +(`usage-index.mjs:1154`), which in turn comes from the `responses` field passed into `addUsage()` at the call site — `1` per Claude assistant turn -(`usage-index.mjs:568-604`), or `rec.responses` (the session's whole response +(`usage-index.mjs:596-608`), or `rec.responses` (the session's whole response count) once per Codex session, passed at the single point Codex calls -`addUsage` (`usage-index.mjs:784-790`). +`addUsage` (`usage-index.mjs:798-804`). **Render:** `bar(name, fmtUsd(cost), fmtTok(tokens)+" · "+fmtNum(responses)+" resp", pct(cost, topModelCost), false)` (`dashboard/client.mjs`), @@ -925,13 +945,13 @@ Codex ≥0.140 maintains its own SQLite thread ledger (`~/.codex/state_N.sqlite` — the `N` is a migration generation, so `codexStateDb` (`codex-state.mjs:30`) globs and takes the newest). `readCodexState` (`:49`) reads per-thread `thread_source` (`user` vs `subagent`) plus `thread_spawn_edges`, and -`applyCodexLedger` (`usage-index.mjs:1469-1479`) overlays that onto parsed +`applyCodexLedger` (`usage-index.mjs:1529-1539`) overlays that onto parsed sessions: a ledger-identified subagent has its token usage stripped — its rollout replays the parent's entire token history (ccusage/ccusage#950 measured up to 91× inflation) — while the session record stays visible. The rollout's own `session_meta.thread_source` sniff remains as the fallback when the ledger is absent or migrated beyond recognition. Codex sessions also carry -`reasoningOutput` (`usage-index.mjs:794`) — reasoning tokens are a **subset** +`reasoningOutput` (`usage-index.mjs:808`) — reasoning tokens are a **subset** of output tokens and are annotation only, never added to any sum. ## 14. Known limitations, restated as a single checklist @@ -978,7 +998,7 @@ commit `540be18` on this branch. Claude's parser passes `responses: 1` per assistant turn (`usage-index.mjs:598`, the current equivalent), but Codex's call passed no such field at all. Because `byModel[model].responses` is summed -directly from each usage row's `responses` field (`usage-index.mjs:1114`, +directly from each usage row's `responses` field (`usage-index.mjs:1154`, `m.responses += row.responses`), **every** Codex model in §10's "Models in Play" list displayed `0 resp` regardless of real token/cost volume or actual `agent_message` count. **Fix:** `parseCodex` now passes `responses: diff --git a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md index 3bb6697..33a9622 100644 --- a/docs/adr/0009-usage-scorecard-local-transcript-analytics.md +++ b/docs/adr/0009-usage-scorecard-local-transcript-analytics.md @@ -11,9 +11,39 @@ host, inference-provider, provenance, and model facts in session rows. ADR-0023 subsequently classified SQLite source failures and made transient OpenCode failures preserve last-good records with explicit degraded source health instead of becoming observed zero usage; the Usage UI now - renders each local source state rather than leaving that evidence API-only. + renders each local source state rather than leaving that evidence API-only. The 2026-08-24 + cross-host telemetry amendment adds an additive common diagnostics envelope and capability states; + it deliberately does not promote Codex's richer public app-server activity items into historical + scorecard metrics until cross-host taxonomy and deduplication rules are answered. - **Deciders:** agentic-kit maintainers +## Amendment — cross-host telemetry evidence and capability states (2026-08-24) + +The public surfaces were re-checked before the implementation: [Claude Code hooks and monitoring](https://code.claude.com/docs/en/hooks), +[Codex app-server](https://github.com/openai/codex/blob/main/codex-rs/app-server/README.md), and +[OpenCode's SDK and message model](https://github.com/anomalyco/opencode/blob/dev/packages/web/src/content/docs/sdk.mdx) +all expose richer runtime activity than a single shared historical transcript schema. Claude exposes +tool hooks and `claude_code.tool` telemetry; Codex exposes typed command, file-change, MCP, and +collaboration items; OpenCode exposes message parts including tool invocations. + +The implemented boundary is therefore additive and evidence-graded: + +- `sourceHealth..diagnostics.common` reports discovered, parsed, usage-bearing, + prompt-bearing, and response-bearing session units, plus observed prompt/response totals, + warnings, and unknown kinds. Unknown kinds are bounded to 32 distinct names; + `unknownKindOverflow` retains additional occurrence volume. +- `sourceHealth..capabilities` reports `supported`, `unsupported`, or `unavailable` for + prompts, responses, tool calls, command executions, file changes, MCP calls, and collaboration. + A supported source with zero observations is not the same as an absent or degraded source. +- Existing source status/reason fields and Codex-specific diagnostics remain unchanged for callers; + the new fields are additive. The Usage dashboard renders the common coverage and capability + states, including the distinction between unsupported and unavailable. +- The historical scorecard still normalizes only prompts/responses across all hosts and the existing + Claude/OpenCode tool-call contract. Codex wire kinds such as `commandExecution` and `fileChange` + remain diagnostic evidence, not Codex-only scorecard counters. A future activity taxonomy requires + maintainer answers on categories, nested-agent semantics, and deduplication before it changes that + boundary. + ## Context Claude Code and Codex both write complete session transcripts to disk — `~/.claude/projects/**/*.jsonl` diff --git a/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md b/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md index d7ca4a5..4d7c2c7 100644 --- a/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md +++ b/docs/adr/0023-fail-closed-operations-and-explicit-degradation.md @@ -168,6 +168,17 @@ shows the full per-field detail. The pill itself uses a solid `var(--panel-2)` b the segmented tab control's own look — no border, state shown via status-text color/weight — rather than the colored-outline chip style originally shipped. +#### 7.1 Cross-host telemetry coverage is additive and evidence-graded (2026-08-24) + +Usage adds `sourceHealth..diagnostics.common` and +`sourceHealth..capabilities` without changing existing status/reason or Codex-specific +diagnostic fields. `supported`, `unsupported`, and `unavailable` are capability states, not +observed counts: a readable source may report a supported zero, while an absent or degraded source +must not render a fabricated zero. The Usage panel therefore says coverage is unavailable when +common evidence cannot be read, and labels the Codex card as transcript coverage because +`codexLedger` remains a separate corrective source folded only into the persistent host-health pill. +The common unknown-kind map is capped at 32 names with overflow volume retained separately. + ### 8. Clean-machine proof is isolated at every mutable boundary Required tests redirect HOME, USERPROFILE, XDG config/state, APPDATA, npm prefix/cache, Brain KB, and diff --git a/src/lib/dashboard/client.mjs b/src/lib/dashboard/client.mjs index a7c1ef9..2d82ce6 100644 --- a/src/lib/dashboard/client.mjs +++ b/src/lib/dashboard/client.mjs @@ -1069,7 +1069,7 @@ export const JS = ` var grp=SOURCE_HEALTH_GROUPS[g],present=[]; for(var p=0; p' @@ -1171,6 +1174,49 @@ export const JS = ` +"open unions whole session spans; summed double-counts overlap."; } + // Host-neutral telemetry is deliberately separate from the scorecard's + // measured totals. A missing common envelope means an older API response, + // not zero observations; the UI says so rather than backfilling a claim. + var TELEMETRY_HOSTS=[ + {key:"claude",label:"Claude"}, + {key:"codex",label:"Codex transcript"}, + {key:"opencode",label:"OpenCode"} + ]; + var TELEMETRY_CATEGORIES=[ + ["prompts","prompts"],["responses","responses"],["toolCalls","tools"], + ["commandExecutions","commands"],["fileChanges","file changes"], + ["mcpCalls","MCP"],["collaboration","collaboration"] + ]; + function renderTelemetryCoverage(health){ + var el=document.getElementById("u-telemetry-grid"); + if(!el)return; + health=health||{}; + el.innerHTML=TELEMETRY_HOSTS.map(function(host){ + var source=health[host.key]||{},status=String(source.status||"not-read"); + var common=source.diagnostics&&source.diagnostics.common; + var counts; + if(!common){ + counts="coverage not reported by this API"; + }else if(status==="ok"){ + counts=fmtNum(common.unitsParsed)+"/"+fmtNum(common.unitsSeen)+" parsed · "+fmtNum(common.prompts)+" prompts · "+fmtNum(common.responses)+" responses"; + }else if(common.unitsSeen>0){ + counts=fmtNum(common.unitsParsed)+"/"+fmtNum(common.unitsSeen)+" parsed · "+fmtNum(common.prompts)+" prompts · "+fmtNum(common.responses)+" responses · partial coverage"; + }else{ + counts="coverage unavailable"+(source.reason?" · "+String(source.reason):""); + } + if(common&&common.warnings&&common.warnings.length) counts+=" · "+common.warnings.join(", "); + var capabilities=source.capabilities||{}; + var caps=TELEMETRY_CATEGORIES.map(function(item){ + var state=String(capabilities[item[0]]||"unavailable"); + return ''+esc(item[1])+" "+esc(state)+""; + }).join(""); + return '
' + +'
'+esc(host.label)+""+esc(status)+"
" + +'
'+esc(counts)+"
" + +'
'+caps+"
"; + }).join(""); + } + function renderScore(d){ var t=d.totals||{}; var cacheShare=pct(t.cacheRead,t.tokens); @@ -1198,6 +1244,7 @@ export const JS = ` return '
' +''+esc(x.day.slice(8))+"
"; }).join(""):'
no days in window.
'; + renderTelemetryCoverage(d.sourceHealth); // Host and inference-provider are independent canonical axes. All three // supported hosts always render (idle/grayed-out when a host has no diff --git a/src/lib/dashboard/page.mjs b/src/lib/dashboard/page.mjs index ddc1daa..67299a0 100644 --- a/src/lib/dashboard/page.mjs +++ b/src/lib/dashboard/page.mjs @@ -424,6 +424,10 @@ export function renderPage({ name, version }) {

cost per day

+
+

telemetry coverage

capabilities · observed locally
+
+

by host

diff --git a/src/lib/dashboard/styles.mjs b/src/lib/dashboard/styles.mjs index cefe084..4906c90 100644 --- a/src/lib/dashboard/styles.mjs +++ b/src/lib/dashboard/styles.mjs @@ -508,6 +508,19 @@ body.gated .band,body.gated .tabbar,body.gated main{display:none} transition:filter .15s ease; } .daybar:hover .db-fill{filter:brightness(1.35)} +.telemetry-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:8px} +.telemetry-card{min-width:0;padding:10px 12px;border:1px solid var(--line);border-radius:var(--r-sm);background:var(--panel-2)} +.telemetry-card .tc-head{display:flex;align-items:center;justify-content:space-between;gap:8px;color:var(--ink);font-size:12px;font-weight:650} +.telemetry-card .tc-status{font-size:9px;font-weight:650;text-transform:uppercase;letter-spacing:.06em;color:var(--ink-dim)} +.telemetry-card[data-status="ok"] .tc-status{color:var(--ok)} +.telemetry-card[data-status="degraded"] .tc-status{color:var(--warn)} +.telemetry-card[data-status="absent"] .tc-status,.telemetry-card[data-status="not-read"] .tc-status{color:var(--ink-dim)} +.telemetry-card .tc-counts{margin-top:5px;color:var(--ink-2);font-size:10px;line-height:1.45} +.telemetry-card .tc-caps{display:flex;flex-wrap:wrap;gap:4px;margin-top:8px} +.telemetry-card .tc-cap{padding:3px 5px;border:1px solid var(--line);border-radius:5px;color:var(--ink-dim);font:9px/1.1 ui-monospace,monospace} +.telemetry-card .tc-cap[data-state="supported"]{border-color:color-mix(in srgb,var(--ok) 42%,var(--line));color:var(--ok)} +.telemetry-card .tc-cap[data-state="unsupported"]{color:var(--ink-dim)} +.telemetry-card .tc-cap[data-state="unavailable"]{border-color:color-mix(in srgb,var(--warn) 42%,var(--line));color:var(--warn)} .db-lab{font-family:var(--mono); font-size:9.5px; color:var(--ink-dim); margin-top:6px} /* punchcard */ diff --git a/src/lib/usage-index.mjs b/src/lib/usage-index.mjs index d76cdf1..42adcc7 100644 --- a/src/lib/usage-index.mjs +++ b/src/lib/usage-index.mjs @@ -36,6 +36,10 @@ import { defaultOpencodeDbPath, listSessionsResult as listOpencodeSessionsResult, parseSession as parseOpencodeSession, sessionExistsResult as opencodeSessionExistsResult, } from './usage-opencode.mjs'; +import { + addTelemetryDiagnostics, emptyTelemetryDiagnostics, finalizeTelemetryDiagnostics, + MAX_TELEMETRY_UNKNOWN_KINDS, recordTelemetryUnit, telemetryCapabilities, +} from './usage-telemetry.mjs'; /** Bump to invalidate every cached entry wholesale. * v2: cached records carry `active` sub-intervals for the idle-gap split. @@ -641,10 +645,20 @@ function codexItemText(item) { function codexParseStats() { return { legacyEvents: 0, itemCompletedEvents: 0, tokenCountEvents: 0, - prompts: 0, responses: 0, unknownItemTypes: {}, + prompts: 0, responses: 0, unknownItemTypes: {}, unknownItemTypeOverflow: 0, }; } +function recordCodexUnknownType(stats, type) { + if (Object.hasOwn(stats.unknownItemTypes, type)) { + stats.unknownItemTypes[type]++; + } else if (Object.keys(stats.unknownItemTypes).length < MAX_TELEMETRY_UNKNOWN_KINDS) { + stats.unknownItemTypes[type] = 1; + } else { + stats.unknownItemTypeOverflow++; + } +} + function codexEvent(payload, stats) { if (payload?.type === 'user_message') { stats.legacyEvents++; @@ -662,7 +676,7 @@ function codexEvent(payload, stats) { if (item?.type === 'AgentMessage') return { kind: 'response', text: codexItemText(item) }; const type = typeof item?.type === 'string' ? item.type : ''; - stats.unknownItemTypes[type] = (stats.unknownItemTypes[type] ?? 0) + 1; + recordCodexUnknownType(stats, type); return null; } @@ -832,7 +846,7 @@ function emptyCodexDiagnostics() { files: 0, cachedFiles: 0, parsedFiles: 0, unparsedFiles: 0, filesWithTokens: 0, filesWithResponses: 0, legacyEvents: 0, itemCompletedEvents: 0, tokenCountEvents: 0, - prompts: 0, responses: 0, unknownItemTypes: {}, warnings: [], + prompts: 0, responses: 0, unknownItemTypes: {}, unknownItemTypeOverflow: 0, warnings: [], }; } @@ -847,8 +861,15 @@ function addCodexParseDiagnostics(target, stats) { target.prompts += stats.prompts; target.responses += stats.responses; for (const [type, count] of Object.entries(stats.unknownItemTypes ?? {})) { - target.unknownItemTypes[type] = (target.unknownItemTypes[type] ?? 0) + count; + if (Object.hasOwn(target.unknownItemTypes, type)) { + target.unknownItemTypes[type] += count; + } else if (Object.keys(target.unknownItemTypes).length < MAX_TELEMETRY_UNKNOWN_KINDS) { + target.unknownItemTypes[type] = count; + } else { + target.unknownItemTypeOverflow += count; + } } + target.unknownItemTypeOverflow += stats.unknownItemTypeOverflow ?? 0; } function finalizeCodexHealth(root, diagnostics) { @@ -857,7 +878,9 @@ function finalizeCodexHealth(root, diagnostics) { const responseFiles = diagnostics.filesWithResponses; if (tokenFiles > 0 && responseFiles === 0) warnings.push('zero-response-yield'); else if (tokenFiles > responseFiles) warnings.push('partial-response-yield'); - if (Object.keys(diagnostics.unknownItemTypes).length) warnings.push('unknown-item-types'); + if (Object.keys(diagnostics.unknownItemTypes).length || diagnostics.unknownItemTypeOverflow > 0) { + warnings.push('unknown-item-types'); + } diagnostics.warnings = warnings; const hasYieldWarning = warnings.includes('zero-response-yield') || warnings.includes('partial-response-yield'); const status = root.status === 'ok' && hasYieldWarning @@ -867,6 +890,21 @@ function finalizeCodexHealth(root, diagnostics) { return { ...root, status, reason, diagnostics }; } +/** Attach the additive host-neutral telemetry contract to source health. + * Existing status/reason and Codex diagnostic keys remain where consumers + * already find them; `diagnostics.common` and `capabilities` are the new + * cross-host surface. */ +function attachTelemetryHealth(host, health, common, diagnostics = health.diagnostics) { + return { + ...health, + capabilities: telemetryCapabilities(host, health.status), + diagnostics: { + ...(diagnostics ?? {}), + common: finalizeTelemetryDiagnostics(common), + }, + }; +} + function defaultRoots() { return { claude: path.join(claudeDir(), 'projects'), @@ -1356,6 +1394,11 @@ async function scan(o = {}) { const entries = {}; const records = []; const codexDiagnostics = emptyCodexDiagnostics(); + const commonDiagnostics = { + claude: emptyTelemetryDiagnostics(), + codex: emptyTelemetryDiagnostics(), + opencode: emptyTelemetryDiagnostics(), + }; const total = candidates.length; let scanned = 0; @@ -1372,6 +1415,15 @@ async function scan(o = {}) { session = parsed ? parsed.session : null; parseStats = parsed?.parseStats ?? null; } + if (commonDiagnostics[c.provider]) { + recordTelemetryUnit(commonDiagnostics[c.provider], session); + if (c.provider === 'codex') { + addTelemetryDiagnostics(commonDiagnostics.codex, { + unknownKinds: parseStats?.unknownItemTypes, + unknownKindOverflow: parseStats?.unknownItemTypeOverflow, + }); + } + } if (c.provider === 'codex') { codexDiagnostics.files++; if (cacheHit) codexDiagnostics.cachedFiles++; @@ -1448,10 +1500,15 @@ async function scan(o = {}) { : { status: observed.error.kind === 'absent' ? 'absent' : 'degraded', reason: observed.error.kind }; } const result = aggregate(applyCodexLedger(records, ledger), { days, now, cutoff, deps }); + const codexSourceHealth = finalizeCodexHealth(codexHealth, codexDiagnostics); + addTelemetryDiagnostics(commonDiagnostics.codex, { + warnings: codexSourceHealth.diagnostics.warnings, + }); result.sourceHealth = { - claude: claudeHealth, - codex: finalizeCodexHealth(codexHealth, codexDiagnostics), - opencode: opencodeHealth, codexLedger: codexLedgerHealth, + claude: attachTelemetryHealth('claude', claudeHealth, commonDiagnostics.claude), + codex: attachTelemetryHealth('codex', codexSourceHealth, commonDiagnostics.codex), + opencode: attachTelemetryHealth('opencode', opencodeHealth, commonDiagnostics.opencode), + codexLedger: codexLedgerHealth, }; return result; } diff --git a/src/lib/usage-telemetry.mjs b/src/lib/usage-telemetry.mjs new file mode 100644 index 0000000..9cc7d4c --- /dev/null +++ b/src/lib/usage-telemetry.mjs @@ -0,0 +1,140 @@ +// usage-telemetry.mjs — the host-neutral evidence contract for the usage +// scorecard. This module describes what a source can report; it does not +// reinterpret host wire events as a different kind of activity. +// +// The first implementation deliberately stops at prompts, responses, and +// parser coverage. Claude and OpenCode can expose normalized tool calls, but +// Codex's current rollout item families do not map one-to-one to that existing +// contract. Activity categories therefore remain explicit capabilities rather +// than being filled with host-specific guesses. + +export const TELEMETRY_CAPABILITY_STATES = Object.freeze([ + 'supported', 'unsupported', 'unavailable', +]); + +/** Categories whose meanings are shared across transcript hosts. */ +export const TELEMETRY_CATEGORIES = Object.freeze([ + 'prompts', 'responses', 'toolCalls', 'commandExecutions', + 'fileChanges', 'mcpCalls', 'collaboration', +]); + +/** Keep future wire-kind growth from becoming an unbounded diagnostics payload. */ +export const MAX_TELEMETRY_UNKNOWN_KINDS = 32; + +/** + * Static capability matrix. `supported` means the source format carries + * enough evidence for this category under the current parser contract; + * `unsupported` means the category is intentionally not claimed. Runtime + * source health can turn a supported category into `unavailable`. + */ +export const HOST_TELEMETRY_CAPABILITIES = Object.freeze({ + claude: Object.freeze({ + prompts: 'supported', responses: 'supported', toolCalls: 'supported', + commandExecutions: 'unsupported', fileChanges: 'unsupported', + mcpCalls: 'unsupported', collaboration: 'unsupported', + }), + codex: Object.freeze({ + prompts: 'supported', responses: 'supported', toolCalls: 'unsupported', + commandExecutions: 'unsupported', fileChanges: 'unsupported', + mcpCalls: 'unsupported', collaboration: 'unsupported', + }), + opencode: Object.freeze({ + prompts: 'supported', responses: 'supported', toolCalls: 'supported', + commandExecutions: 'unsupported', fileChanges: 'unsupported', + mcpCalls: 'unsupported', collaboration: 'unsupported', + }), +}); + +/** Empty host-neutral parser diagnostics. */ +export function emptyTelemetryDiagnostics() { + return { + unitsSeen: 0, + unitsParsed: 0, + unitsWithUsage: 0, + unitsWithPrompts: 0, + unitsWithResponses: 0, + prompts: 0, + responses: 0, + warnings: [], + unknownKinds: {}, + unknownKindOverflow: 0, + }; +} + +/** + * Add one discovered source unit to the common diagnostic envelope. + * A unit is one candidate transcript session in the current scan window. + * `session === null` records that discovery succeeded but parsing did not. + */ +export function recordTelemetryUnit(target, session) { + if (!target || typeof target !== 'object') return target; + target.unitsSeen++; + if (!session) return target; + target.unitsParsed++; + if (Array.isArray(session.usage) && session.usage.length) target.unitsWithUsage++; + if (Number(session.prompts) > 0) target.unitsWithPrompts++; + if (Number(session.responses) > 0) target.unitsWithResponses++; + target.prompts += Number(session.prompts) || 0; + target.responses += Number(session.responses) || 0; + return target; +} + +/** Add bounded parser warnings and unknown wire kinds without duplicates. */ +export function addTelemetryDiagnostics(target, { + warnings = [], unknownKinds = {}, unknownKindOverflow = 0, +} = {}) { + if (!target || typeof target !== 'object') return target; + for (const warning of warnings) { + if (typeof warning === 'string' && warning && !target.warnings.includes(warning)) { + target.warnings.push(warning); + } + } + for (const [kind, count] of Object.entries(unknownKinds ?? {})) { + if (!kind) continue; + if (Object.hasOwn(target.unknownKinds, kind)) { + target.unknownKinds[kind] += Number(count) || 0; + } else if (Object.keys(target.unknownKinds).length < MAX_TELEMETRY_UNKNOWN_KINDS) { + target.unknownKinds[kind] = Number(count) || 0; + } else { + target.unknownKindOverflow += Number(count) || 0; + } + } + target.unknownKindOverflow += Math.max(0, Number(unknownKindOverflow) || 0); + return target; +} + +/** Return a stable, JSON-safe copy of the common envelope. */ +export function finalizeTelemetryDiagnostics(value) { + const out = emptyTelemetryDiagnostics(); + if (!value || typeof value !== 'object') return out; + for (const key of [ + 'unitsSeen', 'unitsParsed', 'unitsWithUsage', 'unitsWithPrompts', + 'unitsWithResponses', 'prompts', 'responses', + ]) out[key] = Math.max(0, Number(value[key]) || 0); + out.warnings = [...new Set((Array.isArray(value.warnings) ? value.warnings : []) + .filter((warning) => typeof warning === 'string' && warning))]; + for (const [kind, count] of Object.entries(value.unknownKinds ?? {})) { + if (typeof kind === 'string' && kind + && Object.keys(out.unknownKinds).length < MAX_TELEMETRY_UNKNOWN_KINDS) { + out.unknownKinds[kind] = Math.max(0, Number(count) || 0); + } else if (typeof kind === 'string' && kind) { + out.unknownKindOverflow += Math.max(0, Number(count) || 0); + } + } + out.unknownKindOverflow += Math.max(0, Number(value.unknownKindOverflow) || 0); + return out; +} + +/** + * Resolve the runtime capability state for one host/source. A source that is + * absent, unreadable, or partially degraded cannot claim a measured zero for + * a category the parser supports; it is `unavailable` instead. + */ +export function telemetryCapabilities(host, sourceStatus = 'unavailable') { + const base = HOST_TELEMETRY_CAPABILITIES[host]; + if (!base) return Object.fromEntries(TELEMETRY_CATEGORIES.map((key) => [key, 'unavailable'])); + return Object.fromEntries(TELEMETRY_CATEGORIES.map((key) => [ + key, + base[key] === 'supported' && sourceStatus !== 'ok' ? 'unavailable' : base[key], + ])); +} diff --git a/tests/dashboard.test.cjs b/tests/dashboard.test.cjs index 42e20e5..fd8351b 100644 --- a/tests/dashboard.test.cjs +++ b/tests/dashboard.test.cjs @@ -470,7 +470,15 @@ async function main() { sourceHealth: { claude: { status: 'ok', reason: null }, codex: { status: 'ok', reason: null }, - opencode: { status: 'degraded', reason: 'busy' }, + opencode: { + status: 'degraded', reason: 'busy', + capabilities: { prompts: 'unavailable', toolCalls: 'unavailable' }, + diagnostics: { common: { + unitsSeen: 0, unitsParsed: 0, unitsWithUsage: 0, + unitsWithPrompts: 0, unitsWithResponses: 0, + prompts: 0, responses: 0, warnings: ['corrupt'], unknownKinds: {}, unknownKindOverflow: 0, + } }, + }, codexLedger: { status: 'ok', reason: null }, }, totals: { sessions: 2, responses: 9, input: 100, output: 200, cacheRead: 900, cacheWrite: 50, tokens: 1250, cost: 12.5, spanMinutes: 90, engagedSeconds: 3600 }, @@ -567,6 +575,10 @@ async function main() { assert(Array.isArray(j.insights) && j.insights.length === 1, 'insights must survive'); assert(j.sourceHealth.opencode.status === 'degraded' && j.sourceHealth.opencode.reason === 'busy', 'source-health evidence must survive the dashboard route'); + assert(j.sourceHealth.opencode.capabilities.toolCalls === 'unavailable', + 'capability states must survive the dashboard route'); + assert(j.sourceHealth.opencode.diagnostics.common.warnings[0] === 'corrupt', + 'common telemetry diagnostics must survive the dashboard route'); assert(spy.calls.readIndex.some((o) => o && o.days === 7), 'days must reach readIndex, got ' + JSON.stringify(spy.calls.readIndex)); }); diff --git a/tests/kit/dashboard-usage-telemetry.test.mjs b/tests/kit/dashboard-usage-telemetry.test.mjs new file mode 100644 index 0000000..6fc4d53 --- /dev/null +++ b/tests/kit/dashboard-usage-telemetry.test.mjs @@ -0,0 +1,25 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { JS } from '../../src/lib/dashboard/client.mjs'; +import { renderPage } from '../../src/lib/dashboard/page.mjs'; +import { CSS } from '../../src/lib/dashboard/styles.mjs'; + +const PAGE = renderPage({ name: 'agentic-kit', version: 'test' }); + +test('usage dashboard includes a visible host-neutral telemetry coverage surface', () => { + assert.match(PAGE, /id="u-telemetry-grid"/); + assert.match(PAGE, />telemetry coverage { + assert.match(JS, /String\(capabilities\[item\[0\]\]\|\|"unavailable"\)/); + assert.match(JS, /data-state=/); + assert.match(JS, /source\.diagnostics&&source\.diagnostics\.common/); + assert.match(JS, /source\.capabilities\|\|\{\}/); +}); diff --git a/tests/kit/usage-index-opencode.test.mjs b/tests/kit/usage-index-opencode.test.mjs index c1fc2d8..7b1058a 100644 --- a/tests/kit/usage-index-opencode.test.mjs +++ b/tests/kit/usage-index-opencode.test.mjs @@ -166,7 +166,11 @@ test('a corrupt OpenCode store preserves last-good usage and surfaces degraded s fs.writeFileSync(sb.dbFile, 'not a sqlite database'); _resetForTest(); const degraded = await buildIndex(opts(sb)); - assert.deepEqual(degraded.sourceHealth.opencode, { status: 'degraded', reason: 'corrupt' }); + assert.equal(degraded.sourceHealth.opencode.status, 'degraded'); + assert.equal(degraded.sourceHealth.opencode.reason, 'corrupt'); + assert.equal(degraded.sourceHealth.opencode.capabilities.prompts, 'unavailable'); + assert.equal(degraded.sourceHealth.opencode.capabilities.toolCalls, 'unavailable'); + assert.equal(degraded.sourceHealth.opencode.diagnostics.common.unitsSeen, 0); assert.equal(degraded.sessions.find((x) => x.id === 'ses_last_good').cost, 0.4, 'a transient source failure must not become an observed zero'); rm(sb.dir); diff --git a/tests/kit/usage-index-v6.test.mjs b/tests/kit/usage-index-v6.test.mjs index 573a59e..f9961fd 100644 --- a/tests/kit/usage-index-v6.test.mjs +++ b/tests/kit/usage-index-v6.test.mjs @@ -132,7 +132,14 @@ test('parseCodex normalizes current item_completed messages and exposes bounded files: 1, cachedFiles: 0, parsedFiles: 1, unparsedFiles: 0, filesWithTokens: 1, filesWithResponses: 1, legacyEvents: 0, itemCompletedEvents: 3, tokenCountEvents: 1, - prompts: 1, responses: 1, unknownItemTypes: { CommandExecution: 1 }, warnings: ['unknown-item-types'], + prompts: 1, responses: 1, unknownItemTypes: { CommandExecution: 1 }, unknownItemTypeOverflow: 0, + warnings: ['unknown-item-types'], + common: { + unitsSeen: 1, unitsParsed: 1, unitsWithUsage: 1, + unitsWithPrompts: 1, unitsWithResponses: 1, + prompts: 1, responses: 1, + warnings: ['unknown-item-types'], unknownKinds: { CommandExecution: 1 }, unknownKindOverflow: 0, + }, }); const detail = await readSession(id, opts(sb)); diff --git a/tests/kit/usage-index.test.mjs b/tests/kit/usage-index.test.mjs index 697e4a1..29d6fd8 100644 --- a/tests/kit/usage-index.test.mjs +++ b/tests/kit/usage-index.test.mjs @@ -843,7 +843,10 @@ test('an empty corpus yields a zeroed Aggregate rather than throwing', async () assert.deepEqual(agg.projectTree, []); // A never-used host (root simply doesn't exist yet) reads as absent, not ok — // "zero sessions" and "we never found the directory" must stay distinguishable. - assert.deepEqual(agg.sourceHealth.claude, { status: 'absent', reason: null }); + assert.equal(agg.sourceHealth.claude.status, 'absent'); + assert.equal(agg.sourceHealth.claude.reason, null); + assert.equal(agg.sourceHealth.claude.capabilities.prompts, 'unavailable'); + assert.equal(agg.sourceHealth.claude.diagnostics.common.unitsSeen, 0); assert.equal(agg.sourceHealth.codex.status, 'absent'); assert.equal(agg.sourceHealth.codex.reason, null); assert.equal(agg.sourceHealth.codex.diagnostics.files, 0); @@ -853,7 +856,12 @@ test('buildIndex reports ok claude/codex root health when the transcript roots e _resetForTest(); const sb = sandbox(); const agg = await buildIndex(opts(sb)); - assert.deepEqual(agg.sourceHealth.claude, { status: 'ok', reason: null }); + assert.equal(agg.sourceHealth.claude.status, 'ok'); + assert.equal(agg.sourceHealth.claude.reason, null); + assert.equal(agg.sourceHealth.claude.capabilities.prompts, 'supported'); + assert.equal(agg.sourceHealth.claude.diagnostics.common.unitsParsed, 3); + assert.equal(agg.sourceHealth.claude.diagnostics.common.prompts, 4); + assert.equal(agg.sourceHealth.claude.diagnostics.common.responses, 5); assert.equal(agg.sourceHealth.codex.status, 'ok'); assert.equal(agg.sourceHealth.codex.reason, null); assert.equal(agg.sourceHealth.codex.diagnostics.files, 1); diff --git a/tests/kit/usage-telemetry.test.mjs b/tests/kit/usage-telemetry.test.mjs new file mode 100644 index 0000000..0dcd3b6 --- /dev/null +++ b/tests/kit/usage-telemetry.test.mjs @@ -0,0 +1,61 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + HOST_TELEMETRY_CAPABILITIES, MAX_TELEMETRY_UNKNOWN_KINDS, TELEMETRY_CATEGORIES, + addTelemetryDiagnostics, emptyTelemetryDiagnostics, + finalizeTelemetryDiagnostics, recordTelemetryUnit, telemetryCapabilities, +} from '../../src/lib/usage-telemetry.mjs'; + +test('the host matrix uses shared meanings and never maps Codex wire items to tool calls', () => { + assert.deepEqual(Object.keys(HOST_TELEMETRY_CAPABILITIES), ['claude', 'codex', 'opencode']); + assert.deepEqual(TELEMETRY_CATEGORIES, [ + 'prompts', 'responses', 'toolCalls', 'commandExecutions', + 'fileChanges', 'mcpCalls', 'collaboration', + ]); + assert.equal(HOST_TELEMETRY_CAPABILITIES.claude.toolCalls, 'supported'); + assert.equal(HOST_TELEMETRY_CAPABILITIES.opencode.toolCalls, 'supported'); + assert.equal(HOST_TELEMETRY_CAPABILITIES.codex.toolCalls, 'unsupported'); +}); + +test('supported categories become unavailable when the source is not readable', () => { + assert.equal(telemetryCapabilities('claude', 'ok').prompts, 'supported'); + assert.equal(telemetryCapabilities('claude', 'ok').toolCalls, 'supported'); + assert.equal(telemetryCapabilities('claude', 'absent').prompts, 'unavailable'); + assert.equal(telemetryCapabilities('claude', 'degraded').toolCalls, 'unavailable'); + assert.equal(telemetryCapabilities('claude', 'absent').commandExecutions, 'unsupported'); + assert.equal(telemetryCapabilities('unknown-host', 'ok').responses, 'unavailable'); +}); + +test('common diagnostics distinguish observed zero from an unavailable source', () => { + const diagnostics = emptyTelemetryDiagnostics(); + recordTelemetryUnit(diagnostics, { prompts: 1, responses: 2, usage: [{ model: 'm' }] }); + recordTelemetryUnit(diagnostics, { prompts: 0, responses: 0, usage: [] }); + recordTelemetryUnit(diagnostics, null); + addTelemetryDiagnostics(diagnostics, { + warnings: ['unknown-item-types', 'unknown-item-types'], + unknownKinds: { CommandExecution: 2 }, + }); + assert.deepEqual(finalizeTelemetryDiagnostics(diagnostics), { + unitsSeen: 3, + unitsParsed: 2, + unitsWithUsage: 1, + unitsWithPrompts: 1, + unitsWithResponses: 1, + prompts: 1, + responses: 2, + warnings: ['unknown-item-types'], + unknownKinds: { CommandExecution: 2 }, + unknownKindOverflow: 0, + }); +}); + +test('unknown-kind diagnostics are bounded and retain overflow volume', () => { + const unknownKinds = Object.fromEntries(Array.from( + { length: MAX_TELEMETRY_UNKNOWN_KINDS + 2 }, (_, i) => [`FutureKind${i}`, 1], + )); + const diagnostics = emptyTelemetryDiagnostics(); + addTelemetryDiagnostics(diagnostics, { unknownKinds }); + const out = finalizeTelemetryDiagnostics(diagnostics); + assert.equal(Object.keys(out.unknownKinds).length, MAX_TELEMETRY_UNKNOWN_KINDS); + assert.equal(out.unknownKindOverflow, 2); +});