From e0ee9bff3c5a6d495fe97a8f52a5f3546a0adfd7 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Sun, 19 Jul 2026 19:24:11 +0100 Subject: [PATCH 1/5] fix(run): preserve terminal usage provenance --- EVENTS.md | 18 ++- packages/cli/src/cli/cmd/run.ts | 54 ++++--- packages/cli/src/session/compaction.ts | 1 + packages/cli/src/session/index.ts | 35 ++-- packages/cli/src/session/message-v2.ts | 1 + packages/cli/src/session/processor.ts | 5 +- packages/cli/src/session/prompt.ts | 13 +- .../cli/test/cli/run-message-terminal.test.ts | 150 ++++++++++++++++++ packages/cli/test/session/compaction.test.ts | 25 +++ .../test/session/prompt-reliability.test.ts | 28 +++- 10 files changed, 288 insertions(+), 42 deletions(-) create mode 100644 packages/cli/test/cli/run-message-terminal.test.ts diff --git a/EVENTS.md b/EVENTS.md index 1363803..6d6e439 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -71,7 +71,7 @@ Emitted once per session, immediately after `session_start` and before the first > **Use case:** The server-side completion gate (aictrl-dev/aictrl #3216) uses `tools[]` to verify that `record_finding` and `record_review_completed` were actually in the model's function list at dispatch time, and `skills[]` to record which skill version produced the review. Detects the "silent success" failure mode where a missing MCP server lets a run complete green without ever having the review tools available. > -> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries. +> **Note on builtin tool filtering:** The `source: "builtin"` entries in `tools[]` reflect the _instance-level superset_ registered in `ToolRegistry`. Per-model filters (e.g. `apply_patch` only on gpt-\*, `codesearch`/`websearch` only for aictrl provider) are applied at dispatch time inside `resolveTools` and are **not** reflected here. Consumers should treat the presence of a builtin tool name as "registered and potentially available", not as "guaranteed to appear in the model's function list". The strong structural guarantee (tool present ↔ tool in dispatch list) applies only to `source: "mcp"` entries. ### `session_complete` @@ -117,11 +117,15 @@ Emitted when an assistant message finishes (one per LLM turn). ```json { "type": "message_complete", + "messageID": "msg_01abc...", "modelID": "claude-sonnet-4-20250514", "providerID": "anthropic", "agent": "default", + "status": "completed", + "usageStatus": "reported", "cost": { "input": 0.003, "output": 0.012, "cache": { "read": 0, "write": 0 } }, "tokens": { + "total": 11360, "input": 1024, "output": 512, "reasoning": 0, @@ -132,10 +136,14 @@ Emitted when an assistant message finishes (one per LLM turn). } ``` -`finish` values: `"tool-calls"` (model wants to call tools), `"end_turn"` (model is done), `"max_tokens"` (output truncated). +- `messageID` (string, **required**) — stable assistant-message identity. A message produces at most one `message_complete`; consumers should use this field rather than timestamps for identity. +- `status` (string, **required**) — `"completed"`, `"error"`, or `"aborted"`. +- `usageStatus` (string, **required**) — `"reported"` when the provider supplied usage, `"missing"` when it did not, or `"estimated"` for an explicitly estimated future source. The CLI does not currently estimate usage. +- `finish` (string, optional) — provider finish reason, such as `"tool-calls"`, `"end_turn"`, or `"max_tokens"`. It can be absent on failed or aborted turns. **`tokens`** (5-way breakdown, mirrors upstream `LLM.Usage`): +- `total` (number) — provider-reported total, or a finite total computed from sufficient reported components. - `input` (number) — raw input tokens billed at the standard input rate. - `output` (number) — output (completion) tokens. - `reasoning` (number) — extended-thinking / reasoning tokens (0 when thinking is off). @@ -144,12 +152,16 @@ Emitted when an assistant message finishes (one per LLM turn). These fields are non-overlapping: a token is counted in exactly one bucket. +`tokens` is `null` when `usageStatus` is `"missing"`. This distinguishes unavailable usage from a provider-reported zero, which has `usageStatus: "reported"` and numeric zero fields. The additional identity, status, and provenance fields are additive to schema v1; successful events retain the existing `modelID`, `providerID`, `agent`, `cost`, `tokens`, `context`, and `finish` fields. + +For compatibility with sessions written before usage provenance was persisted, a legacy terminal message with a finish reason is treated as `"reported"`; one without a finish reason is treated as `"missing"`. + **`context`** (context-window utilization): - `used` (number) — tokens occupying the model's context window this turn: `input + cache.read + cache.write`. - `limit` (number) — the model's total context-window size in tokens, sourced from the models.dev registry (`model.limit.context`). - `ratio` (number) — `used / limit` (≥0; may exceed 1 if usage exceeds the model's registered limit). A value approaching or exceeding 1 signals context-exhaustion risk. -- `null` — emitted when the model's context limit is not known (e.g. unregistered custom endpoint). +- `null` — emitted when the model's context limit is not known (e.g. unregistered custom endpoint), or usage is missing. ### `text` diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 7e17273..3cca76f 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -11,7 +11,7 @@ import { bootstrap } from "../bootstrap" import { EOL } from "os" import { Filesystem } from "../../util/filesystem" import { createAictrlClient } from "@aictrl/sdk" -import type { Message, ToolPart } from "@aictrl/sdk/v2" +import type { ToolPart } from "@aictrl/sdk/v2" import { Provider } from "../../provider/provider" import { Agent } from "../../agent/agent" import { PermissionNext } from "../../permission/next" @@ -480,6 +480,7 @@ export const RunCommand = cmd({ let sessionErrorEmitted = false const startTime = Date.now() const childSessions = new Set() + const emitted = new Set() const seqBySession = new Map() function nextSeq(sid: string): number { const n = (seqBySession.get(sid) ?? 0) + 1 @@ -494,19 +495,27 @@ export const RunCommand = cmd({ if (event.type === "message.updated" && event.properties.info.role === "assistant") { const info = event.properties.info if (args.format === "json") { - if (info.finish) { + if (info.sessionID === sessionID && info.time.completed !== undefined && !emitted.has(info.id)) { + emitted.add(info.id) + const usageStatus = info.usageStatus ?? (info.finish ? "reported" : "missing") // Build 5-way token breakdown mirroring upstream LLM.Usage shape. // info.tokens already carries the full breakdown from StepFinishPart // accumulation — reasoning and cache split are not dropped upstream. - const tokens = { - input: info.tokens.input, - output: info.tokens.output, - reasoning: info.tokens.reasoning, - cache: { - read: info.tokens.cache.read, - write: info.tokens.cache.write, - }, - } + const tokens = + usageStatus === "missing" + ? null + : { + total: + info.tokens.total ?? + info.tokens.input + info.tokens.output + info.tokens.cache.read + info.tokens.cache.write, + input: info.tokens.input, + output: info.tokens.output, + reasoning: info.tokens.reasoning, + cache: { + read: info.tokens.cache.read, + write: info.tokens.cache.write, + }, + } // Context-window utilization: used = input + cache.read + cache.write // (all prompt tokens that occupy the model's context window this turn). @@ -516,25 +525,30 @@ export const RunCommand = cmd({ // undercounts utilization on the first turn of a conversation. // limit comes from the model registry (models.dev). On lookup failure // (or limit===0 for custom models) buildContextWindow returns null. - const contextLimit = await Provider.getModel(info.providerID, info.modelID) - .then((m) => m.limit.context) - .catch((e) => { - if (e instanceof Provider.ModelNotFoundError) return null - throw e - }) - const contextUsed = tokens.input + tokens.cache.read + tokens.cache.write - const context = buildContextWindow(contextLimit, contextUsed) + const contextLimit = args.attach + ? null + : await Provider.getModel(info.providerID, info.modelID) + .then((m) => m.limit.context) + .catch((e) => { + if (e instanceof Provider.ModelNotFoundError) return null + throw e + }) + const contextUsed = tokens ? tokens.input + tokens.cache.read + tokens.cache.write : null + const context = contextUsed === null ? null : buildContextWindow(contextLimit, contextUsed) emit("message_complete", { + messageID: info.id, modelID: info.modelID, providerID: info.providerID, agent: info.agent, + context, // cost is sourced from info.cost which accumulates real per-step costs // from StepFinishPart. Do NOT use the new step.ended event cost field // which emits cost:0 and is reconciled later (the cost:0 trap). cost: info.cost, tokens, - context, + usageStatus, + status: info.error?.name === "MessageAbortedError" ? "aborted" : info.error ? "error" : "completed", finish: info.finish, }) } diff --git a/packages/cli/src/session/compaction.ts b/packages/cli/src/session/compaction.ts index 9245426..bf8a3aa 100644 --- a/packages/cli/src/session/compaction.ts +++ b/packages/cli/src/session/compaction.ts @@ -124,6 +124,7 @@ export namespace SessionCompaction { root: Instance.worktree, }, cost: 0, + usageStatus: "missing", tokens: { output: 0, input: 0, diff --git a/packages/cli/src/session/index.ts b/packages/cli/src/session/index.ts index c9850e3..cd6f3da 100644 --- a/packages/cli/src/session/index.ts +++ b/packages/cli/src/session/index.ts @@ -808,23 +808,32 @@ export namespace Session { metadata: z.custom().optional(), }), (input) => { - const safe = (value: number) => { - if (!Number.isFinite(value)) return 0 + const safe = (value: number | undefined) => { + if (typeof value !== "number" || !Number.isFinite(value)) return 0 return value } + const cacheWrite = + input.metadata?.["anthropic"]?.["cacheCreationInputTokens"] ?? + // @ts-expect-error + input.metadata?.["bedrock"]?.["usage"]?.["cacheWriteInputTokens"] ?? + // @ts-expect-error + input.metadata?.["venice"]?.["usage"]?.["cacheCreationInputTokens"] + const usageStatus = [ + input.usage.totalTokens, + input.usage.inputTokens, + input.usage.outputTokens, + input.usage.reasoningTokens, + input.usage.cachedInputTokens, + cacheWrite, + ].some(Number.isFinite) + ? ("reported" as const) + : ("missing" as const) const inputTokens = safe(input.usage.inputTokens ?? 0) const outputTokens = safe(input.usage.outputTokens ?? 0) const reasoningTokens = safe(input.usage.reasoningTokens ?? 0) const cacheReadInputTokens = safe(input.usage.cachedInputTokens ?? 0) - const cacheWriteInputTokens = safe( - (input.metadata?.["anthropic"]?.["cacheCreationInputTokens"] ?? - // @ts-expect-error - input.metadata?.["bedrock"]?.["usage"]?.["cacheWriteInputTokens"] ?? - // @ts-expect-error - input.metadata?.["venice"]?.["usage"]?.["cacheCreationInputTokens"] ?? - 0) as number, - ) + const cacheWriteInputTokens = safe(cacheWrite as number | undefined) // Providers where inputTokens EXCLUDES cached tokens const excludesCachedTokens = [ @@ -850,9 +859,12 @@ export namespace Session { input.model.api.npm === "@ai-sdk/amazon-bedrock" || input.model.api.npm === "@ai-sdk/google-vertex/anthropic" ) { + if (!Number.isFinite(input.usage.inputTokens) || !Number.isFinite(input.usage.outputTokens)) return return adjustedInputTokens + outputTokens + cacheReadInputTokens + cacheWriteInputTokens } - return input.usage.totalTokens + if (Number.isFinite(input.usage.totalTokens)) return input.usage.totalTokens + if (!Number.isFinite(input.usage.inputTokens) || !Number.isFinite(input.usage.outputTokens)) return + return inputTokens + outputTokens }) const tokens = { @@ -871,6 +883,7 @@ export namespace Session { ? input.model.cost.experimentalOver200K : input.model.cost return { + usageStatus, cost: safe( new Decimal(0) .add(new Decimal(tokens.input).mul(costInfo?.input ?? 0).div(1_000_000)) diff --git a/packages/cli/src/session/message-v2.ts b/packages/cli/src/session/message-v2.ts index 008e3f3..937f6f6 100644 --- a/packages/cli/src/session/message-v2.ts +++ b/packages/cli/src/session/message-v2.ts @@ -429,6 +429,7 @@ export namespace MessageV2 { write: z.number(), }), }), + usageStatus: z.enum(["reported", "missing", "estimated"]).optional(), structured: z.any().optional(), variant: z.string().optional(), finish: z.string().optional(), diff --git a/packages/cli/src/session/processor.ts b/packages/cli/src/session/processor.ts index 4002221..2577169 100644 --- a/packages/cli/src/session/processor.ts +++ b/packages/cli/src/session/processor.ts @@ -251,6 +251,7 @@ export namespace SessionProcessor { input.assistantMessage.finish = value.finishReason input.assistantMessage.cost += usage.cost input.assistantMessage.tokens = usage.tokens + input.assistantMessage.usageStatus = usage.usageStatus await Session.updatePart({ id: Identifier.ascending("part"), reason: value.finishReason, @@ -371,7 +372,6 @@ export namespace SessionProcessor { sessionID: input.assistantMessage.sessionID, error: input.assistantMessage.error, }) - SessionStatus.set(input.sessionID, { type: "idle" }) break } const delay = SessionRetry.delay(attempt, error.name === "APIError" ? error : undefined) @@ -389,7 +389,6 @@ export namespace SessionProcessor { sessionID: input.assistantMessage.sessionID, error: input.assistantMessage.error, }) - SessionStatus.set(input.sessionID, { type: "idle" }) } if (snapshot) { const patch = await Snapshot.patch(snapshot) @@ -423,6 +422,8 @@ export namespace SessionProcessor { } } input.assistantMessage.time.completed = Date.now() + // Persist the terminal message before SessionPrompt's deferred idle + // status; headless consumers stop reading when they observe idle. await Session.updateMessage(input.assistantMessage) if (needsCompaction) return "compact" if (blocked) return "stop" diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 361a352..8e7e998 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -271,7 +271,8 @@ export namespace SessionPrompt { cb.reject(reason) } delete s[sessionID] - SessionStatus.set(sessionID, { type: "idle" }) + // The loop's deferred cleanup publishes idle after the terminal assistant + // update, so event consumers cannot stop before observing the aborted turn. return } @@ -290,7 +291,12 @@ export namespace SessionPrompt { }) } - using _ = defer(() => cancel(sessionID)) + using _ = defer(() => { + cancel(sessionID) + // Keep idle after processor persistence. Headless consumers use this as + // the boundary after which all terminal message events have been queued. + SessionStatus.set(sessionID, { type: "idle" }) + }) // Structured output state // Note: On session resumption, state is reset but outputFormat is preserved @@ -373,6 +379,7 @@ export namespace SessionPrompt { root: Instance.worktree, }, cost: 0, + usageStatus: "missing", tokens: { input: 0, output: 0, @@ -584,6 +591,7 @@ export namespace SessionPrompt { root: Instance.worktree, }, cost: 0, + usageStatus: "missing", tokens: { input: 0, output: 0, @@ -1551,6 +1559,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the created: Date.now(), }, role: "assistant", + usageStatus: "missing", tokens: { input: 0, output: 0, diff --git a/packages/cli/test/cli/run-message-terminal.test.ts b/packages/cli/test/cli/run-message-terminal.test.ts new file mode 100644 index 0000000..66c53b0 --- /dev/null +++ b/packages/cli/test/cli/run-message-terminal.test.ts @@ -0,0 +1,150 @@ +import path from "path" +import { afterEach, describe, expect, test } from "bun:test" + +const cli = path.resolve(import.meta.dir, "../../src/index.ts") +const sessionID = "ses_primary" +const servers: Bun.Server[] = [] + +function message(input: { + id: string + sessionID?: string + usageStatus: "reported" | "missing" + status?: "completed" | "error" | "aborted" + terminal?: boolean + tokens?: { + total?: number + input: number + output: number + reasoning: number + cache: { read: number; write: number } + } +}) { + const status = input.status ?? "completed" + return { + type: "message.updated", + properties: { + info: { + id: input.id, + sessionID: input.sessionID ?? sessionID, + role: "assistant", + time: input.terminal === false ? { created: 1 } : { created: 1, completed: 2 }, + parentID: "msg_user", + modelID: "glm-4.7", + providerID: "zai", + mode: "agent", + agent: "agent", + path: { cwd: "/", root: "/" }, + cost: input.usageStatus === "reported" ? 0.25 : 0, + tokens: input.tokens ?? { + input: 0, + output: 0, + reasoning: 0, + cache: { read: 0, write: 0 }, + }, + usageStatus: input.usageStatus, + finish: status === "completed" ? "stop" : undefined, + error: + status === "completed" + ? undefined + : { + name: status === "aborted" ? "MessageAbortedError" : "ProviderAuthError", + data: { message: `${status} message` }, + }, + }, + }, + } +} + +function server(events: unknown[]) { + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url) + if (req.method === "POST" && url.pathname === "/session") return Response.json({ id: sessionID }) + if (req.method === "GET" && url.pathname === "/config") return Response.json({}) + if (req.method === "POST" && url.pathname.endsWith("/message")) return Response.json({}) + if (req.method === "GET" && url.pathname === "/event") { + return new Response( + events + .concat({ + type: "session.status", + properties: { sessionID, status: { type: "idle" } }, + }) + .map((event) => `data: ${JSON.stringify(event)}\n\n`) + .join(""), + { headers: { "content-type": "text/event-stream" } }, + ) + } + return Response.json({ error: "not found" }, { status: 404 }) + }, + }) + servers.push(server) + return `http://localhost:${server.port}` +} + +afterEach(() => { + servers.splice(0).map((server) => server.stop(true)) +}) + +describe("run --format json terminal assistant telemetry (#93, #45)", () => { + test("emits one scoped terminal event per message with explicit status and usage provenance", async () => { + const zero = message({ id: "msg_zero", usageStatus: "reported" }) + const partial = { + total: 24, + input: 10, + output: 4, + reasoning: 2, + cache: { read: 8, write: 0 }, + } + const events = [ + zero, + zero, + message({ id: "msg_missing", usageStatus: "missing", status: "error" }), + message({ id: "msg_aborted", usageStatus: "missing", status: "aborted" }), + message({ + id: "msg_partial", + usageStatus: "reported", + terminal: false, + tokens: partial, + }), + message({ + id: "msg_partial", + usageStatus: "reported", + status: "error", + tokens: partial, + }), + message({ id: "msg_child", sessionID: "ses_child", usageStatus: "reported" }), + ] + const proc = Bun.spawn(["bun", "run", cli, "run", "--format", "json", "--attach", server(events), "test prompt"], { + cwd: process.cwd(), + stdout: "pipe", + stderr: "pipe", + }) + const stdout = await new Response(proc.stdout).text() + await proc.exited + const output = stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((event) => event.type === "message_complete") + + expect(output.map((event) => event.messageID)).toEqual(["msg_zero", "msg_missing", "msg_aborted", "msg_partial"]) + expect(output.map((event) => event.status)).toEqual(["completed", "error", "aborted", "error"]) + expect(output.map((event) => event.usageStatus)).toEqual(["reported", "missing", "missing", "reported"]) + expect(output[0].tokens).toEqual({ + total: 0, + input: 0, + output: 0, + reasoning: 0, + cache: { read: 0, write: 0 }, + }) + expect(output[1].tokens).toBeNull() + expect(output[3].tokens).toEqual({ + total: 24, + input: 10, + output: 4, + reasoning: 2, + cache: { read: 8, write: 0 }, + }) + }, 20_000) +}) diff --git a/packages/cli/test/session/compaction.test.ts b/packages/cli/test/session/compaction.test.ts index 7cf3ca8..c4f69f3 100644 --- a/packages/cli/test/session/compaction.test.ts +++ b/packages/cli/test/session/compaction.test.ts @@ -410,6 +410,31 @@ describe("session.getUsage", () => { expect(Number.isNaN(result.cost)).toBe(false) }) + test("distinguishes reported zero usage from missing usage", () => { + const model = createModel({ context: 100_000, output: 32_000 }) + const reported = Session.getUsage({ + model, + usage: { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + }, + }) + const missing = Session.getUsage({ + model, + usage: { + inputTokens: undefined, + outputTokens: undefined, + totalTokens: undefined, + }, + }) + + expect(reported.usageStatus).toBe("reported") + expect(reported.tokens.total).toBe(0) + expect(missing.usageStatus).toBe("missing") + expect(missing.tokens.total).toBeUndefined() + }) + test("calculates cost correctly", () => { const model = createModel({ context: 100_000, diff --git a/packages/cli/test/session/prompt-reliability.test.ts b/packages/cli/test/session/prompt-reliability.test.ts index f559c09..502b5be 100644 --- a/packages/cli/test/session/prompt-reliability.test.ts +++ b/packages/cli/test/session/prompt-reliability.test.ts @@ -6,15 +6,13 @@ import { SessionRetry } from "../../src/session/retry" Log.init({ print: false }) const PROMPT_SRC = path.resolve(import.meta.dir, "../../src/session/prompt.ts") +const PROCESSOR_SRC = path.resolve(import.meta.dir, "../../src/session/processor.ts") describe("prompt.ts reliability guards", () => { test("cancel rejects queued callbacks before deleting state", async () => { const source = await Bun.file(PROMPT_SRC).text() // cancel() must iterate callbacks and reject before delete - const cancelFn = source.slice( - source.indexOf("export function cancel("), - source.indexOf("export const LoopInput"), - ) + const cancelFn = source.slice(source.indexOf("export function cancel("), source.indexOf("export const LoopInput")) expect(cancelFn).toBeTruthy() // Must contain a for loop that calls cb.reject or similar expect(cancelFn).toMatch(/for\s*\(.*\bcb\b.*\bcallbacks\b/) @@ -27,6 +25,28 @@ describe("prompt.ts reliability guards", () => { expect(rejectPos).toBeLessThan(deletePos) }) + test("terminal assistant update is queued before idle status", async () => { + const prompt = await Bun.file(PROMPT_SRC).text() + const processor = await Bun.file(PROCESSOR_SRC).text() + const cleanup = prompt.slice( + prompt.indexOf("using _ = defer(() => {"), + prompt.indexOf("// Structured output state"), + ) + const terminal = processor.slice( + processor.indexOf("input.assistantMessage.time.completed = Date.now()"), + processor.indexOf('if (needsCompaction) return "compact"'), + ) + const failure = processor.slice( + processor.indexOf("} catch (e: any) {"), + processor.indexOf("input.assistantMessage.time.completed = Date.now()"), + ) + + expect(cleanup.indexOf("cancel(sessionID)")).toBeLessThan(cleanup.indexOf("SessionStatus.set(sessionID")) + expect(failure).not.toContain('type: "idle"') + expect(terminal).toContain("await Session.updateMessage(input.assistantMessage)") + expect(terminal).not.toContain("SessionStatus.set") + }) + test("dispose handler rejects queued callbacks", async () => { const source = await Bun.file(PROMPT_SRC).text() // The dispose handler is the second argument to Instance.state() From 323e610eb10fa217fe6e0fcd452066f5cdae37d9 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Sun, 19 Jul 2026 19:51:08 +0100 Subject: [PATCH 2/5] fix(run): address terminal telemetry review --- packages/cli/src/cli/cmd/run.ts | 17 +++++-- packages/cli/src/session/prompt.ts | 17 ++++--- .../cli/test/cli/run-message-terminal.test.ts | 19 +++++-- .../test/cli/usage-token-breakdown.test.ts | 2 +- .../test/session/prompt-reliability.test.ts | 50 +++++++++++-------- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 3cca76f..c9fc471 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -13,6 +13,7 @@ import { Filesystem } from "../../util/filesystem" import { createAictrlClient } from "@aictrl/sdk" import type { ToolPart } from "@aictrl/sdk/v2" import { Provider } from "../../provider/provider" +import { ModelsDev } from "../../provider/models" import { Agent } from "../../agent/agent" import { PermissionNext } from "../../permission/next" import { Session } from "../../session" @@ -507,7 +508,11 @@ export const RunCommand = cmd({ : { total: info.tokens.total ?? - info.tokens.input + info.tokens.output + info.tokens.cache.read + info.tokens.cache.write, + info.tokens.input + + info.tokens.output + + info.tokens.reasoning + + info.tokens.cache.read + + info.tokens.cache.write, input: info.tokens.input, output: info.tokens.output, reasoning: info.tokens.reasoning, @@ -525,14 +530,16 @@ export const RunCommand = cmd({ // undercounts utilization on the first turn of a conversation. // limit comes from the model registry (models.dev). On lookup failure // (or limit===0 for custom models) buildContextWindow returns null. - const contextLimit = args.attach - ? null - : await Provider.getModel(info.providerID, info.modelID) + const contextLimit = await (args.attach + ? ModelsDev.get().then( + (providers) => providers[info.providerID]?.models[info.modelID]?.limit.context ?? null, + ) + : Provider.getModel(info.providerID, info.modelID) .then((m) => m.limit.context) .catch((e) => { if (e instanceof Provider.ModelNotFoundError) return null throw e - }) + })) const contextUsed = tokens ? tokens.input + tokens.cache.read + tokens.cache.write : null const context = contextUsed === null ? null : buildContextWindow(contextLimit, contextUsed) diff --git a/packages/cli/src/session/prompt.ts b/packages/cli/src/session/prompt.ts index 8e7e998..96f8ff9 100644 --- a/packages/cli/src/session/prompt.ts +++ b/packages/cli/src/session/prompt.ts @@ -68,6 +68,7 @@ export namespace SessionPrompt { string, { abort: AbortController + loop: boolean callbacks: { resolve(input: MessageV2.WithParts): void reject(reason?: any): void @@ -239,12 +240,13 @@ export namespace SessionPrompt { return parts } - function start(sessionID: string) { + function start(sessionID: string, loop = false) { const s = state() if (s[sessionID]) return const controller = new AbortController() s[sessionID] = { abort: controller, + loop, callbacks: [], } return controller.signal @@ -254,6 +256,7 @@ export namespace SessionPrompt { const s = state() if (!s[sessionID]) return + s[sessionID].loop = true return s[sessionID].abort.signal } @@ -263,7 +266,7 @@ export namespace SessionPrompt { const match = s[sessionID] if (!match) { SessionStatus.set(sessionID, { type: "idle" }) - return + return true } match.abort.abort() const reason = new Error("Session cancelled") @@ -271,9 +274,9 @@ export namespace SessionPrompt { cb.reject(reason) } delete s[sessionID] - // The loop's deferred cleanup publishes idle after the terminal assistant - // update, so event consumers cannot stop before observing the aborted turn. - return + if (match.loop) return false + SessionStatus.set(sessionID, { type: "idle" }) + return true } export const LoopInput = z.object({ @@ -283,7 +286,7 @@ export namespace SessionPrompt { export const loop = fn(LoopInput, async (input) => { const { sessionID, resume_existing } = input - const abort = resume_existing ? resume(sessionID) : start(sessionID) + const abort = resume_existing ? resume(sessionID) : start(sessionID, true) if (!abort) { return new Promise((resolve, reject) => { const callbacks = state()[sessionID].callbacks @@ -292,7 +295,7 @@ export namespace SessionPrompt { } using _ = defer(() => { - cancel(sessionID) + if (cancel(sessionID)) return // Keep idle after processor persistence. Headless consumers use this as // the boundary after which all terminal message events have been queued. SessionStatus.set(sessionID, { type: "idle" }) diff --git a/packages/cli/test/cli/run-message-terminal.test.ts b/packages/cli/test/cli/run-message-terminal.test.ts index 66c53b0..0f4b55d 100644 --- a/packages/cli/test/cli/run-message-terminal.test.ts +++ b/packages/cli/test/cli/run-message-terminal.test.ts @@ -2,6 +2,7 @@ import path from "path" import { afterEach, describe, expect, test } from "bun:test" const cli = path.resolve(import.meta.dir, "../../src/index.ts") +const models = path.resolve(import.meta.dir, "../tool/fixtures/models-api.json") const sessionID = "ses_primary" const servers: Bun.Server[] = [] @@ -90,7 +91,6 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { test("emits one scoped terminal event per message with explicit status and usage provenance", async () => { const zero = message({ id: "msg_zero", usageStatus: "reported" }) const partial = { - total: 24, input: 10, output: 4, reasoning: 2, @@ -117,11 +117,19 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { ] const proc = Bun.spawn(["bun", "run", cli, "run", "--format", "json", "--attach", server(events), "test prompt"], { cwd: process.cwd(), + env: { + ...process.env, + AICTRL_MODELS_PATH: models, + }, stdout: "pipe", stderr: "pipe", }) - const stdout = await new Response(proc.stdout).text() - await proc.exited + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(code, stderr).toBe(0) const output = stdout .split("\n") .filter(Boolean) @@ -146,5 +154,10 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { reasoning: 2, cache: { read: 8, write: 0 }, }) + expect(output[3].context).toEqual({ + used: 18, + limit: 204_800, + ratio: 18 / 204_800, + }) }, 20_000) }) diff --git a/packages/cli/test/cli/usage-token-breakdown.test.ts b/packages/cli/test/cli/usage-token-breakdown.test.ts index cf68baf..9634b74 100644 --- a/packages/cli/test/cli/usage-token-breakdown.test.ts +++ b/packages/cli/test/cli/usage-token-breakdown.test.ts @@ -101,7 +101,7 @@ describe("message_complete emit block shape (source-verified, #86)", () => { const source = await Bun.file(RUN_SRC).text() const emitIdx = source.indexOf('emit("message_complete"') expect(emitIdx).toBeGreaterThan(-1) - const blockStart = Math.max(0, emitIdx - 1500) + const blockStart = Math.max(0, emitIdx - 2000) const block = source.slice(blockStart, emitIdx + 200) expect(block).toContain("reasoning") expect(block).toContain("cache") diff --git a/packages/cli/test/session/prompt-reliability.test.ts b/packages/cli/test/session/prompt-reliability.test.ts index 502b5be..35e55ba 100644 --- a/packages/cli/test/session/prompt-reliability.test.ts +++ b/packages/cli/test/session/prompt-reliability.test.ts @@ -2,12 +2,15 @@ import path from "path" import { describe, expect, test } from "bun:test" import { Log } from "../../src/util/log" import { SessionRetry } from "../../src/session/retry" +import { Instance } from "../../src/project/instance" +import { Bus } from "../../src/bus" +import { SessionStatus } from "../../src/session/status" +import { SessionPrompt } from "../../src/session/prompt" +import { Session } from "../../src/session" Log.init({ print: false }) const PROMPT_SRC = path.resolve(import.meta.dir, "../../src/session/prompt.ts") -const PROCESSOR_SRC = path.resolve(import.meta.dir, "../../src/session/processor.ts") - describe("prompt.ts reliability guards", () => { test("cancel rejects queued callbacks before deleting state", async () => { const source = await Bun.file(PROMPT_SRC).text() @@ -25,26 +28,31 @@ describe("prompt.ts reliability guards", () => { expect(rejectPos).toBeLessThan(deletePos) }) - test("terminal assistant update is queued before idle status", async () => { - const prompt = await Bun.file(PROMPT_SRC).text() - const processor = await Bun.file(PROCESSOR_SRC).text() - const cleanup = prompt.slice( - prompt.indexOf("using _ = defer(() => {"), - prompt.indexOf("// Structured output state"), - ) - const terminal = processor.slice( - processor.indexOf("input.assistantMessage.time.completed = Date.now()"), - processor.indexOf('if (needsCompaction) return "compact"'), - ) - const failure = processor.slice( - processor.indexOf("} catch (e: any) {"), - processor.indexOf("input.assistantMessage.time.completed = Date.now()"), - ) + test("shell-owned cancellation publishes idle", async () => { + await Instance.provide({ + directory: path.join(import.meta.dir, "../.."), + fn: async () => { + const session = await Session.create({}) + const statuses: string[] = [] + const unsub = Bus.subscribe(SessionStatus.Event.Status, (event) => { + if (event.properties.sessionID === session.id) statuses.push(event.properties.status.type) + }) + + await SessionPrompt.shell({ + sessionID: session.id, + agent: "build", + model: { + providerID: "zai", + modelID: "glm-4.7", + }, + command: "printf test", + }) - expect(cleanup.indexOf("cancel(sessionID)")).toBeLessThan(cleanup.indexOf("SessionStatus.set(sessionID")) - expect(failure).not.toContain('type: "idle"') - expect(terminal).toContain("await Session.updateMessage(input.assistantMessage)") - expect(terminal).not.toContain("SessionStatus.set") + unsub() + expect(statuses).toContain("idle") + await Session.remove(session.id) + }, + }) }) test("dispose handler rejects queued callbacks", async () => { From 72145fba3c0f8d2a334f4c8ab2a2623e25079cb1 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Sun, 19 Jul 2026 20:12:07 +0100 Subject: [PATCH 3/5] fix(run): harden usage provenance --- EVENTS.md | 3 +- packages/cli/src/cli/cmd/run.ts | 18 +++-- packages/cli/src/session/index.ts | 7 +- .../cli/test/cli/run-message-terminal.test.ts | 80 +++++++++++++++++-- .../test/cli/usage-token-breakdown.test.ts | 7 +- packages/cli/test/session/compaction.test.ts | 21 +++++ 6 files changed, 114 insertions(+), 22 deletions(-) diff --git a/EVENTS.md b/EVENTS.md index 6d6e439..2b41488 100644 --- a/EVENTS.md +++ b/EVENTS.md @@ -112,7 +112,8 @@ Emitted immediately before `session_complete` when the session terminates abnorm ### `message_complete` -Emitted when an assistant message finishes (one per LLM turn). +Emitted when a primary-session assistant message finishes (one per LLM turn). Child-session assistant messages are +excluded; use the child-session lifecycle events when tracking subagents. ```json { diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index c9fc471..0ecb211 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -508,11 +508,13 @@ export const RunCommand = cmd({ : { total: info.tokens.total ?? - info.tokens.input + - info.tokens.output + - info.tokens.reasoning + - info.tokens.cache.read + - info.tokens.cache.write, + (info.usageStatus === undefined + ? info.tokens.input + + info.tokens.output + + info.tokens.reasoning + + info.tokens.cache.read + + info.tokens.cache.write + : undefined), input: info.tokens.input, output: info.tokens.output, reasoning: info.tokens.reasoning, @@ -531,9 +533,9 @@ export const RunCommand = cmd({ // limit comes from the model registry (models.dev). On lookup failure // (or limit===0 for custom models) buildContextWindow returns null. const contextLimit = await (args.attach - ? ModelsDev.get().then( - (providers) => providers[info.providerID]?.models[info.modelID]?.limit.context ?? null, - ) + ? ModelsDev.get() + .then((providers) => providers[info.providerID]?.models[info.modelID]?.limit.context ?? null) + .catch(() => null) : Provider.getModel(info.providerID, info.modelID) .then((m) => m.limit.context) .catch((e) => { diff --git a/packages/cli/src/session/index.ts b/packages/cli/src/session/index.ts index cd6f3da..49560c0 100644 --- a/packages/cli/src/session/index.ts +++ b/packages/cli/src/session/index.ts @@ -808,7 +808,7 @@ export namespace Session { metadata: z.custom().optional(), }), (input) => { - const safe = (value: number | undefined) => { + const safe = (value: unknown) => { if (typeof value !== "number" || !Number.isFinite(value)) return 0 return value } @@ -833,7 +833,7 @@ export namespace Session { const reasoningTokens = safe(input.usage.reasoningTokens ?? 0) const cacheReadInputTokens = safe(input.usage.cachedInputTokens ?? 0) - const cacheWriteInputTokens = safe(cacheWrite as number | undefined) + const cacheWriteInputTokens = safe(cacheWrite) // Providers where inputTokens EXCLUDES cached tokens const excludesCachedTokens = [ @@ -863,8 +863,7 @@ export namespace Session { return adjustedInputTokens + outputTokens + cacheReadInputTokens + cacheWriteInputTokens } if (Number.isFinite(input.usage.totalTokens)) return input.usage.totalTokens - if (!Number.isFinite(input.usage.inputTokens) || !Number.isFinite(input.usage.outputTokens)) return - return inputTokens + outputTokens + return }) const tokens = { diff --git a/packages/cli/test/cli/run-message-terminal.test.ts b/packages/cli/test/cli/run-message-terminal.test.ts index 0f4b55d..3e552de 100644 --- a/packages/cli/test/cli/run-message-terminal.test.ts +++ b/packages/cli/test/cli/run-message-terminal.test.ts @@ -3,13 +3,14 @@ import { afterEach, describe, expect, test } from "bun:test" const cli = path.resolve(import.meta.dir, "../../src/index.ts") const models = path.resolve(import.meta.dir, "../tool/fixtures/models-api.json") +const eventsDoc = path.resolve(import.meta.dir, "../../../../EVENTS.md") const sessionID = "ses_primary" const servers: Bun.Server[] = [] function message(input: { id: string sessionID?: string - usageStatus: "reported" | "missing" + usageStatus?: "reported" | "missing" status?: "completed" | "error" | "aborted" terminal?: boolean tokens?: { @@ -89,7 +90,17 @@ afterEach(() => { describe("run --format json terminal assistant telemetry (#93, #45)", () => { test("emits one scoped terminal event per message with explicit status and usage provenance", async () => { - const zero = message({ id: "msg_zero", usageStatus: "reported" }) + const zero = message({ + id: "msg_zero", + usageStatus: "reported", + tokens: { + total: 0, + input: 0, + output: 0, + reasoning: 0, + cache: { read: 0, write: 0 }, + }, + }) const partial = { input: 10, output: 4, @@ -113,6 +124,10 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { status: "error", tokens: partial, }), + message({ + id: "msg_legacy", + tokens: partial, + }), message({ id: "msg_child", sessionID: "ses_child", usageStatus: "reported" }), ] const proc = Bun.spawn(["bun", "run", cli, "run", "--format", "json", "--attach", server(events), "test prompt"], { @@ -136,9 +151,15 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { .map((line) => JSON.parse(line)) .filter((event) => event.type === "message_complete") - expect(output.map((event) => event.messageID)).toEqual(["msg_zero", "msg_missing", "msg_aborted", "msg_partial"]) - expect(output.map((event) => event.status)).toEqual(["completed", "error", "aborted", "error"]) - expect(output.map((event) => event.usageStatus)).toEqual(["reported", "missing", "missing", "reported"]) + expect(output.map((event) => event.messageID)).toEqual([ + "msg_zero", + "msg_missing", + "msg_aborted", + "msg_partial", + "msg_legacy", + ]) + expect(output.map((event) => event.status)).toEqual(["completed", "error", "aborted", "error", "completed"]) + expect(output.map((event) => event.usageStatus)).toEqual(["reported", "missing", "missing", "reported", "reported"]) expect(output[0].tokens).toEqual({ total: 0, input: 0, @@ -148,16 +169,63 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { }) expect(output[1].tokens).toBeNull() expect(output[3].tokens).toEqual({ - total: 24, input: 10, output: 4, reasoning: 2, cache: { read: 8, write: 0 }, }) + expect(output[4].tokens.total).toBe(24) expect(output[3].context).toEqual({ used: 18, limit: 204_800, ratio: 18 / 204_800, }) }, 20_000) + + test("degrades to null context when the attached model registry is unavailable", async () => { + const proc = Bun.spawn( + [ + "bun", + "run", + cli, + "run", + "--format", + "json", + "--attach", + server([message({ id: "msg_registry", usageStatus: "reported" })]), + "test prompt", + ], + { + cwd: process.cwd(), + env: { + ...process.env, + AICTRL_MODELS_PATH: path.resolve(import.meta.dir, "missing-models.json"), + AICTRL_MODELS_URL: "http://127.0.0.1:1", + }, + stdout: "pipe", + stderr: "pipe", + }, + ) + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + expect(code, stderr).toBe(0) + const output = stdout + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .find((event) => event.type === "message_complete") + + expect(output.messageID).toBe("msg_registry") + expect(output.context).toBeNull() + }, 20_000) + + test("documents primary-session scope", async () => { + const doc = await Bun.file(eventsDoc).text() + const section = doc.slice(doc.indexOf("### `message_complete`"), doc.indexOf("### `text`")) + expect(section).toContain("primary-session") + expect(section).toContain("Child-session assistant messages are") + }) }) diff --git a/packages/cli/test/cli/usage-token-breakdown.test.ts b/packages/cli/test/cli/usage-token-breakdown.test.ts index 9634b74..c7c4bd8 100644 --- a/packages/cli/test/cli/usage-token-breakdown.test.ts +++ b/packages/cli/test/cli/usage-token-breakdown.test.ts @@ -181,10 +181,11 @@ describe("EVENTS.md documents token breakdown and context (#86)", () => { test("EVENTS.md documents null as the unknown-limit sentinel", async () => { const doc = await Bun.file(EVENTS_MD).text() - const idx = doc.indexOf("message_complete") + const idx = doc.indexOf("### `message_complete`") expect(idx).toBeGreaterThan(-1) - // null sentinel doc is ~1593 chars after message_complete heading; use 2000 window - const section = doc.slice(idx, idx + 2000) + const end = doc.indexOf("### `text`", idx) + expect(end).toBeGreaterThan(idx) + const section = doc.slice(idx, end) // The documented contract: null = context limit not known expect(section).toContain("null") }) diff --git a/packages/cli/test/session/compaction.test.ts b/packages/cli/test/session/compaction.test.ts index c4f69f3..47f7416 100644 --- a/packages/cli/test/session/compaction.test.ts +++ b/packages/cli/test/session/compaction.test.ts @@ -435,6 +435,27 @@ describe("session.getUsage", () => { expect(missing.tokens.total).toBeUndefined() }) + test("omits total when a generic provider reports only partial components", () => { + const model = createModel({ context: 100_000, output: 32_000, npm: "@ai-sdk/openai" }) + const result = Session.getUsage({ + model, + usage: { + inputTokens: 10, + outputTokens: 5, + totalTokens: undefined, + reasoningTokens: 2, + }, + }) + + expect(result.usageStatus).toBe("reported") + expect(result.tokens).toMatchObject({ + input: 10, + output: 5, + reasoning: 2, + }) + expect(result.tokens.total).toBeUndefined() + }) + test("calculates cost correctly", () => { const model = createModel({ context: 100_000, From 69958f8bc9f9c3411b456a8f7c39b42993f89440 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Sun, 19 Jul 2026 20:16:36 +0100 Subject: [PATCH 4/5] test(run): make registry failure deterministic --- packages/cli/src/cli/cmd/run.ts | 12 ++++-- .../cli/test/cli/run-message-terminal.test.ts | 41 ++----------------- 2 files changed, 12 insertions(+), 41 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 0ecb211..3c90347 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -108,6 +108,14 @@ export function buildContextWindow( } } +export async function attachedContextLimit( + providerID: string, + modelID: string, + providers = ModelsDev.get(), +): Promise { + return providers.then((providers) => providers[providerID]?.models[modelID]?.limit.context ?? null).catch(() => null) +} + function glob(info: ToolProps) { const root = info.input.path ?? "" const title = `Glob "${info.input.pattern}"` @@ -533,9 +541,7 @@ export const RunCommand = cmd({ // limit comes from the model registry (models.dev). On lookup failure // (or limit===0 for custom models) buildContextWindow returns null. const contextLimit = await (args.attach - ? ModelsDev.get() - .then((providers) => providers[info.providerID]?.models[info.modelID]?.limit.context ?? null) - .catch(() => null) + ? attachedContextLimit(info.providerID, info.modelID) : Provider.getModel(info.providerID, info.modelID) .then((m) => m.limit.context) .catch((e) => { diff --git a/packages/cli/test/cli/run-message-terminal.test.ts b/packages/cli/test/cli/run-message-terminal.test.ts index 3e552de..6179ce3 100644 --- a/packages/cli/test/cli/run-message-terminal.test.ts +++ b/packages/cli/test/cli/run-message-terminal.test.ts @@ -1,5 +1,6 @@ import path from "path" import { afterEach, describe, expect, test } from "bun:test" +import { attachedContextLimit } from "../../src/cli/cmd/run" const cli = path.resolve(import.meta.dir, "../../src/index.ts") const models = path.resolve(import.meta.dir, "../tool/fixtures/models-api.json") @@ -183,44 +184,8 @@ describe("run --format json terminal assistant telemetry (#93, #45)", () => { }, 20_000) test("degrades to null context when the attached model registry is unavailable", async () => { - const proc = Bun.spawn( - [ - "bun", - "run", - cli, - "run", - "--format", - "json", - "--attach", - server([message({ id: "msg_registry", usageStatus: "reported" })]), - "test prompt", - ], - { - cwd: process.cwd(), - env: { - ...process.env, - AICTRL_MODELS_PATH: path.resolve(import.meta.dir, "missing-models.json"), - AICTRL_MODELS_URL: "http://127.0.0.1:1", - }, - stdout: "pipe", - stderr: "pipe", - }, - ) - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]) - expect(code, stderr).toBe(0) - const output = stdout - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)) - .find((event) => event.type === "message_complete") - - expect(output.messageID).toBe("msg_registry") - expect(output.context).toBeNull() - }, 20_000) + expect(await attachedContextLimit("zai", "glm-4.7", Promise.reject(new Error("registry unavailable")))).toBeNull() + }) test("documents primary-session scope", async () => { const doc = await Bun.file(eventsDoc).text() From 48036a9e68aa405974496110e7a109ca1f6d0047 Mon Sep 17 00:00:00 2001 From: Bulat Yapparov Date: Mon, 20 Jul 2026 08:47:10 +0100 Subject: [PATCH 5/5] refactor(run): clarify terminal usage emission --- packages/cli/src/cli/cmd/run.ts | 67 +++++++++++-------- .../test/cli/usage-token-breakdown.test.ts | 13 ++-- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/packages/cli/src/cli/cmd/run.ts b/packages/cli/src/cli/cmd/run.ts index 3c90347..5d15ce4 100644 --- a/packages/cli/src/cli/cmd/run.ts +++ b/packages/cli/src/cli/cmd/run.ts @@ -17,6 +17,7 @@ import { ModelsDev } from "../../provider/models" import { Agent } from "../../agent/agent" import { PermissionNext } from "../../permission/next" import { Session } from "../../session" +import type { MessageV2 } from "../../session/message-v2" import { SessionPrompt } from "../../session/prompt" import { Config } from "../../config/config" import { GlobalBus } from "../../bus/global" @@ -116,6 +117,38 @@ export async function attachedContextLimit( return providers.then((providers) => providers[providerID]?.models[modelID]?.limit.context ?? null).catch(() => null) } +type TerminalUsageInfo = Pick + +function legacyTotal(info: TerminalUsageInfo) { + if (info.usageStatus !== undefined) return + return ( + info.tokens.input + info.tokens.output + info.tokens.reasoning + info.tokens.cache.read + info.tokens.cache.write + ) +} + +function terminalUsage(info: TerminalUsageInfo) { + const usageStatus = info.usageStatus ?? (info.finish ? "reported" : "missing") + if (usageStatus === "missing") { + return { + usageStatus, + tokens: null, + } + } + return { + usageStatus, + tokens: { + total: info.tokens.total ?? legacyTotal(info), + input: info.tokens.input, + output: info.tokens.output, + reasoning: info.tokens.reasoning, + cache: { + read: info.tokens.cache.read, + write: info.tokens.cache.write, + }, + }, + } +} + function glob(info: ToolProps) { const root = info.input.path ?? "" const title = `Glob "${info.input.pattern}"` @@ -506,31 +539,7 @@ export const RunCommand = cmd({ if (args.format === "json") { if (info.sessionID === sessionID && info.time.completed !== undefined && !emitted.has(info.id)) { emitted.add(info.id) - const usageStatus = info.usageStatus ?? (info.finish ? "reported" : "missing") - // Build 5-way token breakdown mirroring upstream LLM.Usage shape. - // info.tokens already carries the full breakdown from StepFinishPart - // accumulation — reasoning and cache split are not dropped upstream. - const tokens = - usageStatus === "missing" - ? null - : { - total: - info.tokens.total ?? - (info.usageStatus === undefined - ? info.tokens.input + - info.tokens.output + - info.tokens.reasoning + - info.tokens.cache.read + - info.tokens.cache.write - : undefined), - input: info.tokens.input, - output: info.tokens.output, - reasoning: info.tokens.reasoning, - cache: { - read: info.tokens.cache.read, - write: info.tokens.cache.write, - }, - } + const usage = terminalUsage(info) // Context-window utilization: used = input + cache.read + cache.write // (all prompt tokens that occupy the model's context window this turn). @@ -548,7 +557,9 @@ export const RunCommand = cmd({ if (e instanceof Provider.ModelNotFoundError) return null throw e })) - const contextUsed = tokens ? tokens.input + tokens.cache.read + tokens.cache.write : null + const contextUsed = usage.tokens + ? usage.tokens.input + usage.tokens.cache.read + usage.tokens.cache.write + : null const context = contextUsed === null ? null : buildContextWindow(contextLimit, contextUsed) emit("message_complete", { @@ -561,8 +572,8 @@ export const RunCommand = cmd({ // from StepFinishPart. Do NOT use the new step.ended event cost field // which emits cost:0 and is reconciled later (the cost:0 trap). cost: info.cost, - tokens, - usageStatus, + tokens: usage.tokens, + usageStatus: usage.usageStatus, status: info.error?.name === "MessageAbortedError" ? "aborted" : info.error ? "error" : "completed", finish: info.finish, }) diff --git a/packages/cli/test/cli/usage-token-breakdown.test.ts b/packages/cli/test/cli/usage-token-breakdown.test.ts index c7c4bd8..af48818 100644 --- a/packages/cli/test/cli/usage-token-breakdown.test.ts +++ b/packages/cli/test/cli/usage-token-breakdown.test.ts @@ -97,12 +97,13 @@ describe("message_complete emit block shape (source-verified, #86)", () => { // that cannot be covered by pure unit-testing buildContextWindow. const RUN_SRC = path.resolve(import.meta.dir, "../../src/cli/cmd/run.ts") - test("emit block passes tokens with reasoning and cache read/write fields", async () => { + test("terminal usage helper preserves reasoning and cache read/write fields", async () => { const source = await Bun.file(RUN_SRC).text() - const emitIdx = source.indexOf('emit("message_complete"') - expect(emitIdx).toBeGreaterThan(-1) - const blockStart = Math.max(0, emitIdx - 2000) - const block = source.slice(blockStart, emitIdx + 200) + const start = source.indexOf("function terminalUsage(") + const end = source.indexOf("function glob(", start) + expect(start).toBeGreaterThan(-1) + expect(end).toBeGreaterThan(start) + const block = source.slice(start, end) expect(block).toContain("reasoning") expect(block).toContain("cache") expect(block).toContain("read") @@ -154,7 +155,7 @@ describe("message_complete emit block shape (source-verified, #86)", () => { const source = await Bun.file(RUN_SRC).text() const contextUsedIdx = source.indexOf("const contextUsed =") expect(contextUsedIdx).toBeGreaterThan(-1) - const line = source.slice(contextUsedIdx, contextUsedIdx + 100) + const line = source.slice(contextUsedIdx, contextUsedIdx + 160) expect(line).toContain("cache.write") }) })