diff --git a/.changeset/resumable-streams.md b/.changeset/resumable-streams.md new file mode 100644 index 000000000..35b6ebfd7 --- /dev/null +++ b/.changeset/resumable-streams.md @@ -0,0 +1,53 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-client': minor +'@tanstack/ai-durable-stream': minor +--- + +Resumable streams: reconnect to an in-flight SSE **or NDJSON** response without +re-running the provider. + +`toServerSentEventsResponse` and `toHttpResponse` both accept a +`durability: { adapter, batch }` option. The adapter (`StreamDurability`) +records every chunk to an ordered log before delivery and tags each event with +an opaque, adapter-owned offset — an SSE `id:` line, or the `id` of an NDJSON +`{ id, chunk }` envelope (NDJSON has no native event-id). A reconnect +(`Last-Event-ID`) or an explicit `?offset` read replays strictly after that +offset from the log — the lazy provider stream is never iterated on resume. +Producers terminalize the log on cancellation and failure (`RUN_ERROR` append + +- `close()`) and on completion when the source stream emits its own terminal + event (`chat()` always does), so readers are never parked on a dead run. + +Two adapters ship: `memoryStream(request)` in `@tanstack/ai` (process-local, +for development and tests) and the new `@tanstack/ai-durable-stream` package, +a Durable Streams protocol adapter for production backends. + +For the `GET` handler that a reload or a second tab reconnects to, +`resumeServerSentEventsResponse({ adapter })` and `resumeHttpResponse({ adapter })` +replay a run straight from the durability log. They need no producer stream and +return a 400 when the request carries no resume offset. + +On the client, all four HTTP adapters are now resumable — `fetchServerSentEvents`, +`fetchHttpStream`, `xhrServerSentEvents`, and `xhrHttpStream`. Each tracks the +per-event offset, auto-reconnects with `Last-Event-ID`, de-duplicates the +replayed prefix, and exposes `joinRun(runId)` to attach to an in-flight or +finished run from the start (read-only GET with `offset=-1`). Untagged streams +behave exactly as before. A durable run that ends with no terminal event and no +forward progress now throws `DurableStreamIncompleteError` instead of hanging. + +Reconnection and durability are bounded so failures surface rather than hang or +loop: + +- `memoryStream` evicts completed logs after a grace window (unbounded growth + is gone); resuming an expired/unknown run throws, and a from-start join to a + run that never produces fails after `MemoryStreamOptions.firstChunkDeadlineMs`. +- all four HTTP adapters accept `reconnect: { maxAttempts, delayMs }` — a + throttle plus a ceiling on CONSECUTIVE no-progress reconnects (default 5; + forward progress resets it) that fails with the new `StreamReconnectLimitError` + instead of reconnecting endlessly, without penalizing a healthy long-lived run. +- `durableStream` accepts `reconnect: { maxReadFailures, delayMs }` to bound its + read-retry loop, and `server` is now optional when `fetch` is provided (e.g. a + Cloudflare service binding). +- `toServerSentEventsResponse` accepts `debug` to record durability terminal / + close failures server-side, where a replaying joiner cannot observe them. diff --git a/docs/chat/connection-adapters.md b/docs/chat/connection-adapters.md index a67e17e6f..96ff035f9 100644 --- a/docs/chat/connection-adapters.md +++ b/docs/chat/connection-adapters.md @@ -73,13 +73,65 @@ import { useChat, fetchServerSentEvents } from "@tanstack/ai-react"; const { messages } = useChat({ connection: fetchServerSentEvents("/api/chat", { - body: { provider: "openai", model: "gpt-5.1" }, + body: { provider: "openai", model: "gpt-5.5" }, }), }); ``` > **Tip:** `body` and `forwardedProps` populate the same wire field. Use `body` for static defaults, the `forwardedProps` constructor option (or per-`sendMessage` `data`) for dynamic values. Runtime values always win. +### Resumable SSE + +`fetchServerSentEvents` watches SSE `id:` values. If a connection drops after +receiving an id, it reconnects with `Last-Event-ID` and de-duplicates the +replayed prefix. `joinRun(runId)` performs a read-only GET with `offset=-1` and +the run id, replaying an in-flight or finished run from the start. + +The ids only appear when the server passes a durability adapter to +`toServerSentEventsResponse`. They are opaque tokens owned by that adapter; the +chat client does not create, parse, or persist them. Without ids, behavior is +identical to a plain single fetch. See +[Resumable Streams](../resumable-streams/overview). + +Your route needs a `GET` handler alongside `POST` for `joinRun` (second tab or +reload) to work. `POST` handles fresh runs and auto-reconnects (it re-sends the +same body with `Last-Event-ID`); `GET` replays a known run from the start: + +```typescript +import { + chat, + chatParamsFromRequest, + memoryStream, + resumeServerSentEventsResponse, + toServerSentEventsResponse, +} from "@tanstack/ai"; +import { openaiText } from "@tanstack/ai-openai"; + +export async function POST(request: Request) { + const { messages, threadId, runId } = await chatParamsFromRequest(request); + const stream = chat({ adapter: openaiText("gpt-5.5"), messages, threadId, runId }); + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + }); +} + +// joinRun hits GET ?offset=-1&runId=... (replay only, no messages sent). +export async function GET(request: Request) { + return resumeServerSentEventsResponse({ adapter: memoryStream(request) }); +} +``` + +The `GET` handler calls no provider: on a replay the durability adapter's +`resumeFrom()` is non-null (from `?offset`), so the log is replayed instead. +`resumeServerSentEventsResponse` returns a 400 when the request has no resume +offset. Use `resumeHttpResponse` for the NDJSON adapters. + +`fetchHttpStream` and `xhrHttpStream` resume the same way over NDJSON, where the +offset rides in an `{ id, chunk }` envelope (see below) instead of an SSE `id:` +line. Enable it by passing a durability adapter to `toHttpResponse`. +`xhrServerSentEvents` resumes over SSE exactly like `fetchServerSentEvents` +(paired with `toServerSentEventsResponse` and its `id:` lines). + ## HTTP Streaming (NDJSON) For environments that don't speak SSE — some edge runtimes, certain mobile WebViews, or anywhere a proxy strips `text/event-stream` — use raw newline-delimited JSON. The wire format is one JSON `StreamChunk` per line: @@ -92,7 +144,9 @@ const { messages } = useChat({ }); ``` -Server-side, write each chunk as `JSON.stringify(chunk) + "\n"` to the response body. Options (`url`, `headers`, `body`, `fetchClient`, dynamic functions) match `fetchServerSentEvents` exactly. +Server-side, write each chunk as `JSON.stringify(chunk) + "\n"` to the response body (or use `toHttpResponse(stream)`). Options (`url`, `headers`, `body`, `fetchClient`, dynamic functions) match `fetchServerSentEvents` exactly. + +`fetchHttpStream` is also resumable: pass a durability adapter to `toHttpResponse` and each line becomes an `{ id, chunk }` envelope. A dropped connection reconnects with `Last-Event-ID`, de-duplicates the replayed prefix, and `joinRun(runId)` attaches to an existing run. Same guarantees as [Resumable SSE](#resumable-sse), over NDJSON. ## React Native and Expo @@ -129,6 +183,10 @@ const chat = useChat({ }); ``` +Mobile connections drop often, so this is where resumability pays off most. +Both XHR adapters reconnect and `joinRun` when the server adds a durability +adapter. See [Resumable Streams](../resumable-streams/overview). + Use `xhrServerSentEvents()` when your server returns `text/event-stream` via `toServerSentEventsResponse()`: @@ -203,7 +261,7 @@ export const chatFn = createServerFn({ method: "POST" }) .inputValidator((data: { messages: Array }) => data) .handler(({ data }) => toServerSentEventsResponse( - chat({ adapter: openaiText("gpt-5.1"), messages: data.messages }), + chat({ adapter: openaiText("gpt-5.5"), messages: data.messages }), ), ); ``` @@ -292,20 +350,30 @@ function websocketConnection(url: string): SubscribeConnectionAdapter { return { async *subscribe(abortSignal) { - while (!abortSignal?.aborted && !closed) { - const buffered = queue.shift(); - if (buffered !== undefined) { - yield buffered; - continue; - } - const chunk = await new Promise((resolve) => { - pending = resolve; - abortSignal?.addEventListener("abort", () => resolve(null), { - once: true, + // Register the abort listener once (not per-iteration) so it can't + // accumulate on a long-lived socket. + const onAbort = () => deliver(null); + abortSignal?.addEventListener("abort", onAbort, { once: true }); + try { + while (!abortSignal?.aborted) { + // Drain buffered chunks BEFORE honoring `closed`: a burst of messages + // followed by a close event (common within one macrotask) must still + // deliver the queued chunks (including a trailing RUN_FINISHED), + // otherwise the client would hang waiting for a terminal it dropped. + const buffered = queue.shift(); + if (buffered !== undefined) { + yield buffered; + continue; + } + if (closed) return; + const chunk = await new Promise((resolve) => { + pending = resolve; }); - }); - if (chunk === null) return; - yield chunk; + if (chunk === null) return; + yield chunk; + } + } finally { + abortSignal?.removeEventListener("abort", onAbort); } }, diff --git a/docs/config.json b/docs/config.json index cf22278f5..7e37f4cb1 100644 --- a/docs/config.json +++ b/docs/config.json @@ -166,7 +166,8 @@ { "label": "Connection Adapters", "to": "chat/connection-adapters", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-07-17" }, { "label": "Thinking & Reasoning", @@ -180,6 +181,26 @@ } ] }, + { + "label": "Resumable Streams", + "children": [ + { + "label": "Overview", + "to": "resumable-streams/overview", + "addedAt": "2026-07-17" + }, + { + "label": "Advanced", + "to": "resumable-streams/advanced", + "addedAt": "2026-07-17" + }, + { + "label": "Custom Durability Adapter", + "to": "resumable-streams/custom-adapter", + "addedAt": "2026-07-17" + } + ] + }, { "label": "Protocol", "children": [ diff --git a/docs/resumable-streams/advanced.md b/docs/resumable-streams/advanced.md new file mode 100644 index 000000000..640e172b4 --- /dev/null +++ b/docs/resumable-streams/advanced.md @@ -0,0 +1,250 @@ +--- +title: Resumable Streams (Advanced) +id: advanced +description: "The durability contract, terminal and error handling, reconnect tuning, joinRun, Cloudflare deployment, and process-death recovery for resumable streams." +keywords: + - stream durability + - durableStream options + - joinRun + - StreamReconnectLimitError + - DurableStreamIncompleteError + - process death + - cloudflare durable streams +--- + +# Resumable Streams: Advanced + +The [Overview](./overview) covers the common case: pick an adapter, wrap your +response, add a `GET` handler. This page covers the rest. + +## durableStream options + +`durableStream(request, options)` talks to an external +[Durable Streams](https://durablestreams.com) backend: + +```ts +import { + chat, + chatParamsFromRequest, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { durableStream } from '@tanstack/ai-durable-stream' +import { openaiText } from '@tanstack/ai-openai' +// Your token source. +import { getDurableStreamsToken } from './auth' + +const durableOptions = { + server: 'https://streams.example.com', + streamPrefix: 'chat-runs', + headers: async () => ({ + Authorization: `Bearer ${await getDurableStreamsToken()}`, + }), +} + +export async function POST(request: Request) { + const { messages, threadId, runId } = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages, + threadId, + runId, + }) + return toServerSentEventsResponse(stream, { + durability: { adapter: durableStream(request, durableOptions), batch: 32 }, + }) +} +``` + +- `headers` takes a static object for fixed credentials or an async resolver for + rotating tokens. The resolver runs for every create, append, read, and close. +- `batch` controls how many chunks are buffered per log append (default 32). +- The backend must return a non-empty `Stream-Next-Offset` header on create, + append, and close. A missing header fails loudly. The adapter never guesses an + offset. + +## Attaching to a run by id + +Reconnect after a drop is automatic. To attach to a run from the start on +purpose (a second tab, or a full reload where you already know the run id), call +`joinRun`. It performs a read-only `GET` with `offset=-1`, which is why the +server needs the `GET` handler from the Overview. That handler is just +`resumeServerSentEventsResponse({ adapter })` (or `resumeHttpResponse` for +NDJSON): it replays from the log and returns a 400 if the request has no resume +offset. + +```ts +import { fetchServerSentEvents } from '@tanstack/ai-client' + +async function attach(runId: string) { + const connection = fetchServerSentEvents('/api/chat') + for await (const chunk of connection.joinRun(runId)) { + console.log(chunk) + } +} +``` + +All four HTTP adapters (`fetchServerSentEvents`, `fetchHttpStream`, +`xhrServerSentEvents`, `xhrHttpStream`) expose `joinRun`. + +## Completion, stop, and errors + +The producer awaits `close()` on every in-process exit: normal completion, +`stop()` or response cancellation, provider iteration errors, and caught +server-side durability failures. + +Cancellation and provider failure also append a terminal `RUN_ERROR` before +closing, so a reconnecting or joining client sees a terminal instead of hanging. +If appending that terminal or closing fails, the cause is logged server-side by +default (a joiner only ever sees a generic incomplete error, so the server log +is where the real cause lives). Pass `debug` to route it to your own logger: + +```ts +import { memoryStream, toServerSentEventsResponse } from '@tanstack/ai' +import type { StreamChunk } from '@tanstack/ai' + +function respond(request: Request, stream: AsyncIterable) { + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + debug: true, // or { logger } for a custom Logger + }) +} +``` + +A durable source must end with its own terminal event +(`RUN_FINISHED`/`RUN_ERROR`). On normal completion the log is terminalized only +if the source emitted a terminal; without one, a durable consumer reconnects +once, makes no progress, and fails with `DurableStreamIncompleteError`. `chat()` +always emits `RUN_FINISHED`, so this only affects hand-rolled streams. + +## memoryStream in production + +`memoryStream` is for development and single-process deployments. Two reasons it +does not fit production: + +1. The log lives in one process's memory, so a reconnect that lands on a + different worker finds nothing. +2. The producer and the delivery socket are the same process. A mid-stream + client disconnect aborts the producer and writes a terminal `RUN_ERROR` to + the log, so a later reconnect replays the partial content plus that error + rather than resuming a still-running response. + +Completed runs are evicted after a grace window, so resuming an expired or +unknown run fails loudly instead of hanging, and a from-start join to a run that +never produces fails after `firstChunkDeadlineMs`. Live resume of a run that is +still producing after a disconnect needs a backend whose producer outlives the +socket (see [Process death](#process-death)). + +## Reconnection bounding + +A dropped connection resumes from the last offset. A transport error retries as +long as an offset is held, even if that attempt delivered only the replayed +overlap. A durable run that ends cleanly without a terminal and makes no forward +progress fails with `DurableStreamIncompleteError`. Only a non-durable (untagged) +stream that ends cleanly counts as a completed run. The distinction is +deliberate: a clean close means the server ended the response, so a durable +transport must never end an empty long-poll window while the producer is alive. + +The client throttles between attempts and bounds reconnection with `maxAttempts`, +failing with `StreamReconnectLimitError`. The ceiling counts only consecutive +reconnects that deliver no new events; forward progress resets it to zero. A +healthy long run, even one behind a proxy that rolls the socket after every +event, never approaches it. It fires only when a run is genuinely stuck. + +```ts +import { fetchServerSentEvents } from '@tanstack/ai-client' + +function makeConnection() { + return fetchServerSentEvents('/api/chat', { + reconnect: { maxAttempts: 5, delayMs: 250 }, // defaults shown + }) +} +``` + +`durableStream` bounds its own read loop the same way. After a mid-window body +read failure it retries from the last valid position, capping consecutive +failures (`reconnect: { maxReadFailures: 10, delayMs: 250 }`). Normal long-poll +advancement is never throttled. + +## Offset ownership + +`StreamDurability` owns its offset format. Core only passes returned +values back to that adapter and writes them to the wire (an SSE `id:` field, or +the `id` of an NDJSON `{ id, chunk }` envelope). For every appended batch: + +1. core calls `append(chunks)` before forwarding the chunks; +2. the adapter returns exactly one offset per chunk, in order; +3. core rejects missing, extra, empty, or whitespace/CR/LF-containing offsets; +4. a resume reads strictly after the supplied offset. + +Core never derives an offset from an array index and never stamps one onto the +`StreamChunk`. See [Custom Durability Adapter](./custom-adapter) to build your +own. + +## Cloudflare Durable Streams + +[Durable Streams](https://durablestreams.com) ships a Cloudflare Workers plus +Durable Objects backend that speaks this protocol, so `durableStream` talks to +it directly with no new adapter. + +When your TanStack AI endpoint also runs on Workers, reach the backend over a +[service binding](https://developers.cloudflare.com/workers/runtime-apis/bindings/service-bindings/) +instead of a public URL. The adapter's injectable `fetch` routes every request +through the binding, so traffic stays on Cloudflare's network and the binding +(not a bearer token) authorizes the call: + +```ts +import { durableStream } from '@tanstack/ai-durable-stream' + +interface Env { + // A service binding to the deployed Durable Streams Worker. + DURABLE_STREAMS: { fetch: typeof fetch } +} + +function cloudflareAdapter(request: Request, env: Env) { + // No `server` needed: the binding routes by path, so the adapter uses an + // internal placeholder base and only the `/streams/...` path matters. + return durableStream(request, { + streamPrefix: 'chat-runs', + fetch: env.DURABLE_STREAMS.fetch.bind(env.DURABLE_STREAMS), + }) +} +``` + +If the backend runs elsewhere, point `server` at the Worker's public URL +instead: + +```ts +import { durableStream } from '@tanstack/ai-durable-stream' + +function urlAdapter(request: Request) { + return durableStream(request, { + server: 'https://durable-streams.example.workers.dev', + streamPrefix: 'chat-runs', + }) +} +``` + +Running on a Durable Object also satisfies the lease/reaper described under +[Process death](#process-death): a DO alarm can terminalize a run whose producer +died, so a reconnecting client sees a terminal state instead of waiting forever. + +## Process death + +A process that has already terminated cannot run cleanup code, so literal +process death cannot be guaranteed by `finally` or `close()` alone. Production +backends should add a lease/reaper: + +1. the producer acquires or renews a lease while writing; +2. a timer, alarm, or background worker detects expired leases; +3. the reaper records an aborted terminal state and closes the log; +4. readers observe that terminal state instead of waiting forever. + +This belongs to the durability service or deployment, not the in-process +response helper. + +## Delivery is not state + +The durability log replays chunks. It is not a queryable source of truth for +thread messages or conversation history. It answers "what did this run stream?", +not "what has this user said?". Keep authoritative state in your own storage. +See [Persistence](../chat/persistence) for the client-side options. diff --git a/docs/resumable-streams/custom-adapter.md b/docs/resumable-streams/custom-adapter.md new file mode 100644 index 000000000..e73384dcf --- /dev/null +++ b/docs/resumable-streams/custom-adapter.md @@ -0,0 +1,157 @@ +--- +title: Custom Durability Adapter +id: custom-adapter +description: "Back resumable streams with your own store (Redis, Postgres, a queue) by implementing the four-method StreamDurability contract." +keywords: + - custom durability adapter + - StreamDurability + - resumable streams + - redis durable stream + - postgres durable stream + - delivery durability +--- + +# Custom Durability Adapter + +You have a store you want streams to survive on: Redis, Postgres, a queue, +Electric, an object store. By the end of this page you have a `StreamDurability` +adapter that plugs into `toServerSentEventsResponse` / `toHttpResponse`, so a +client can reconnect to an in-flight run without re-running the model. + +Core never understands your store. It only round-trips opaque offset strings you +hand it. You implement four methods: + +| Method | Job | +| --- | --- | +| `resumeFrom()` | Return the resume offset from this request, or `null` for a fresh run. | +| `append(chunks)` | Persist a batch before delivery; return one offset per chunk, in order. | +| `read(offset, signal)` | Replay chunks strictly after `offset`. | +| `close()` | Mark the run complete and wake any parked readers. | + +## The rules that matter + +Get these wrong and resume breaks in subtle ways: + +- **Offsets are opaque, unique, and round-trip-safe.** Return a distinct offset + per chunk. It travels on an SSE `id:` line or inside an NDJSON `{ id, chunk }` + envelope, so it must survive that: core rejects an empty offset, one + containing `NUL`/CR/LF, one with leading or trailing whitespace, or a + duplicate. +- **`read` replays strictly *after* the offset**, oldest first, and stops at the + first `RUN_FINISHED` / `RUN_ERROR`. +- **`read` must never end the response empty while the run is still producing.** + Park (wait for the next append) instead. A clean end with no new data tells + the client the run is over; if it isn't, the client fails with + `DurableStreamIncompleteError`. Honor the abort `signal` so a gone client + stops the wait. +- You do not handle ordering or append-before-deliver. Core buffers, calls + `append`, and only forwards a chunk once you return its offset. + +## Implement it + +Write the adapter against your store's operations. Here it is over an +append-only per-run log you provide; swap `RunLog` for your backend: + +```ts ignore +import { EventType } from '@tanstack/ai' +import type { StreamChunk, StreamDurability } from '@tanstack/ai' + +// Your backend, one append-only log per run. Back it with Redis Streams, a +// Postgres table, a queue. Anything that returns a stable cursor per entry. +interface RunLog { + append: (chunks: Array) => Promise> + readAfter: ( + cursor: string | null, + ) => Promise> + isComplete: () => Promise + waitForChange: (signal?: AbortSignal) => Promise + markComplete: () => Promise +} + +function isTerminal(chunk: StreamChunk): boolean { + return chunk.type === EventType.RUN_FINISHED || chunk.type === EventType.RUN_ERROR +} + +export function customDurability( + request: Request, + openLog: (runId: string) => RunLog, +): StreamDurability { + const url = new URL(request.url) + // The resume offset: native SSE reconnect header first, then a join's ?offset. + const resume = + request.headers.get('Last-Event-ID') ?? url.searchParams.get('offset') + // Your adapter owns run identity. A real backend decodes the runId from the + // resume offset; this example takes it from ?runId or mints a fresh one. + const runId = url.searchParams.get('runId') ?? crypto.randomUUID() + const log = openLog(runId) + + return { + resumeFrom: () => resume, + append: (chunks) => log.append(chunks), + close: () => log.markComplete(), + read: async function* (offset, signal) { + // '-1' / 'now' are the from-start / from-tail join sentinels. + let cursor: string | null = offset === '-1' ? null : offset + for (;;) { + if (signal?.aborted) return + const entries = await log.readAfter(cursor) + for (const entry of entries) { + cursor = entry.cursor + yield { offset: entry.cursor, chunk: entry.chunk } + if (isTerminal(entry.chunk)) return + } + if (await log.isComplete()) return + // Park. Do NOT end the response here while the producer is alive. + await log.waitForChange(signal) + } + }, + } +} +``` + +Wire it up exactly like the built-in adapters: + +```ts +import { chat, chatParamsFromRequest, toServerSentEventsResponse } from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +// Your modules: the adapter above, and your backend's per-run log factory. +import { customDurability } from './durability' +import { openRunLog } from './run-log' + +export async function POST(request: Request) { + const { messages, threadId, runId } = await chatParamsFromRequest(request) + const stream = chat({ adapter: openaiText('gpt-5.5'), messages, threadId, runId }) + return toServerSentEventsResponse(stream, { + durability: { adapter: customDurability(request, openRunLog) }, + }) +} +``` + +For NDJSON, swap `toServerSentEventsResponse` for `toHttpResponse`. The adapter +is identical; only the wire encoding changes. + +## Type your offsets (optional) + +`StreamDurability` is generic over the offset string. Brand it so a +raw string can't be passed where one of your offsets is expected: + +```ts +import type { StreamDurability } from '@tanstack/ai' + +type MyOffset = string & { readonly __brand: 'MyOffset' } + +// Your adapter is then StreamDurability; append/read/resumeFrom all +// speak MyOffset, and a plain string won't type-check where one is expected. +type MyAdapter = StreamDurability +``` + +Core still treats the value as opaque; the brand only tightens your own code. + +## Terminalization is on you + +Core awaits `close()` on every producer exit (normal completion, cancellation, +and failure) and appends a terminal `RUN_ERROR` on cancel/failure before +closing. Your `close()` must make `read`'s `isComplete()` return `true` and wake +parked readers, so a caught-up reader stops rather than hanging. If your backend +producer can die without running `close()` (process crash), add a lease/reaper +that terminalizes abandoned logs. See [Process death](./advanced#process-death). diff --git a/docs/resumable-streams/overview.md b/docs/resumable-streams/overview.md new file mode 100644 index 000000000..01f2b856f --- /dev/null +++ b/docs/resumable-streams/overview.md @@ -0,0 +1,143 @@ +--- +title: Resumable Streams +id: overview +description: "Reconnect to an in-flight AI response without re-running the model. Plug in a durability adapter, add a GET handler, and streams survive refreshes and dropped connections." +keywords: + - resumable streams + - resume stream + - reconnect sse + - reconnect ndjson + - delivery durability + - durable streams + - last-event-id +--- + +# Resumable Streams + +A resumable stream lets a client reconnect to an in-flight response after a page +refresh, a dropped connection, or a suspended tab, without calling the provider +again. + +You turn it on by plugging a **durability adapter** into your streaming +response. The adapter records every chunk to an ordered log before delivery and +tags each event with an opaque offset. On reconnect the client resends the last +offset and the server replays from the log instead of re-running the model. + +Three steps: pick an adapter, wrap your response with it, add a `GET` handler. + +## 1. Pick an adapter + +- `memoryStream(request)` from `@tanstack/ai` keeps the log in process memory. + Zero setup, ideal for development. Single process only. +- `durableStream(request, options)` from `@tanstack/ai-durable-stream` writes to + an external [Durable Streams](https://durablestreams.com) backend. Use this in + production, where requests span many processes. + +Using a different store (Redis, Postgres, a queue)? Implement the four-method +`StreamDurability` interface: see [Custom Durability Adapter](./custom-adapter). + +## 2. Wrap your server response + +Pass the adapter as `durability` to `toServerSentEventsResponse` (SSE) or +`toHttpResponse` (NDJSON). Add a `GET` handler so a reload or a second tab can +re-attach to a run: + +```ts +import { + chat, + chatParamsFromRequest, + memoryStream, + resumeServerSentEventsResponse, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' + +export async function POST(request: Request) { + const { messages, threadId, runId } = await chatParamsFromRequest(request) + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages, + threadId, + runId, + }) + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + }) +} + +export async function GET(request: Request) { + // Replays the run from the durability log. No model call happens here. + return resumeServerSentEventsResponse({ adapter: memoryStream(request) }) +} +``` + +For production, swap `memoryStream(request)` for +`durableStream(request, options)`. Everything else stays the same. + +> **One gotcha:** on a dropped connection the client reconnects by re-sending +> the same `POST`. The model is not re-run (the log is replayed), but any side +> effects your handler runs around the stream (saving the user's message, +> creating a run row, counting usage) would fire a second time. Guard them +> behind a resume check so they only run on a fresh request. + +The adapter already knows whether this is a resume: `resumeFrom()` returns the +offset on a reconnect and `null` on a fresh request. Build the adapter once, +check it, then reuse it for the response: + +```ts +import { + chat, + chatParamsFromRequest, + memoryStream, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' +// Your own one-time side effects. +import { countUsage, saveUserMessage } from './db' + +export async function POST(request: Request) { + const durability = memoryStream(request) + const { messages, threadId, runId } = await chatParamsFromRequest(request) + + // null on a fresh request, non-null on a reconnect. Do one-time work once. + if (durability.resumeFrom() === null) { + await saveUserMessage(threadId, messages) + await countUsage(runId) + } + + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages, + threadId, + runId, + }) + return toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + }) +} +``` + +## 3. Client: nothing to wire + +Reconnect is automatic. Use `useChat` with any HTTP connection adapter and a +dropped connection resumes itself: + +```tsx +import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' + +export function Chat() { + const chat = useChat({ + connection: fetchServerSentEvents('/api/chat'), + }) + + return +} +``` + +For NDJSON, swap `fetchServerSentEvents` for `fetchHttpStream` (with the server +on `toHttpResponse`). The XHR adapters (`xhrServerSentEvents`, `xhrHttpStream`) +work the same way, for runtimes without streaming `fetch`. + +That covers the common case. For the durability contract, terminal and error +handling, reconnect tuning, attaching to a run by id, Cloudflare deployment, and +production process-death concerns, see [Advanced](./advanced). diff --git a/examples/ts-react-chat/src/components/Header.tsx b/examples/ts-react-chat/src/components/Header.tsx index 709c2905e..9c71d1577 100644 --- a/examples/ts-react-chat/src/components/Header.tsx +++ b/examples/ts-react-chat/src/components/Header.tsx @@ -12,12 +12,13 @@ import { Home, Image, Layers, - Menu, LayoutGrid, - Mic, + Menu, MessageSquare, + Mic, Music, Plug, + RefreshCw, Server, Sparkles, Video, @@ -298,6 +299,19 @@ export default function Header() { Server Function Chat + setIsOpen(false)} + className="flex items-center gap-3 p-3 rounded-lg hover:bg-gray-800 transition-colors mb-2" + activeProps={{ + className: + 'flex items-center gap-3 p-3 rounded-lg bg-cyan-600 hover:bg-cyan-700 transition-colors mb-2', + }} + > + + Resumable Streams + + setIsOpen(false)} diff --git a/examples/ts-react-chat/src/routeTree.gen.ts b/examples/ts-react-chat/src/routeTree.gen.ts index 9d3ea3e22..0f05c32c4 100644 --- a/examples/ts-react-chat/src/routeTree.gen.ts +++ b/examples/ts-react-chat/src/routeTree.gen.ts @@ -13,6 +13,7 @@ import { Route as TypesafeToolsRouteImport } from './routes/typesafe-tools' import { Route as ThreadsRouteImport } from './routes/threads' import { Route as ServerFnChatRouteImport } from './routes/server-fn-chat' import { Route as SandboxesRouteImport } from './routes/sandboxes' +import { Route as ResumableRouteImport } from './routes/resumable' import { Route as RealtimeRouteImport } from './routes/realtime' import { Route as QueueingRouteImport } from './routes/queueing' import { Route as McpDemoRouteImport } from './routes/mcp-demo' @@ -38,6 +39,7 @@ import { Route as ApiSummarizeRouteImport } from './routes/api.summarize' import { Route as ApiStructuredOutputRouteImport } from './routes/api.structured-output' import { Route as ApiStructuredChatRouteImport } from './routes/api.structured-chat' import { Route as ApiSandboxTriageRouteImport } from './routes/api.sandbox-triage' +import { Route as ApiResumableRouteImport } from './routes/api.resumable' import { Route as ApiMcpStatusRouteImport } from './routes/api.mcp-status' import { Route as ApiMcpPoolRouteImport } from './routes/api.mcp-pool' import { Route as ApiMcpManualRouteImport } from './routes/api.mcp-manual' @@ -76,6 +78,11 @@ const SandboxesRoute = SandboxesRouteImport.update({ path: '/sandboxes', getParentRoute: () => rootRouteImport, } as any) +const ResumableRoute = ResumableRouteImport.update({ + id: '/resumable', + path: '/resumable', + getParentRoute: () => rootRouteImport, +} as any) const RealtimeRoute = RealtimeRouteImport.update({ id: '/realtime', path: '/realtime', @@ -204,6 +211,11 @@ const ApiSandboxTriageRoute = ApiSandboxTriageRouteImport.update({ path: '/api/sandbox-triage', getParentRoute: () => rootRouteImport, } as any) +const ApiResumableRoute = ApiResumableRouteImport.update({ + id: '/api/resumable', + path: '/api/resumable', + getParentRoute: () => rootRouteImport, +} as any) const ApiMcpStatusRoute = ApiMcpStatusRouteImport.update({ id: '/api/mcp-status', path: '/api/mcp-status', @@ -301,6 +313,7 @@ export interface FileRoutesByFullPath { '/mcp-demo': typeof McpDemoRoute '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -316,6 +329,7 @@ export interface FileRoutesByFullPath { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -349,6 +363,7 @@ export interface FileRoutesByTo { '/mcp-demo': typeof McpDemoRoute '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -364,6 +379,7 @@ export interface FileRoutesByTo { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -398,6 +414,7 @@ export interface FileRoutesById { '/mcp-demo': typeof McpDemoRoute '/queueing': typeof QueueingRoute '/realtime': typeof RealtimeRoute + '/resumable': typeof ResumableRoute '/sandboxes': typeof SandboxesRoute '/server-fn-chat': typeof ServerFnChatRoute '/threads': typeof ThreadsRoute @@ -413,6 +430,7 @@ export interface FileRoutesById { '/api/mcp-manual': typeof ApiMcpManualRoute '/api/mcp-pool': typeof ApiMcpPoolRoute '/api/mcp-status': typeof ApiMcpStatusRoute + '/api/resumable': typeof ApiResumableRoute '/api/sandbox-triage': typeof ApiSandboxTriageRoute '/api/structured-chat': typeof ApiStructuredChatRoute '/api/structured-output': typeof ApiStructuredOutputRoute @@ -448,6 +466,7 @@ export interface FileRouteTypes { | '/mcp-demo' | '/queueing' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -463,6 +482,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -496,6 +516,7 @@ export interface FileRouteTypes { | '/mcp-demo' | '/queueing' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -511,6 +532,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -544,6 +566,7 @@ export interface FileRouteTypes { | '/mcp-demo' | '/queueing' | '/realtime' + | '/resumable' | '/sandboxes' | '/server-fn-chat' | '/threads' @@ -559,6 +582,7 @@ export interface FileRouteTypes { | '/api/mcp-manual' | '/api/mcp-pool' | '/api/mcp-status' + | '/api/resumable' | '/api/sandbox-triage' | '/api/structured-chat' | '/api/structured-output' @@ -593,6 +617,7 @@ export interface RootRouteChildren { McpDemoRoute: typeof McpDemoRoute QueueingRoute: typeof QueueingRoute RealtimeRoute: typeof RealtimeRoute + ResumableRoute: typeof ResumableRoute SandboxesRoute: typeof SandboxesRoute ServerFnChatRoute: typeof ServerFnChatRoute ThreadsRoute: typeof ThreadsRoute @@ -608,6 +633,7 @@ export interface RootRouteChildren { ApiMcpManualRoute: typeof ApiMcpManualRoute ApiMcpPoolRoute: typeof ApiMcpPoolRoute ApiMcpStatusRoute: typeof ApiMcpStatusRoute + ApiResumableRoute: typeof ApiResumableRoute ApiSandboxTriageRoute: typeof ApiSandboxTriageRoute ApiStructuredChatRoute: typeof ApiStructuredChatRoute ApiStructuredOutputRoute: typeof ApiStructuredOutputRoute @@ -661,6 +687,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof SandboxesRouteImport parentRoute: typeof rootRouteImport } + '/resumable': { + id: '/resumable' + path: '/resumable' + fullPath: '/resumable' + preLoaderRoute: typeof ResumableRouteImport + parentRoute: typeof rootRouteImport + } '/realtime': { id: '/realtime' path: '/realtime' @@ -836,6 +869,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiSandboxTriageRouteImport parentRoute: typeof rootRouteImport } + '/api/resumable': { + id: '/api/resumable' + path: '/api/resumable' + fullPath: '/api/resumable' + preLoaderRoute: typeof ApiResumableRouteImport + parentRoute: typeof rootRouteImport + } '/api/mcp-status': { id: '/api/mcp-status' path: '/api/mcp-status' @@ -969,6 +1009,7 @@ const rootRouteChildren: RootRouteChildren = { McpDemoRoute: McpDemoRoute, QueueingRoute: QueueingRoute, RealtimeRoute: RealtimeRoute, + ResumableRoute: ResumableRoute, SandboxesRoute: SandboxesRoute, ServerFnChatRoute: ServerFnChatRoute, ThreadsRoute: ThreadsRoute, @@ -984,6 +1025,7 @@ const rootRouteChildren: RootRouteChildren = { ApiMcpManualRoute: ApiMcpManualRoute, ApiMcpPoolRoute: ApiMcpPoolRoute, ApiMcpStatusRoute: ApiMcpStatusRoute, + ApiResumableRoute: ApiResumableRoute, ApiSandboxTriageRoute: ApiSandboxTriageRoute, ApiStructuredChatRoute: ApiStructuredChatRoute, ApiStructuredOutputRoute: ApiStructuredOutputRoute, diff --git a/examples/ts-react-chat/src/routes/api.resumable.ts b/examples/ts-react-chat/src/routes/api.resumable.ts new file mode 100644 index 000000000..074c7002e --- /dev/null +++ b/examples/ts-react-chat/src/routes/api.resumable.ts @@ -0,0 +1,63 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + chat, + chatParamsFromRequestBody, + memoryStream, + resumeServerSentEventsResponse, + toServerSentEventsResponse, +} from '@tanstack/ai' +import { openaiText } from '@tanstack/ai-openai' + +/** + * Resumable-streams demo endpoint. + * + * The `memoryStream(request)` durability sink records every chunk to an ordered + * log before delivery and tags each SSE event with an opaque `id:` offset, so a + * dropped connection reconnects (`Last-Event-ID`) and resumes from the log + * without re-running the model. The `useChat` client on /resumable needs no + * extra code for this; it just uses this route's connection. + * + * `memoryStream` is process-local, which is fine for a single-process demo. In + * production, swap it for `durableStream(request, { server })` from + * `@tanstack/ai-durable-stream` backed by Cloudflare Durable Streams, Electric, + * or any Durable Streams protocol server. + */ + +export const Route = createFileRoute('/api/resumable')({ + server: { + handlers: { + // Fresh run (no `Last-Event-ID`) → produce with the model and append to + // the log. Reconnect (client re-sends with `Last-Event-ID`) → the sink + // replays strictly after that offset and the lazy `chat()` stream below + // is never iterated. + POST: async ({ request }) => { + const abortController = new AbortController() + const params = await chatParamsFromRequestBody(await request.json()) + + const stream = chat({ + adapter: openaiText('gpt-5.5'), + messages: params.messages, + threadId: params.threadId, + runId: params.runId, + abortController, + }) + + return toServerSentEventsResponse(stream, { + durability: { adapter: memoryStream(request) }, + abortController, + }) + }, + + // Replay a run from the start (`?offset=-1&runId=…`) for an explicit + // `joinRun` (a second tab or a full reload). Read-only: no messages are + // sent and no model is called. This demo relies on automatic POST + // reconnect and does not call joinRun, but a durable route should expose + // this so an attach-by-id resume is possible. + GET: ({ request }) => { + return resumeServerSentEventsResponse({ + adapter: memoryStream(request), + }) + }, + }, + }, +}) diff --git a/examples/ts-react-chat/src/routes/resumable.tsx b/examples/ts-react-chat/src/routes/resumable.tsx new file mode 100644 index 000000000..81cf86f1a --- /dev/null +++ b/examples/ts-react-chat/src/routes/resumable.tsx @@ -0,0 +1,118 @@ +import { createFileRoute } from '@tanstack/react-router' +import { useState } from 'react' +import { fetchServerSentEvents } from '@tanstack/ai-client' +import { useChat } from '@tanstack/ai-react' + +export const Route = createFileRoute('/resumable')({ + component: ResumablePage, +}) + +// A durable connection. The /api/resumable route records every chunk to a +// durability log (see its POST handler) and exposes a GET replay handler, so a +// dropped or rolled-over connection reconnects and resumes the same response +// automatically. Nothing on the client opts in beyond using useChat with it. +const connection = fetchServerSentEvents('/api/resumable') + +function ResumablePage() { + const { messages, sendMessage, isLoading, connectionStatus } = useChat({ + connection, + }) + const [input, setInput] = useState( + 'Write a short haiku about durable streams.', + ) + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault() + const text = input.trim() + if (!text || isLoading) return + setInput('') + void sendMessage(text) + } + + return ( +
+

Resumable streams

+

+ This chat talks to a durability-backed endpoint. Because the server + records each chunk and exposes a GET replay handler, a dropped or + rolled-over connection reconnects and resumes the same response + automatically, with no client code beyond useChat. Send a + message, then kill the network for a moment: the stream picks up where + it left off instead of restarting the model. +

+ +
+ connection: {connectionStatus} +
+ +
+ {messages.map((message) => ( +
+
{message.role}
+ {message.parts.map((part, index) => + part.type === 'text' && part.content ? ( +

+ {part.content} +

+ ) : null, + )} +
+ ))} +
+ +
+ setInput(e.target.value)} + placeholder="Ask something…" + style={{ flex: 1, padding: 8 }} + /> + +
+
+ ) +} + +const page: React.CSSProperties = { + maxWidth: 720, + margin: '0 auto', + padding: 24, + fontFamily: 'system-ui, sans-serif', +} + +const bubble: React.CSSProperties = { + borderRadius: 8, + padding: '10px 14px', + maxWidth: '85%', +} + +const userBubble: React.CSSProperties = { + ...bubble, + alignSelf: 'flex-end', + background: '#eef2ff', +} + +const assistantBubble: React.CSSProperties = { + ...bubble, + alignSelf: 'flex-start', + background: '#f6f6f6', +} + +const roleLabel: React.CSSProperties = { + fontSize: 11, + textTransform: 'uppercase', + letterSpacing: '0.05em', + color: '#999', + marginBottom: 4, +} diff --git a/packages/ai-client/src/connection-adapters.ts b/packages/ai-client/src/connection-adapters.ts index 3c4010047..e8fbb4c68 100644 --- a/packages/ai-client/src/connection-adapters.ts +++ b/packages/ai-client/src/connection-adapters.ts @@ -47,6 +47,103 @@ export class StreamTruncatedError extends Error { } } +class StreamReadError extends Error { + constructor(cause: unknown) { + super('Stream response body read failed', { cause }) + this.name = 'StreamReadError' + } +} + +/** + * Thrown when a durable (id-tagged) run's stream ends with no terminal event + * and a reconnect makes no forward progress — the run cannot complete, so the + * consumer must not be left silently hanging on a stream that just stops. + */ +export class DurableStreamIncompleteError extends Error { + constructor() { + super( + 'Durable run ended without a terminal event and could not resume — the run did not complete.', + ) + this.name = 'DurableStreamIncompleteError' + } +} + +/** + * Thrown when a durable run exceeds its reconnect ceiling. Bounds the + * otherwise-unbounded reconnect loop so a flapping producer (or a proxy that + * rolls the socket after every event) surfaces a failure instead of + * reconnecting without end. + */ +export class StreamReconnectLimitError extends Error { + constructor(attempts: number) { + super( + `Durable run exceeded its reconnect ceiling of ${attempts} attempts — giving up.`, + ) + this.name = 'StreamReconnectLimitError' + } +} + +/** + * Reconnect bounding for resumable streams. A constant throttle delay prevents a + * hot loop against the origin, and the ceiling bounds a pathologically failing + * run — but only counts CONSECUTIVE reconnects that made no forward progress. + */ +export interface ReconnectOptions { + /** + * Ceiling on the number of CONSECUTIVE reconnects that deliver no new events, + * before failing with {@link StreamReconnectLimitError}. The counter resets to + * zero whenever a reconnect makes forward progress, so a healthy long run — + * even one behind a proxy that rolls the socket after every event — never + * approaches it; the ceiling only fires when the run is genuinely stuck + * (reconnecting repeatedly without receiving anything new). Default 5. + */ + maxAttempts?: number + /** Delay between reconnect attempts, in ms, to avoid hammering. Default 250. */ + delayMs?: number +} + +interface ResolvedReconnectOptions { + maxAttempts: number + delayMs: number +} + +function resolveReconnectOptions( + options: ReconnectOptions | undefined, +): ResolvedReconnectOptions { + const maxAttempts = options?.maxAttempts ?? 5 + const delayMs = options?.delayMs ?? 250 + // Reject non-finite / negative bounds up front: a NaN or Infinity maxAttempts + // would make the ceiling ineffective (unbounded reconnects), and a non-finite + // delayMs would remove throttling. Fail loudly on misconfiguration. + if (!Number.isInteger(maxAttempts) || maxAttempts < 0) { + throw new Error( + `Invalid reconnect.maxAttempts: ${maxAttempts}. Must be a non-negative integer.`, + ) + } + if (!Number.isFinite(delayMs) || delayMs < 0) { + throw new Error( + `Invalid reconnect.delayMs: ${delayMs}. Must be a non-negative finite number.`, + ) + } + return { maxAttempts, delayMs } +} + +/** Resolve after `ms`, or immediately once `signal` aborts. Never rejects. */ +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0 || signal?.aborted) return Promise.resolve() + return new Promise((resolve) => { + const onAbort = () => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + function generateRunId(prefix: string): string { return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}` } @@ -88,6 +185,21 @@ function mergeHeaders( return customHeaders } +function withSearchParams(url: string, values: Record): string { + const hashIndex = url.indexOf('#') + const hash = hashIndex === -1 ? '' : url.slice(hashIndex) + const withoutHash = hashIndex === -1 ? url : url.slice(0, hashIndex) + const queryIndex = withoutHash.indexOf('?') + const base = + queryIndex === -1 ? withoutHash : withoutHash.slice(0, queryIndex) + const search = new URLSearchParams( + queryIndex === -1 ? '' : withoutHash.slice(queryIndex + 1), + ) + for (const [key, value] of Object.entries(values)) search.set(key, value) + const query = search.toString() + return `${base}${query.length === 0 ? '' : `?${query}`}${hash}` +} + /** * Read lines from a stream (newline-delimited) */ @@ -100,7 +212,14 @@ async function* readStreamLines( let buffer = '' while (!abortSignal?.aborted) { - const { done, value } = await reader.read() + let result: ReadableStreamReadResult + try { + result = await reader.read() + } catch (error) { + if (abortSignal?.aborted) return + throw new StreamReadError(error) + } + const { done, value } = result if (done) break buffer += decoder.decode(value, { stream: true }) @@ -110,12 +229,22 @@ async function* readStreamLines( buffer = lines.pop() || '' for (const line of lines) { - if (line.trim()) { - yield line + // Strip a trailing CR so a CRLF stream matches the LF path (and the + // XHR reader). Without this an exact-equality check like the `[DONE]` + // sentinel in linesToSSEEvents would miss `data: [DONE]\r`. + const normalized = line.endsWith('\r') ? line.slice(0, -1) : line + if (normalized.trim()) { + yield normalized } } } + // Flush the decoder: a connection cut mid-multibyte-character leaves bytes + // held inside the streaming TextDecoder. Draining them here (as U+FFFD) + // makes the trailing-buffer check below see the incomplete tail and report + // truncation instead of silently swallowing it. + buffer += decoder.decode() + // A non-empty trailing buffer means the connection was cut mid-line. // Surface this as an error so the chat client transitions to 'error' // state instead of silently presenting a partial stream as success. @@ -129,36 +258,71 @@ async function* readStreamLines( } } +/** A parsed stream chunk paired with its adapter-owned delivery offset (if any). */ +interface StreamEvent { + chunk: StreamChunk + id?: string +} + +/** + * Type guard for a durable NDJSON envelope `{ id, chunk }`. NDJSON has no + * native event-id field, so durability rides the offset inside the payload. + * A bare `StreamChunk` always has a top-level `type`, and the envelope never + * does, so the two forms are unambiguous — a non-durable line stays bare. + */ +function isNdjsonEnvelope( + value: unknown, +): value is { id: string; chunk: StreamChunk } { + return ( + typeof value === 'object' && + value !== null && + 'chunk' in value && + 'id' in value && + typeof (value as { id: unknown }).id === 'string' && + !('type' in value) + ) +} + /** - * Yield StreamChunks parsed from an SSE Response body. + * Parse SSE-format lines into stream events, pairing each chunk with the `id:` + * offset of the event it arrived on. Shared by the fetch- and XHR-backed SSE + * adapters so both track delivery offsets identically. * * Accepts either `data: {...}` lines or bare JSON lines. Skips comments * starting with `:` (proxies and CDNs inject these as keepalives) and the - * `event:` / `id:` / `retry:` SSE control fields. A `[DONE]` sentinel is - * treated as a terminal event: a synthesized RUN_FINISHED is yielded using - * the most recent upstream `threadId` / `runId`, ensuring the consumer sees - * a clean terminal event with real correlation ids. + * `event:` / `retry:` SSE control fields. A `[DONE]` sentinel is treated as a + * terminal event: a synthesized RUN_FINISHED is yielded using the most recent + * upstream `threadId` / `runId` (falling back to `fallbackIds`), so the + * consumer sees a clean terminal event with real correlation ids. * * A JSON parse failure throws — the consumer surfaces it as an error. */ -async function* responseToSSEChunks( - response: Response, - abortSignal?: AbortSignal, -): AsyncGenerator { - if (!response.ok) { - throw new Error( - `HTTP error! status: ${response.status} ${response.statusText}`, - ) - } - const reader = getResponseStreamReader(response) +async function* linesToSSEEvents( + lines: AsyncIterable, + fallbackIds?: { threadId?: string; runId?: string }, +): AsyncGenerator { let lastThreadId: string | undefined let lastRunId: string | undefined let lastModel: string | undefined - for await (const line of readStreamLines(reader, abortSignal)) { + let pendingId: string | undefined + for await (const line of lines) { + if (line === 'id' || line.startsWith('id:')) { + // SSE spec: strip a single leading space after the colon, preserve the + // rest verbatim so an opaque adapter offset round-trips exactly (do NOT + // trim, which would mangle a legitimate offset). An empty value is kept as + // '' and resets the resume cursor downstream (see resumableStream). + const rawId = line === 'id' ? '' : line.slice(3) + pendingId = rawId.startsWith(' ') ? rawId.slice(1) : rawId + continue + } + // Assumes the durability wire emits one `id:` immediately followed by one + // `data:` per event (both shipped sinks do). `pendingId` attaches to the + // next data line and is cleared after it; blank-line event boundaries are + // stripped upstream, so a hand-rolled server that emits an id-only event or + // a persistent `id:` across events is not supported here. if ( line.startsWith(':') || line.startsWith('event:') || - line.startsWith('id:') || line.startsWith('retry:') ) { continue @@ -167,13 +331,13 @@ async function* responseToSSEChunks( if (data === '[DONE]') { const synthetic: RunFinishedEvent = { type: EventType.RUN_FINISHED, - threadId: lastThreadId ?? '', - runId: lastRunId ?? '', + threadId: lastThreadId ?? fallbackIds?.threadId ?? '', + runId: lastRunId ?? fallbackIds?.runId ?? '', model: lastModel ?? '', timestamp: Date.now(), finishReason: 'stop', } - yield synthetic + yield { chunk: synthetic } return } const chunk = JSON.parse(data) as StreamChunk @@ -186,10 +350,250 @@ async function* responseToSSEChunks( if ('model' in chunk && typeof chunk.model === 'string') { lastModel = chunk.model } + const id = pendingId + pendingId = undefined + yield { chunk, ...(id !== undefined ? { id } : {}) } + } +} + +/** + * Parse NDJSON-format lines into stream events. Durable streams emit each line + * as an `{ id, chunk }` envelope carrying the delivery offset; non-durable + * streams emit bare chunks. Both are auto-detected (see {@link isNdjsonEnvelope}), + * so an untagged stream behaves exactly as a plain single fetch used to. + */ +async function* linesToNdjsonEvents( + lines: AsyncIterable, +): AsyncGenerator { + for await (const line of lines) { + const parsed = JSON.parse(line) as unknown + if (isNdjsonEnvelope(parsed)) { + yield { chunk: parsed.chunk, id: parsed.id } + } else { + yield { chunk: parsed as StreamChunk } + } + } +} + +function assertResponseOk(response: Response): void { + if (!response.ok) { + throw new Error( + `HTTP error! status: ${response.status} ${response.statusText}`, + ) + } +} + +/** Yield SSE stream events (chunk + offset) from a fetch Response body. */ +async function* responseToSSEEvents( + response: Response, + abortSignal?: AbortSignal, + fallbackIds?: { threadId?: string; runId?: string }, +): AsyncGenerator { + assertResponseOk(response) + const reader = getResponseStreamReader(response) + yield* linesToSSEEvents(readStreamLines(reader, abortSignal), fallbackIds) +} + +/** Yield NDJSON stream events (chunk + offset) from a fetch Response body. */ +async function* responseToNdjsonEvents( + response: Response, + abortSignal?: AbortSignal, +): AsyncGenerator { + assertResponseOk(response) + const reader = getResponseStreamReader(response) + yield* linesToNdjsonEvents(readStreamLines(reader, abortSignal)) +} + +async function* responseToSSEChunks( + response: Response, + abortSignal?: AbortSignal, +): AsyncGenerator { + for await (const { chunk } of responseToSSEEvents(response, abortSignal)) { yield chunk } } +/** + * A re-issuable event source. Given extra headers (a `Last-Event-ID` on a + * reconnect) and an abort signal, it opens the transport and yields stream + * events. {@link resumableStream} calls it once per attempt, so each call MUST + * open a fresh underlying request (a new fetch or a new XHR). + */ +type StreamEventSource = ( + extraHeaders: Record, + abortSignal?: AbortSignal, +) => AsyncIterable + +/** + * Build a fetch-backed {@link StreamEventSource}. `parseResponse` decodes the + * body into events (SSE or NDJSON) — the reconnect engine is identical for both. + */ +function fetchEventSource( + fetchClient: typeof globalThis.fetch, + url: string, + requestInit: RequestInit, + parseResponse: ( + response: Response, + abortSignal?: AbortSignal, + ) => AsyncIterable, +): StreamEventSource { + return async function* (extraHeaders, abortSignal) { + let response: Response + try { + response = await fetchClient(url, { + ...requestInit, + headers: { + ...(requestInit.headers as Record | undefined), + ...extraHeaders, + }, + ...(abortSignal ? { signal: abortSignal } : {}), + }) + } catch (error) { + // A fetch REJECTION (device offline, DNS blip, connection refused) is a + // recoverable transport failure, not a fatal one — surface it as + // StreamReadError so resumableStream retries from the last offset, mirroring + // the XHR path (whose onerror wraps the same way). On a genuine abort this + // wraps the AbortError too, but that's harmless: resumableStream checks + // `abortSignal.aborted` first and returns, so the wrapped error's type is + // never inspected. Without an offset (initial connect / non-durable), it + // still surfaces as a hard failure. + throw new StreamReadError(error) + } + yield* parseResponse(response, abortSignal) + } +} + +/** + * Drive a {@link StreamEventSource} with native-style resumability. Each event's + * adapter-owned delivery offset (its `id`) is remembered; if the connection + * drops or ends before a terminal event, the source is re-opened with a + * `Last-Event-ID` header so the server replays strictly after the last offset. + * Already-seen offsets are de-duped, so an overlapping replay is safe. + * + * When the server does NOT tag events (no durability), no offset is ever seen, + * so no reconnect happens — behaviour is identical to a plain single request. + * This engine is transport-agnostic: fetch/XHR × SSE/NDJSON all share it, the + * only difference being the {@link StreamEventSource} they pass in. + */ +async function* resumableStream( + openEventSource: StreamEventSource, + abortSignal?: AbortSignal, + reconnectOptions?: ReconnectOptions, +): AsyncGenerator { + // Retains every delivered offset for the run's lifetime. Intentionally bounded + // by run length (not evicted): a conforming server replays strictly after the + // acknowledged offset, so this only needs to catch the single boundary event + // on reconnect, but keeping the full set keeps de-dup correct even if a server + // replays a wider overlap. + const seen = new Set() + let lastEventId: string | undefined + const reconnect = resolveReconnectOptions(reconnectOptions) + let reconnectAttempts = 0 + + // Throttle before re-issuing the request, and enforce the total ceiling so a + // producer that keeps dropping after each event is bounded rather than + // reconnecting forever. + // Bound only CONSECUTIVE no-progress reconnects. A reconnect that made forward + // progress resets the counter, so a healthy long run (even one whose socket + // rolls after every event) never approaches the ceiling; it fires only when + // the run is genuinely stuck — reconnecting repeatedly with nothing new. + async function waitBeforeReconnect(madeProgress: boolean): Promise { + if (madeProgress) { + reconnectAttempts = 0 + } else { + reconnectAttempts += 1 + if (reconnectAttempts > reconnect.maxAttempts) { + throw new StreamReconnectLimitError(reconnect.maxAttempts) + } + } + await abortableDelay(reconnect.delayMs, abortSignal) + } + + for (;;) { + if (abortSignal?.aborted) return + const extraHeaders: Record = + lastEventId !== undefined ? { 'Last-Event-ID': lastEventId } : {} + + let sawTerminal = false + let progressed = false + try { + for await (const { chunk, id } of openEventSource( + extraHeaders, + abortSignal, + )) { + if (id !== undefined) { + if (id === '') { + // SSE spec: an empty `id:` resets the resume cursor. Drop the last + // offset and clear the de-dupe set; the chunk itself still delivers. + lastEventId = undefined + seen.clear() + } else { + if (seen.has(id)) continue + seen.add(id) + lastEventId = id + } + } + progressed = true + if (chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR') { + sawTerminal = true + } + yield chunk + if (sawTerminal) return + } + } catch (error) { + if (abortSignal?.aborted) return + // A transport drop is resumable once we hold an offset — retry from it, + // even if THIS attempt made no new progress. A caught-up run whose parked + // long-poll socket drops (or a proxy that drops just after replaying the + // de-duped overlap) is transient, not fatal; the consecutive-no-progress + // ceiling in waitBeforeReconnect already bounds a genuinely stuck flapper, + // so a per-attempt progress requirement here would only convert + // recoverable drops into hard failures on flaky (mobile/edge) networks. + // Without an offset (a non-durable stream), surface the failure. + if ( + (error instanceof StreamTruncatedError || + error instanceof StreamReadError) && + lastEventId !== undefined + ) { + await waitBeforeReconnect(progressed) + continue + } + throw error + } + + if (abortSignal?.aborted) return + + if (lastEventId !== undefined) { + // A durable (id-tagged) run. + if (progressed) { + // Clean end WITHOUT a terminal event but we advanced — the producer is + // still going (or the socket rolled over). Reconnect from the last + // offset (backing off to avoid a hot loop against the origin). Progress + // resets the no-progress ceiling. + await waitBeforeReconnect(true) + continue + } + // Ended without a terminal event AND made no forward progress on this + // pass: the run cannot complete. Surface an error rather than returning + // silently, which would leave the consumer with neither a terminal event + // nor a failure. + // + // Invariant this relies on: a durable transport must never surface an + // empty long-poll window as a CLEAN end while the producer is still + // alive. Both shipped backends honor it — memoryStream parks until data + // or completion, and durableStream keeps one continuous response across + // windows — so this fires only on a genuinely complete-but-unterminated + // log. A custom StreamDurability transport that ends a response empty + // mid-run would trip this; keep the response open until data or terminal. + throw new DurableStreamIncompleteError() + } + + // A non-durable (untagged) stream that ended cleanly. Legitimate — the + // upper layer synthesizes a terminal event. Stop. + return + } +} + /** * Per-send context provided by the chat client to the connection adapter. * The adapter combines this with serialized messages to build a full @@ -221,6 +625,22 @@ export interface ConnectConnectionAdapter { ) => AsyncIterable } +/** + * A {@link ConnectConnectionAdapter} that also supports joining an existing run + * (a second tab, or re-attaching after a full reload) via `joinRun`, replaying + * the ordered stream from the start off the server's delivery-durability sink. + */ +export interface ResumableConnectConnectionAdapter extends ConnectConnectionAdapter { + /** + * Join an in-flight or finished run by id, replaying from the start + * (`?offset=-1`). Read-only — sends no messages. + */ + joinRun: ( + runId: string, + abortSignal?: AbortSignal, + ) => AsyncIterable +} + export interface SubscribeConnectionAdapter { /** * Subscribe to stream chunks. @@ -372,22 +792,30 @@ export function normalizeConnectionAdapter( } } catch (err) { if (!abortSignal?.aborted && !hasTerminalEvent) { - const message = - err instanceof Error ? err.message : 'Unknown error in connect()' - const synthetic: RunErrorEvent = { - type: EventType.RUN_ERROR, - threadId: requireSyntheticId( - upstreamThreadId ?? runContext?.threadId, - 'threadId', - ), - runId: requireSyntheticId( - upstreamRunId ?? runContext?.runId, - 'runId', - ), - timestamp: Date.now(), - message, + // Guard synthesis: requireSyntheticId throws when no id is available, + // and that must not replace the original `err` we are about to + // rethrow. If we can't synthesize a terminal, the real failure still + // surfaces below. + try { + const message = + err instanceof Error ? err.message : 'Unknown error in connect()' + const synthetic: RunErrorEvent = { + type: EventType.RUN_ERROR, + threadId: requireSyntheticId( + upstreamThreadId ?? runContext?.threadId, + 'threadId', + ), + runId: requireSyntheticId( + upstreamRunId ?? runContext?.runId, + 'runId', + ), + timestamp: Date.now(), + message, + } + push(synthetic) + } catch { + // fall through to rethrow the original error } - push(synthetic) } throw err } @@ -404,6 +832,8 @@ export interface FetchConnectionOptions { signal?: AbortSignal body?: Record fetchClient?: typeof globalThis.fetch + /** Bounding for resumable-SSE reconnection (throttle delay + attempt ceiling). */ + reconnect?: ReconnectOptions } /** @@ -415,6 +845,8 @@ export interface XhrConnectionOptions { signal?: AbortSignal body?: Record xhrFactory?: () => XMLHttpRequest + /** Bounding for resumable reconnection (throttle delay + attempt ceiling). */ + reconnect?: ReconnectOptions } type ResolvedConnectionOptions = Pick< @@ -482,7 +914,7 @@ function buildRunAgentInputBody( * const connection = fetchServerSentEvents('/api/chat', async () => ({ * body: { * provider: 'openai', - * model: 'gpt-4o', + * model: 'gpt-5.5', * } * })); * ``` @@ -492,7 +924,7 @@ export function fetchServerSentEvents( options: | FetchConnectionOptions | (() => FetchConnectionOptions | Promise) = {}, -): ConnectConnectionAdapter { +): ResumableConnectConnectionAdapter { return { async *connect(messages, data, abortSignal, runContext) { // Resolve URL and options if they are functions @@ -524,15 +956,76 @@ export function fetchServerSentEvents( // under `exactOptionalPropertyTypes`), so spread it conditionally // rather than passing `undefined` explicitly. const signal = abortSignal || resolvedOptions.signal - const response = await fetchClient(resolvedUrl, { - method: 'POST', - headers: requestHeaders, - body: JSON.stringify(requestBody), - credentials: resolvedOptions.credentials || 'same-origin', - ...(signal ? { signal } : {}), + const requestUrl = runContext?.runId + ? withSearchParams(resolvedUrl, { runId: runContext.runId }) + : resolvedUrl + + // Resumable SSE: if the server tags events with `id:` offsets (delivery + // durability), a dropped/rolled-over connection auto-reconnects with a + // `Last-Event-ID` header and de-dupes the replayed prefix. With no tags, + // this is a single plain fetch. + yield* resumableStream( + fetchEventSource( + fetchClient, + requestUrl, + { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(requestBody), + credentials: resolvedOptions.credentials || 'same-origin', + }, + // Thread the run's ids so a `[DONE]`-terminating server that doesn't + // stamp them onto events still yields a correlated terminal (parity + // with the XHR adapter's xhrSSEParser). + (response, sseSignal) => + responseToSSEEvents(response, sseSignal, { + ...(runContext?.threadId !== undefined + ? { threadId: runContext.threadId } + : {}), + ...(runContext?.runId !== undefined + ? { runId: runContext.runId } + : {}), + }), + ), + signal, + resolvedOptions.reconnect, + ) + }, + async *joinRun(runId, abortSignal) { + // Read an in-flight or finished run from the start. `?offset=-1` tells the + // server's delivery-durability sink to replay from the beginning; `runId` + // identifies which run. This is a read-only GET — no messages are sent. + const resolvedUrl = typeof url === 'function' ? url() : url + const resolvedOptions = + typeof options === 'function' ? await options() : options + + const joinUrl = withSearchParams(resolvedUrl, { + offset: '-1', + runId, }) - yield* responseToSSEChunks(response, abortSignal) + const requestHeaders: Record = { + ...mergeHeaders(resolvedOptions.headers), + } + const fetchClient = resolvedOptions.fetchClient ?? fetch + const signal = abortSignal || resolvedOptions.signal + + yield* resumableStream( + fetchEventSource( + fetchClient, + joinUrl, + { + method: 'GET', + headers: requestHeaders, + credentials: resolvedOptions.credentials || 'same-origin', + }, + // A `[DONE]` during a join correlates to the joined run id. + (response, sseSignal) => + responseToSSEEvents(response, sseSignal, { runId }), + ), + signal, + resolvedOptions.reconnect, + ) }, } } @@ -566,7 +1059,7 @@ export function fetchServerSentEvents( * const connection = fetchHttpStream('/api/chat', async () => ({ * body: { * provider: 'openai', - * model: 'gpt-4o', + * model: 'gpt-5.5', * } * })); * ``` @@ -576,7 +1069,7 @@ export function fetchHttpStream( options: | FetchConnectionOptions | (() => FetchConnectionOptions | Promise) = {}, -): ConnectConnectionAdapter { +): ResumableConnectConnectionAdapter { return { async *connect(messages, data, abortSignal, runContext) { // Resolve URL and options if they are functions @@ -608,26 +1101,60 @@ export function fetchHttpStream( // under `exactOptionalPropertyTypes`), so spread it conditionally // rather than passing `undefined` explicitly. const signal = abortSignal || resolvedOptions.signal - const response = await fetchClient(resolvedUrl, { - method: 'POST', - headers: requestHeaders, - body: JSON.stringify(requestBody), - credentials: resolvedOptions.credentials || 'same-origin', - ...(signal ? { signal } : {}), - }) + const requestUrl = runContext?.runId + ? withSearchParams(resolvedUrl, { runId: runContext.runId }) + : resolvedUrl + + // Resumable NDJSON: if the server envelopes each line with an + // `{ id, chunk }` offset (delivery durability), a dropped/rolled-over + // connection auto-reconnects with a `Last-Event-ID` header and de-dupes + // the replayed prefix. With bare lines (no durability), this is a single + // plain fetch — identical to before. + yield* resumableStream( + fetchEventSource( + fetchClient, + requestUrl, + { + method: 'POST', + headers: requestHeaders, + body: JSON.stringify(requestBody), + credentials: resolvedOptions.credentials || 'same-origin', + }, + responseToNdjsonEvents, + ), + signal, + resolvedOptions.reconnect, + ) + }, + async *joinRun(runId, abortSignal) { + // Read an in-flight or finished run from the start. `?offset=-1` tells the + // server's delivery-durability sink to replay from the beginning; `runId` + // identifies which run. This is a read-only GET — no messages are sent. + const resolvedUrl = typeof url === 'function' ? url() : url + const resolvedOptions = + typeof options === 'function' ? await options() : options - if (!response.ok) { - throw new Error( - `HTTP error! status: ${response.status} ${response.statusText}`, - ) + const joinUrl = withSearchParams(resolvedUrl, { offset: '-1', runId }) + const requestHeaders: Record = { + ...mergeHeaders(resolvedOptions.headers), } + const fetchClient = resolvedOptions.fetchClient ?? fetch + const signal = abortSignal || resolvedOptions.signal - // Parse raw HTTP stream (newline-delimited JSON) - const reader = getResponseStreamReader(response) - - for await (const line of readStreamLines(reader, abortSignal)) { - yield JSON.parse(line) as StreamChunk - } + yield* resumableStream( + fetchEventSource( + fetchClient, + joinUrl, + { + method: 'GET', + headers: requestHeaders, + credentials: resolvedOptions.credentials || 'same-origin', + }, + responseToNdjsonEvents, + ), + signal, + resolvedOptions.reconnect, + ) }, } } @@ -705,7 +1232,10 @@ function readXhrLines( const finish = () => { enqueueDelta() - if (xhr.status < 200 || xhr.status >= 300) { + // Tolerate a transient status === 0 (matches enqueueDelta): a real non-2xx + // is an error, but status 0 here is not — treat the trailing buffer as a + // truncation check instead of fabricating a bogus "status: 0" error. + if (xhr.status !== 0 && (xhr.status < 200 || xhr.status >= 300)) { error = new Error(`XHR error! status: ${xhr.status} ${xhr.statusText}`) } else if (buffer.trim() && !aborted) { error = new StreamTruncatedError() @@ -720,7 +1250,10 @@ function readXhrLines( } xhr.onload = finish xhr.onerror = () => { - error = new Error('XHR request failed') + // Surface as StreamReadError so a durable (id-tagged) run whose socket + // drops mid-stream is eligible for auto-reconnect, matching the fetch path. + // A non-durable run has no offset, so resumableStream rethrows it as-is. + error = new StreamReadError(new Error('XHR request failed')) done = true wake() } @@ -787,9 +1320,11 @@ function createConfiguredXhrRequest( messages: Array | Array, data: Record | undefined, runContext: RunAgentInputContext | undefined, + method: string = 'POST', + extraHeaders: Record = {}, ): ConfiguredXhrRequest { const xhr = options.xhrFactory?.() ?? createDefaultXMLHttpRequest() - xhr.open('POST', url) + xhr.open(method, url) if (options.withCredentials !== undefined) { xhr.withCredentials = options.withCredentials } @@ -797,6 +1332,8 @@ function createConfiguredXhrRequest( const requestHeaders: Record = { 'Content-Type': 'application/json', ...mergeHeaders(options.headers), + // Reconnect offset (`Last-Event-ID`) wins over static headers. + ...extraHeaders, } for (const [name, value] of Object.entries(requestHeaders)) { @@ -819,105 +1356,170 @@ async function resolveXhrConnectionOptions( return typeof options === 'function' ? await options() : options } +/** + * Build an XHR-backed {@link StreamEventSource}. `parseLines` decodes the raw + * newline-delimited body into events (SSE or NDJSON); the reconnect engine is + * shared with the fetch adapters. A fresh XHR is opened per attempt, so a + * `Last-Event-ID` reconnect header (via `extraHeaders`) is applied at open time. + */ +function xhrEventSource( + url: string, + options: XhrConnectionOptions, + method: string, + messages: Array | Array, + data: Record | undefined, + runContext: RunAgentInputContext | undefined, + parseLines: (lines: AsyncIterable) => AsyncIterable, +): StreamEventSource { + return async function* (extraHeaders, abortSignal) { + const request = createConfiguredXhrRequest( + url, + options, + messages, + data, + runContext, + method, + extraHeaders, + ) + const lines = readXhrLines(request.xhr, abortSignal) + if (abortSignal?.aborted) { + await lines.next() + return + } + // A read-only join is a bodyless GET; a run POSTs the RunAgentInput payload. + request.xhr.send(method === 'GET' ? null : request.body) + try { + yield* parseLines(lines) + } finally { + // Tear the socket down on an early exit (terminal reached or reconnect + // break) so late bytes stop downloading. When the abort signal fired, + // `readXhrLines` already aborted — skip here to avoid a double abort(). + if (!abortSignal?.aborted) request.xhr.abort() + } + } +} + +/** SSE line parser bound to the run's ids for a `[DONE]` fallback. */ +function xhrSSEParser(runContext: RunAgentInputContext | undefined) { + const fallbackIds: { threadId?: string; runId?: string } = { + ...(runContext?.threadId !== undefined + ? { threadId: runContext.threadId } + : {}), + ...(runContext?.runId !== undefined ? { runId: runContext.runId } : {}), + } + return (lines: AsyncIterable) => linesToSSEEvents(lines, fallbackIds) +} + /** * Create an XMLHttpRequest-backed Server-Sent Events connection adapter. + * + * Resumable: against a durable (`id:`-tagged) server response, a dropped socket + * auto-reconnects with `Last-Event-ID` and de-dupes the replayed prefix, and + * `joinRun` attaches to an existing run from the start. A non-durable response + * is a single plain request, exactly as before. */ export function xhrServerSentEvents( url: string | (() => string), options: XhrConnectionOptionsResolver = {}, -): ConnectConnectionAdapter { +): ResumableConnectConnectionAdapter { return { async *connect(messages, data, abortSignal, runContext) { const resolvedUrl = typeof url === 'function' ? url() : url const resolvedOptions = await resolveXhrConnectionOptions(options) const signal = abortSignal || resolvedOptions.signal - const request = createConfiguredXhrRequest( - resolvedUrl, - resolvedOptions, - messages, - data, - runContext, + const requestUrl = runContext?.runId + ? withSearchParams(resolvedUrl, { runId: runContext.runId }) + : resolvedUrl + yield* resumableStream( + xhrEventSource( + requestUrl, + resolvedOptions, + 'POST', + messages, + data, + runContext, + xhrSSEParser(runContext), + ), + signal, + resolvedOptions.reconnect, + ) + }, + async *joinRun(runId, abortSignal) { + const resolvedUrl = typeof url === 'function' ? url() : url + const resolvedOptions = await resolveXhrConnectionOptions(options) + const signal = abortSignal || resolvedOptions.signal + const joinUrl = withSearchParams(resolvedUrl, { offset: '-1', runId }) + yield* resumableStream( + xhrEventSource( + joinUrl, + resolvedOptions, + 'GET', + [], + undefined, + undefined, + // A `[DONE]` during a join correlates to the joined run id (parity + // with fetchServerSentEvents.joinRun). + (lines) => linesToSSEEvents(lines, { runId }), + ), + signal, + resolvedOptions.reconnect, ) - const lines = readXhrLines(request.xhr, signal) - if (signal?.aborted) { - await lines.next() - return - } - request.xhr.send(request.body) - let lastThreadId: string | undefined - let lastRunId: string | undefined - let lastModel: string | undefined - - for await (const line of lines) { - if ( - line.startsWith(':') || - line.startsWith('event:') || - line.startsWith('id:') || - line.startsWith('retry:') - ) { - continue - } - - const chunkData = parseSseDataLine(line) - if (chunkData === '[DONE]') { - const synthetic: RunFinishedEvent = { - type: EventType.RUN_FINISHED, - threadId: lastThreadId ?? runContext?.threadId ?? '', - runId: lastRunId ?? runContext?.runId ?? '', - model: lastModel ?? '', - timestamp: Date.now(), - finishReason: 'stop', - } - request.xhr.abort() - yield synthetic - return - } - - const chunk = JSON.parse(chunkData) as StreamChunk - if ('threadId' in chunk && typeof chunk.threadId === 'string') { - lastThreadId = chunk.threadId - } - if ('runId' in chunk && typeof chunk.runId === 'string') { - lastRunId = chunk.runId - } - if ('model' in chunk && typeof chunk.model === 'string') { - lastModel = chunk.model - } - yield chunk - } }, } } /** * Create an XMLHttpRequest-backed newline-delimited JSON stream adapter. + * + * Resumable: against a durable (envelope-tagged) server response, a dropped + * socket auto-reconnects with `Last-Event-ID` and de-dupes the replayed prefix, + * and `joinRun` attaches to an existing run from the start. A non-durable + * (bare-line) response is a single plain request, exactly as before. */ export function xhrHttpStream( url: string | (() => string), options: XhrConnectionOptionsResolver = {}, -): ConnectConnectionAdapter { +): ResumableConnectConnectionAdapter { return { async *connect(messages, data, abortSignal, runContext) { const resolvedUrl = typeof url === 'function' ? url() : url const resolvedOptions = await resolveXhrConnectionOptions(options) const signal = abortSignal || resolvedOptions.signal - const request = createConfiguredXhrRequest( - resolvedUrl, - resolvedOptions, - messages, - data, - runContext, + const requestUrl = runContext?.runId + ? withSearchParams(resolvedUrl, { runId: runContext.runId }) + : resolvedUrl + yield* resumableStream( + xhrEventSource( + requestUrl, + resolvedOptions, + 'POST', + messages, + data, + runContext, + linesToNdjsonEvents, + ), + signal, + resolvedOptions.reconnect, + ) + }, + async *joinRun(runId, abortSignal) { + const resolvedUrl = typeof url === 'function' ? url() : url + const resolvedOptions = await resolveXhrConnectionOptions(options) + const signal = abortSignal || resolvedOptions.signal + const joinUrl = withSearchParams(resolvedUrl, { offset: '-1', runId }) + yield* resumableStream( + xhrEventSource( + joinUrl, + resolvedOptions, + 'GET', + [], + undefined, + undefined, + linesToNdjsonEvents, + ), + signal, + resolvedOptions.reconnect, ) - const lines = readXhrLines(request.xhr, signal) - if (signal?.aborted) { - await lines.next() - return - } - request.xhr.send(request.body) - - for await (const line of lines) { - yield JSON.parse(line) as StreamChunk - } }, } } diff --git a/packages/ai-client/src/index.ts b/packages/ai-client/src/index.ts index 705be1c15..41605186c 100644 --- a/packages/ai-client/src/index.ts +++ b/packages/ai-client/src/index.ts @@ -95,9 +95,13 @@ export { stream, rpcStream, StreamTruncatedError, + DurableStreamIncompleteError, + StreamReconnectLimitError, type ConnectConnectionAdapter, type ConnectionAdapter, type FetchConnectionOptions, + type ReconnectOptions, + type ResumableConnectConnectionAdapter, type RunAgentInputContext, type SubscribeConnectionAdapter, type XhrConnectionOptions, diff --git a/packages/ai-client/tests/connection-adapters-resumable-transports.test.ts b/packages/ai-client/tests/connection-adapters-resumable-transports.test.ts new file mode 100644 index 000000000..2aa08fda9 --- /dev/null +++ b/packages/ai-client/tests/connection-adapters-resumable-transports.test.ts @@ -0,0 +1,367 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventType } from '@tanstack/ai/client' +import { + fetchHttpStream, + xhrHttpStream, + xhrServerSentEvents, +} from '../src/connection-adapters' +import type { StreamChunk } from '@tanstack/ai/client' + +/** + * Resumability across the NON-SSE-fetch transports. The reconnect engine + * (offset tracking, de-dupe, Last-Event-ID resend, terminal detection, ceiling) + * is exercised exhaustively over fetch-SSE in `connection-adapters-resumable`. + * All four adapters share that engine, so these focus on the two things that + * differ per transport: the NDJSON `{ id, chunk }` envelope wire format, and the + * XHR transport re-issuing a fresh request with `Last-Event-ID` on reconnect. + */ + +function contentChunk(delta: string): object { + return { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'm', + model: 'test', + timestamp: 0, + delta, + content: delta, + } +} + +function finishedChunk(): object { + return { + type: EventType.RUN_FINISHED, + threadId: 't', + runId: 'r', + model: 'test', + timestamp: 0, + finishReason: 'stop', + } +} + +/** A durable NDJSON line: an `{ id, chunk }` envelope carrying the offset. */ +function envelopeLine(id: string, chunk: object): string { + return `${JSON.stringify({ id, chunk })}\n` +} + +/** A non-durable NDJSON line: a bare chunk, no offset. */ +function bareLine(chunk: object): string { + return `${JSON.stringify(chunk)}\n` +} + +function ndjsonResponse(body: string): Response { + return new Response(body, { + headers: { 'Content-Type': 'application/x-ndjson' }, + }) +} + +function deltasOf(chunks: Array): Array { + return chunks + .filter((c) => c.type === EventType.TEXT_MESSAGE_CONTENT) + .map((c) => c.delta) +} + +describe('resumable NDJSON (fetchHttpStream)', () => { + it('reconnects with Last-Event-ID and de-dupes already-seen chunks', async () => { + const fetchClient = vi.fn(async (url, init) => { + expect(String(url)).toBe('/api/chat?runId=r') + if (fetchClient.mock.calls.length === 1) { + // First response: 3 enveloped lines, then a clean end with no terminal. + return ndjsonResponse( + envelopeLine('run@1', contentChunk('1')) + + envelopeLine('run@2', contentChunk('2')) + + envelopeLine('run@3', contentChunk('3')), + ) + } + // Reconnect: server replays seq 3 (must be de-duped), then tail + terminal. + expect(new Headers(init?.headers).get('Last-Event-ID')).toBe('run@3') + return ndjsonResponse( + envelopeLine('run@3', contentChunk('3')) + + envelopeLine('run@4', contentChunk('4')) + + envelopeLine('run@5', finishedChunk()), + ) + }) + + const adapter = fetchHttpStream('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect(deltasOf(chunks)).toEqual(['1', '2', '3', '4']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('joinRun opens the stream from the start with ?offset=-1', async () => { + const fetchClient = vi.fn(async () => + ndjsonResponse(envelopeLine('run@1', finishedChunk())), + ) + const adapter = fetchHttpStream('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.joinRun('run-x')) { + chunks.push(chunk) + } + + const calledUrl = String(fetchClient.mock.calls[0]![0]) + expect(calledUrl).toContain('offset=-1') + expect(calledUrl).toContain('runId=run-x') + expect(chunks.map((c) => c.type)).toContain(EventType.RUN_FINISHED) + }) + + it('treats a bare-line (non-durable) stream as a single fetch with no reconnect', async () => { + const fetchClient = vi.fn(async () => + ndjsonResponse( + bareLine(contentChunk('1')) + + bareLine(contentChunk('2')) + + bareLine(finishedChunk()), + ), + ) + const adapter = fetchHttpStream('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect(deltasOf(chunks)).toEqual(['1', '2']) + // No offsets were ever seen, so a clean end is final — no reconnect. + expect(fetchClient).toHaveBeenCalledTimes(1) + }) +}) + +type XhrEventHandler = ((event: ProgressEvent) => void) | null + +/** A push-driven fake XHR sufficient for the resumable transport tests. */ +class FakeXhr { + method: string | undefined + url: string | undefined + responseText = '' + status = 200 + statusText = 'OK' + withCredentials = false + onprogress: XhrEventHandler = null + onload: XhrEventHandler = null + onerror: XhrEventHandler = null + onabort: XhrEventHandler = null + onloadend: XhrEventHandler = null + readonly requestHeaders: Record = {} + readonly abort = vi.fn(() => { + this.onabort?.({ type: 'abort' } as ProgressEvent) + this.onloadend?.({ type: 'loadend' } as ProgressEvent) + }) + readonly send = vi.fn() + + open(method: string, url: string): void { + this.method = method + this.url = url + } + + setRequestHeader(name: string, value: string): void { + this.requestHeaders[name] = value + } + + progress(text: string): void { + this.responseText += text + this.onprogress?.({ type: 'progress' } as ProgressEvent) + } + + /** Clean end of the current response (socket rolled over, no terminal). */ + end(): void { + this.onload?.({ type: 'load' } as ProgressEvent) + this.onloadend?.({ type: 'loadend' } as ProgressEvent) + } + + /** A transport error mid-stream (socket dropped). */ + error(): void { + this.onerror?.({ type: 'error' } as ProgressEvent) + this.onloadend?.({ type: 'loadend' } as ProgressEvent) + } +} + +/** A factory that hands out a fresh FakeXhr per open, tracked in `xhrs`. */ +function createXhrQueue(): { + xhrFactory: () => XMLHttpRequest + xhrs: Array +} { + const xhrs: Array = [] + return { + xhrs, + xhrFactory: () => { + const xhr = new FakeXhr() + xhrs.push(xhr) + return xhr as unknown as XMLHttpRequest + }, + } +} + +/** + * Yield to the macrotask queue so the reconnect loop's microtask chain (read → + * de-dupe → throttle → re-open) fully settles and the next XHR is created. + */ +async function flush(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)) +} + +describe('resumable XHR transports', () => { + it('xhrHttpStream reconnects with Last-Event-ID across a fresh XHR (NDJSON envelopes)', async () => { + const queue = createXhrQueue() + const adapter = xhrHttpStream('/api/chat', { + xhrFactory: queue.xhrFactory, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + })() + + await flush() + const first = queue.xhrs[0]! + expect(first.url).toBe('/api/chat?runId=r') + first.progress( + envelopeLine('run@1', contentChunk('1')) + + envelopeLine('run@2', contentChunk('2')), + ) + first.end() // clean end, no terminal → reconnect + + await flush() + const second = queue.xhrs[1]! + expect(second.requestHeaders['Last-Event-ID']).toBe('run@2') + second.progress( + envelopeLine('run@2', contentChunk('2')) + // replayed, de-duped + envelopeLine('run@3', contentChunk('3')) + + envelopeLine('run@4', finishedChunk()), + ) + second.end() + + await done + expect(deltasOf(chunks)).toEqual(['1', '2', '3']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + expect(queue.xhrs).toHaveLength(2) + }) + + it('xhrHttpStream reconnects after an onerror socket drop (held offset)', async () => { + // Proves the full XHR error→reconnect chain: readXhrLines.onerror wraps the + // failure as StreamReadError, which reaches resumableStream's retry branch + // and re-issues with Last-Event-ID (fetch/XHR reconnect parity). + const queue = createXhrQueue() + const adapter = xhrHttpStream('/api/chat', { + xhrFactory: queue.xhrFactory, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + })() + + await flush() + queue.xhrs[0]!.progress(envelopeLine('run@1', contentChunk('1'))) + queue.xhrs[0]!.error() // socket drops after delivering run@1 + + await flush() + expect(queue.xhrs[1]!.requestHeaders['Last-Event-ID']).toBe('run@1') + queue.xhrs[1]!.progress( + envelopeLine('run@2', contentChunk('2')) + + envelopeLine('run@3', finishedChunk()), + ) + queue.xhrs[1]!.end() + + await done + expect(deltasOf(chunks)).toEqual(['1', '2']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + expect(queue.xhrs).toHaveLength(2) + }) + + it('xhrServerSentEvents is resumable too (durable SSE over XHR)', async () => { + const queue = createXhrQueue() + const adapter = xhrServerSentEvents('/api/chat', { + xhrFactory: queue.xhrFactory, + reconnect: { delayMs: 0 }, + }) + + const sse = (id: string, chunk: object): string => + `id: ${id}\ndata: ${JSON.stringify(chunk)}\n\n` + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + })() + + await flush() + queue.xhrs[0]!.progress(sse('run@1', contentChunk('1'))) + queue.xhrs[0]!.end() + + await flush() + expect(queue.xhrs[1]!.requestHeaders['Last-Event-ID']).toBe('run@1') + queue.xhrs[1]!.progress( + sse('run@1', contentChunk('1')) + sse('run@2', finishedChunk()), + ) + queue.xhrs[1]!.end() + + await done + expect(deltasOf(chunks)).toEqual(['1']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + }) + + it('xhrHttpStream joinRun issues a GET from the start with ?offset=-1', async () => { + const queue = createXhrQueue() + const adapter = xhrHttpStream('/api/chat', { + xhrFactory: queue.xhrFactory, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.joinRun('run-x')) { + chunks.push(chunk) + } + })() + + await flush() + const xhr = queue.xhrs[0]! + expect(xhr.method).toBe('GET') + expect(xhr.url).toContain('offset=-1') + expect(xhr.url).toContain('runId=run-x') + xhr.progress(envelopeLine('run@1', finishedChunk())) + xhr.end() + + await done + expect(chunks.map((c) => c.type)).toContain(EventType.RUN_FINISHED) + }) +}) diff --git a/packages/ai-client/tests/connection-adapters-resumable.test.ts b/packages/ai-client/tests/connection-adapters-resumable.test.ts new file mode 100644 index 000000000..4d3beb434 --- /dev/null +++ b/packages/ai-client/tests/connection-adapters-resumable.test.ts @@ -0,0 +1,575 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventType } from '@tanstack/ai/client' +import { + DurableStreamIncompleteError, + StreamReconnectLimitError, + fetchServerSentEvents, +} from '../src/connection-adapters' +import type { StreamChunk } from '@tanstack/ai/client' + +function sseResponse(body: string): Response { + return new Response(body, { + headers: { 'Content-Type': 'text/event-stream' }, + }) +} + +function failingSseResponse(body: string, error: Error): Response { + const bytes = new TextEncoder().encode(body) + let sent = false + return new Response( + new ReadableStream( + { + pull(controller) { + if (!sent) { + sent = true + controller.enqueue(bytes) + return + } + controller.error(error) + }, + }, + { highWaterMark: 0 }, + ), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) +} + +function contentEvent(id: string, delta: string): string { + const chunk = { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'm', + model: 'test', + timestamp: 0, + delta, + content: delta, + } + return `id: ${id}\ndata: ${JSON.stringify(chunk)}\n\n` +} + +function finishedEvent(id: string): string { + const chunk = { + type: EventType.RUN_FINISHED, + threadId: 't', + runId: 'r', + model: 'test', + timestamp: 0, + finishReason: 'stop', + } + return `id: ${id}\ndata: ${JSON.stringify(chunk)}\n\n` +} + +/** An event carrying an EMPTY `id:` (SSE reset), plus its data line. */ +function emptyIdEvent(delta: string): string { + const chunk = { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'm', + model: 'test', + timestamp: 0, + delta, + content: delta, + } + return `id:\ndata: ${JSON.stringify(chunk)}\n\n` +} + +describe('resumable SSE connection adapter', () => { + it('reconnects with Last-Event-ID and de-dupes already-seen chunks', async () => { + const fetchClient = vi.fn(async (url, init) => { + expect(String(url)).toBe('/api/chat?runId=r') + if (fetchClient.mock.calls.length === 1) { + // First response: 3 tagged chunks, then the connection closes with no + // terminal event (a mid-stream drop). + return sseResponse( + contentEvent('run@1', '1') + + contentEvent('run@2', '2') + + contentEvent('run@3', '3'), + ) + } + // Second response (reconnect): server replays from the offset — it + // re-sends seq 3 (must be de-duped), then the tail + terminal. + expect(new Headers(init?.headers).get('Last-Event-ID')).toBe('run@3') + return sseResponse( + contentEvent('run@3', '3') + + contentEvent('run@4', '4') + + finishedEvent('run@5'), + ) + }) + + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + const deltas = chunks + .filter((c) => c.type === EventType.TEXT_MESSAGE_CONTENT) + .map((c) => c.delta) + expect(deltas).toEqual(['1', '2', '3', '4']) + expect(chunks[chunks.length - 1]?.type).toBe(EventType.RUN_FINISHED) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('preserves query parameters while replacing a stale runId', async () => { + const fetchClient = vi.fn(async () => + sseResponse(finishedEvent('run@1')), + ) + const adapter = fetchServerSentEvents( + '/api/chat?provider=openai&runId=stale#response', + { fetchClient }, + ) + + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'current' }, + )) { + // drain + } + + expect(String(fetchClient.mock.calls[0]![0])).toBe( + '/api/chat?provider=openai&runId=current#response', + ) + }) + + it('joinRun opens the stream from the start with ?offset=-1', async () => { + const fetchClient = vi.fn(async () => + sseResponse(finishedEvent('run@1')), + ) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.joinRun('run-x')) { + chunks.push(chunk) + } + + const calledUrl = String(fetchClient.mock.calls[0]![0]) + expect(calledUrl).toContain('offset=-1') + expect(calledUrl).toContain('runId=run-x') + expect(chunks.map((c) => c.type)).toContain(EventType.RUN_FINISHED) + }) + + // A durable (id-tagged) run that ends with no terminal event and makes no + // forward progress on reconnect must surface an error, not silently return + // leaving the consumer with neither a terminal nor a failure. + it('surfaces an error when a durable run ends without a terminal and cannot progress', async () => { + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + // First pass: two tagged content events, then a clean end with NO + // terminal — the adapter reconnects from run@2. + return sseResponse( + contentEvent('run@1', '1') + contentEvent('run@2', '2'), + ) + } + // Reconnect: server replays nothing new (no progress) and still no + // terminal — the run cannot complete. + return sseResponse('') + }) + + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const deltas: Array = [] + await expect(async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + }).rejects.toBeInstanceOf(DurableStreamIncompleteError) + + // The consumer still received everything delivered before the failure. + expect(deltas).toEqual(['1', '2']) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('reconnects after a body reader failure and resumes exactly once', async () => { + const fetchClient = vi.fn(async (_url, init) => { + if (fetchClient.mock.calls.length === 1) { + return failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected'), + ) + } + expect(new Headers(init?.headers).get('Last-Event-ID')).toBe('run@1') + return sseResponse( + contentEvent('run@1', '1') + + contentEvent('run@2', '2') + + finishedEvent('run@3'), + ) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2']) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) + + it('retries a reconnect whose fetch() itself rejects (connection-establishment failure)', async () => { + // A fetch rejection (offline / DNS blip / connection refused) on a reconnect + // must be treated as a recoverable transport drop, not a fatal error, once + // an offset is held — matching the XHR path. Without wrapping the rejection + // as StreamReadError it would be a raw TypeError and hard-fail. + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + // Clean end with progress, no terminal → reconnect. + return sseResponse(contentEvent('run@1', '1')) + } + if (fetchClient.mock.calls.length === 2) { + throw new TypeError('Failed to fetch') + } + return sseResponse(contentEvent('run@2', '2') + finishedEvent('run@3')) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + expect(fetchClient).toHaveBeenCalledTimes(3) + }) + + it('surfaces a first-attempt fetch() rejection (no offset held) as a hard failure', async () => { + // With no offset yet, a fetch rejection is not recoverable — it must surface. + const fetchClient = vi.fn(async () => { + throw new TypeError('Failed to fetch') + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toThrow() + expect(fetchClient).toHaveBeenCalledTimes(1) + }) + + it('retries a transport drop that replayed only the de-duped overlap (no new progress that attempt)', async () => { + // A caught-up run whose reconnect replays only the already-seen boundary + // event and then the socket drops must retry from the offset, not fail + // hard — the drop is transient and we still hold a valid resume point. + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + // First pass: one new event, then the socket drops. + return failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected'), + ) + } + if (fetchClient.mock.calls.length === 2) { + // Reconnect: replays ONLY the de-duped overlap (run@1), then drops + // again before any new event — this attempt makes no forward progress. + return failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected again'), + ) + } + // Final reconnect delivers the tail + terminal. + return sseResponse( + contentEvent('run@1', '1') + + contentEvent('run@2', '2') + + finishedEvent('run@3'), + ) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + // The no-progress overlap-only drop was retried, not surfaced as an error. + expect(fetchClient).toHaveBeenCalledTimes(3) + }) + + it('bounds reconnection with a consecutive-no-progress ceiling', async () => { + // Every pass replays only the already-seen boundary event then drops — no + // new events ever arrive, so the run is genuinely stuck. The ceiling counts + // these consecutive no-progress reconnects and fails. + const fetchClient = vi.fn(async () => + failingSseResponse( + contentEvent('run@1', 'x'), + new TypeError('socket disconnected'), + ), + ) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { maxAttempts: 3, delayMs: 0 }, + }) + + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toBeInstanceOf(StreamReconnectLimitError) + + // fetch #1 delivers run@1 (progress) then drops → resets the counter; fetches + // #2-#5 each replay only the de-duped run@1 then drop (no progress), so the + // 4th such reconnect (after fetch #5) trips maxAttempts=3. + expect(fetchClient).toHaveBeenCalledTimes(5) + }) + + it('does not count progress-making reconnects toward the ceiling', async () => { + // A deliberately low ceiling, but every pass delivers a NEW event before + // dropping, so the no-progress counter keeps resetting and the run completes + // instead of failing — a healthy socket-per-event run must not be bounded. + let pass = 0 + const fetchClient = vi.fn(async () => { + pass += 1 + if (pass <= 4) { + return failingSseResponse( + contentEvent(`run@${pass}`, String(pass)), + new TypeError('socket rolled'), + ) + } + return sseResponse(finishedEvent('run@final')) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { maxAttempts: 2, delayMs: 0 }, + }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + // 4 progress-then-drop reconnects, each resetting the ceiling of 2, then a + // clean finish — never trips the limit. + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2', '3', '4']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + expect(fetchClient).toHaveBeenCalledTimes(5) + }) + + it('rejects invalid reconnect bounds (non-finite maxAttempts / delayMs)', async () => { + const fetchClient = vi.fn(async () => + sseResponse(finishedEvent('run@1')), + ) + for (const reconnect of [ + { maxAttempts: Number.NaN }, + { maxAttempts: Number.POSITIVE_INFINITY }, + { maxAttempts: -1 }, + { delayMs: Number.POSITIVE_INFINITY }, + { delayMs: -5 }, + ]) { + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect, + }) + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toThrow(/Invalid reconnect\./) + } + }) + + it('delivers an empty-id event and does not track "" as a durable offset', async () => { + // A tagged event, then an event with an empty `id:` (SSE reset), then the + // terminal — all in one response. The empty-id chunk must be delivered (not + // dropped) and '' must not be recorded as an offset. + const fetchClient = vi.fn(async () => + sseResponse( + contentEvent('run@1', '1') + emptyIdEvent('2') + finishedEvent('run@3'), + ), + ) + const adapter = fetchServerSentEvents('/api/chat', { fetchClient }) + + const chunks: Array = [] + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + + expect( + chunks + .filter((chunk) => chunk.type === EventType.TEXT_MESSAGE_CONTENT) + .map((chunk) => chunk.delta), + ).toEqual(['1', '2']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + // Terminal reached on the first response — no reconnect. + expect(fetchClient).toHaveBeenCalledTimes(1) + }) + + it('stops reconnecting promptly when aborted during the throttle delay', async () => { + const controller = new AbortController() + let pass = 0 + const fetchClient = vi.fn(async () => { + pass += 1 + return sseResponse(contentEvent(`run@${pass}`, 'x')) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + signal: controller.signal, + reconnect: { delayMs: 10_000 }, + }) + + const chunks: Array = [] + const done = (async () => { + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + } + })() + + // Let the first pass finish and enter the 10s throttle, then abort — the + // delay must resolve immediately rather than stalling for 10s. + await new Promise((resolve) => setTimeout(resolve, 20)) + controller.abort() + await done + + expect(chunks).toHaveLength(1) + expect(fetchClient).toHaveBeenCalledTimes(1) + }) + + it('does not reconnect a body reader after the caller aborts', async () => { + const fetchClient = vi.fn(async () => + failingSseResponse( + contentEvent('run@1', '1'), + new TypeError('socket disconnected'), + ), + ) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + const controller = new AbortController() + const chunks: Array = [] + + for await (const chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + controller.signal, + { threadId: 't', runId: 'r' }, + )) { + chunks.push(chunk) + controller.abort() + } + + expect(chunks).toHaveLength(1) + expect(fetchClient).toHaveBeenCalledOnce() + }) + + it('does not retry HTTP setup failures after earlier progress', async () => { + const fetchClient = vi.fn(async () => { + if (fetchClient.mock.calls.length === 1) { + return sseResponse(contentEvent('run@1', '1')) + } + return new Response(null, { status: 503, statusText: 'Unavailable' }) + }) + const adapter = fetchServerSentEvents('/api/chat', { + fetchClient, + reconnect: { delayMs: 0 }, + }) + + await expect(async () => { + for await (const _chunk of adapter.connect( + [{ role: 'user', content: 'hi' }], + undefined, + undefined, + { threadId: 't', runId: 'r' }, + )) { + // drain + } + }).rejects.toThrow(/503 Unavailable/) + expect(fetchClient).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/ai-client/tests/connection-adapters-xhr.test.ts b/packages/ai-client/tests/connection-adapters-xhr.test.ts index c69430f5a..6ed57dbac 100644 --- a/packages/ai-client/tests/connection-adapters-xhr.test.ts +++ b/packages/ai-client/tests/connection-adapters-xhr.test.ts @@ -95,7 +95,9 @@ describe('xhr connection adapters', () => { await nextTick() expect(xhr.method).toBe('POST') - expect(xhr.url).toBe('/api/chat') + // The run id is threaded into the URL so a durable server can correlate + // a reconnect/join to the same run (matches the fetch SSE adapter). + expect(xhr.url).toBe('/api/chat?runId=run-1') expect(xhr.withCredentials).toBe(true) expect(xhr.requestHeaders).toMatchObject({ 'Content-Type': 'application/json', @@ -175,6 +177,12 @@ describe('xhr connection adapters', () => { done: false, value: { type: EventType.RUN_FINISHED }, }) + // Completing iteration tears the socket down so late bytes stop + // downloading (the terminal event's yield pauses before cleanup). + await expect(iterator.next()).resolves.toEqual({ + done: true, + value: undefined, + }) expect(xhr.abort).toHaveBeenCalledTimes(1) }) @@ -257,13 +265,10 @@ describe('xhr connection adapters', () => { done: true, value: undefined, }) - expect(xhr.abort).toHaveBeenCalledTimes(1) + // An already-aborted signal short-circuits before any request is opened, + // so no XHR is created, sent, or aborted. + expect(xhr.abort).not.toHaveBeenCalled() expect(xhr.send).not.toHaveBeenCalled() - expect(xhr.onprogress).toBeNull() - expect(xhr.onload).toBeNull() - expect(xhr.onerror).toBeNull() - expect(xhr.onabort).toBeNull() - expect(xhr.onloadend).toBeNull() }) it('maps AbortSignal to xhr.abort, stops output, and cleans up handlers', async () => { @@ -341,7 +346,11 @@ describe('xhr connection adapters', () => { await nextTick() failure.xhr.error() - await expect(failureNext).rejects.toThrow('XHR request failed') + // A network onerror surfaces as StreamReadError (so a durable run can + // reconnect); the original cause is preserved on `.cause`. + await expect(failureNext).rejects.toThrow( + 'Stream response body read failed', + ) expect(failureRemoveEventListener).toHaveBeenCalledWith( 'abort', expect.any(Function), @@ -427,13 +436,9 @@ describe('xhr connection adapters', () => { done: true, value: undefined, }) - expect(xhr.abort).toHaveBeenCalledTimes(1) + // An already-aborted signal short-circuits before any request is opened. + expect(xhr.abort).not.toHaveBeenCalled() expect(xhr.send).not.toHaveBeenCalled() - expect(xhr.onprogress).toBeNull() - expect(xhr.onload).toBeNull() - expect(xhr.onerror).toBeNull() - expect(xhr.onabort).toBeNull() - expect(xhr.onloadend).toBeNull() }) }) }) diff --git a/packages/ai-durable-stream/package.json b/packages/ai-durable-stream/package.json new file mode 100644 index 000000000..72cfeffcd --- /dev/null +++ b/packages/ai-durable-stream/package.json @@ -0,0 +1,50 @@ +{ + "name": "@tanstack/ai-durable-stream", + "version": "0.0.0", + "description": "Delivery durability for TanStack AI over the durable-streams HTTP protocol — a resumable StreamDurability transport sink (append/read/resume) that stores zero delivery events itself.", + "author": "", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-durable-stream" + }, + "keywords": [ + "ai", + "tanstack", + "durable-streams", + "delivery-durability", + "resumable", + "sse" + ], + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "eslint ./src --fix", + "test:build": "publint --strict", + "test:eslint": "eslint ./src", + "test:lib": "vitest", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.0.14" + } +} diff --git a/packages/ai-durable-stream/src/durable-stream.ts b/packages/ai-durable-stream/src/durable-stream.ts new file mode 100644 index 000000000..bb264b46a --- /dev/null +++ b/packages/ai-durable-stream/src/durable-stream.ts @@ -0,0 +1,739 @@ +import type { StreamChunk, StreamDurability } from '@tanstack/ai' + +declare const durableStreamCursorBrand: unique symbol + +/** A validated, versioned offset produced by this adapter. */ +type DurableStreamCursor = string & { + readonly [durableStreamCursorBrand]: true +} + +/** Adapter offsets also include the Durable Streams protocol sentinels. */ +export type DurableStreamOffset = DurableStreamCursor | '-1' | 'now' + +export interface DurableStreamOptions { + /** + * Base URL of the Durable Streams server (no trailing slash needed). + * Optional when `fetch` is supplied — e.g. a Cloudflare service binding that + * ignores the host and dispatches to the bound Worker by path — in which case + * an internal placeholder base is used and only the `/streams/...` path + * matters. + */ + server?: string + /** Stream-name prefix. Defaults to `runs`. */ + streamPrefix?: string + /** Fetch implementation. Defaults to the global fetch. */ + fetch?: typeof globalThis.fetch + /** + * Headers applied to every create, append, read, and close request. A + * resolver is called for every request so credentials can rotate. + */ + headers?: HeadersInit | (() => HeadersInit | Promise) + /** + * Bounding for the read reconnect loop. After a response-body read failure + * mid-window, `read` retries from the last valid position; these cap + * consecutive retries and throttle them so a persistently failing backend + * surfaces the error instead of looping without end. Normal window + * advancement (long-poll) is never throttled. + */ + reconnect?: { + /** + * Consecutive body-read-failure retries before surfacing the underlying + * read error. Default 10. + */ + maxReadFailures?: number + /** Delay between read retries, in ms. Default 250. */ + delayMs?: number + } + /** + * Timeout (ms) for a single create / append / close request to the backend. + * A stalled backend would otherwise hang chunk delivery or terminalization + * indefinitely. Default 30000. Long-poll `read` window advancement is NOT + * bounded by this — a caught-up reader may legitimately wait. + */ + operationTimeoutMs?: number +} + +/** Resolve after `ms`, or immediately once `signal` aborts. Never rejects. */ +function abortableDelay(ms: number, signal?: AbortSignal): Promise { + if (ms <= 0 || signal?.aborted) return Promise.resolve() + return new Promise((resolve) => { + const onAbort = () => { + clearTimeout(timer) + resolve() + } + const timer = setTimeout(() => { + signal?.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal?.addEventListener('abort', onAbort, { once: true }) + }) +} + +export class DurableStreamError extends Error { + override name = 'DurableStreamError' + + constructor(message: string) { + super(`durableStream: ${message}`) + } +} + +interface SseEvent { + event?: string + data?: string +} + +interface WireRecord { + v: 1 + seq: number + chunk: StreamChunk +} + +interface CursorPayload { + v: 1 + backendOffset: string + seq: number +} + +interface ControlFrame { + streamNextOffset: string + streamCursor?: string + upToDate?: boolean + streamClosed?: boolean +} + +const CURSOR_PREFIX = 'tanstack-ai-ds:v1:' +const READ_ABORTED = Symbol('read aborted') + +class ResponseBodyReadFailure extends Error { + override name = 'ResponseBodyReadFailure' + + constructor(readonly readError: unknown) { + super('response body read failed') + } +} + +function assertTransportField(value: string, name: string): string { + if (value.trim().length === 0 || /[\r\n]/.test(value)) { + throw new DurableStreamError( + `${name} must be non-empty and contain no CR/LF`, + ) + } + return value +} + +function assertRunId(value: string): string { + return assertTransportField(value, 'runId') +} + +function isDurableStreamCursor(value: string): value is DurableStreamCursor { + return value.startsWith(CURSOR_PREFIX) +} + +function isCursorPayload(value: unknown): value is CursorPayload { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'backendOffset' in value && + typeof value.backendOffset === 'string' && + 'seq' in value && + typeof value.seq === 'number' && + Number.isSafeInteger(value.seq) && + value.seq > 0 + ) +} + +function encodeCursor(payload: CursorPayload): DurableStreamCursor { + assertTransportField(payload.backendOffset, 'backend offset') + if (!Number.isSafeInteger(payload.seq) || payload.seq < 1) { + throw new DurableStreamError(`invalid record sequence: ${payload.seq}`) + } + const cursor = `${CURSOR_PREFIX}${encodeURIComponent(JSON.stringify(payload))}` + if (!isDurableStreamCursor(cursor)) { + throw new DurableStreamError('failed to encode cursor') + } + return cursor +} + +function decodeCursor(cursor: string): CursorPayload { + if (!isDurableStreamCursor(cursor)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + let parsed: unknown + try { + parsed = JSON.parse(decodeURIComponent(cursor.slice(CURSOR_PREFIX.length))) + } catch { + throw new DurableStreamError('invalid or unsupported resume offset') + } + if (!isCursorPayload(parsed)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + assertTransportField(parsed.backendOffset, 'backend offset') + return parsed +} + +function safeSearchParam(request: Request, key: string): string | null { + try { + return new URL(request.url).searchParams.get(key) + } catch { + return null + } +} + +function parseResumeOffset(raw: string | null): DurableStreamOffset | null { + if (raw === null || raw === '-1' || raw === 'now') return raw + decodeCursor(raw) + if (!isDurableStreamCursor(raw)) { + throw new DurableStreamError('invalid or unsupported resume offset') + } + return raw +} + +async function* readLines( + body: ReadableStream, + signal?: AbortSignal, +): AsyncGenerator { + const reader = body.getReader() + const decoder = new TextDecoder() + let buffer = '' + let completed = false + let cancelled = false + let readFailed = false + try { + for (;;) { + let result: ReadableStreamReadResult | typeof READ_ABORTED + try { + result = await readWithAbort(reader, signal) + } catch (error) { + readFailed = true + throw new ResponseBodyReadFailure(error) + } + if (result === READ_ABORTED) { + cancelled = true + await reader.cancel(signal?.reason) + return + } + if (result.done) { + completed = true + break + } + buffer += decoder.decode(result.value, { stream: true }) + const parts = buffer.split('\n') + buffer = parts.pop() ?? '' + for (const raw of parts) { + yield raw.endsWith('\r') ? raw.slice(0, -1) : raw + } + } + buffer += decoder.decode() + if (buffer.length > 0) { + yield buffer.endsWith('\r') ? buffer.slice(0, -1) : buffer + } + } finally { + try { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- readFailed is set in the catch before its throw; CFA can't see that from finally + if (!completed && !cancelled && !readFailed) await reader.cancel() + } finally { + reader.releaseLock() + } + } +} + +function readWithAbort( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined, +): Promise | typeof READ_ABORTED> { + if (!signal) return reader.read() + if (signal.aborted) return Promise.resolve(READ_ABORTED) + + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener('abort', onAbort) + resolve(READ_ABORTED) + } + signal.addEventListener('abort', onAbort, { once: true }) + reader.read().then( + (result) => { + signal.removeEventListener('abort', onAbort) + resolve(result) + }, + (error: unknown) => { + signal.removeEventListener('abort', onAbort) + reject(error) + }, + ) + }) +} + +async function* parseSseEvents( + body: ReadableStream, + signal?: AbortSignal, +): AsyncGenerator { + let current: SseEvent = {} + let hasField = false + + for await (const line of readLines(body, signal)) { + if (line === '') { + if (hasField) yield current + current = {} + hasField = false + continue + } + if (line.startsWith(':')) continue + + const colon = line.indexOf(':') + const field = colon === -1 ? line : line.slice(0, colon) + let value = colon === -1 ? '' : line.slice(colon + 1) + if (value.startsWith(' ')) value = value.slice(1) + if (field === 'event') { + current.event = value + hasField = true + } else if (field === 'data') { + current.data = + current.data === undefined ? value : `${current.data}\n${value}` + hasField = true + } + } + if (hasField) yield current +} + +function isStreamChunk(value: unknown): value is StreamChunk { + return ( + typeof value === 'object' && + value !== null && + 'type' in value && + typeof value.type === 'string' + ) +} + +function isWireRecord(value: unknown): value is WireRecord { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'seq' in value && + typeof value.seq === 'number' && + Number.isSafeInteger(value.seq) && + value.seq > 0 && + 'chunk' in value && + isStreamChunk(value.chunk) + ) +} + +function parseDataRecords(data: string | undefined): Array { + if (data === undefined) { + throw new DurableStreamError('data event had no payload') + } + let parsed: unknown + try { + parsed = JSON.parse(data) + } catch { + throw new DurableStreamError('data event contained invalid JSON') + } + if (!Array.isArray(parsed)) { + throw new DurableStreamError('data event payload must be a JSON array') + } + const records: Array = [] + for (const value of parsed) { + if (!isWireRecord(value)) { + throw new DurableStreamError('data event contained an invalid record') + } + records.push(value) + } + return records +} + +function optionalBoolean( + value: object, + name: 'upToDate' | 'streamClosed', +): boolean | undefined { + const field = + name === 'upToDate' + ? 'upToDate' in value + ? value.upToDate + : undefined + : 'streamClosed' in value + ? value.streamClosed + : undefined + if (field === undefined) return undefined + if (typeof field !== 'boolean') { + throw new DurableStreamError(`control field ${name} must be boolean`) + } + return field +} + +function parseControlFrame(data: string | undefined): ControlFrame { + if (data === undefined) { + throw new DurableStreamError('control event had no payload') + } + let parsed: unknown + try { + parsed = JSON.parse(data) + } catch { + throw new DurableStreamError('control event contained invalid JSON') + } + if (typeof parsed !== 'object' || parsed === null) { + throw new DurableStreamError('control event payload must be an object') + } + if ( + !('streamNextOffset' in parsed) || + typeof parsed.streamNextOffset !== 'string' + ) { + throw new DurableStreamError( + 'control event requires string streamNextOffset', + ) + } + const streamNextOffset = assertTransportField( + parsed.streamNextOffset, + 'control streamNextOffset', + ) + let streamCursor: string | undefined + if ('streamCursor' in parsed) { + if (typeof parsed.streamCursor !== 'string') { + throw new DurableStreamError('control streamCursor must be a string') + } + streamCursor = assertTransportField( + parsed.streamCursor, + 'control streamCursor', + ) + } + const upToDate = optionalBoolean(parsed, 'upToDate') + const streamClosed = optionalBoolean(parsed, 'streamClosed') + if (streamClosed !== true && streamCursor === undefined) { + throw new DurableStreamError( + 'open control event requires string streamCursor', + ) + } + return { + streamNextOffset, + ...(streamCursor === undefined ? {} : { streamCursor }), + ...(upToDate === undefined ? {} : { upToDate }), + ...(streamClosed === undefined ? {} : { streamClosed }), + } +} + +function requireNextOffset(response: Response, operation: string): string { + const offset = response.headers.get('Stream-Next-Offset') + if (offset === null || offset.trim().length === 0) { + throw new DurableStreamError( + `${operation} response missing non-empty Stream-Next-Offset`, + ) + } + return assertTransportField(offset, `${operation} Stream-Next-Offset`) +} + +function httpFailure( + operation: string, + response: Response, +): DurableStreamError { + return new DurableStreamError( + `failed to ${operation} (${response.status} ${response.statusText})`, + ) +} + +/** External-URL Durable Streams protocol adapter. */ +export function durableStream( + request: Request, + options: DurableStreamOptions, +): StreamDurability { + const fetchFn = options.fetch ?? globalThis.fetch + if (options.server === undefined && options.fetch === undefined) { + throw new DurableStreamError( + 'server is required unless a fetch implementation is provided', + ) + } + // When a custom fetch routes by path (e.g. a service binding), the host is + // irrelevant; a reserved `.internal` base parses without ever resolving. + const rawServer = options.server ?? 'https://durable-streams.internal' + assertTransportField(rawServer, 'server URL') + try { + void new URL(rawServer) + } catch { + throw new DurableStreamError( + `invalid server URL: ${JSON.stringify(rawServer)}`, + ) + } + const server = rawServer.replace(/\/+$/, '') + const maxReadFailures = options.reconnect?.maxReadFailures ?? 10 + const readRetryDelayMs = options.reconnect?.delayMs ?? 250 + const operationTimeoutMs = options.operationTimeoutMs ?? 30_000 + + // create / append / close go through this so a stalled backend can't hang the + // operation forever. Each call gets a fresh timeout; long-poll `read` calls + // deliberately do NOT use it (they may wait for the producer to advance). + const fetchWithTimeout = async ( + url: string | URL, + init: RequestInit, + ): Promise => { + const controller = new AbortController() + const timer = setTimeout(() => { + controller.abort( + new DurableStreamError( + `request exceeded operationTimeoutMs (${operationTimeoutMs}ms)`, + ), + ) + }, operationTimeoutMs) + try { + return await fetchFn(url, { ...init, signal: controller.signal }) + } finally { + clearTimeout(timer) + } + } + const prefix = assertTransportField( + options.streamPrefix ?? 'runs', + 'streamPrefix', + ) + const rawResumeOffset = + request.headers.get('Last-Event-ID') ?? safeSearchParam(request, 'offset') + const resumeOffset = parseResumeOffset(rawResumeOffset) + const requestedRunId = safeSearchParam(request, 'runId') + if (resumeOffset !== null && requestedRunId === null) { + throw new DurableStreamError('resume offset requires a runId') + } + const runId = assertRunId(requestedRunId ?? crypto.randomUUID()) + + const streamUrl = `${server}/streams/${encodeURIComponent(`${prefix}/${runId}`)}` + let createPromise: Promise | undefined + let appendTailOffset: string | undefined + let nextSeq = 1 + const producerId = crypto.randomUUID() + const producerEpoch = 0 + let producerSeq = 0 + let closePromise: Promise | undefined + + const resolveHeaders = async (required?: HeadersInit): Promise => { + const configured = + typeof options.headers === 'function' + ? await options.headers() + : options.headers + const headers = new Headers(configured) + if (required) { + new Headers(required).forEach((value, key) => headers.set(key, value)) + } + return headers + } + + const ensureCreated = (): Promise => { + if (createPromise) return createPromise + createPromise = (async () => { + const response = await fetchWithTimeout(streamUrl, { + method: 'PUT', + headers: await resolveHeaders({ 'Content-Type': 'application/json' }), + }) + if (!response.ok) throw httpFailure('create stream', response) + const offset = requireNextOffset(response, 'create') + appendTailOffset = offset + return offset + })().catch((error: unknown) => { + createPromise = undefined + throw error + }) + return createPromise + } + + return { + resumeFrom: () => resumeOffset, + append: async (chunks) => { + if (chunks.length === 0) return [] + const batchStartOffset = appendTailOffset ?? (await ensureCreated()) + const firstSeq = nextSeq + const records = chunks.map( + (chunk, index): WireRecord => ({ + v: 1, + seq: firstSeq + index, + chunk, + }), + ) + nextSeq += records.length + const requestProducerSeq = producerSeq + producerSeq += 1 + const requestInit: RequestInit = { + method: 'POST', + headers: await resolveHeaders({ + 'Content-Type': 'application/json', + 'Producer-Id': producerId, + 'Producer-Epoch': String(producerEpoch), + 'Producer-Seq': String(requestProducerSeq), + }), + body: JSON.stringify(records), + } + let response: Response + try { + response = await fetchWithTimeout(streamUrl, requestInit) + } catch (firstError) { + try { + response = await fetchWithTimeout(streamUrl, requestInit) + } catch (retryError) { + throw new AggregateError( + [firstError, retryError], + 'durableStream: append failed before its outcome could be confirmed', + ) + } + } + if (!response.ok) throw httpFailure('append', response) + const nextOffset = requireNextOffset(response, 'append') + appendTailOffset = nextOffset + return records.map((record) => + encodeCursor({ + v: 1, + backendOffset: batchStartOffset, + seq: record.seq, + }), + ) + }, + close: () => { + if (closePromise) return closePromise + closePromise = (async () => { + await ensureCreated() + const response = await fetchWithTimeout(streamUrl, { + method: 'POST', + headers: await resolveHeaders({ 'Stream-Closed': 'true' }), + }) + if (!response.ok) throw httpFailure('close', response) + const nextOffset = requireNextOffset(response, 'close') + if (response.headers.get('Stream-Closed')?.toLowerCase() !== 'true') { + throw new DurableStreamError( + 'close response missing Stream-Closed: true', + ) + } + appendTailOffset = nextOffset + })().catch((error: unknown) => { + closePromise = undefined + throw error + }) + return closePromise + }, + read: async function* (offset, signal) { + let backendOffset: string + let deliveredThroughSeq = 0 + if (offset === '-1' || offset === 'now') { + backendOffset = offset + } else { + const cursor = decodeCursor(offset) + backendOffset = cursor.backendOffset + deliveredThroughSeq = cursor.seq + } + let streamCursor: string | undefined + let consecutiveReadFailures = 0 + + for (;;) { + if (signal?.aborted) return + const requestOffset = backendOffset + const requestCursor = streamCursor + const url = new URL(streamUrl) + url.searchParams.set('offset', backendOffset) + url.searchParams.set('live', 'sse') + if (streamCursor !== undefined) { + url.searchParams.set('cursor', streamCursor) + } + + let response: Response + try { + response = await fetchFn(url, { + method: 'GET', + headers: await resolveHeaders(), + signal, + }) + } catch (error) { + if (signal?.aborted) return + throw error + } + if (!response.ok) throw httpFailure('read', response) + if (!response.body) { + throw new DurableStreamError('read response had no body') + } + + let dataStartOffset = backendOffset + let sawControl = false + let dataAwaitingControl = false + let yieldedData = false + // Guards intra-response ordering: seqs must strictly increase across the + // whole response (including across data frames and control frames — seq + // is per-run, not per-window). Starts at 0 so a legitimate replay of + // already-delivered records still passes, then the dedup below drops + // them; the throw catches a genuinely malformed [seq 2, seq 1] or a + // duplicate seq that would otherwise be silently discarded. + let previousResponseSeq = 0 + try { + for await (const event of parseSseEvents(response.body, signal)) { + if (signal?.aborted) return + if (event.event === 'data') { + dataAwaitingControl = true + for (const record of parseDataRecords(event.data)) { + if (record.seq <= previousResponseSeq) { + throw new DurableStreamError( + 'data records must have strictly increasing sequences', + ) + } + previousResponseSeq = record.seq + if (record.seq <= deliveredThroughSeq) continue + deliveredThroughSeq = record.seq + yieldedData = true + yield { + offset: encodeCursor({ + v: 1, + backendOffset: dataStartOffset, + seq: record.seq, + }), + chunk: record.chunk, + } + } + continue + } + if (event.event === 'control') { + const control = parseControlFrame(event.data) + backendOffset = control.streamNextOffset + streamCursor = control.streamCursor + dataStartOffset = backendOffset + sawControl = true + dataAwaitingControl = false + if (control.streamClosed === true) return + continue + } + throw new DurableStreamError( + `unexpected SSE event type: ${JSON.stringify(event.event)}`, + ) + } + } catch (error) { + if (signal?.aborted) return + if (error instanceof ResponseBodyReadFailure) { + if ( + yieldedData || + (sawControl && + (backendOffset !== requestOffset || + streamCursor !== requestCursor)) + ) { + // Made progress before the body failed — retry from the last valid + // position, but cap consecutive failures and throttle so a + // persistently failing backend surfaces the error, not a hot loop. + consecutiveReadFailures += 1 + if (consecutiveReadFailures > maxReadFailures) + throw error.readError + await abortableDelay(readRetryDelayMs, signal) + continue + } + throw error.readError + } + throw error + } + + // A window read to completion (no body failure) clears the streak; only + // consecutive failures accumulate toward the ceiling. + consecutiveReadFailures = 0 + + if (signal?.aborted) return + if (dataAwaitingControl || !sawControl) { + throw new DurableStreamError( + 'read SSE window ended without a matching control event', + ) + } + if (backendOffset === requestOffset && streamCursor === requestCursor) { + throw new DurableStreamError( + 'read SSE window ended without advancing offset or cursor', + ) + } + } + }, + } +} diff --git a/packages/ai-durable-stream/src/index.ts b/packages/ai-durable-stream/src/index.ts new file mode 100644 index 000000000..857485aae --- /dev/null +++ b/packages/ai-durable-stream/src/index.ts @@ -0,0 +1,10 @@ +/** + * `@tanstack/ai-durable-stream` — a {@link StreamDurability} backend over the + * durable-streams HTTP protocol, for production-grade delivery durability. + */ +export { durableStream } from './durable-stream' +export { DurableStreamError } from './durable-stream' +export type { + DurableStreamOffset, + DurableStreamOptions, +} from './durable-stream' diff --git a/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts b/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts new file mode 100644 index 000000000..576151967 --- /dev/null +++ b/packages/ai-durable-stream/tests/durable-stream-types.test-d.ts @@ -0,0 +1,16 @@ +import { expectTypeOf } from 'vitest' +import { durableStream } from '../src' +import type { DurableStreamOffset } from '../src' + +declare const durability: ReturnType +declare const offset: DurableStreamOffset + +expectTypeOf( + durability.resumeFrom(), +).toEqualTypeOf() +durability.read(offset) +durability.read('-1') +durability.read('now') + +// @ts-expect-error arbitrary strings are not validated adapter cursors +durability.read('unvalidated-offset') diff --git a/packages/ai-durable-stream/tests/durable-stream.test.ts b/packages/ai-durable-stream/tests/durable-stream.test.ts new file mode 100644 index 000000000..993109a2d --- /dev/null +++ b/packages/ai-durable-stream/tests/durable-stream.test.ts @@ -0,0 +1,1185 @@ +import { describe, expect, it, vi } from 'vitest' +import { EventType, toServerSentEventsResponse } from '@tanstack/ai' +import { durableStream } from '../src' +import type { StreamChunk } from '@tanstack/ai' +import type { DurableStreamOffset } from '../src' + +interface WireRecord { + v: 1 + seq: number + chunk: StreamChunk +} + +interface CapturedRequest { + url: URL + method: string + headers: Headers + body?: string + signal?: AbortSignal | null +} + +interface StoredBatch { + startOffset: string + nextOffset: string + records: Array +} + +interface ProtocolServerOptions { + createOffset?: string + appendOffsets?: Array + closeResponse?: () => Promise + readStatus?: number + loseAppendResponses?: number +} + +function textChunk(delta: string) { + return { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: 'message-1', + delta, + timestamp: 0, + } as const +} + +function textStream(deltas: Array): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + for (const delta of deltas) yield textChunk(delta) + }, + } +} + +function isStreamChunk(value: unknown): value is StreamChunk { + return ( + typeof value === 'object' && + value !== null && + 'type' in value && + typeof value.type === 'string' + ) +} + +function isWireRecord(value: unknown): value is WireRecord { + return ( + typeof value === 'object' && + value !== null && + 'v' in value && + value.v === 1 && + 'seq' in value && + typeof value.seq === 'number' && + 'chunk' in value && + isStreamChunk(value.chunk) + ) +} + +function parseRecords(body: string | undefined) { + if (body === undefined) throw new Error('Expected an append body') + const parsed: unknown = JSON.parse(body) + if (!Array.isArray(parsed)) throw new Error('Expected a JSON record array') + const records: Array = [] + for (const value of parsed) { + if (!isWireRecord(value)) throw new Error('Invalid versioned wire record') + records.push(value) + } + return records +} + +function requestUrl(input: Parameters[0]) { + if (input instanceof Request) return new URL(input.url) + return new URL(input.toString()) +} + +function createHeaders(offset: string | null, closed = false) { + const headers = new Headers() + if (offset !== null) headers.set('Stream-Next-Offset', offset) + if (closed) headers.set('Stream-Closed', 'true') + return headers +} + +function dataEvent(records: Array) { + return `event: data\ndata: ${JSON.stringify(records)}\n\n` +} + +function controlEvent(control: { + streamNextOffset: string + streamCursor?: string + upToDate?: boolean + streamClosed?: boolean +}) { + return `event: control\ndata: ${JSON.stringify(control)}\n\n` +} + +function makeProtocolServer(options: ProtocolServerOptions = {}) { + const requests: Array = [] + const batches: Array = [] + const createOffset = + options.createOffset ?? 'origin::partition/A?cursor=%2F+==' + let tailOffset = createOffset + let closed = false + let appendIndex = 0 + let lostAppendResponses = options.loseAppendResponses ?? 0 + const producerResponses = new Map() + + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + const method = (init?.method ?? 'GET').toUpperCase() + const headers = new Headers(init?.headers) + const body = init?.body === undefined ? undefined : String(init.body) + requests.push({ + url, + method, + headers, + ...(body === undefined ? {} : { body }), + signal: init?.signal, + }) + + if (method === 'PUT') { + return new Response(null, { + status: 201, + headers: createHeaders(createOffset), + }) + } + + if (method === 'POST' && headers.get('Stream-Closed') === 'true') { + if (options.closeResponse) { + const response = await options.closeResponse() + if (response.ok) closed = true + return response + } + closed = true + return new Response(null, { + status: 204, + headers: createHeaders(tailOffset, true), + }) + } + + if (method === 'POST') { + const records = parseRecords(body) + const producerId = headers.get('Producer-Id') + const producerEpoch = headers.get('Producer-Epoch') + const producerSeq = headers.get('Producer-Seq') + const producerHeaders = [producerId, producerEpoch, producerSeq] + if ( + producerHeaders.some((value) => value !== null) && + producerHeaders.some((value) => value === null) + ) { + return new Response(null, { status: 400 }) + } + const producerKey = + producerId === null || producerEpoch === null || producerSeq === null + ? undefined + : `${producerId}:${producerEpoch}:${producerSeq}` + const deduplicatedOffset = + producerKey === undefined + ? undefined + : producerResponses.get(producerKey) + if (deduplicatedOffset !== undefined) { + return new Response(null, { + status: 204, + headers: createHeaders(deduplicatedOffset), + }) + } + const configuredOffset = options.appendOffsets?.[appendIndex] + const nextOffset = + configuredOffset === undefined + ? `opaque::next/${appendIndex}?token=%2B==` + : configuredOffset + appendIndex += 1 + if (nextOffset !== null && nextOffset.length > 0) { + batches.push({ startOffset: tailOffset, nextOffset, records }) + tailOffset = nextOffset + if (producerKey !== undefined) { + producerResponses.set(producerKey, nextOffset) + } + } + if (lostAppendResponses > 0) { + lostAppendResponses -= 1 + throw new TypeError('append response was lost') + } + return new Response(null, { + status: 204, + headers: createHeaders(nextOffset), + }) + } + + if (options.readStatus !== undefined) { + return new Response(null, { status: options.readStatus }) + } + + const requestedOffset = url.searchParams.get('offset') ?? '-1' + let firstBatch = 0 + if (requestedOffset === 'now' || requestedOffset === tailOffset) { + firstBatch = batches.length + } else if (requestedOffset !== '-1') { + firstBatch = batches.findIndex( + (batch) => batch.startOffset === requestedOffset, + ) + if (firstBatch === -1) return new Response(null, { status: 400 }) + } + + let sse = '' + for (let index = firstBatch; index < batches.length; index += 1) { + const batch = batches[index] + if (!batch) continue + const isFinal = index === batches.length - 1 + sse += dataEvent(batch.records) + sse += controlEvent({ + streamNextOffset: batch.nextOffset, + ...(!closed || !isFinal + ? { streamCursor: `collapse::${index}?edge=%2F` } + : {}), + ...(isFinal ? { upToDate: true } : {}), + ...(closed && isFinal ? { streamClosed: true } : {}), + }) + } + if (firstBatch === batches.length) { + sse += controlEvent({ + streamNextOffset: tailOffset, + ...(closed + ? { streamClosed: true } + : { streamCursor: 'collapse::tail', upToDate: true }), + }) + } + return new Response(sse, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + }) + + return { + fetchStub, + requests, + batches, + createOffset, + closeCount: () => + requests.filter( + (request) => + request.method === 'POST' && + request.headers.get('Stream-Closed') === 'true', + ).length, + } +} + +function requestWithMethod(requests: Array, method: string) { + const request = requests.find((candidate) => candidate.method === method) + if (!request) throw new Error(`Expected a ${method} request`) + return request +} + +async function readBody(response: Response) { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +function parseTransportEvents(body: string) { + return body + .split('\n\n') + .filter((block) => block.length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing transport data in ${block}`) + const parsed: unknown = JSON.parse(data) + return { id, data: parsed } + }) +} + +function deltaFrom(value: unknown) { + if ( + typeof value !== 'object' || + value === null || + !('delta' in value) || + typeof value.delta !== 'string' + ) { + throw new Error('Expected a text chunk') + } + return value.delta +} + +function deferred() { + let resolve = (_value: T): void => undefined + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +describe('durableStream official HTTP protocol', () => { + it('appends one versioned batch and returns one adapter offset per record', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-batch'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + + const offsets = await durability.append([ + textChunk('a'), + textChunk('b'), + textChunk('c'), + ]) + + expect(offsets).toHaveLength(3) + expect(new Set(offsets).size).toBe(3) + expect(server.requests.map((request) => request.method)).toEqual([ + 'PUT', + 'POST', + ]) + expect( + requestWithMethod(server.requests, 'PUT').headers.get('Content-Type'), + ).toBe('application/json') + const appendRequest = requestWithMethod(server.requests, 'POST') + expect(appendRequest.headers.get('Content-Type')).toBe('application/json') + expect(parseRecords(appendRequest.body)).toEqual([ + { v: 1, seq: 1, chunk: textChunk('a') }, + { v: 1, seq: 2, chunk: textChunk('b') }, + { v: 1, seq: 3, chunk: textChunk('c') }, + ]) + }) + + it('retries an ambiguously committed append with one producer tuple', async () => { + const server = makeProtocolServer({ loseAppendResponses: 1 }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-idempotent-producer'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield textChunk('persisted-before-failure') + throw new Error('provider failed') + }, + } + + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability }, + }), + ) + + const appendRequests = server.requests.filter( + (request) => + request.method === 'POST' && + request.headers.get('Stream-Closed') !== 'true', + ) + expect( + appendRequests.map((request) => request.headers.get('Producer-Seq')), + ).toEqual(['0', '0', '1']) + expect( + new Set( + appendRequests.map((request) => request.headers.get('Producer-Id')), + ).size, + ).toBe(1) + expect(server.batches).toHaveLength(2) + + const replayed: Array = [] + for await (const { chunk } of durability.read('-1')) replayed.push(chunk) + expect(replayed.map((chunk) => chunk.type)).toEqual([ + EventType.TEXT_MESSAGE_CONTENT, + EventType.RUN_ERROR, + ]) + }) + + it('applies static auth headers to create, append, close, and read', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-static-auth'), + { + server: 'https://ds.test', + fetch: server.fetchStub, + headers: { + Authorization: 'Bearer static-token', + 'Content-Type': 'text/plain', + 'Stream-Closed': 'false', + }, + }, + ) + + await durability.append([textChunk('secured')]) + await durability.close() + for await (const _entry of durability.read('-1')) { + // drain + } + + expect(server.requests).toHaveLength(4) + expect( + server.requests.map((request) => request.headers.get('Authorization')), + ).toEqual(Array.from({ length: 4 }, () => 'Bearer static-token')) + expect(server.requests[0]?.headers.get('Content-Type')).toBe( + 'application/json', + ) + expect(server.requests[1]?.headers.get('Content-Type')).toBe( + 'application/json', + ) + expect(server.requests[2]?.headers.get('Stream-Closed')).toBe('true') + }) + + it('resolves rotating async auth headers for every protocol request', async () => { + const server = makeProtocolServer() + let token = 0 + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-rotating-auth'), + { + server: 'https://ds.test', + fetch: server.fetchStub, + headers: async () => ({ Authorization: `Bearer token-${++token}` }), + }, + ) + + await durability.append([textChunk('secured')]) + await durability.close() + for await (const _entry of durability.read('-1')) { + // drain + } + + expect( + server.requests.map((request) => request.headers.get('Authorization')), + ).toEqual([ + 'Bearer token-1', + 'Bearer token-2', + 'Bearer token-3', + 'Bearer token-4', + ]) + }) + + it.each([ + ['missing', null], + ['empty', ''], + ])('fails when an append returns a %s next offset', async (_name, offset) => { + const server = makeProtocolServer({ appendOffsets: [offset] }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-bad-offset'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + + await expect(durability.append([textChunk('x')])).rejects.toThrow( + /Stream-Next-Offset/, + ) + }) + + it('parses conforming id-less data and control events', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-idless'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('a'), textChunk('b')]) + await durability.close() + + const received: Array<{ offset: DurableStreamOffset; delta: string }> = [] + for await (const entry of durability.read('-1')) { + received.push({ + offset: entry.offset, + delta: + entry.chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? entry.chunk.delta + : entry.chunk.type, + }) + } + + expect(received.map((entry) => entry.delta)).toEqual(['a', 'b']) + expect(new Set(received.map((entry) => entry.offset)).size).toBe(2) + }) + + it('rejects a read response whose record sequences are not strictly increasing', async () => { + const fetchStub = vi.fn(async (_input, init) => { + const method = (init?.method ?? 'GET').toUpperCase() + if (method !== 'GET') { + return new Response(null, { + status: 200, + headers: createHeaders('origin::p/A'), + }) + } + const body = + dataEvent([ + { v: 1, seq: 2, chunk: textChunk('b') }, + { v: 1, seq: 1, chunk: textChunk('a') }, + ]) + + controlEvent({ streamNextOffset: 'origin::p/A', streamClosed: true }) + return new Response(body, { + status: 200, + headers: createHeaders('origin::p/A'), + }) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?offset=-1&runId=run-nonmono'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain until the out-of-order record throws + } + }).rejects.toThrow(/strictly increasing sequences/) + }) + + it('times out a stalled close via operationTimeoutMs', async () => { + const fetchStub = vi.fn((_input, init) => { + const method = (init?.method ?? 'GET').toUpperCase() + if (method === 'PUT') { + return Promise.resolve( + new Response(null, { + status: 200, + headers: createHeaders('origin::p/A'), + }), + ) + } + // The close POST hangs; only the timeout's abort resolves it. + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(init.signal?.reason ?? new Error('aborted')), + { once: true }, + ) + }) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-timeout'), + { server: 'https://ds.test', fetch: fetchStub, operationTimeoutMs: 20 }, + ) + + await expect(durability.close()).rejects.toThrow(/operationTimeoutMs/) + }) + + it('reconnects an open SSE window with control offset and cursor', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + const headers = new Headers(init?.headers) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers, + signal: init?.signal, + }) + readNumber += 1 + const record = { + v: 1, + seq: readNumber, + chunk: textChunk(String(readNumber)), + } satisfies WireRecord + const nextOffset = `opaque::window/${readNumber}?token=%2F` + return new Response( + dataEvent([record]) + + controlEvent({ + streamNextOffset: nextOffset, + ...(readNumber === 1 + ? { streamCursor: 'collapse::window-1', upToDate: true } + : { streamClosed: true }), + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-rollover'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) + deltas.push(chunk.delta) + } + + expect(deltas).toEqual(['1', '2']) + expect(requests[1]?.url.searchParams.get('offset')).toBe( + 'opaque::window/1?token=%2F', + ) + expect(requests[1]?.url.searchParams.get('cursor')).toBe( + 'collapse::window-1', + ) + }) + + it('reconnects after a body read failure from the last valid control', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers: new Headers(init?.headers), + signal: init?.signal, + }) + readNumber += 1 + + if (readNumber === 1) { + let pullNumber = 0 + const body = new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([ + { v: 1, seq: 1, chunk: textChunk('before-failure') }, + ]) + + controlEvent({ + streamNextOffset: 'opaque::after/1?token=%2F', + streamCursor: 'collapse::after-1', + upToDate: true, + }), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }) + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + } + + return new Response( + dataEvent([{ v: 1, seq: 2, chunk: textChunk('after-reconnect') }]) + + controlEvent({ + streamNextOffset: 'opaque::after/2?token=%2F', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-body-reconnect'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + + expect(deltas).toEqual(['before-failure', 'after-reconnect']) + expect(requests).toHaveLength(2) + expect(requests[1]?.url.searchParams.get('offset')).toBe( + 'opaque::after/1?token=%2F', + ) + expect(requests[1]?.url.searchParams.get('cursor')).toBe( + 'collapse::after-1', + ) + }) + + it('caps consecutive body-read failures and surfaces the read error', async () => { + let readNumber = 0 + const fetchStub = vi.fn(async () => { + readNumber += 1 + const seq = readNumber + let pullNumber = 0 + const body = new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([{ v: 1, seq, chunk: textChunk(`d${seq}`) }]) + + controlEvent({ + // Advance the offset every window so each pass makes real + // progress before the body fails. + streamNextOffset: `opaque::after/${readNumber}?token=%2F`, + streamCursor: `collapse::${readNumber}`, + upToDate: true, + }), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }) + return new Response(body, { + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + }) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-read-cap'), + { + server: 'https://ds.test', + fetch: fetchStub, + reconnect: { maxReadFailures: 3, delayMs: 0 }, + }, + ) + + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/socket read failed/) + + // Initial read + 3 permitted retries; the 4th failure trips the ceiling. + expect(fetchStub).toHaveBeenCalledTimes(4) + }) + + it('reconnects from the same control after data is read before a body failure', async () => { + const requests: Array = [] + let readNumber = 0 + const fetchStub = vi.fn(async (input, init) => { + const url = requestUrl(input) + requests.push({ + url, + method: (init?.method ?? 'GET').toUpperCase(), + headers: new Headers(init?.headers), + signal: init?.signal, + }) + readNumber += 1 + + if (readNumber === 1) { + return new Response( + dataEvent([{ v: 1, seq: 1, chunk: textChunk('before-drop') }]) + + controlEvent({ + streamNextOffset: 'opaque::stable/window?token=%2F', + streamCursor: 'collapse::stable-window', + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + } + + if (readNumber === 2) { + let pullNumber = 0 + return new Response( + new ReadableStream({ + pull(controller) { + pullNumber += 1 + if (pullNumber === 1) { + controller.enqueue( + new TextEncoder().encode( + dataEvent([ + { v: 1, seq: 2, chunk: textChunk('during-drop') }, + ]), + ), + ) + return + } + controller.error(new TypeError('socket read failed')) + }, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + } + + return new Response( + dataEvent([ + { v: 1, seq: 2, chunk: textChunk('during-drop') }, + { v: 1, seq: 3, chunk: textChunk('after-reconnect') }, + ]) + + controlEvent({ + streamNextOffset: 'opaque::tail/window?token=%3D', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ) + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-data-body-reconnect'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const deltas: Array = [] + for await (const { chunk } of durability.read('-1')) { + if (chunk.type === EventType.TEXT_MESSAGE_CONTENT) { + deltas.push(chunk.delta) + } + } + + expect(deltas).toEqual(['before-drop', 'during-drop', 'after-reconnect']) + expect(requests).toHaveLength(3) + expect(requests[2]?.url.searchParams.get('offset')).toBe( + requests[1]?.url.searchParams.get('offset'), + ) + expect(requests[2]?.url.searchParams.get('cursor')).toBe( + requests[1]?.url.searchParams.get('cursor'), + ) + expect(requests[2]?.url.searchParams.get('offset')).toBe( + 'opaque::stable/window?token=%2F', + ) + expect(requests[2]?.url.searchParams.get('cursor')).toBe( + 'collapse::stable-window', + ) + }) + + it('fails loudly when a control event omits streamNextOffset', async () => { + const fetchStub = vi.fn( + async () => + new Response( + 'event: control\ndata: {"upToDate":true,"streamCursor":"c"}\n\n', + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-bad-control'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/streamNextOffset/) + }) + + it('requires a separate runId when resuming from an adapter offset', async () => { + const server = makeProtocolServer() + const producer = durableStream( + new Request('https://app.test/api/chat?runId=run-resume'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const [resumeOffset] = await producer.append([textChunk('x')]) + if (!resumeOffset) throw new Error('Expected a resume offset') + + expect(() => + durableStream( + new Request('https://app.test/api/chat', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ), + ).toThrow(/resume offset requires a runId/) + + expect( + durableStream( + new Request('https://app.test/api/chat?runId=run-resume', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ).resumeFrom(), + ).toBe(resumeOffset) + }) + + it('rejects CR/LF injection in run ids, prefixes, cursors, and controls', async () => { + expect(() => + durableStream( + new Request( + `https://app.test/api/chat?runId=${encodeURIComponent('bad\nrun')}`, + ), + { server: 'https://ds.test' }, + ), + ).toThrow(/CR\/LF/) + expect(() => + durableStream(new Request('https://app.test/api/chat'), { + server: 'https://ds.test', + streamPrefix: 'bad\rprefix', + }), + ).toThrow(/CR\/LF/) + + const forgedCursor = `tanstack-ai-ds:v1:${encodeURIComponent( + JSON.stringify({ + v: 1, + backendOffset: 'bad\noffset', + seq: 1, + }), + )}` + expect(() => + durableStream( + new Request('https://app.test/api/chat', { + headers: { 'Last-Event-ID': forgedCursor }, + }), + { server: 'https://ds.test' }, + ), + ).toThrow(/CR\/LF/) + + const fetchStub = vi.fn( + async () => + new Response( + controlEvent({ + streamNextOffset: 'bad\ncontrol-offset', + streamClosed: true, + }), + { status: 200, headers: { 'Content-Type': 'text/event-stream' } }, + ), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-control-injection'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + await expect(async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + }).rejects.toThrow(/CR\/LF/) + }) + + it('omits server when a routing fetch (e.g. a service binding) is provided', async () => { + const server = makeProtocolServer() + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-binding'), + { fetch: server.fetchStub }, + ) + + const [offset] = await durability.append([textChunk('x')]) + expect(offset).toBeTruthy() + + // The adapter routed through the provided fetch using an internal base URL, + // so only the `/streams/...` path is meaningful. + const firstCall = server.fetchStub.mock.calls[0] + if (!firstCall) throw new Error('Expected the adapter to call fetch') + expect(String(firstCall[0])).toContain('/streams/') + }) + + it('requires server when no fetch is provided', () => { + expect(() => + durableStream( + new Request('https://app.test/api/chat?runId=run-no-fetch'), + {}, + ), + ).toThrow(/server is required unless a fetch/) + }) + + it('awaits external close and sends the protocol close header', async () => { + const closing = deferred() + const server = makeProtocolServer({ + closeResponse: () => closing.promise, + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-close'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + const closePromise = durability.close() + let settled = false + void closePromise.then(() => { + settled = true + }) + await vi.waitFor(() => expect(server.closeCount()).toBe(1)) + expect(settled).toBe(false) + const closeRequest = server.requests.at(-1) + expect(closeRequest?.headers.get('Stream-Closed')).toBe('true') + + closing.resolve( + new Response(null, { + status: 204, + headers: createHeaders('opaque::closed', true), + }), + ) + await expect(closePromise).resolves.toBeUndefined() + }) + + it('surfaces close and read HTTP failures', async () => { + const closeServer = makeProtocolServer({ + closeResponse: async () => new Response(null, { status: 503 }), + }) + const closeDurability = durableStream( + new Request('https://app.test/api/chat?runId=run-close-error'), + { server: 'https://ds.test', fetch: closeServer.fetchStub }, + ) + await closeDurability.append([textChunk('x')]) + await expect(closeDurability.close()).rejects.toThrow(/failed to close/) + + const readServer = makeProtocolServer({ readStatus: 502 }) + const readDurability = durableStream( + new Request('https://app.test/api/chat?runId=run-read-error'), + { server: 'https://ds.test', fetch: readServer.fetchStub }, + ) + await expect(async () => { + for await (const _entry of readDurability.read('-1')) { + // drain + } + }).rejects.toThrow(/failed to read/) + }) + + it.each([ + ['next offset', createHeaders(null, true)], + ['closed state', createHeaders('opaque::closed', false)], + ])( + 'rejects a successful close response missing its %s', + async (_name, headers) => { + const server = makeProtocolServer({ + closeResponse: async () => new Response(null, { status: 204, headers }), + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-invalid-close'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + await expect(durability.close()).rejects.toThrow(/close response/i) + }, + ) + + it('retries close after a transient failure', async () => { + let closeAttempt = 0 + const server = makeProtocolServer({ + closeResponse: async () => { + closeAttempt += 1 + return closeAttempt === 1 + ? new Response(null, { status: 503 }) + : new Response(null, { + status: 204, + headers: createHeaders('opaque::closed', true), + }) + }, + }) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-close-retry'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + await durability.append([textChunk('x')]) + + await expect(durability.close()).rejects.toThrow(/failed to close/i) + await expect(durability.close()).resolves.toBeUndefined() + expect(server.closeCount()).toBe(2) + }) + + it.each([ + [ + 'closed control', + controlEvent({ streamNextOffset: 'opaque::closed', streamClosed: true }), + ], + ['parse failure', 'event: unexpected\ndata: {}\n\n'], + ])('cancels the replay body after %s', async (_name, body) => { + const cancel = vi.fn(() => undefined) + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(body)) + }, + cancel, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) + const fetchStub = vi.fn(async () => response) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-cancel-body'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + + const consume = async () => { + for await (const _entry of durability.read('-1')) { + // drain + } + } + if (_name === 'parse failure') { + await expect(consume()).rejects.toThrow(/unexpected SSE event type/) + } else { + await expect(consume()).resolves.toBeUndefined() + } + expect(cancel).toHaveBeenCalledOnce() + }) + + it( + 'cancels a pending replay body when aborted', + { timeout: 500 }, + async () => { + const cancel = vi.fn(() => undefined) + const response = new Response( + new ReadableStream({ cancel }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ) + const fetchStub = vi.fn(async () => response) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-abort-body'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + const controller = new AbortController() + const iterator = durability + .read('-1', controller.signal) + [Symbol.asyncIterator]() + const next = iterator.next() + + await vi.waitFor(() => expect(fetchStub).toHaveBeenCalledOnce()) + controller.abort() + + await expect(next).resolves.toEqual({ done: true, value: undefined }) + expect(cancel).toHaveBeenCalledOnce() + }, + ) + + it('propagates read abort and ends the iterator cleanly', async () => { + const fetchStub = vi.fn( + async (_input, init) => + await new Promise((_resolve, reject) => { + const signal = init?.signal + if (!signal) throw new Error('Expected a read abort signal') + signal.addEventListener( + 'abort', + () => reject(new DOMException('Aborted', 'AbortError')), + { once: true }, + ) + }), + ) + const durability = durableStream( + new Request('https://app.test/api/chat?runId=run-abort'), + { server: 'https://ds.test', fetch: fetchStub }, + ) + const controller = new AbortController() + const iterator = durability + .read('-1', controller.signal) + [Symbol.asyncIterator]() + const next = iterator.next() + + await vi.waitFor(() => expect(fetchStub).toHaveBeenCalledOnce()) + controller.abort() + await expect(next).resolves.toEqual({ done: true, value: undefined }) + }) +}) + +describe('durableStream exact-once resume', () => { + it('resumes mid-way through a coalesced data batch without gaps or duplicates', async () => { + const server = makeProtocolServer({ + createOffset: 'opaque::batch-start/A?token=%2F+==', + appendOffsets: ['opaque::batch-end/Z?token=%3D'], + }) + const full = ['a', 'b', 'c', 'd', 'e', 'f'] + const producer = durableStream( + new Request('https://app.test/api/chat?runId=run-exact-once'), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const produced = parseTransportEvents( + await readBody( + toServerSentEventsResponse(textStream(full), { + durability: { adapter: producer }, + }), + ), + ) + + expect(server.batches).toHaveLength(1) + expect(produced.every((event) => event.id !== undefined)).toBe(true) + const beforeDrop = produced.slice(0, 2) + const resumeOffset = beforeDrop.at(-1)?.id + if (!resumeOffset) throw new Error('Expected a resume offset') + expect(decodeURIComponent(resumeOffset)).not.toContain('runId') + const exploding: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next() { + throw new Error('input stream must not be iterated on resume') + }, + } + }, + } + const reconnect = durableStream( + new Request('https://app.test/api/chat?runId=run-exact-once', { + headers: { 'Last-Event-ID': resumeOffset }, + }), + { server: 'https://ds.test', fetch: server.fetchStub }, + ) + const afterDrop = parseTransportEvents( + await readBody( + toServerSentEventsResponse(exploding, { + durability: { adapter: reconnect }, + }), + ), + ) + + expect( + [...beforeDrop, ...afterDrop].map((event) => deltaFrom(event.data)), + ).toEqual(full) + const replayRequest = server.requests.find( + (request) => request.method === 'GET', + ) + expect(replayRequest?.url.searchParams.get('offset')).toBe( + server.createOffset, + ) + expect(server.closeCount()).toBe(1) + }) +}) diff --git a/packages/ai-durable-stream/tsconfig.json b/packages/ai-durable-stream/tsconfig.json new file mode 100644 index 000000000..c38689f4e --- /dev/null +++ b/packages/ai-durable-stream/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-durable-stream/vite.config.ts b/packages/ai-durable-stream/vite.config.ts new file mode 100644 index 000000000..11f5b20b7 --- /dev/null +++ b/packages/ai-durable-stream/vite.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/packages/ai/skills/ai-core/chat-experience/SKILL.md b/packages/ai/skills/ai-core/chat-experience/SKILL.md index 91840c4ed..b6a114345 100644 --- a/packages/ai/skills/ai-core/chat-experience/SKILL.md +++ b/packages/ai/skills/ai-core/chat-experience/SKILL.md @@ -16,6 +16,7 @@ sources: - 'TanStack/ai:docs/chat/connection-adapters.md' - 'TanStack/ai:docs/chat/thinking-content.md' - 'TanStack/ai:docs/advanced/multimodal-content.md' + - 'TanStack/ai:docs/resumable-streams/overview.md' --- # Chat Experience @@ -41,7 +42,7 @@ export const Route = createFileRoute('/api/chat')({ const { messages } = body const stream = chat({ - adapter: openaiText('gpt-5.2'), + adapter: openaiText('gpt-5.5'), messages, systemPrompts: ['You are a helpful assistant.'], abortController, @@ -148,6 +149,16 @@ const stream = chat({ return toServerSentEventsResponse(stream, { abortController }) ``` +To make the SSE response resumable (reconnect after a drop/refresh without +re-running the provider), pass a delivery-durability adapter: +`toServerSentEventsResponse(stream, { durability: { adapter: memoryStream(request) } })` +(`memoryStream` from `@tanstack/ai` is process-local, for dev/tests) or +`durableStream(request, { server })` from `@tanstack/ai-durable-stream` +(Durable Streams protocol, production). Each SSE event gets an opaque +adapter-owned `id:`; `fetchServerSentEvents` auto-reconnects with +`Last-Event-ID` and exposes `joinRun(runId)` to replay a run from the start. +See `docs/resumable-streams/overview.md`. + **Client:** ```typescript @@ -317,7 +328,7 @@ import { chat, toHttpResponse } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' const stream = chat({ - adapter: openaiText('gpt-5.2'), + adapter: openaiText('gpt-5.5'), messages, abortController, }) @@ -338,6 +349,13 @@ const { messages, sendMessage } = useChat({ The only difference is swapping `toServerSentEventsResponse` / `fetchServerSentEvents` for `toHttpResponse` / `fetchHttpStream`. Everything else stays identical. +This includes resumability: pass the same `durability` adapter to +`toHttpResponse(stream, { durability: { adapter: memoryStream(request) } })` and +each NDJSON line becomes an `{ id, chunk }` envelope. `fetchHttpStream` +auto-reconnects with `Last-Event-ID`, de-dupes the replayed prefix, and exposes +`joinRun(runId)` — the same guarantees as resumable SSE. The XHR adapters +(`xhrServerSentEvents` / `xhrHttpStream`) are resumable too. + ### 6. MCP Tool Discovery via `chat({ mcp })` Pass `mcp` to let `chat()` own discovery **and** lifecycle for one or more MCP @@ -475,12 +493,12 @@ sendMessage('Never mind, do this instead', { whenBusy: 'interrupt' }) // WRONG import { streamText } from 'ai' import { openai } from '@ai-sdk/openai' -const result = streamText({ model: openai('gpt-4o'), messages }) +const result = streamText({ model: openai('gpt-5.5'), messages }) // CORRECT import { chat } from '@tanstack/ai' import { openaiText } from '@tanstack/ai-openai' -const stream = chat({ adapter: openaiText('gpt-5.2'), messages }) +const stream = chat({ adapter: openaiText('gpt-5.5'), messages }) ``` ### b. CRITICAL: Using Vercel createOpenAI() provider pattern @@ -489,12 +507,12 @@ const stream = chat({ adapter: openaiText('gpt-5.2'), messages }) // WRONG import { createOpenAI } from '@ai-sdk/openai' const openai = createOpenAI({ apiKey }) -streamText({ model: openai('gpt-4o'), messages }) +streamText({ model: openai('gpt-5.5'), messages }) // CORRECT import { openaiText } from '@tanstack/ai-openai' import { chat } from '@tanstack/ai' -chat({ adapter: openaiText('gpt-5.2'), messages }) +chat({ adapter: openaiText('gpt-5.5'), messages }) ``` ### c. CRITICAL: Using monolithic openai() instead of openaiText() @@ -502,11 +520,11 @@ chat({ adapter: openaiText('gpt-5.2'), messages }) ```typescript // WRONG import { openai } from '@tanstack/ai-openai' -chat({ adapter: openai(), model: 'gpt-5.2', messages }) +chat({ adapter: openai(), model: 'gpt-5.5', messages }) // CORRECT import { openaiText } from '@tanstack/ai-openai' -chat({ adapter: openaiText('gpt-5.2'), messages }) +chat({ adapter: openaiText('gpt-5.5'), messages }) ``` The monolithic `openai()` adapter is deprecated. Use tree-shakeable adapters: @@ -528,10 +546,10 @@ return toServerSentEventsResponse(stream, { abortController }) ```typescript // WRONG -chat({ adapter: openaiText(), model: 'gpt-5.2', messages }) +chat({ adapter: openaiText(), model: 'gpt-5.5', messages }) // CORRECT -chat({ adapter: openaiText('gpt-5.2'), messages }) +chat({ adapter: openaiText('gpt-5.5'), messages }) ``` The model is passed to the adapter factory, not to `chat()`. diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 102987c02..0ecdaf8f4 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -75,10 +75,16 @@ export { streamToText, toServerSentEventsStream, toServerSentEventsResponse, + resumeServerSentEventsResponse, toHttpStream, toHttpResponse, + resumeHttpResponse, } from './stream-to-response' +// Delivery durability (transport layer) +export { memoryStream } from './stream-durability' +export type { MemoryStreamOptions, StreamDurability } from './stream-durability' + // Tool call management export { ToolCallManager } from './activities/chat/tools/tool-calls' diff --git a/packages/ai/src/stream-durability.ts b/packages/ai/src/stream-durability.ts new file mode 100644 index 000000000..538f4c91a --- /dev/null +++ b/packages/ai/src/stream-durability.ts @@ -0,0 +1,334 @@ +import type { StreamChunk } from './types' + +/** + * A pluggable delivery-durability backend. + * + * Offsets are owned by the adapter and opaque to the transport. The generic + * parameter lets an adapter retain a branded string type across append, read, + * and resume without requiring core to understand its cursor format. + */ +export interface StreamDurability { + /** Return the adapter offset captured from the request, or null for a producer. */ + resumeFrom: () => TOffset | null + /** + * Persist a batch before it is delivered and return exactly one resumable + * offset for each chunk, in the same order. + */ + append: (chunks: Array) => Promise> + /** Replay chunks strictly after the supplied adapter-owned offset. */ + read: ( + offset: TOffset, + signal?: AbortSignal, + ) => AsyncIterable<{ offset: TOffset; chunk: StreamChunk }> + /** + * Terminalize the producer log and unblock live readers. Core awaits this + * for every producer exit, including completion, cancellation, and failure. + */ + close: () => Promise +} + +const MEMORY_OFFSET_PREFIX = 'memory:v1:' + +interface MemoryOffset { + runId: string + seq: number +} + +function encodeMemoryOffset(runId: string, seq: number): string { + return `${MEMORY_OFFSET_PREFIX}${encodeURIComponent(runId)}:${seq}` +} + +function decodeMemoryOffset(offset: string): MemoryOffset { + if (!offset.startsWith(MEMORY_OFFSET_PREFIX)) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + const encoded = offset.slice(MEMORY_OFFSET_PREFIX.length) + const separator = encoded.lastIndexOf(':') + if (separator === -1) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + const runId = decodeURIComponent(encoded.slice(0, separator)) + const seq = Number(encoded.slice(separator + 1)) + if (!Number.isSafeInteger(seq) || seq < 1) { + throw new Error(`Invalid memory stream offset: ${offset}`) + } + return { runId, seq } +} + +function readResumeOffset(request: Request): string | null { + const header = request.headers.get('Last-Event-ID') + if (header) return header + try { + return new URL(request.url).searchParams.get('offset') + } catch { + return null + } +} + +function readRunId(request: Request): string | null { + try { + return new URL(request.url).searchParams.get('runId') + } catch { + return null + } +} + +function assertValidRunId(runId: string): string { + if (runId.length === 0 || /[\r\n]/.test(runId)) { + throw new Error( + `Invalid runId (must be non-empty and contain no CR/LF): ${JSON.stringify(runId)}`, + ) + } + return runId +} + +function resolveMemoryRunId( + request: Request, + resumeOffset: string | null, +): string { + if ( + resumeOffset !== null && + resumeOffset !== '-1' && + resumeOffset !== 'now' + ) { + return assertValidRunId(decodeMemoryOffset(resumeOffset).runId) + } + const requestedRunId = readRunId(request) + return requestedRunId === null + ? crypto.randomUUID() + : assertValidRunId(requestedRunId) +} + +function memoryThreshold(offset: string, runId: string, tail: number): number { + if (offset === '-1') return -1 + if (offset === 'now') return tail + const decoded = decodeMemoryOffset(offset) + if (decoded.runId !== runId) { + throw new Error( + `Memory stream offset belongs to run ${JSON.stringify(decoded.runId)}, not ${JSON.stringify(runId)}`, + ) + } + return decoded.seq +} + +function isTerminalChunk(chunk: StreamChunk): boolean { + return chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR' +} + +interface MemoryEntry { + seq: number + offset: string + chunk: StreamChunk +} + +interface MemoryLog { + entries: Array + complete: boolean + /** Epoch ms when the log was terminalized; undefined while still producing. */ + completedAt: number | undefined + waiters: Array<() => void> +} + +/** + * Bounds for the in-process log store. `memoryStream` is the dev/single-process + * backend; without eviction its module-global Map would grow without bound on a + * long-lived server (one retained chunk buffer per run, forever). Completed logs + * are swept after a grace window — late resumers/joiners still work briefly — + * and a hard cap drops the oldest completed logs under pressure. Active + * (incomplete) logs are never evicted, so an in-flight run is never dropped. + */ +const MAX_MEMORY_RUNS = 1024 +const COMPLETED_LOG_TTL_MS = 5 * 60_000 + +/** + * How long a from-start join (`-1` / `now`) waits for a run's first chunk before + * failing. Bounds the "joined a run that never produces" case so a consumer + * gets a surfaced error instead of an indefinitely-open, event-less connection. + */ +const DEFAULT_FIRST_CHUNK_DEADLINE_MS = 30_000 + +/** Options for the in-process delivery-durability backend. */ +export interface MemoryStreamOptions { + /** + * Milliseconds a from-start join waits for the run's first chunk before + * throwing. Defaults to {@link DEFAULT_FIRST_CHUNK_DEADLINE_MS}. + */ + firstChunkDeadlineMs?: number +} + +const memoryLogs = new Map() + +/** + * Evict completed logs past their grace window, then, if still over the cap, + * drop the oldest completed logs (the Map preserves insertion order) until back + * under the cap. Never touches an incomplete (in-flight) log. + */ +function sweepMemoryLogs(now: number): void { + for (const [id, log] of memoryLogs) { + if ( + log.complete && + log.completedAt !== undefined && + now - log.completedAt > COMPLETED_LOG_TTL_MS + ) { + memoryLogs.delete(id) + } + } + if (memoryLogs.size <= MAX_MEMORY_RUNS) return + for (const [id, log] of memoryLogs) { + if (memoryLogs.size <= MAX_MEMORY_RUNS) break + if (log.complete) memoryLogs.delete(id) + } +} + +function getOrCreateLog(id: string): MemoryLog { + let log = memoryLogs.get(id) + if (!log) { + sweepMemoryLogs(Date.now()) + log = { entries: [], complete: false, completedAt: undefined, waiters: [] } + memoryLogs.set(id, log) + } + return log +} + +function markComplete(log: MemoryLog): void { + if (!log.complete) { + log.complete = true + log.completedAt = Date.now() + } +} + +function wakeWaiters(log: MemoryLog): void { + const waiters = log.waiters + log.waiters = [] + for (const wake of waiters) wake() +} + +/** + * The zero-infrastructure delivery-durability backend. Its versioned cursor is + * deliberately private: callers and core only pass the returned string back. + * + * Logs live in a process-global map, so this backend is for development, tests, + * and single-process deployments only. Completed runs are evicted after a grace + * window (see {@link COMPLETED_LOG_TTL_MS}); a resume of an evicted or unknown + * run fails loudly rather than hanging. + */ +export function memoryStream( + request: Request, + options: MemoryStreamOptions = {}, +): StreamDurability { + const resumeOffset = readResumeOffset(request) + const runId = resolveMemoryRunId(request, resumeOffset) + const firstChunkDeadlineMs = + options.firstChunkDeadlineMs ?? DEFAULT_FIRST_CHUNK_DEADLINE_MS + + return { + resumeFrom: () => resumeOffset, + append: (chunks) => { + const log = getOrCreateLog(runId) + const firstSeq = (log.entries.at(-1)?.seq ?? 0) + 1 + const offsets = chunks.map((chunk, index) => { + const seq = firstSeq + index + const offset = encodeMemoryOffset(runId, seq) + log.entries.push({ seq, offset, chunk }) + if (isTerminalChunk(chunk)) markComplete(log) + return offset + }) + wakeWaiters(log) + return Promise.resolve(offsets) + }, + close: () => { + const log = getOrCreateLog(runId) + markComplete(log) + wakeWaiters(log) + return Promise.resolve() + }, + read: async function* (offset, signal) { + const isFromStartJoin = offset === '-1' || offset === 'now' + + // Peek, never getOrCreateLog. A concrete resume offset for an absent run + // means the run was evicted (or never lived in this process) and will not + // reappear — fail WITHOUT inserting a log. Inserting here would leave a + // permanent empty, never-completed log (sweep only reclaims complete + // ones), so client-supplied offsets could grow the map without bound and + // defeat the eviction this backend relies on. + let log = memoryLogs.get(runId) + if (log === undefined || (log.entries.length === 0 && !log.complete)) { + if (!isFromStartJoin) { + throw new Error( + `Unknown or expired memory stream run: ${JSON.stringify(runId)}`, + ) + } + // A from-start join may legitimately attach before the producer creates + // the log (second-tab race); create it so a later append reuses the + // same entry. If no producer ever arrives, the first-chunk deadline + // below deletes this phantom before rejecting. + log = getOrCreateLog(runId) + } + + const threshold = memoryThreshold( + offset, + runId, + log.entries.at(-1)?.seq ?? 0, + ) + let index = 0 + + for (;;) { + while (index < log.entries.length) { + const entry = log.entries[index] + index += 1 + if (entry && entry.seq > threshold) { + yield { offset: entry.offset, chunk: entry.chunk } + if (isTerminalChunk(entry.chunk)) return + } + } + if (log.complete || signal?.aborted) return + + // Bound only the wait for the very first chunk: once a run has produced + // anything, its producer owns termination and a caught-up reader may + // legitimately park indefinitely between chunks. + const deadlineForFirstChunk = + log.entries.length === 0 ? firstChunkDeadlineMs : undefined + + await new Promise((resolve, reject) => { + let timer: ReturnType | undefined + const cleanup = () => { + if (timer !== undefined) clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + const waiterIndex = log.waiters.indexOf(wake) + if (waiterIndex !== -1) log.waiters.splice(waiterIndex, 1) + } + const onAbort = () => { + cleanup() + resolve() + } + const wake = () => { + cleanup() + resolve() + } + log.waiters.push(wake) + signal?.addEventListener('abort', onAbort, { once: true }) + if (deadlineForFirstChunk !== undefined) { + timer = setTimeout(() => { + cleanup() + // No producer ever created data for this joined run. Drop the + // phantom log we created above so it does not linger uncollected + // (it is empty and will never be marked complete). + if ( + log.entries.length === 0 && + !log.complete && + memoryLogs.get(runId) === log + ) { + memoryLogs.delete(runId) + } + reject( + new Error( + `Memory stream run produced no data within ${deadlineForFirstChunk}ms: ${JSON.stringify(runId)}`, + ), + ) + }, deadlineForFirstChunk) + } + }) + } + }, + } +} diff --git a/packages/ai/src/stream-to-response.ts b/packages/ai/src/stream-to-response.ts index 9850f4d60..09fda5b98 100644 --- a/packages/ai/src/stream-to-response.ts +++ b/packages/ai/src/stream-to-response.ts @@ -1,4 +1,9 @@ import { toRunErrorPayload } from './activities/error-payload' +import { EventType } from './types' +import { resolveDebugOption } from './logger/resolve' +import type { InternalLogger } from './logger/internal-logger' +import type { DebugOption } from './logger/types' +import type { StreamDurability } from './stream-durability' import type { StreamChunk } from './types' /** @@ -13,8 +18,7 @@ import type { StreamChunk } from './types' * @example * ```typescript * const stream = chat({ - * adapter: openaiText(), - * model: 'gpt-4o', + * adapter: openaiText('gpt-5.5'), * messages: [{ role: 'user', content: 'Hello!' }] * }); * const text = await streamToText(stream); @@ -35,6 +39,151 @@ export async function streamToText( return accumulatedContent } +interface RecordedFailure { + error: unknown +} + +function errorMessage(error: unknown): string { + return toRunErrorPayload(error).message +} + +function combineFailures( + primary: unknown, + secondary: unknown, + phase: string, +): unknown { + if (primary === secondary) return primary + const errors = + primary instanceof AggregateError + ? [...primary.errors, secondary] + : [primary, secondary] + return new AggregateError( + errors, + `${errorMessage(primary)}; ${phase}: ${errorMessage(secondary)}`, + ) +} + +function runErrorChunk( + error: unknown, +): Extract { + const payload = toRunErrorPayload(error) + return { + type: EventType.RUN_ERROR, + timestamp: Date.now(), + message: payload.message, + ...(payload.code === undefined ? {} : { code: payload.code }), + error: payload, + } +} + +function isAborted(signal: AbortSignal): boolean { + return signal.aborted +} + +function needsTerminalPersistence( + terminalPersisted: boolean, + cancelled: boolean, + failed: boolean, +): boolean { + return !terminalPersisted && (cancelled || failed) +} + +function toEncodedStream( + stream: AsyncIterable, + abortController: AbortController | undefined, + encodeChunk: (chunk: StreamChunk, index: number) => Uint8Array, + encodeError: (error: unknown) => Uint8Array, +): ReadableStream { + const cancellation = abortController ?? new AbortController() + let iterator: AsyncIterator | undefined + let iteratorCleanup: Promise | undefined + let pumpPromise: Promise = Promise.resolve() + let pumpFailure: RecordedFailure | undefined + let cancelled = false + + const recordPumpFailure = (error: unknown, phase: string): void => { + pumpFailure = { + error: + pumpFailure === undefined + ? error + : combineFailures(pumpFailure.error, error, phase), + } + } + + const closeIterator = (): Promise => { + iteratorCleanup ??= (async () => { + if (iterator?.return) await iterator.return() + })() + return iteratorCleanup + } + + return new ReadableStream({ + start(controller) { + iterator = stream[Symbol.asyncIterator]() + pumpPromise = (async () => { + let index = 0 + let iteratorDone = false + + try { + while (!isAborted(cancellation.signal)) { + const result = await iterator.next() + if (result.done) { + iteratorDone = true + break + } + if (isAborted(cancellation.signal)) break + controller.enqueue(encodeChunk(result.value, index)) + index += 1 + } + } catch (error) { + recordPumpFailure(error, 'stream iteration failed') + } finally { + if (!iteratorDone) { + try { + await closeIterator() + } catch (error) { + recordPumpFailure(error, 'iterator cleanup failed') + } + } + + if ( + !cancelled && + !isAborted(cancellation.signal) && + pumpFailure !== undefined + ) { + controller.enqueue(encodeError(pumpFailure.error)) + } + if (!cancelled) controller.close() + } + })().catch((error: unknown) => { + recordPumpFailure(error, 'stream pump failed') + }) + }, + async cancel(reason) { + cancelled = true + if (!isAborted(cancellation.signal)) cancellation.abort(reason) + + let cancellationFailure: RecordedFailure | undefined + try { + await closeIterator() + } catch (error) { + cancellationFailure = { error } + } + await pumpPromise + + if (pumpFailure !== undefined && cancellationFailure !== undefined) { + throw combineFailures( + pumpFailure.error, + cancellationFailure.error, + 'iterator cancellation failed', + ) + } + if (pumpFailure !== undefined) throw pumpFailure.error + if (cancellationFailure !== undefined) throw cancellationFailure.error + }, + }) +} + /** * Convert a StreamChunk async iterable to a ReadableStream in Server-Sent Events format * @@ -45,58 +194,304 @@ export async function streamToText( * * @param stream - AsyncIterable of StreamChunks from chat() * @param abortController - Optional AbortController to abort when stream is cancelled + * @param getId - Optional per-chunk durability offset; when present, each event gets an `id:` line * @returns ReadableStream in Server-Sent Events format */ export function toServerSentEventsStream( stream: AsyncIterable, abortController?: AbortController, + getId?: (chunk: StreamChunk, index: number) => string | undefined, ): ReadableStream { const encoder = new TextEncoder() + return toEncodedStream( + stream, + abortController, + (chunk, index) => { + const id = getId?.(chunk, index) + const idLine = id === undefined ? '' : `id: ${id}\n` + return encoder.encode(`${idLine}data: ${JSON.stringify(chunk)}\n\n`) + }, + (error) => + encoder.encode(`data: ${JSON.stringify(runErrorChunk(error))}\n\n`), + ) +} - return new ReadableStream({ - async start(controller) { - try { - for await (const chunk of stream) { - // Check if stream was cancelled/aborted - if (abortController?.signal.aborted) { - break - } +/** Default number of chunks buffered before a durability `append`. */ +const DEFAULT_DURABILITY_BATCH = 32 + +/** + * Resolve and validate the durability batch size. A non-positive-integer (0, + * negative, fractional, or `NaN`) is rejected rather than clamped: silently + * `Math.max(1, …)`-ing a `NaN` used to disable size-based flushing entirely + * (`length >= NaN` is always false), which is a subtle footgun. + */ +function resolveBatchSize(batch: number | undefined): number { + if (batch === undefined) return DEFAULT_DURABILITY_BATCH + if (!Number.isInteger(batch) || batch <= 0) { + throw new Error( + `Invalid durability batch size: ${batch}. Must be a positive integer.`, + ) + } + return batch +} + +/** + * Boundaries at which the batching producer flushes early, regardless of the + * batch size — terminal events and tool-call ends. Flushing here keeps the + * durability log promptly consistent at semantically meaningful points. + */ +function isDurabilityFlushBoundary(chunk: StreamChunk): boolean { + return ( + chunk.type === 'RUN_FINISHED' || + chunk.type === 'RUN_ERROR' || + chunk.type === 'TOOL_CALL_END' + ) +} + +/** + * Build the delivery-durable source iterable for a transport helper. + * + * - **Resume** (`resumeFrom()` non-null): replay strictly after the offset, + * reading only from the durability log. The input `stream` is NEVER iterated, + * so `chat()`'s lazy iterator never fires the provider — the untouched + * generator is simply GC'd. This is what makes resume free of re-invocation. + * - **Fresh** (`resumeFrom()` null): iterate `stream`, buffering up to `batch` + * chunks (flushing early at terminal / tool-call boundaries), `append` each + * batch to the log, then forward. Appending BEFORE forwarding guarantees a + * reconnecting client can always replay exactly what it already saw. + * + * The returned `getId` maps each forwarded chunk to the exact opaque offset + * returned by the durability adapter for the SSE `id:` line. + */ +function durableStreamSource( + stream: AsyncIterable, + durability: StreamDurability, + options: { + abortController: AbortController + batch?: number + logger?: InternalLogger + }, +): { + source: AsyncIterable + getId: (chunk: StreamChunk) => string | undefined +} { + const resumeOffset = durability.resumeFrom() + const batchSize = resolveBatchSize(options.batch) + const abortController = options.abortController + const logger = options.logger + const idByChunk = new WeakMap() + const seenOffsets = new Set() + const getId = (chunk: StreamChunk): string | undefined => idByChunk.get(chunk) + + const validateOffset = (offset: TOffset): void => { + // Reject NUL/CR/LF (would corrupt the SSE `id:` line) and any offset that + // is not invariant under the wire round-trip. The SSE client reads the id + // with `.trim()`, so an offset with leading/trailing whitespace would come + // back changed and no longer match on reconnect — fail loud here rather + // than silently mis-resuming. (NDJSON carries the offset inside the JSON + // envelope and is unaffected, but the contract must hold for both wires.) + if ( + offset.length === 0 || + /[\0\r\n]/.test(offset) || + offset !== offset.trim() + ) { + throw new Error( + `Invalid durability offset for SSE id: ${JSON.stringify(offset)}`, + ) + } + if (seenOffsets.has(offset)) { + throw new Error( + `Durability adapter must return a unique offset per chunk: ${JSON.stringify(offset)}`, + ) + } + seenOffsets.add(offset) + } - // Send each chunk as Server-Sent Events format - controller.enqueue( - encoder.encode(`data: ${JSON.stringify(chunk)}\n\n`), - ) + async function* produce(): AsyncIterable { + let batch: Array = [] + let terminalPersisted = false + // Whether a terminal event was actually delivered LIVE to the consumer (as + // opposed to only appended to the log). Distinguishes "the run already ended + // on the wire" from "the log has a terminal but the consumer never saw one", + // which governs whether a late durability-cleanup failure may be rethrown. + // Only ever assigned inside the nested flush() closure, which TS's + // control-flow analysis can't observe (see the disable at the read site). + let terminalForwarded = false + let failure: RecordedFailure | undefined + let terminalCause: unknown + let hasTerminalCause = false + + const recordFailure = (error: unknown, phase: string): void => { + failure = { + error: + failure === undefined + ? error + : combineFailures(failure.error, error, phase), + } + } + + async function* flush(): AsyncIterable { + if (batch.length === 0) return + const toForward = batch + batch = [] + // Tag each chunk with the exact backend offset. Requiring one opaque + // token per chunk preserves exact-once resume at any batch size. + const offsets = await durability.append(toForward) + if (offsets.length !== toForward.length) { + throw new Error( + `Durability append returned ${offsets.length} offsets for ${toForward.length} chunks`, + ) + } + toForward.forEach((chunk, i) => { + const offset = offsets[i] + if (offset === undefined) { + throw new Error(`Durability append omitted offset at index ${i}`) } + validateOffset(offset) + idByChunk.set(chunk, offset) + }) + if ( + toForward.some( + (chunk) => + chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR', + ) + ) { + terminalPersisted = true + } + for (const chunk of toForward) { + if (chunk.type === 'RUN_FINISHED' || chunk.type === 'RUN_ERROR') { + terminalForwarded = true + } + yield chunk + } + } - controller.close() - } catch (error: unknown) { - // Don't send error if aborted - if (abortController?.signal.aborted) { - controller.close() - return + try { + if (isAborted(abortController.signal)) return + for await (const chunk of stream) { + if (isAborted(abortController.signal)) break + batch.push(chunk) + if (batch.length >= batchSize || isDurabilityFlushBoundary(chunk)) { + yield* flush() } + } + if (!isAborted(abortController.signal)) yield* flush() + } catch (error) { + terminalCause = error + hasTerminalCause = true + recordFailure(error, 'producer failed') + // The provider stream threw. Persist a terminal RUN_ERROR to the + // durability log so a resumer / joiner learns the run failed (otherwise + // the log ends with no terminal and they wait forever). Flush any + // buffered chunks first, then append the terminal WITHOUT forwarding it + // live — the transport layer synthesizes the live RUN_ERROR on rethrow, + // so forwarding here too would double-emit. + if (!isAborted(abortController.signal)) { + try { + yield* flush() + } catch (flushError) { + recordFailure(flushError, 'flushing buffered chunks failed') + } + } + } finally { + const cancelled = isAborted(abortController.signal) - // Send error event (AG-UI RUN_ERROR) - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - type: 'RUN_ERROR', - timestamp: Date.now(), - error: toRunErrorPayload(error), - })}\n\n`, - ), - ) - controller.close() + // Persist any buffered-but-unflushed chunks before terminalizing, so a + // joiner replaying the log sees everything produced up to a disconnect + // rather than a truncated prefix. On the abort path the streaming loop + // broke before its trailing flush; drain flush() here for its persistence + // side effect only (the delivery socket is gone, so the yielded chunks are + // discarded). The normal and provider-throw paths already flushed, so + // `batch` is empty for them and this is a no-op. + if (batch.length > 0) { + try { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const _chunk of flush()) { + // persist-only: nothing consumes these + } + } catch (flushError) { + recordFailure(flushError, 'flushing buffered chunks on exit failed') + } } - }, - cancel() { - // When the ReadableStream is cancelled (e.g., client disconnects), - // abort the underlying stream - if (abortController) { - abortController.abort() + + if ( + needsTerminalPersistence(terminalPersisted, cancelled, hasTerminalCause) + ) { + // Prefer the real provider error even when the delivery socket was also + // aborted: if the run genuinely failed, a joiner should see that cause, + // not a generic AbortError that masks it. AbortError is only used for a + // pure cancellation with no underlying failure. + const cause = hasTerminalCause ? terminalCause : { name: 'AbortError' } + try { + await durability.append([runErrorChunk(cause)]) + terminalPersisted = true + } catch (terminalError) { + // Rethrown to the live consumer below, but a joiner replaying the log + // only ever sees a generic incomplete error — so record the real + // cause server-side where an operator can act on it. + logger?.errors('persisting terminal RUN_ERROR failed', { + error: terminalError, + }) + recordFailure(terminalError, 'persisting terminal RUN_ERROR failed') + } } - }, - }) + + try { + await durability.close() + } catch (closeError) { + // A failed close leaves the durable log unterminated for joiners; the + // live consumer gets the rethrow, but log it for the joiner's sake. + logger?.errors('closing durability stream failed', { + error: closeError, + }) + recordFailure(closeError, 'closing durability stream failed') + } + + // Rethrow a terminalization/close failure to the live consumer ONLY when + // no terminal reached it yet — the transport then synthesizes a live + // RUN_ERROR so the consumer isn't left without a terminal. If a terminal + // was already forwarded (the run ended on the wire), a late failure is a + // server-side cleanup issue; rethrowing it would append a contradictory + // second terminal (RUN_ERROR after RUN_FINISHED) on the wire. Suppress the + // rethrow, but never let the cause vanish — record it server-side, the + // same as the close / terminal-append failures above. (This also covers a + // provider that throws AFTER emitting its own terminal, whose error is + // otherwise neither delivered nor logged.) + if (failure !== undefined) { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- terminalForwarded is set only inside the flush() closure, which TS CFA narrows away here + if (!terminalForwarded) { + // eslint-disable-next-line no-unsafe-finally + throw failure.error + } + logger?.errors( + 'durability failure after a terminal event was forwarded', + { + error: failure.error, + }, + ) + } + } + } + + async function* replay(offset: TOffset): AsyncIterable { + // Thread the consumer's abort signal into the read so a live-tailing join + // (a mid-stream reconnect) that is aborted — or that hit a runId with no + // in-process producer — stops parking and ends instead of hanging forever. + for await (const { offset: eventOffset, chunk } of durability.read( + offset, + abortController.signal, + )) { + if (isAborted(abortController.signal)) break + validateOffset(eventOffset) + idByChunk.set(chunk, eventOffset) + yield chunk + } + } + + return { + source: resumeOffset !== null ? replay(resumeOffset) : produce(), + getId, + } } /** @@ -107,21 +502,42 @@ export function toServerSentEventsStream( * - Each chunk is followed by "\n\n" * - Stream ends when the underlying iterable is exhausted (RUN_FINISHED is the terminal event) * + * Pass a `durability` sink (`memoryStream(request)` / `durableStream(request)`) + * to make the stream resumable: fresh runs are appended to the log and each SSE + * event is tagged with an `id:` offset; a reconnect (native `Last-Event-ID`) or + * a `?offset` join replays from the log without re-running the producer. `batch` + * controls how many chunks are buffered per `append` (default 32). + * * @param stream - AsyncIterable of StreamChunks from chat() - * @param init - Optional Response initialization options (including `abortController`) + * @param init - Optional Response initialization options (including `abortController`, `durability` with its optional `batch`, and `debug`) * @returns Response in Server-Sent Events format * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); - * return toServerSentEventsResponse(stream, { abortController }); + * export async function POST(request: Request) { + * const stream = chat({ adapter: openaiText('gpt-5.5'), messages: [...] }); + * return toServerSentEventsResponse(stream, { durability: { adapter: memoryStream(request) } }); + * } * ``` */ -export function toServerSentEventsResponse( +export function toServerSentEventsResponse( stream: AsyncIterable, - init?: ResponseInit & { abortController?: AbortController }, + init?: ResponseInit & { + abortController?: AbortController + durability?: { adapter: StreamDurability; batch?: number } + /** + * Customize logging for durability failure paths (terminal-append and + * close). These failures are always logged server-side by default (the + * `errors` category is on even without `debug`, via a `ConsoleLogger`); + * pass `debug` to route them to a custom `Logger` or raise verbosity. A + * joiner replaying the log only ever sees a generic incomplete error, so + * server-side logging is where the real cause is recoverable. + */ + debug?: DebugOption + }, ): Response { - const { headers, abortController, ...responseInit } = init ?? {} + const { headers, abortController, durability, debug, ...responseInit } = + init ?? {} // Start with default SSE headers const mergedHeaders = new Headers({ @@ -139,12 +555,77 @@ export function toServerSentEventsResponse( }) } - return new Response(toServerSentEventsStream(stream, abortController), { + let body: ReadableStream + if (durability) { + const deliveryAbortController = abortController ?? new AbortController() + const { source, getId } = durableStreamSource(stream, durability.adapter, { + abortController: deliveryAbortController, + batch: durability.batch, + // `errors` category is on by default even when `debug` is undefined, so + // durability terminal-append / close failures always surface server-side — + // including on the client-disconnect path where there is no live consumer. + logger: resolveDebugOption(debug), + }) + body = toServerSentEventsStream(source, deliveryAbortController, getId) + } else { + body = toServerSentEventsStream(stream, abortController) + } + + return new Response(body, { ...responseInit, headers: mergedHeaders, }) } +/** + * A resume is served entirely from the durability log, so there is no producer + * to iterate. This empty source satisfies the response helpers' signature; on a + * resume `durableStreamSource` replays from the log and never touches it. + */ +function emptyDurableSource(): AsyncIterable { + return (async function* () {})() +} + +/** Shared options for the resume-only response helpers. */ +type ResumeResponseOptions = ResponseInit & { + adapter: StreamDurability + batch?: number + debug?: DebugOption +} + +const NO_RESUME_OFFSET = + 'No resume offset provided (expected a Last-Event-ID header or an ?offset query parameter).' + +/** + * Serve a resumable run from its durability log over Server-Sent Events, without + * re-running the model. Use this in a `GET` handler so a reload or a second tab + * can re-attach to an in-flight or finished run. + * + * The adapter (`memoryStream(request)` / `durableStream(request)`) captures the + * resume offset from the request. If there is none (no `Last-Event-ID` header + * and no `?offset`), there is nothing to replay and this returns a 400. + * + * @example + * ```typescript + * export async function GET(request: Request) { + * return resumeServerSentEventsResponse({ adapter: memoryStream(request) }); + * } + * ``` + */ +export function resumeServerSentEventsResponse( + options: ResumeResponseOptions, +): Response { + const { adapter, batch, debug, ...responseInit } = options + if (adapter.resumeFrom() === null) { + return new Response(NO_RESUME_OFFSET, { status: 400 }) + } + return toServerSentEventsResponse(emptyDurableSource(), { + ...responseInit, + durability: { adapter, batch }, + debug, + }) +} + /** * Convert a StreamChunk async iterable to a ReadableStream in HTTP stream format (newline-delimited JSON) * @@ -154,13 +635,20 @@ export function toServerSentEventsResponse( * * This format is compatible with `fetchHttpStream` connection adapter. * + * When `getId` is supplied (delivery durability), each chunk is emitted as an + * envelope `{"id":"","chunk":{…}}` instead of a bare chunk. NDJSON has + * no native event-id field like SSE's `id:` line, so the resumable offset rides + * inside the payload. Untagged chunks (no id) stay bare, so a non-durable + * stream is byte-identical to before and the client auto-detects either form. + * * @param stream - AsyncIterable of StreamChunks from chat() * @param abortController - Optional AbortController to abort when stream is cancelled + * @param getId - Optional per-chunk durability offset; when present, chunks are envelope-encoded * @returns ReadableStream in HTTP stream format (newline-delimited JSON) * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); + * const stream = chat({ adapter: openaiText('gpt-5.5'), messages: [...] }); * const readableStream = toHttpStream(stream); * // Use with Response for HTTP streaming (not SSE) * return new Response(readableStream, { @@ -171,51 +659,20 @@ export function toServerSentEventsResponse( export function toHttpStream( stream: AsyncIterable, abortController?: AbortController, + getId?: (chunk: StreamChunk, index: number) => string | undefined, ): ReadableStream { const encoder = new TextEncoder() - - return new ReadableStream({ - async start(controller) { - try { - for await (const chunk of stream) { - // Check if stream was cancelled/aborted - if (abortController?.signal.aborted) { - break - } - - // Send each chunk as newline-delimited JSON - controller.enqueue(encoder.encode(`${JSON.stringify(chunk)}\n`)) - } - - controller.close() - } catch (error: unknown) { - // Don't send error if aborted - if (abortController?.signal.aborted) { - controller.close() - return - } - - // Send error event (AG-UI RUN_ERROR) - controller.enqueue( - encoder.encode( - `${JSON.stringify({ - type: 'RUN_ERROR', - timestamp: Date.now(), - error: toRunErrorPayload(error), - })}\n`, - ), - ) - controller.close() - } - }, - cancel() { - // When the ReadableStream is cancelled (e.g., client disconnects), - // abort the underlying stream - if (abortController) { - abortController.abort() - } + return toEncodedStream( + stream, + abortController, + (chunk, index) => { + const id = getId?.(chunk, index) + const line = + id === undefined ? JSON.stringify(chunk) : JSON.stringify({ id, chunk }) + return encoder.encode(`${line}\n`) }, - }) + (error) => encoder.encode(`${JSON.stringify(runErrorChunk(error))}\n`), + ) } /** @@ -227,21 +684,103 @@ export function toHttpStream( * * This format is compatible with `fetchHttpStream` connection adapter. * + * Pass a `durability` sink (`memoryStream(request)` / `durableStream(request)`) + * to make the stream resumable: fresh runs are appended to the log and each + * NDJSON line is emitted as an `{ id, chunk }` envelope carrying an opaque + * offset; a reconnect (native `Last-Event-ID` header) or a `?offset` join + * replays from the log without re-running the producer. `batch` controls how + * many chunks are buffered per `append` (default 32). This shares the exact + * `durableStreamSource` used by `toServerSentEventsResponse` — only the wire + * encoding differs. + * * @param stream - AsyncIterable of StreamChunks from chat() - * @param init - Optional Response initialization options (including `abortController`) + * @param init - Optional Response initialization options (including `abortController`, `durability` with its optional `batch`, and `debug`) * @returns Response in HTTP stream format (newline-delimited JSON) * * @example * ```typescript - * const stream = chat({ adapter: openaiText(), model: "gpt-4o", messages: [...] }); - * return toHttpResponse(stream, { abortController }); + * export async function POST(request: Request) { + * const stream = chat({ adapter: openaiText('gpt-5.5'), messages: [...] }); + * return toHttpResponse(stream, { durability: { adapter: memoryStream(request) } }); + * } * ``` */ -export function toHttpResponse( +export function toHttpResponse( stream: AsyncIterable, - init?: ResponseInit & { abortController?: AbortController }, + init?: ResponseInit & { + abortController?: AbortController + durability?: { adapter: StreamDurability; batch?: number } + /** + * Customize logging for durability failure paths (terminal-append and + * close). These failures are always logged server-side by default (the + * `errors` category is on even without `debug`, via a `ConsoleLogger`); + * pass `debug` to route them to a custom `Logger` or raise verbosity. A + * joiner replaying the log only ever sees a generic incomplete error, so + * server-side logging is where the real cause is recoverable. + */ + debug?: DebugOption + }, ): Response { - return new Response(toHttpStream(stream, init?.abortController), { - ...init, + const { abortController, durability, debug, headers, ...responseInit } = + init ?? {} + + // Default to a streaming NDJSON content type (with no-cache), overridable by + // user headers. Without an explicit streaming type some intermediaries buffer + // the response, defeating incremental delivery. Mirrors the SSE helper. + const mergedHeaders = new Headers({ + 'Content-Type': 'application/x-ndjson', + 'Cache-Control': 'no-cache', + }) + if (headers) { + const userHeaders = new Headers(headers) + userHeaders.forEach((value, key) => { + mergedHeaders.set(key, value) + }) + } + + let body: ReadableStream + if (durability) { + const deliveryAbortController = abortController ?? new AbortController() + const { source, getId } = durableStreamSource(stream, durability.adapter, { + abortController: deliveryAbortController, + batch: durability.batch, + // Errors-on-by-default logger (see toServerSentEventsResponse). + logger: resolveDebugOption(debug), + }) + body = toHttpStream(source, deliveryAbortController, getId) + } else { + body = toHttpStream(stream, abortController) + } + + return new Response(body, { + ...responseInit, + headers: mergedHeaders, + }) +} + +/** + * Serve a resumable run from its durability log over NDJSON, without re-running + * the model. The NDJSON counterpart of {@link resumeServerSentEventsResponse}; + * pair it with a `toHttpResponse` producer. Returns a 400 when the request + * carries no resume offset (no `Last-Event-ID` header and no `?offset`). + * + * @example + * ```typescript + * export async function GET(request: Request) { + * return resumeHttpResponse({ adapter: memoryStream(request) }); + * } + * ``` + */ +export function resumeHttpResponse( + options: ResumeResponseOptions, +): Response { + const { adapter, batch, debug, ...responseInit } = options + if (adapter.resumeFrom() === null) { + return new Response(NO_RESUME_OFFSET, { status: 400 }) + } + return toHttpResponse(emptyDurableSource(), { + ...responseInit, + durability: { adapter, batch }, + debug, }) } diff --git a/packages/ai/tests/stream-delivery-contract.test.ts b/packages/ai/tests/stream-delivery-contract.test.ts new file mode 100644 index 000000000..f4e9620d7 --- /dev/null +++ b/packages/ai/tests/stream-delivery-contract.test.ts @@ -0,0 +1,273 @@ +import { describe, expect, it, vi } from 'vitest' +import { toServerSentEventsResponse } from '../src/stream-to-response' +import { ev } from './test-utils' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +function deferred(): { + promise: Promise + resolve: () => void +} { + let resolve = (): void => undefined + const promise = new Promise((done) => { + resolve = done + }) + return { promise, resolve } +} + +async function readBody(response: Response): Promise { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +function parseEvents(body: string): Array<{ id?: string; chunk: StreamChunk }> { + return body + .split('\n\n') + .filter((block) => block.length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing SSE data line in ${block}`) + return { ...(id === undefined ? {} : { id }), chunk: JSON.parse(data) } + }) +} + +function oneChunkStream(): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + yield ev.textContent('hello') + }, + } +} + +describe('delivery durability contract', () => { + it('forwards an adapter-owned replay offset unchanged and never closes the producer', async () => { + const resumeOffset = 'backend:v3:resume/token?partition=west' + const replayOffset = 'backend:v3:event/token#17' + const close = vi.fn(async () => undefined) + const durability = { + resumeFrom: () => resumeOffset, + append: async () => [], + read: async function* (offset: string) { + expect(offset).toBe(resumeOffset) + yield { + offset: replayOffset, + chunk: ev.textContent('replayed'), + } + }, + close, + } satisfies StreamDurability + + const response = toServerSentEventsResponse(oneChunkStream(), { + durability: { adapter: durability }, + }) + const events = parseEvents(await readBody(response)) + + expect(events.map((event) => event.id)).toEqual([replayOffset]) + expect(close).not.toHaveBeenCalled() + }) + + it('awaits producer close before completing a normal response', async () => { + const closing = deferred() + const close = vi.fn(() => closing.promise) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => + chunks.map((_, index) => `backend:normal:${index}`), + read: async function* () {}, + close, + } satisfies StreamDurability + const bodyPromise = readBody( + toServerSentEventsResponse(oneChunkStream(), { + durability: { adapter: durability, batch: 1 }, + }), + ) + let bodySettled = false + void bodyPromise.then(() => { + bodySettled = true + }) + + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + expect(bodySettled).toBe(false) + + closing.resolve() + await expect(bodyPromise).resolves.toContain('backend:normal:0') + }) + + it('persists an aborted RUN_ERROR and awaits close when the reader cancels', async () => { + const abortController = new AbortController() + const closing = deferred() + const sourceClosed = deferred() + const appended: Array = [] + const close = vi.fn(() => closing.promise) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + return chunks.map((_, index) => `backend:cancel:${index}`) + }, + read: async function* () {}, + close, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + try { + yield ev.textContent('before cancel') + if (!abortController.signal.aborted) { + await new Promise((resolve) => { + abortController.signal.addEventListener( + 'abort', + () => resolve(), + { + once: true, + }, + ) + }) + } + } finally { + sourceClosed.resolve() + } + }, + } + const response = toServerSentEventsResponse(source, { + abortController, + durability: { adapter: durability, batch: 1 }, + }) + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + + await reader.read() + const cancelPromise = reader.cancel() + let cancelSettled = false + void cancelPromise.then(() => { + cancelSettled = true + }) + + await vi.waitFor(() => expect(abortController.signal.aborted).toBe(true)) + await vi.waitFor(() => { + expect(appended.at(-1)?.type).toBe('RUN_ERROR') + }) + const terminal = appended.at(-1) + expect(terminal).toMatchObject({ + type: 'RUN_ERROR', + message: 'Request aborted', + code: 'aborted', + error: { message: 'Request aborted', code: 'aborted' }, + }) + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()) + expect(cancelSettled).toBe(false) + + closing.resolve() + await expect(cancelPromise).resolves.toBeUndefined() + await sourceClosed.promise + }) + + it('preserves a provider error when producer close also fails', async () => { + const providerError = new Error('provider exploded') + const closeError = new Error('close exploded') + const appended: Array = [] + const close = vi.fn(async () => { + throw closeError + }) + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + return chunks.map((_, index) => `backend:error:${index}`) + }, + read: async function* () {}, + close, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('before error') + throw providerError + }, + } + + const events = parseEvents( + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability, batch: 1 }, + }), + ), + ) + const persistedTerminal = appended.find( + (chunk) => chunk.type === 'RUN_ERROR', + ) + const liveTerminal = events.find( + (event) => event.chunk.type === 'RUN_ERROR', + )?.chunk + + expect(persistedTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: 'provider exploded', + }) + expect(liveTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: expect.stringContaining('provider exploded'), + }) + expect(liveTerminal).toMatchObject({ + error: { + message: expect.stringContaining('close exploded'), + }, + }) + }) + + it('aggregates provider, terminal persistence, and close failures', async () => { + const appended: Array = [] + const durability = { + resumeFrom: () => null, + append: async (chunks: Array) => { + appended.push(...chunks) + if (chunks.some((chunk) => chunk.type === 'RUN_ERROR')) { + throw new Error('terminal persistence exploded') + } + return chunks.map((_, index) => `backend:aggregate:${index}`) + }, + read: async function* () {}, + close: async () => { + throw new Error('aggregate close exploded') + }, + } satisfies StreamDurability + const source: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('before aggregate') + throw new Error('aggregate provider exploded') + }, + } + + const events = parseEvents( + await readBody( + toServerSentEventsResponse(source, { + durability: { adapter: durability, batch: 1 }, + }), + ), + ) + const liveTerminal = events.find( + (event) => event.chunk.type === 'RUN_ERROR', + )?.chunk + + expect(appended.at(-1)?.type).toBe('RUN_ERROR') + expect(liveTerminal).toMatchObject({ + type: 'RUN_ERROR', + message: expect.stringContaining('aggregate provider exploded'), + error: { + message: expect.stringContaining('terminal persistence exploded'), + }, + }) + expect(liveTerminal).toMatchObject({ + error: { + message: expect.stringContaining('aggregate close exploded'), + }, + }) + }) +}) diff --git a/packages/ai/tests/stream-durability-types.test-d.ts b/packages/ai/tests/stream-durability-types.test-d.ts new file mode 100644 index 000000000..182622bee --- /dev/null +++ b/packages/ai/tests/stream-durability-types.test-d.ts @@ -0,0 +1,36 @@ +import { expectTypeOf } from 'vitest' +import { + toHttpResponse, + toServerSentEventsResponse, +} from '../src/stream-to-response' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +declare const backendOffsetBrand: unique symbol +type BackendOffset = string & { + readonly [backendOffsetBrand]: true +} + +declare const durability: StreamDurability +declare const stream: AsyncIterable + +expectTypeOf(durability.resumeFrom()).toEqualTypeOf() +expectTypeOf(durability.append).returns.toEqualTypeOf< + Promise> +>() + +toServerSentEventsResponse(stream, { + durability: { adapter: durability }, +}) + +// NDJSON delivery is durable too: the branded adapter threads through +// `toHttpResponse` exactly as it does through the SSE helper. +toHttpResponse(stream, { + durability: { adapter: durability }, +}) + +// @ts-expect-error raw strings cannot be passed to a branded-offset adapter +durability.read('raw-offset') + +type HttpResponseOptions = NonNullable[1]> +expectTypeOf().toHaveProperty('durability') diff --git a/packages/ai/tests/stream-durability.test.ts b/packages/ai/tests/stream-durability.test.ts new file mode 100644 index 000000000..657d96870 --- /dev/null +++ b/packages/ai/tests/stream-durability.test.ts @@ -0,0 +1,294 @@ +import { describe, expect, it, vi } from 'vitest' +import { memoryStream } from '../src/stream-durability' +import { EventType } from '../src/types' +import { ev } from './test-utils' +import type { StreamChunk } from '../src/types' + +function label(chunk: StreamChunk): string { + return chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? chunk.delta + : `[${chunk.type}]` +} + +async function readLabels( + stream: AsyncIterable<{ offset: string; chunk: StreamChunk }>, +): Promise> { + const labels: Array = [] + for await (const { chunk } of stream) labels.push(label(chunk)) + return labels +} + +describe('memoryStream', () => { + it('returns opaque per-chunk offsets and replays them unchanged', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat', { method: 'POST' }), + ) + + expect(durability.resumeFrom()).toBeNull() + const offsets = await durability.append([ + ev.textContent('a'), + ev.textContent('b'), + ev.textContent('c'), + ]) + expect(offsets).toHaveLength(3) + expect(new Set(offsets).size).toBe(3) + await durability.close() + + const replayedOffsets: Array = [] + const replayedLabels: Array = [] + for await (const entry of durability.read('-1')) { + replayedOffsets.push(entry.offset) + replayedLabels.push(label(entry.chunk)) + } + expect(replayedOffsets).toEqual(offsets) + expect(replayedLabels).toEqual(['a', 'b', 'c']) + }) + + it('resumes strictly after an adapter-owned Last-Event-ID', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-resume', { + method: 'POST', + }), + ) + const offsets = await producer.append([ + ev.textContent('a'), + ev.textContent('b'), + ev.textContent('c'), + ]) + await producer.close() + + const reconnect = memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': offsets[1] ?? '' }, + }), + ) + expect(reconnect.resumeFrom()).toBe(offsets[1]) + + const entries = [] + const resumeOffset = reconnect.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + for await (const entry of reconnect.read(resumeOffset)) entries.push(entry) + expect(entries.map((entry) => entry.offset)).toEqual([offsets[2]]) + expect(entries.map((entry) => label(entry.chunk))).toEqual(['c']) + }) + + it('reads an opaque offset from the query string', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-query', { + method: 'POST', + }), + ) + const offsets = await producer.append([ + ev.textContent('x'), + ev.textContent('y'), + ]) + await producer.close() + + const joiner = memoryStream( + new Request( + `https://example.test/api/chat?offset=${encodeURIComponent(offsets[0] ?? '')}`, + { method: 'POST' }, + ), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + expect(await readLabels(joiner.read(resumeOffset))).toEqual(['y']) + }) + + it('live-tails a from-start join through the producer terminal', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-live', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('a'), ev.textContent('b')]) + + const joiner = memoryStream( + new Request('https://example.test/api/chat?runId=run-live&offset=-1', { + method: 'POST', + }), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + await new Promise((resolve) => setTimeout(resolve, 10)) + expect(received).toEqual(['a', 'b']) + + await producer.append([ev.textContent('c'), ev.textContent('d')]) + await producer.append([ev.runFinished()]) + await done + expect(received).toEqual(['a', 'b', 'c', 'd', '[RUN_FINISHED]']) + }) + + it('supports an adapter-owned tail sentinel for future writes', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-tail', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('old')]) + const joiner = memoryStream( + new Request('https://example.test/api/chat?runId=run-tail&offset=now', { + method: 'POST', + }), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + await new Promise((resolve) => setTimeout(resolve, 10)) + await producer.append([ev.textContent('new'), ev.runFinished()]) + await done + expect(received).toEqual(['new', '[RUN_FINISHED]']) + }) + + it('ends a parked reader when its signal aborts', async () => { + const controller = new AbortController() + const joiner = memoryStream( + new Request( + 'https://example.test/api/chat?runId=never-produced&offset=-1', + { method: 'POST' }, + ), + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + const iterated = readLabels(joiner.read(resumeOffset, controller.signal)) + + await new Promise((resolve) => setTimeout(resolve, 10)) + controller.abort() + await expect(iterated).resolves.toEqual([]) + }) + + it('fails a from-start join that never receives data before the deadline', async () => { + const joiner = memoryStream( + new Request( + 'https://example.test/api/chat?runId=run-no-producer&offset=-1', + { method: 'POST' }, + ), + { firstChunkDeadlineMs: 20 }, + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + + await expect(readLabels(joiner.read(resumeOffset))).rejects.toThrow( + /produced no data within 20ms/, + ) + }) + + it('does not apply the first-chunk deadline once a run has produced data', async () => { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-slow-tail', { + method: 'POST', + }), + ) + await producer.append([ev.textContent('a')]) + + const joiner = memoryStream( + new Request( + 'https://example.test/api/chat?runId=run-slow-tail&offset=-1', + { + method: 'POST', + }, + ), + { firstChunkDeadlineMs: 20 }, + ) + const resumeOffset = joiner.resumeFrom() + if (resumeOffset === null) throw new Error('Expected a resume offset') + + const received: Array = [] + const done = (async () => { + for await (const { chunk } of joiner.read(resumeOffset)) { + received.push(label(chunk)) + } + })() + + // Well past the 20ms first-chunk deadline: a caught-up reader keeps parking + // because the run already produced data. + await new Promise((resolve) => setTimeout(resolve, 60)) + expect(received).toEqual(['a']) + + await producer.append([ev.textContent('b'), ev.runFinished()]) + await done + expect(received).toEqual(['a', 'b', '[RUN_FINISHED]']) + }) + + it('fails a resume of an evicted run rather than hanging', async () => { + vi.useFakeTimers() + vi.setSystemTime(0) + try { + const producer = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + }), + ) + const offsets = await producer.append([ev.textContent('a')]) + await producer.append([ev.runFinished()]) + await producer.close() + const resumeFrom = offsets[0] + if (resumeFrom === undefined) throw new Error('Expected an offset') + + // Within the grace window the completed run still resumes. + const early = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + headers: { 'Last-Event-ID': resumeFrom }, + }), + ) + expect(await readLabels(early.read(resumeFrom))).toEqual([ + '[RUN_FINISHED]', + ]) + + // Past the grace window, creating a new log sweeps the completed one, and + // resuming the evicted run surfaces an error instead of parking. + vi.setSystemTime(6 * 60_000) + await memoryStream( + new Request('https://example.test/api/chat?runId=run-sweep-trigger', { + method: 'POST', + }), + ).append([ev.textContent('x')]) + const late = memoryStream( + new Request('https://example.test/api/chat?runId=run-evictable', { + method: 'POST', + headers: { 'Last-Event-ID': resumeFrom }, + }), + ) + await expect(readLabels(late.read(resumeFrom))).rejects.toThrow( + /Unknown or expired memory stream run/, + ) + } finally { + vi.useRealTimers() + } + }) + + it('rejects invalid run ids and offsets loudly', () => { + expect(() => + memoryStream( + new Request( + `https://example.test/api/chat?runId=${encodeURIComponent('evil\ninjected')}`, + { method: 'POST' }, + ), + ), + ).toThrow(/Invalid runId/) + + expect(() => + memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': 'another-backend:cursor' }, + }), + ), + ).toThrow(/Invalid memory stream offset/) + }) +}) diff --git a/packages/ai/tests/stream-to-response-durability.test.ts b/packages/ai/tests/stream-to-response-durability.test.ts new file mode 100644 index 000000000..f0a5f4343 --- /dev/null +++ b/packages/ai/tests/stream-to-response-durability.test.ts @@ -0,0 +1,706 @@ +import { describe, expect, it, vi } from 'vitest' +import { memoryStream } from '../src/stream-durability' +import { + resumeHttpResponse, + resumeServerSentEventsResponse, + toHttpResponse, + toServerSentEventsResponse, +} from '../src/stream-to-response' +import { EventType } from '../src/types' +import { ev } from './test-utils' +import type { StreamDurability } from '../src/stream-durability' +import type { StreamChunk } from '../src/types' + +function fiveChunkStream(): { + stream: AsyncIterable + iterated: () => boolean +} { + let started = false + const stream: AsyncIterable = { + async *[Symbol.asyncIterator]() { + started = true + for (const delta of ['1', '2', '3', '4', '5']) { + yield ev.textContent(delta) + } + }, + } + return { stream, iterated: () => started } +} + +async function readBody(response: Response): Promise { + if (!response.body) throw new Error('Expected a response body') + const reader = response.body.getReader() + const decoder = new TextDecoder() + let body = '' + for (;;) { + const result = await reader.read() + if (result.done) return body + body += decoder.decode(result.value) + } +} + +interface ParsedSseEvent { + id?: string + data: unknown +} + +function parseSseEvents(body: string): Array { + return body + .split('\n\n') + .filter((block) => block.trim().length > 0) + .map((block) => { + const lines = block.split('\n') + const id = lines.find((line) => line.startsWith('id: '))?.slice(4) + const data = lines.find((line) => line.startsWith('data: '))?.slice(6) + if (!data) throw new Error(`Missing SSE data line in ${block}`) + return { ...(id === undefined ? {} : { id }), data: JSON.parse(data) } + }) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function field(event: ParsedSseEvent, name: string): unknown { + if (!isRecord(event.data)) throw new Error('Expected an object SSE payload') + return event.data[name] +} + +function label(chunk: StreamChunk): string { + return chunk.type === EventType.TEXT_MESSAGE_CONTENT + ? chunk.delta + : `[${chunk.type}]` +} + +function fixedOffsetDurability( + offsets: Array, +): StreamDurability { + return { + resumeFrom: () => null, + append: async () => offsets, + close: async () => undefined, + async *read() { + // No replay is needed by these validation tests. + }, + } +} + +describe('toServerSentEventsResponse with durability', () => { + it('appends a fresh run and tags every event with its adapter offset', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-fresh', { + method: 'POST', + }), + ) + const { stream, iterated } = fiveChunkStream() + + const events = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + }), + ), + ) + const eventOffsets = events.map((event) => event.id) + + expect(iterated()).toBe(true) + expect(events).toHaveLength(5) + expect(eventOffsets.every((offset) => offset !== undefined)).toBe(true) + expect(new Set(eventOffsets).size).toBe(5) + + const loggedOffsets: Array = [] + const loggedLabels: Array = [] + for await (const entry of durability.read('-1')) { + loggedOffsets.push(entry.offset) + loggedLabels.push(label(entry.chunk)) + } + expect(loggedOffsets).toEqual(eventOffsets) + expect(loggedLabels).toEqual(['1', '2', '3', '4', '5']) + }) + + it('logs a durability close failure server-side when debug is enabled', async () => { + const closeError = new Error('close boom') + let seq = 0 + const durability: StreamDurability = { + resumeFrom: () => null, + append: async (chunks) => chunks.map(() => `off-${seq++}`), + close: async () => { + throw closeError + }, + async *read() { + // Not exercised by this test. + }, + } + const errorLog = vi.fn() + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: errorLog, + } + const { stream } = fiveChunkStream() + + // The rethrown close failure lands in-band as an error event; the consumer + // drains cleanly. The point is that the cause is recorded server-side, where + // a joiner (who only sees a generic incomplete error) cannot observe it. + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + debug: { logger }, + }), + ) + + expect(errorLog).toHaveBeenCalledWith( + expect.stringContaining('closing durability stream failed'), + expect.objectContaining({ error: closeError }), + ) + }) + + it('replays opaque IDs from the log without iterating the input stream', async () => { + const { stream } = fiveChunkStream() + const produced = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { + adapter: memoryStream( + new Request( + 'https://example.test/api/chat?runId=response-replay', + { + method: 'POST', + }, + ), + ), + }, + }), + ), + ) + const resumeOffset = produced[1]?.id + if (!resumeOffset) throw new Error('Expected a replay offset') + const exploding: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next() { + throw new Error('input stream must not be iterated on resume') + }, + } + }, + } + + const replayed = parseSseEvents( + await readBody( + toServerSentEventsResponse(exploding, { + durability: { + adapter: memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': resumeOffset }, + }), + ), + }, + }), + ), + ) + + expect(replayed.map((event) => event.id)).toEqual( + produced.slice(2).map((event) => event.id), + ) + expect(replayed.map((event) => field(event, 'delta'))).toEqual([ + '3', + '4', + '5', + ]) + }) + + it('batches appends to at most the configured batch size', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-batch', { + method: 'POST', + }), + ) + const appendSpy = vi.spyOn(durability, 'append') + const { stream } = fiveChunkStream() + + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability, batch: 2 }, + }), + ) + + const batchSizes = appendSpy.mock.calls.map(([chunks]) => chunks.length) + expect(batchSizes.every((size) => size <= 2)).toBe(true) + expect(batchSizes.reduce((sum, size) => sum + size, 0)).toBe(5) + }) + + it('rejects a non-positive-integer batch size', () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-bad-batch', { + method: 'POST', + }), + ) + const { stream } = fiveChunkStream() + for (const batch of [0, -1, 1.5, NaN]) { + expect(() => + toServerSentEventsResponse(stream, { + durability: { adapter: durability, batch }, + }), + ).toThrow(/Invalid durability batch size/) + } + }) + + it('rejects duplicate offsets before emitting distinct chunks', async () => { + const { stream } = fiveChunkStream() + const response = toServerSentEventsResponse(stream, { + durability: { + adapter: fixedOffsetDurability(Array.from({ length: 5 }, () => 'same')), + }, + }) + + const events = parseSseEvents(await readBody(response)) + expect(events).toHaveLength(1) + expect(events[0]?.id).toBeUndefined() + expect(field(events[0]!, 'type')).toBe(EventType.RUN_ERROR) + expect(field(events[0]!, 'message')).toMatch(/unique.*offset/i) + }) + + it('rejects SSE offsets containing U+0000', async () => { + const response = toServerSentEventsResponse(textStreamWithOneChunk(), { + durability: { adapter: fixedOffsetDurability(['bad\0offset']) }, + }) + + const events = parseSseEvents(await readBody(response)) + expect(events).toHaveLength(1) + expect(events[0]?.id).toBeUndefined() + expect(field(events[0]!, 'type')).toBe(EventType.RUN_ERROR) + expect(field(events[0]!, 'message')).toMatch(/Invalid durability offset/) + }) + + it('persists a terminal RUN_ERROR before closing when the source throws', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=response-error', { + method: 'POST', + }), + ) + const throwing: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('1') + throw new Error('provider exploded') + }, + } + + const liveEvents = parseSseEvents( + await readBody( + toServerSentEventsResponse(throwing, { + durability: { adapter: durability }, + }), + ), + ) + expect(liveEvents.map((event) => field(event, 'type'))).toContain( + 'RUN_ERROR', + ) + + const logged: Array = [] + for await (const { chunk } of durability.read('-1')) logged.push(chunk) + expect(logged.map((chunk) => chunk.type)).toEqual([ + 'TEXT_MESSAGE_CONTENT', + 'RUN_ERROR', + ]) + expect(logged.at(-1)).toMatchObject({ + message: 'provider exploded', + }) + }) +}) + +/** + * Parse an NDJSON body into the same `{ id?, data }` shape as `parseSseEvents`. + * A durable line is an `{ id, chunk }` envelope; a non-durable line is a bare + * chunk — both are auto-detected, mirroring the client parser. + */ +function parseNdjsonEvents(body: string): Array { + return body + .split('\n') + .filter((line) => line.trim().length > 0) + .map((line) => { + const parsed = JSON.parse(line) as Record + // Mirror the production `isNdjsonEnvelope` discriminator exactly: an + // envelope carries `id` + `chunk` and no top-level `type`. + if ( + 'chunk' in parsed && + 'id' in parsed && + typeof parsed.id === 'string' && + !('type' in parsed) + ) { + return { id: parsed.id, data: parsed.chunk } + } + return { data: parsed } + }) +} + +describe('toHttpResponse with durability', () => { + it('appends a fresh run and envelopes every line with its adapter offset', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=ndjson-fresh', { + method: 'POST', + }), + ) + const { stream, iterated } = fiveChunkStream() + + const events = parseNdjsonEvents( + await readBody( + toHttpResponse(stream, { durability: { adapter: durability } }), + ), + ) + const eventOffsets = events.map((event) => event.id) + + expect(iterated()).toBe(true) + expect(events).toHaveLength(5) + expect(eventOffsets.every((offset) => offset !== undefined)).toBe(true) + expect(new Set(eventOffsets).size).toBe(5) + expect(events.map((event) => field(event, 'delta'))).toEqual([ + '1', + '2', + '3', + '4', + '5', + ]) + + const loggedOffsets: Array = [] + for await (const entry of durability.read('-1')) { + loggedOffsets.push(entry.offset) + } + expect(loggedOffsets).toEqual(eventOffsets) + }) + + it('replays opaque IDs from the log without iterating the input stream', async () => { + const { stream } = fiveChunkStream() + const produced = parseNdjsonEvents( + await readBody( + toHttpResponse(stream, { + durability: { + adapter: memoryStream( + new Request('https://example.test/api/chat?runId=ndjson-replay', { + method: 'POST', + }), + ), + }, + }), + ), + ) + const resumeOffset = produced[1]?.id + if (!resumeOffset) throw new Error('Expected a replay offset') + const exploding: AsyncIterable = { + [Symbol.asyncIterator]() { + return { + next() { + throw new Error('input stream must not be iterated on resume') + }, + } + }, + } + + const replayed = parseNdjsonEvents( + await readBody( + toHttpResponse(exploding, { + durability: { + adapter: memoryStream( + new Request('https://example.test/api/chat', { + method: 'POST', + headers: { 'Last-Event-ID': resumeOffset }, + }), + ), + }, + }), + ), + ) + + expect(replayed.map((event) => event.id)).toEqual( + produced.slice(2).map((event) => event.id), + ) + expect(replayed.map((event) => field(event, 'delta'))).toEqual([ + '3', + '4', + '5', + ]) + }) + + it('persists a terminal RUN_ERROR before closing when the source throws', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=ndjson-error', { + method: 'POST', + }), + ) + const throwing: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('1') + throw new Error('provider exploded') + }, + } + + const liveEvents = parseNdjsonEvents( + await readBody( + toHttpResponse(throwing, { durability: { adapter: durability } }), + ), + ) + expect(liveEvents.map((event) => field(event, 'type'))).toContain( + 'RUN_ERROR', + ) + + const logged: Array = [] + for await (const { chunk } of durability.read('-1')) logged.push(chunk) + expect(logged.map((chunk) => chunk.type)).toEqual([ + 'TEXT_MESSAGE_CONTENT', + 'RUN_ERROR', + ]) + expect(logged.at(-1)).toMatchObject({ message: 'provider exploded' }) + }) + + it('emits bare chunk lines (no envelope) when no durability is configured', async () => { + const { stream } = fiveChunkStream() + const events = parseNdjsonEvents(await readBody(toHttpResponse(stream))) + + expect(events).toHaveLength(5) + expect(events.every((event) => event.id === undefined)).toBe(true) + expect(events.map((event) => field(event, 'delta'))).toEqual([ + '1', + '2', + '3', + '4', + '5', + ]) + }) +}) + +function runFinished(): StreamChunk { + return { + type: EventType.RUN_FINISHED, + threadId: 't', + runId: 'r', + model: 'm', + finishReason: 'stop', + timestamp: 0, + } +} + +describe('durability producer robustness', () => { + it('flushes buffered chunks to the durable log before the terminal on abort', async () => { + const durability = memoryStream( + new Request('https://example.test/api/chat?runId=h1-abort', { + method: 'POST', + }), + ) + const abortController = new AbortController() + // Yields 3 chunks (buffered under the default batch), then aborts before + // ending. produce breaks on the abort check before buffering '4', so '4' is + // dropped but '1'..'3' must still be persisted to the log on the exit path. + const stream: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('1') + yield ev.textContent('2') + yield ev.textContent('3') + abortController.abort() + yield ev.textContent('4') + }, + } + + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability, batch: 32 }, + abortController, + }), + ) + + const logged: Array = [] + for await (const { chunk } of durability.read('-1')) logged.push(chunk) + const deltas = logged.flatMap((chunk) => + chunk.type === EventType.TEXT_MESSAGE_CONTENT ? [chunk.delta] : [], + ) + expect(deltas).toEqual(['1', '2', '3']) + expect(logged.at(-1)?.type).toBe(EventType.RUN_ERROR) + }) + + it('does not emit a second contradictory terminal when close() fails after a forwarded RUN_FINISHED', async () => { + let seq = 0 + const durability: StreamDurability = { + resumeFrom: () => null, + append: async (chunks) => chunks.map(() => `off-${seq++}`), + close: async () => { + throw new Error('close boom') + }, + async *read() { + // Not exercised. + }, + } + const stream: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield ev.textContent('hi') + yield runFinished() + }, + } + + const events = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + }), + ), + ) + const terminals = events.filter((event) => { + const type = field(event, 'type') + return type === EventType.RUN_FINISHED || type === EventType.RUN_ERROR + }) + // Exactly one terminal (the forwarded RUN_FINISHED). The close() failure is + // recorded server-side, not appended as a contradictory RUN_ERROR. + expect(terminals).toHaveLength(1) + expect(field(terminals[0]!, 'type')).toBe(EventType.RUN_FINISHED) + }) + + it('logs (does not rethrow) a producer error thrown after a terminal was forwarded', async () => { + const errorLog = vi.fn() + const logger = { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: errorLog, + } + let seq = 0 + const durability: StreamDurability = { + resumeFrom: () => null, + append: async (chunks) => chunks.map(() => `off-${seq++}`), + close: async () => undefined, + async *read() { + // Not exercised. + }, + } + const stream: AsyncIterable = { + async *[Symbol.asyncIterator]() { + yield runFinished() + throw new Error('provider exploded after terminal') + }, + } + + const events = parseSseEvents( + await readBody( + toServerSentEventsResponse(stream, { + durability: { adapter: durability }, + debug: { logger }, + }), + ), + ) + // Exactly one terminal on the wire (the forwarded RUN_FINISHED) — the + // post-terminal producer error is NOT re-emitted as a contradictory + // RUN_ERROR... + const terminals = events.filter((event) => { + const type = field(event, 'type') + return type === EventType.RUN_FINISHED || type === EventType.RUN_ERROR + }) + expect(terminals).toHaveLength(1) + expect(field(terminals[0]!, 'type')).toBe(EventType.RUN_FINISHED) + // ...but it must still be recorded server-side rather than vanishing. + expect(errorLog).toHaveBeenCalledWith( + expect.stringContaining('after a terminal event was forwarded'), + expect.objectContaining({ error: expect.any(Error) }), + ) + }) + + it('rejects SSE offsets with surrounding whitespace', async () => { + const response = toServerSentEventsResponse(textStreamWithOneChunk(), { + durability: { adapter: fixedOffsetDurability([' padded ']) }, + }) + const events = parseSseEvents(await readBody(response)) + expect(events).toHaveLength(1) + expect(field(events[0]!, 'type')).toBe(EventType.RUN_ERROR) + expect(field(events[0]!, 'message')).toMatch(/Invalid durability offset/) + }) + + it('toHttpResponse defaults Content-Type to application/x-ndjson, overridable by the caller', async () => { + const res = toHttpResponse(fiveChunkStream().stream) + expect(res.headers.get('Content-Type')).toBe('application/x-ndjson') + expect(res.headers.get('Cache-Control')).toBe('no-cache') + + const overridden = toHttpResponse(fiveChunkStream().stream, { + headers: { 'Content-Type': 'application/json' }, + }) + expect(overridden.headers.get('Content-Type')).toBe('application/json') + }) +}) + +function textStreamWithOneChunk(): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + yield ev.textContent('one') + }, + } +} + +// Produce a fresh run into the process-global memory log under `runId`, so a +// later resume/join can replay it. +async function seedRun(runId: string): Promise { + const producer = memoryStream( + new Request(`https://example.test/api/chat?runId=${runId}`, { + method: 'POST', + }), + ) + await readBody( + toServerSentEventsResponse(fiveChunkStream().stream, { + durability: { adapter: producer }, + }), + ) +} + +describe('resume response helpers', () => { + it('resumeServerSentEventsResponse replays a run from its log without a producer', async () => { + await seedRun('resume-sse') + const join = memoryStream( + new Request('https://example.test/api/chat?runId=resume-sse&offset=-1'), + ) + + const events = parseSseEvents( + await readBody(resumeServerSentEventsResponse({ adapter: join })), + ) + + expect(events.map((event) => field(event, 'delta'))).toEqual([ + '1', + '2', + '3', + '4', + '5', + ]) + expect(events.every((event) => event.id !== undefined)).toBe(true) + }) + + it('resumeHttpResponse replays a run over NDJSON', async () => { + await seedRun('resume-ndjson') + const join = memoryStream( + new Request( + 'https://example.test/api/chat?runId=resume-ndjson&offset=-1', + ), + ) + + const events = parseNdjsonEvents( + await readBody(resumeHttpResponse({ adapter: join })), + ) + + expect(events.map((event) => field(event, 'delta'))).toEqual([ + '1', + '2', + '3', + '4', + '5', + ]) + }) + + it('returns 400 when the request carries no resume offset', async () => { + const noOffset = memoryStream( + new Request('https://example.test/api/chat?runId=no-offset'), + ) + const sse = resumeServerSentEventsResponse({ adapter: noOffset }) + const ndjson = resumeHttpResponse({ + adapter: memoryStream( + new Request('https://example.test/api/chat?runId=no-offset-2'), + ), + }) + + expect(sse.status).toBe(400) + expect(ndjson.status).toBe(400) + expect(await sse.text()).toMatch(/No resume offset/) + }) +}) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5d92af110..adbeee227 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1658,6 +1658,15 @@ importers: specifier: ^2.11.10 version: 2.11.10(solid-js@1.9.10)(vite@7.3.3(@types/node@24.10.3)(jiti@2.6.1)(less@4.6.6)(lightningcss@1.30.2)(sass@1.101.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)) + packages/ai-durable-stream: + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.0.14 + version: 4.0.14(vitest@4.1.4) + packages/ai-elevenlabs: dependencies: '@elevenlabs/client': diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 14ffbf330..8abc6f917 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -45,6 +45,7 @@ import { Route as ApiMcpAppsChatRouteImport } from './routes/api.mcp-apps-chat' import { Route as ApiMcpAppsCallRouteImport } from './routes/api.mcp-apps-call' import { Route as ApiLazyToolsWireRouteImport } from './routes/api.lazy-tools-wire' import { Route as ApiImageRouteImport } from './routes/api.image' +import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiChatRouteImport } from './routes/api.chat' import { Route as ApiAudioRouteImport } from './routes/api.audio' import { Route as ApiArktypeToolWireRouteImport } from './routes/api.arktype-tool-wire' @@ -242,6 +243,11 @@ const ApiImageRoute = ApiImageRouteImport.update({ path: '/api/image', getParentRoute: () => rootRouteImport, } as any) +const ApiDurableDeliveryRoute = ApiDurableDeliveryRouteImport.update({ + id: '/api/durable-delivery', + path: '/api/durable-delivery', + getParentRoute: () => rootRouteImport, +} as any) const ApiChatRoute = ApiChatRouteImport.update({ id: '/api/chat', path: '/api/chat', @@ -323,6 +329,7 @@ export interface FileRoutesByFullPath { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -373,6 +380,7 @@ export interface FileRoutesByTo { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -424,6 +432,7 @@ export interface FileRoutesById { '/api/arktype-tool-wire': typeof ApiArktypeToolWireRoute '/api/audio': typeof ApiAudioRouteWithChildren '/api/chat': typeof ApiChatRoute + '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/image': typeof ApiImageRouteWithChildren '/api/lazy-tools-wire': typeof ApiLazyToolsWireRoute '/api/mcp-apps-call': typeof ApiMcpAppsCallRoute @@ -476,6 +485,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -526,6 +536,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -576,6 +587,7 @@ export interface FileRouteTypes { | '/api/arktype-tool-wire' | '/api/audio' | '/api/chat' + | '/api/durable-delivery' | '/api/image' | '/api/lazy-tools-wire' | '/api/mcp-apps-call' @@ -627,6 +639,7 @@ export interface RootRouteChildren { ApiArktypeToolWireRoute: typeof ApiArktypeToolWireRoute ApiAudioRoute: typeof ApiAudioRouteWithChildren ApiChatRoute: typeof ApiChatRoute + ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiImageRoute: typeof ApiImageRouteWithChildren ApiLazyToolsWireRoute: typeof ApiLazyToolsWireRoute ApiMcpAppsCallRoute: typeof ApiMcpAppsCallRoute @@ -908,6 +921,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiImageRouteImport parentRoute: typeof rootRouteImport } + '/api/durable-delivery': { + id: '/api/durable-delivery' + path: '/api/durable-delivery' + fullPath: '/api/durable-delivery' + preLoaderRoute: typeof ApiDurableDeliveryRouteImport + parentRoute: typeof rootRouteImport + } '/api/chat': { id: '/api/chat' path: '/api/chat' @@ -1072,6 +1092,7 @@ const rootRouteChildren: RootRouteChildren = { ApiArktypeToolWireRoute: ApiArktypeToolWireRoute, ApiAudioRoute: ApiAudioRouteWithChildren, ApiChatRoute: ApiChatRoute, + ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiImageRoute: ApiImageRouteWithChildren, ApiLazyToolsWireRoute: ApiLazyToolsWireRoute, ApiMcpAppsCallRoute: ApiMcpAppsCallRoute, diff --git a/testing/e2e/src/routes/api.durable-delivery.ts b/testing/e2e/src/routes/api.durable-delivery.ts new file mode 100644 index 000000000..fb32f9519 --- /dev/null +++ b/testing/e2e/src/routes/api.durable-delivery.ts @@ -0,0 +1,123 @@ +import { createFileRoute } from '@tanstack/react-router' +import { + memoryStream, + resumeHttpResponse, + resumeServerSentEventsResponse, + toHttpResponse, + toServerSentEventsResponse, +} from '@tanstack/ai' +import type { StreamChunk } from '@tanstack/ai' + +/** + * A provider-free delivery-durability harness route. It streams a FIXED + * sequence of AG-UI events through the transport helper's `durability` sink + * (`memoryStream`), so the delivery e2e can assert disconnect→reconnect→ordered + * resume and second-tab join deterministically, with no LLM in the loop. + * + * - `POST` with no offset → fresh run: produce + append the fixed sequence, + * tagging each event with an opaque adapter-owned offset. + * - `POST` with `Last-Event-ID` → reconnect: replay strictly after the offset + * from the log (the fixed sequence is never re-produced). + * - `GET ?offset=-1&runId=…` → second-tab join: replay from the start. + * + * `?transport=ndjson` switches the wire encoding from SSE to newline-delimited + * JSON (each durable line is an `{ id, chunk }` envelope). The durability layer + * — logging, offsets, resume, terminalization — is identical for both. + */ +// Emits bare TEXT_MESSAGE_CONTENT chunks without TEXT_MESSAGE_START/END +// bracketing: this harness deliberately exercises raw chunk delivery + resume, +// not UIMessage reassembly. The durability layer terminalizes on RUN_FINISHED +// (emitted below), which is all resume/join needs. +function fixedRun(threadId: string, runId: string): AsyncIterable { + return (async function* () { + yield { + type: 'RUN_STARTED', + threadId, + runId, + timestamp: Date.now(), + } as StreamChunk + for (let i = 1; i <= 5; i++) { + yield { + type: 'TEXT_MESSAGE_CONTENT', + messageId: 'm', + model: 'fixed', + delta: String(i), + content: String(i), + timestamp: Date.now(), + } as StreamChunk + } + yield { + type: 'RUN_FINISHED', + threadId, + runId, + model: 'fixed', + finishReason: 'stop', + timestamp: Date.now(), + } as StreamChunk + })() +} + +function durableRun(request: Request) { + const url = new URL(request.url) + const runId = url.searchParams.get('runId') ?? crypto.randomUUID() + url.searchParams.set('runId', runId) + // On a reconnect (Last-Event-ID present), memoryStream resolves the real run + // from the offset itself and ignores this URL runId — so a freshly minted + // random id here does NOT name the run being served and must not be + // advertised via X-Run-Id. + const isResume = request.headers.get('Last-Event-ID') !== null + return { + durability: memoryStream(new Request(url, request)), + runId, + advertiseRunId: isResume ? undefined : runId, + } +} + +function withRunId(response: Response, runId: string | undefined): Response { + if (runId !== undefined) response.headers.set('X-Run-Id', runId) + return response +} + +function isNdjson(request: Request): boolean { + try { + return new URL(request.url).searchParams.get('transport') === 'ndjson' + } catch { + return false + } +} + +/** Build the durable response in the requested wire encoding (SSE or NDJSON). */ +function durableResponse( + request: Request, + runId: string, + durability: ReturnType, + batch?: number, +): Response { + const stream = fixedRun('thread-durable', runId) + const durabilityOption = { adapter: durability, ...(batch ? { batch } : {}) } + return isNdjson(request) + ? toHttpResponse(stream, { durability: durabilityOption }) + : toServerSentEventsResponse(stream, { durability: durabilityOption }) +} + +export const Route = createFileRoute('/api/durable-delivery')({ + server: { + handlers: { + POST: async ({ request }) => { + const { durability, runId, advertiseRunId } = durableRun(request) + return withRunId( + durableResponse(request, runId, durability, 2), + advertiseRunId, + ) + }, + GET: async ({ request }) => { + // A join replays from the log, so no producer stream is built here. + const { durability, advertiseRunId } = durableRun(request) + const response = isNdjson(request) + ? resumeHttpResponse({ adapter: durability }) + : resumeServerSentEventsResponse({ adapter: durability }) + return withRunId(response, advertiseRunId) + }, + }, + }, +}) diff --git a/testing/e2e/tests/delivery-durability.spec.ts b/testing/e2e/tests/delivery-durability.spec.ts new file mode 100644 index 000000000..9f179b685 --- /dev/null +++ b/testing/e2e/tests/delivery-durability.spec.ts @@ -0,0 +1,192 @@ +import { expect, test } from '@playwright/test' + +/** + * Delivery durability (transport layer) over real HTTP. + * + * These exercise the server-side `durability` sink end-to-end: a fresh run is + * appended to the log and each SSE event is tagged with an opaque adapter-owned + * offset; a reconnect (native `Last-Event-ID`) or a `?offset` join replays from + * the log without re-producing. + * + * We test only the **producer-alive** path deterministically (the log is fully + * written by the first request). The **producer-dead** case — where the client + * disconnects mid-run and the producer process is torn down with it — cannot be + * resumed by the log alone; it needs the producer to outlive the socket + * (`waitUntil` / durable object / queue), which is a per-platform deployment + * concern, not something this transport can guarantee. The client-side + * auto-reconnect (Last-Event-ID resend + de-dupe) is covered by unit tests in + * `@tanstack/ai-client`. + * + * Exempt from the aimock policy: the `api.durable-delivery` harness route + * streams a fixed AG-UI sequence and never reaches an LLM provider's HTTP layer, + * so there is nothing to mock. + */ + +interface SseEvent { + id?: string + data: unknown +} + +function parseSse(body: string): Array { + return body + .split('\n\n') + .filter((block) => block.trim().length > 0) + .map((block) => { + const lines = block.split('\n') + const idLine = lines.find((l) => l.startsWith('id:')) + const dataLine = lines.find((l) => l.startsWith('data:')) + const rawData = dataLine + ? dataLine.slice(dataLine.indexOf(':') + 1).trim() + : '' + return { + ...(idLine ? { id: idLine.slice(idLine.indexOf(':') + 1).trim() } : {}), + data: JSON.parse(rawData), + } + }) +} + +function eventType(e: SseEvent): string { + return (e.data as { type: string }).type +} + +function contentDeltas(events: Array): Array { + return events + .filter((e) => eventType(e) === 'TEXT_MESSAGE_CONTENT') + .map((e) => (e.data as { delta: string }).delta) +} + +test.describe('delivery durability', () => { + test('disconnect → reconnect resumes the ordered stream exactly once', async ({ + request, + }) => { + // Fresh run: produce + persist the full ordered sequence. + const produce = await request.post('/api/durable-delivery', { data: {} }) + expect(produce.ok()).toBeTruthy() + const produced = parseSse(await produce.text()) + + // Sequence: RUN_STARTED(1), content 1..5 (seq 2..6), RUN_FINISHED(7). + expect(contentDeltas(produced)).toEqual(['1', '2', '3', '4', '5']) + expect(produced.every((e) => e.id !== undefined)).toBeTruthy() + + // Client received through seq 3 (RUN_STARTED, content "1", content "2") + // then dropped. Reconnect from that offset via native Last-Event-ID. + const resumeFrom = produced[2]!.id! + const reconnect = await request.post('/api/durable-delivery', { + headers: { 'Last-Event-ID': resumeFrom }, + data: {}, + }) + expect(reconnect.ok()).toBeTruthy() + const resumed = parseSse(await reconnect.text()) + + // Exactly the tail, in order, exactly once — content "3","4","5" then RUN_FINISHED. + expect(contentDeltas(resumed)).toEqual(['3', '4', '5']) + expect(eventType(resumed[resumed.length - 1]!)).toBe('RUN_FINISHED') + // Adapter offsets are opaque; no replayed event may reuse the acknowledged + // resume offset. + expect(resumed.map((event) => event.id)).not.toContain(resumeFrom) + }) + + test('a second tab joins an existing run from the start', async ({ + request, + }) => { + const produce = await request.post('/api/durable-delivery', { data: {} }) + const produced = parseSse(await produce.text()) + const runId = produce.headers()['x-run-id'] + if (!runId) throw new Error('Missing X-Run-Id response header') + + // Join from the start with ?offset=-1 (a fresh tab attaching to the run). + const join = await request.get( + `/api/durable-delivery?offset=-1&runId=${encodeURIComponent(runId)}`, + ) + expect(join.ok()).toBeTruthy() + const joined = parseSse(await join.text()) + + expect(contentDeltas(joined)).toEqual(['1', '2', '3', '4', '5']) + expect(eventType(joined[0]!)).toBe('RUN_STARTED') + expect(eventType(joined[joined.length - 1]!)).toBe('RUN_FINISHED') + }) +}) + +/** + * The same delivery-durability guarantees over the NDJSON wire encoding + * (`?transport=ndjson`). NDJSON has no native event-id, so each durable line is + * an `{ id, chunk }` envelope carrying the opaque offset. The durability layer + * is shared with SSE — only the parsing differs. + */ +function parseNdjson(body: string): Array { + return body + .split('\n') + .filter((line) => line.trim().length > 0) + .map((line) => { + const parsed = JSON.parse(line) as Record + // Mirror the production `isNdjsonEnvelope` discriminator exactly: an + // envelope carries `id` + `chunk` and no top-level `type`. + if ( + 'chunk' in parsed && + 'id' in parsed && + typeof parsed.id === 'string' && + !('type' in parsed) + ) { + return { id: parsed.id, data: parsed.chunk } + } + return { data: parsed } + }) +} + +test.describe('delivery durability (ndjson)', () => { + test('disconnect → reconnect resumes the ordered stream exactly once', async ({ + request, + }) => { + const produce = await request.post( + '/api/durable-delivery?transport=ndjson', + { + data: {}, + }, + ) + expect(produce.ok()).toBeTruthy() + const produced = parseNdjson(await produce.text()) + + expect(contentDeltas(produced)).toEqual(['1', '2', '3', '4', '5']) + expect(produced.every((e) => e.id !== undefined)).toBeTruthy() + + const resumeFrom = produced[2]!.id! + const reconnect = await request.post( + '/api/durable-delivery?transport=ndjson', + { + headers: { 'Last-Event-ID': resumeFrom }, + data: {}, + }, + ) + expect(reconnect.ok()).toBeTruthy() + const resumed = parseNdjson(await reconnect.text()) + + expect(contentDeltas(resumed)).toEqual(['3', '4', '5']) + expect(eventType(resumed[resumed.length - 1]!)).toBe('RUN_FINISHED') + expect(resumed.map((event) => event.id)).not.toContain(resumeFrom) + }) + + test('a second tab joins an existing run from the start', async ({ + request, + }) => { + const produce = await request.post( + '/api/durable-delivery?transport=ndjson', + { + data: {}, + }, + ) + const produced = parseNdjson(await produce.text()) + const runId = produce.headers()['x-run-id'] + if (!runId) throw new Error('Missing X-Run-Id response header') + expect(produced.length).toBeGreaterThan(0) + + const join = await request.get( + `/api/durable-delivery?transport=ndjson&offset=-1&runId=${encodeURIComponent(runId)}`, + ) + expect(join.ok()).toBeTruthy() + const joined = parseNdjson(await join.text()) + + expect(contentDeltas(joined)).toEqual(['1', '2', '3', '4', '5']) + expect(eventType(joined[0]!)).toBe('RUN_STARTED') + expect(eventType(joined[joined.length - 1]!)).toBe('RUN_FINISHED') + }) +})